pool/rdma-core
SHA256
2
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Accepting request 680356 from home:NMoreyChaisemartin:branches:science:HPC

Browse Source 
- Add patches to enable Broadcom's 57500 RoCE adapter (bsc#1125651):
  - Update-kernel-headers.patch
  - bnxt_re-lib-Enable-Broadcom-s-57500-RoCE-adapter.patch

OBS-URL: https://build.opensuse.org/request/show/680356
OBS-URL: https://build.opensuse.org/package/show/science:HPC/rdma-core?expand=0&rev=102
This commit is contained in:
Nicolas Morey-Chaisemartin 2019-03-01 07:19:27 +00:00 committed by Git OBS Bridge
parent e01f00321a
commit 1f23dc1c70
4 changed files with 492 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

202
Update-kernel-headers.patch Normal file
View File

@ -0,0 +1,202 @@
commit a1b591edcd84f52fc61ee945c0454f1933b1aeab
Author: Yishai Hadas <yishaih@mellanox.com>
Date: Sun Feb 10 10:43:12 2019 +0200
Update kernel headers
To commit 2c1619edef61 ("IB/cma: Define option to set ack timeout and
pack tos_set")
Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
diff --git kernel-headers/rdma/bnxt_re-abi.h kernel-headers/rdma/bnxt_re-abi.h
index a7a6111e50c7..dc52e3cf574c 100644
--- kernel-headers/rdma/bnxt_re-abi.h
+++ kernel-headers/rdma/bnxt_re-abi.h
@@ -44,6 +44,14 @@
#define BNXT_RE_ABI_VERSION 1
+#define BNXT_RE_CHIP_ID0_CHIP_NUM_SFT 0x00
+#define BNXT_RE_CHIP_ID0_CHIP_REV_SFT 0x10
+#define BNXT_RE_CHIP_ID0_CHIP_MET_SFT 0x18
+
+enum {
+ BNXT_RE_UCNTX_CMASK_HAVE_CCTX = 0x1ULL
+};
+
struct bnxt_re_uctx_resp {
__u32 dev_id;
__u32 max_qp;
@@ -51,6 +59,9 @@ struct bnxt_re_uctx_resp {
__u32 cqe_sz;
__u32 max_cqd;
__u32 rsvd;
+ __aligned_u64 comp_mask;
+ __u32 chip_id0;
+ __u32 chip_id1;
};
/*
diff --git kernel-headers/rdma/ib_user_verbs.h kernel-headers/rdma/ib_user_verbs.h
index 480d9a60b68e..0474c7400268 100644
--- kernel-headers/rdma/ib_user_verbs.h
+++ kernel-headers/rdma/ib_user_verbs.h
@@ -270,6 +270,8 @@ struct ib_uverbs_ex_query_device_resp {
struct ib_uverbs_tm_caps tm_caps;
struct ib_uverbs_cq_moderation_caps cq_moderation_caps;
__aligned_u64 max_dm_size;
+ __u32 xrc_odp_caps;
+ __u32 reserved;
};
struct ib_uverbs_query_port {
diff --git kernel-headers/rdma/rdma_netlink.h kernel-headers/rdma/rdma_netlink.h
index 2e18b77a817f..3a9e681e4257 100644
--- kernel-headers/rdma/rdma_netlink.h
+++ kernel-headers/rdma/rdma_netlink.h
@@ -5,8 +5,7 @@
#include <linux/types.h>
enum {
- RDMA_NL_RDMA_CM = 1,
- RDMA_NL_IWCM,
+ RDMA_NL_IWCM = 2,
RDMA_NL_RSVD,
RDMA_NL_LS, /* RDMA Local Services */
RDMA_NL_NLDEV, /* RDMA device interface */
@@ -14,8 +13,7 @@ enum {
};
enum {
- RDMA_NL_GROUP_CM = 1,
- RDMA_NL_GROUP_IWPM,
+ RDMA_NL_GROUP_IWPM = 2,
RDMA_NL_GROUP_LS,
RDMA_NL_NUM_GROUPS
};
@@ -24,15 +22,17 @@ enum {
#define RDMA_NL_GET_OP(type) (type & ((1 << 10) - 1))
#define RDMA_NL_GET_TYPE(client, op) ((client << 10) + op)
-enum {
- RDMA_NL_RDMA_CM_ID_STATS = 0,
- RDMA_NL_RDMA_CM_NUM_OPS
-};
+/* The minimum version that the iwpm kernel supports */
+#define IWPM_UABI_VERSION_MIN 3
+/* The latest version that the iwpm kernel supports */
+#define IWPM_UABI_VERSION 4
+
+/* iwarp port mapper message flags */
enum {
- RDMA_NL_RDMA_CM_ATTR_SRC_ADDR = 1,
- RDMA_NL_RDMA_CM_ATTR_DST_ADDR,
- RDMA_NL_RDMA_CM_NUM_ATTR,
+
+ /* Do not map the port for this IWPM request */
+ IWPM_FLAGS_NO_PORT_MAP = (1 << 0),
};
/* iwarp port mapper op-codes */
@@ -45,6 +45,7 @@ enum {
RDMA_NL_IWPM_HANDLE_ERR,
RDMA_NL_IWPM_MAPINFO,
RDMA_NL_IWPM_MAPINFO_NUM,
+ RDMA_NL_IWPM_HELLO,
RDMA_NL_IWPM_NUM_OPS
};
@@ -83,20 +84,38 @@ enum {
IWPM_NLA_MANAGE_MAPPING_UNSPEC = 0,
IWPM_NLA_MANAGE_MAPPING_SEQ,
IWPM_NLA_MANAGE_ADDR,
- IWPM_NLA_MANAGE_MAPPED_LOC_ADDR,
+ IWPM_NLA_MANAGE_FLAGS,
+ IWPM_NLA_MANAGE_MAPPING_MAX
+};
+
+enum {
+ IWPM_NLA_RMANAGE_MAPPING_UNSPEC = 0,
+ IWPM_NLA_RMANAGE_MAPPING_SEQ,
+ IWPM_NLA_RMANAGE_ADDR,
+ IWPM_NLA_RMANAGE_MAPPED_LOC_ADDR,
+ /* The following maintains bisectability of rdma-core */
+ IWPM_NLA_MANAGE_MAPPED_LOC_ADDR = IWPM_NLA_RMANAGE_MAPPED_LOC_ADDR,
IWPM_NLA_RMANAGE_MAPPING_ERR,
IWPM_NLA_RMANAGE_MAPPING_MAX
};
-#define IWPM_NLA_MANAGE_MAPPING_MAX 3
-#define IWPM_NLA_QUERY_MAPPING_MAX 4
#define IWPM_NLA_MAPINFO_SEND_MAX 3
+#define IWPM_NLA_REMOVE_MAPPING_MAX 3
enum {
IWPM_NLA_QUERY_MAPPING_UNSPEC = 0,
IWPM_NLA_QUERY_MAPPING_SEQ,
IWPM_NLA_QUERY_LOCAL_ADDR,
IWPM_NLA_QUERY_REMOTE_ADDR,
+ IWPM_NLA_QUERY_FLAGS,
+ IWPM_NLA_QUERY_MAPPING_MAX,
+};
+
+enum {
+ IWPM_NLA_RQUERY_MAPPING_UNSPEC = 0,
+ IWPM_NLA_RQUERY_MAPPING_SEQ,
+ IWPM_NLA_RQUERY_LOCAL_ADDR,
+ IWPM_NLA_RQUERY_REMOTE_ADDR,
IWPM_NLA_RQUERY_MAPPED_LOC_ADDR,
IWPM_NLA_RQUERY_MAPPED_REM_ADDR,
IWPM_NLA_RQUERY_MAPPING_ERR,
@@ -114,6 +133,7 @@ enum {
IWPM_NLA_MAPINFO_UNSPEC = 0,
IWPM_NLA_MAPINFO_LOCAL_ADDR,
IWPM_NLA_MAPINFO_MAPPED_ADDR,
+ IWPM_NLA_MAPINFO_FLAGS,
IWPM_NLA_MAPINFO_MAX
};
@@ -132,6 +152,12 @@ enum {
IWPM_NLA_ERR_MAX
};
+enum {
+ IWPM_NLA_HELLO_UNSPEC = 0,
+ IWPM_NLA_HELLO_ABI_VERSION,
+ IWPM_NLA_HELLO_MAX
+};
+
/*
* Local service operations:
* RESOLVE - The client requests the local service to resolve a path.
diff --git kernel-headers/rdma/rdma_user_cm.h kernel-headers/rdma/rdma_user_cm.h
index 0d1e78ebad05..e42940a215a3 100644
--- kernel-headers/rdma/rdma_user_cm.h
+++ kernel-headers/rdma/rdma_user_cm.h
@@ -300,6 +300,10 @@ enum {
RDMA_OPTION_ID_TOS = 0,
RDMA_OPTION_ID_REUSEADDR = 1,
RDMA_OPTION_ID_AFONLY = 2,
+ RDMA_OPTION_ID_ACK_TIMEOUT = 3
+};
+
+enum {
RDMA_OPTION_IB_PATH = 1
};
diff --git kernel-headers/rdma/rdma_user_rxe.h kernel-headers/rdma/rdma_user_rxe.h
index 44ef6a3b7afc..aae2e696bb38 100644
--- kernel-headers/rdma/rdma_user_rxe.h
+++ kernel-headers/rdma/rdma_user_rxe.h
@@ -58,8 +58,7 @@ struct rxe_global_route {
struct rxe_av {
__u8 port_num;
__u8 network_type;
- __u16 reserved1;
- __u32 reserved2;
+ __u8 dmac[6];
struct rxe_global_route grh;
union {
struct sockaddr_in _sockaddr_in;

278
bnxt_re-lib-Enable-Broadcom-s-57500-RoCE-adapter.patch Normal file
View File

@ -0,0 +1,278 @@
commit 885a8d3a3e2aa4e68f56c4c7d17abde437e0328b
Author: Devesh Sharma <devesh.sharma@broadcom.com>
Date: Sun Jan 13 14:36:13 2019 -0500
bnxt_re/lib: Enable Broadcom's 57500 RoCE adapter
This is to add Broadcom's 57500 series of adapters support
to RoCE from libbnxt_re. Listing below the significant changes
done as part of the patch.
- Added the pci-id of the basic gen-p5 chip.
- Adjust psn search memory allocation to suite new search
psn structure.
- Added chip context structure to select the appropriate
execution flow in data-path and control path.
- Fill psn search area as per new or older chip execution
flow.
- removed duplicate declaration of BNXT_RE_ABI_VERSION macro
Signed-off-by: Devesh Sharma <devesh.sharma@broadcom.com>
diff --git providers/bnxt_re/bnxt_re-abi.h providers/bnxt_re/bnxt_re-abi.h
index 65d048d3b00d..c6998e85b174 100644
--- providers/bnxt_re/bnxt_re-abi.h
+++ providers/bnxt_re/bnxt_re-abi.h
@@ -43,8 +43,6 @@
#include <rdma/bnxt_re-abi.h>
#include <kernel-abi/bnxt_re-abi.h>
-#define BNXT_RE_ABI_VERSION 1
-
#define BNXT_RE_FULL_FLAG_DELTA 0x80
DECLARE_DRV_CMD(ubnxt_re_pd, IB_USER_VERBS_CMD_ALLOC_PD,
@@ -246,6 +244,14 @@ struct bnxt_re_psns {
__le32 flg_npsn;
};
+struct bnxt_re_psns_ext {
+ __u32 opc_spsn;
+ __u32 flg_npsn;
+ __u16 st_slot_idx;
+ __u16 rsvd0;
+ __u32 rsvd1;
+};
+
struct bnxt_re_sge {
__le64 pa;
__le32 lkey;
diff --git providers/bnxt_re/main.c providers/bnxt_re/main.c
index 1cd4d8804128..d171748e4d1d 100644
--- providers/bnxt_re/main.c
+++ providers/bnxt_re/main.c
@@ -74,6 +74,7 @@ static const struct verbs_match_ent cna_table[] = {
CNA(BROADCOM, 0x16EF), /* BCM57416 NPAR */
CNA(BROADCOM, 0x16F0), /* BCM58730 */
CNA(BROADCOM, 0x16F1), /* BCM57452 */
+ CNA(BROADCOM, 0x1750), /* BCM57500 */
CNA(BROADCOM, 0xD800), /* BCM880xx VF */
CNA(BROADCOM, 0xD802), /* BCM58802 */
CNA(BROADCOM, 0xD804), /* BCM8804 SR */
@@ -108,6 +109,11 @@ static const struct verbs_context_ops bnxt_re_cntx_ops = {
.destroy_ah = bnxt_re_destroy_ah
};
+bool bnxt_re_is_chip_gen_p5(struct bnxt_re_chip_ctx *cctx)
+{
+ return cctx->chip_num == CHIP_NUM_57500;
+}
+
/* Context Init functions */
static struct verbs_context *bnxt_re_alloc_context(struct ibv_device *vdev,
int cmd_fd,
@@ -133,6 +139,14 @@ static struct verbs_context *bnxt_re_alloc_context(struct ibv_device *vdev,
dev->pg_size = resp.pg_size;
dev->cqe_size = resp.cqe_sz;
dev->max_cq_depth = resp.max_cqd;
+ if (resp.comp_mask & BNXT_RE_UCNTX_CMASK_HAVE_CCTX) {
+ cntx->cctx.chip_num = resp.chip_id0 & 0xFFFF;
+ cntx->cctx.chip_rev = (resp.chip_id0 >>
+ BNXT_RE_CHIP_ID0_CHIP_REV_SFT) & 0xFF;
+ cntx->cctx.chip_metal = (resp.chip_id0 >>
+ BNXT_RE_CHIP_ID0_CHIP_MET_SFT) &
+ 0xFF;
+ }
pthread_spin_init(&cntx->fqlock, PTHREAD_PROCESS_PRIVATE);
/* mmap shared page. */
cntx->shpg = mmap(NULL, dev->pg_size, PROT_READ | PROT_WRITE,
diff --git providers/bnxt_re/main.h providers/bnxt_re/main.h
index 0b5c749fba6a..be5734960475 100644
--- providers/bnxt_re/main.h
+++ providers/bnxt_re/main.h
@@ -54,7 +54,14 @@
#define DEV "bnxt_re : "
-#define BNXT_RE_UD_QP_HW_STALL 0x400000
+#define BNXT_RE_UD_QP_HW_STALL 0x400000
+
+#define CHIP_NUM_57500 0x1750
+struct bnxt_re_chip_ctx {
+ __u16 chip_num;
+ __u8 chip_rev;
+ __u8 chip_metal;
+};
struct bnxt_re_dpi {
__u32 dpindx;
@@ -81,6 +88,7 @@ struct bnxt_re_cq {
};
struct bnxt_re_wrid {
+ struct bnxt_re_psns_ext *psns_ext;
struct bnxt_re_psns *psns;
uint64_t wrid;
uint32_t bytes;
@@ -111,6 +119,7 @@ struct bnxt_re_srq {
struct bnxt_re_qp {
struct ibv_qp ibvqp;
+ struct bnxt_re_chip_ctx *cctx;
struct bnxt_re_queue *sqq;
struct bnxt_re_wrid *swrid;
struct bnxt_re_queue *rqq;
@@ -155,6 +164,7 @@ struct bnxt_re_context {
struct verbs_context ibvctx;
uint32_t dev_id;
uint32_t max_qp;
+ struct bnxt_re_chip_ctx cctx;
uint32_t max_srq;
struct bnxt_re_dpi udpi;
void *shpg;
@@ -162,6 +172,9 @@ struct bnxt_re_context {
pthread_spinlock_t fqlock;
};
+/* Chip context related functions */
+bool bnxt_re_is_chip_gen_p5(struct bnxt_re_chip_ctx *cctx);
+
/* DB ring functions used internally*/
void bnxt_re_ring_rq_db(struct bnxt_re_qp *qp);
void bnxt_re_ring_sq_db(struct bnxt_re_qp *qp);
diff --git providers/bnxt_re/verbs.c providers/bnxt_re/verbs.c
index 7786d24718d3..bec382b36838 100644
--- providers/bnxt_re/verbs.c
+++ providers/bnxt_re/verbs.c
@@ -844,9 +844,11 @@ static void bnxt_re_free_queues(struct bnxt_re_qp *qp)
static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp,
struct ibv_qp_init_attr *attr,
uint32_t pg_size) {
+ struct bnxt_re_psns_ext *psns_ext;
struct bnxt_re_queue *que;
struct bnxt_re_psns *psns;
uint32_t psn_depth;
+ uint32_t psn_size;
int ret, indx;
que = qp->sqq;
@@ -857,11 +859,12 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp,
que->diff = que->depth - attr->cap.max_send_wr;
/* psn_depth extra entries of size que->stride */
- psn_depth = (que->depth * sizeof(struct bnxt_re_psns)) /
- que->stride;
- if ((que->depth * sizeof(struct bnxt_re_psns)) % que->stride)
+ psn_size = bnxt_re_is_chip_gen_p5(qp->cctx) ?
+ sizeof(struct bnxt_re_psns_ext) :
+ sizeof(struct bnxt_re_psns);
+ psn_depth = (que->depth * psn_size) / que->stride;
+ if ((que->depth * psn_size) % que->stride)
psn_depth++;
-
que->depth += psn_depth;
/* PSN-search memory is allocated without checking for
* QP-Type. Kenrel driver do not map this memory if it
@@ -875,6 +878,7 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp,
que->depth -= psn_depth;
/* start of spsn space sizeof(struct bnxt_re_psns) each. */
psns = (que->va + que->stride * que->depth);
+ psns_ext = (struct bnxt_re_psns_ext *)psns;
pthread_spin_init(&que->qlock, PTHREAD_PROCESS_PRIVATE);
qp->swrid = calloc(que->depth, sizeof(struct bnxt_re_wrid));
if (!qp->swrid) {
@@ -884,6 +888,13 @@ static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp,
for (indx = 0 ; indx < que->depth; indx++, psns++)
qp->swrid[indx].psns = psns;
+ if (bnxt_re_is_chip_gen_p5(qp->cctx)) {
+ for (indx = 0 ; indx < que->depth; indx++, psns_ext++) {
+ qp->swrid[indx].psns_ext = psns_ext;
+ qp->swrid[indx].psns = (struct bnxt_re_psns *)psns_ext;
+ }
+ }
+
qp->cap.max_swr = que->depth;
if (qp->rqq) {
@@ -931,6 +942,7 @@ struct ibv_qp *bnxt_re_create_qp(struct ibv_pd *ibvpd,
if (bnxt_re_alloc_queue_ptr(qp, attr))
goto fail;
/* alloc queues */
+ qp->cctx = &cntx->cctx;
if (bnxt_re_alloc_queues(qp, attr, dev->pg_size))
goto failq;
/* Fill ibv_cmd */
@@ -1094,26 +1106,36 @@ static int bnxt_re_build_sge(struct bnxt_re_sge *sge, struct ibv_sge *sg_list,
return length;
}
-static void bnxt_re_fill_psns(struct bnxt_re_qp *qp, struct bnxt_re_psns *psns,
+static void bnxt_re_fill_psns(struct bnxt_re_qp *qp, struct bnxt_re_wrid *wrid,
uint8_t opcode, uint32_t len)
{
- uint32_t pkt_cnt = 0, nxt_psn;
+ uint32_t opc_spsn = 0, flg_npsn = 0;
+ struct bnxt_re_psns_ext *psns_ext;
+ uint32_t pkt_cnt = 0, nxt_psn = 0;
+ struct bnxt_re_psns *psns;
+
+ psns = wrid->psns;
+ psns_ext = wrid->psns_ext;
- memset(psns, 0, sizeof(*psns));
if (qp->qptyp == IBV_QPT_RC) {
- psns->opc_spsn = htole32(qp->sq_psn & BNXT_RE_PSNS_SPSN_MASK);
+ opc_spsn = qp->sq_psn & BNXT_RE_PSNS_SPSN_MASK;
pkt_cnt = (len / qp->mtu);
if (len % qp->mtu)
pkt_cnt++;
if (len == 0)
pkt_cnt = 1;
nxt_psn = ((qp->sq_psn + pkt_cnt) & BNXT_RE_PSNS_NPSN_MASK);
- psns->flg_npsn = htole32(nxt_psn);
+ flg_npsn = nxt_psn;
qp->sq_psn = nxt_psn;
}
opcode = bnxt_re_ibv_wr_to_wc_opcd(opcode);
- psns->opc_spsn |= htole32(((opcode & BNXT_RE_PSNS_OPCD_MASK) <<
- BNXT_RE_PSNS_OPCD_SHIFT));
+ opc_spsn |= (((uint32_t)opcode & BNXT_RE_PSNS_OPCD_MASK) <<
+ BNXT_RE_PSNS_OPCD_SHIFT);
+ memset(psns, 0, sizeof(*psns));
+ psns->opc_spsn = htole32(opc_spsn);
+ psns->flg_npsn = htole32(flg_npsn);
+ if (bnxt_re_is_chip_gen_p5(qp->cctx))
+ psns_ext->st_slot_idx = 0;
}
static void bnxt_re_fill_wrid(struct bnxt_re_wrid *wrid, struct ibv_send_wr *wr,
@@ -1235,10 +1257,9 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
{
struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp);
struct bnxt_re_queue *sq = qp->sqq;
- struct bnxt_re_bsqe *hdr;
struct bnxt_re_wrid *wrid;
- struct bnxt_re_psns *psns;
uint8_t is_inline = false;
+ struct bnxt_re_bsqe *hdr;
int ret = 0, bytes = 0;
bool ring_db = false;
void *sqe;
@@ -1268,7 +1289,6 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
sqe = (void *)(sq->va + (sq->tail * sq->stride));
wrid = &qp->swrid[sq->tail];
- psns = wrid->psns;
memset(sqe, 0, bnxt_re_get_sqe_sz());
hdr = sqe;
@@ -1318,7 +1338,7 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
}
bnxt_re_fill_wrid(wrid, wr, bytes, qp->cap.sqsig);
- bnxt_re_fill_psns(qp, psns, wr->opcode, bytes);
+ bnxt_re_fill_psns(qp, wrid, wr->opcode, bytes);
bnxt_re_incr_tail(sq);
qp->wqe_cnt++;
wr = wr->next;

7
rdma-core.changes
View File

@ -1,3 +1,10 @@
-------------------------------------------------------------------
Fri Mar 1 07:07:49 UTC 2019 - Nicolas Morey-Chaisemartin <nmoreychaisemartin@suse.com>
- Add patches to enable Broadcom's 57500 RoCE adapter (bsc#1125651):
- Update-kernel-headers.patch
- bnxt_re-lib-Enable-Broadcom-s-57500-RoCE-adapter.patch
-------------------------------------------------------------------
Wed Feb 6 08:37:03 UTC 2019 - nmoreychaisemartin@suse.com

6
rdma-core.spec
View File

@ -12,7 +12,7 @@
# license that conforms to the Open Source Definition (Version 1.9)
# published by the Open Source Initiative.
# Please submit bugfixes or comments via http://bugs.opensuse.org/
# Please submit bugfixes or comments via https://bugs.opensuse.org/
#
@ -54,6 +54,8 @@ Source: rdma-core-%{version}%{git_ver}.tar.gz
Source1: baselibs.conf
Source2: prebuild-pandoc.sh
Source3: prebuilt-pandoc.tgz
Patch0: Update-kernel-headers.patch
Patch1: bnxt_re-lib-Enable-Broadcom-s-57500-RoCE-adapter.patch
BuildRequires: binutils
BuildRequires: cmake >= 2.8.11
BuildRequires: gcc
@ -341,6 +343,8 @@ easy, object-oriented access to IB verbs.
%prep
%setup -q -n %{name}-%{version}%{git_ver}
%patch0
%patch1
#Extract prebuilt pandoc file in the buildlib directory
(cd buildlib && tar xf %{S:3})