pool/rdma-core
SHA256
4
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
rdma-core/Revert-libnes-Remove-libnes-from-rdma-core.patch
Nicolas Morey-Chaisemartin 53b40e150c Accepting request 763762 from home:NMoreyChaisemartin:branches:science:HPC 
- Update to v27.0 (bsc#1160338)
  - No release notes available
- Dropped patches merged upstream:
  - bnxt_re-lib-Add-remaining-pci-ids-for-gen-P5-devices.patch
  - bnxt_re-lib-Recognize-additional-5750x-device-ID-s.patch
- Add patches to support older HW (nes, cxgb3) that was dropped upstream
  - Revert-Update-kernel-headers.patch
  - Revert-libcxgb3-Remove-libcxgb3-from-rdma-core.patch
  - Revert-libnes-Remove-libnes-from-rdma-core.patch

OBS-URL: https://build.opensuse.org/request/show/763762
OBS-URL: https://build.opensuse.org/package/show/science:HPC/rdma-core?expand=0&rev=126
2020-01-13 06:59:37 +00:00

2377 lines
71 KiB
Diff
Raw Blame History

commit 7e7a6f829acc1044b53d58afd88d0c10d6b8acaf
Author: Nicolas Morey-Chaisemartin <nmoreychaisemartin@suse.com>
Date: Fri Jan 10 09:07:34 2020 +0100
Revert "libnes: Remove libnes from rdma-core"
This reverts commit 4daf5c91c1296683924cb9668c3d879da072756b.
diff --git CMakeLists.txt CMakeLists.txt
index 364b4be0a04e..b3ff156956a0 100644
--- CMakeLists.txt
+++ CMakeLists.txt
@@ -625,6 +625,7 @@ add_subdirectory(providers/mlx4/man)
add_subdirectory(providers/mlx5)
add_subdirectory(providers/mlx5/man)
add_subdirectory(providers/mthca)
+add_subdirectory(providers/nes) # NO SPARSE
add_subdirectory(providers/ocrdma)
add_subdirectory(providers/qedr)
add_subdirectory(providers/vmw_pvrdma)
diff --git MAINTAINERS MAINTAINERS
index 70c8f6759982..8030891d96a4 100644
--- MAINTAINERS
+++ MAINTAINERS
@@ -141,6 +141,11 @@ H: Roland Dreier <roland@topspin.com>
S: Supported
F: providers/mthca/
+NES USERSPACE PROVIDER (for iw_nes.ko)
+M: Tatyana Nikolova <Tatyana.E.Nikolova@intel.com>
+S: Supported
+F: providers/nes/
+
OCRDMA USERSPACE PROVIDER (for ocrdma.ko)
M: Devesh Sharma <Devesh.sharma@broadcom.com>
S: Supported
diff --git README.md README.md
index ba509ef0cce2..24eee90cb8b7 100644
--- README.md
+++ README.md
@@ -23,6 +23,7 @@ is included:
- mlx4_ib.ko
- mlx5_ib.ko
- ib_mthca.ko
+ - iw_nes.ko
- ocrdma.ko
- qedr.ko
- rdma_rxe.ko
diff --git debian/control debian/control
index 25abf9b588be..738b2d6da39d 100644
--- debian/control
+++ debian/control
@@ -102,6 +102,7 @@ Description: User space provider drivers for libibverbs
- mlx4: Mellanox ConnectX-3 InfiniBand HCAs
- mlx5: Mellanox Connect-IB/X-4+ InfiniBand HCAs
- mthca: Mellanox InfiniBand HCAs
+ - nes: Intel NetEffect NE020-based iWARP adapters
- ocrdma: Emulex OneConnect RDMA/RoCE device
- qedr: QLogic QL4xxx RoCE HCAs
- rxe: A software implementation of the RoCE protocol
diff --git debian/copyright debian/copyright
index 6f86d1c86e21..db4951993bd8 100644
--- debian/copyright
+++ debian/copyright
@@ -197,6 +197,11 @@ Copyright: 2004-2005, Topspin Communications.
2005, Mellanox Technologies Ltd.
License: BSD-MIT or GPL-2
+Files: providers/nes/*
+Copyright: 2006-2010, Intel Corporation.
+ 2006, Open Grid Computing, Inc.
+License: BSD-MIT or GPL-2
+
Files: providers/ocrdma/*
Copyright: 2008-2013, Emulex.
License: BSD-2-clause or GPL-2
diff --git kernel-boot/rdma-description.rules kernel-boot/rdma-description.rules
index 48a7cede9bc8..4ea59ba1977b 100644
--- kernel-boot/rdma-description.rules
+++ kernel-boot/rdma-description.rules
@@ -24,6 +24,7 @@ DRIVERS=="hfi1", ENV{ID_RDMA_OPA}="1"
# Hardware that supports iWarp
DRIVERS=="cxgb4", ENV{ID_RDMA_IWARP}="1"
DRIVERS=="i40e", ENV{ID_RDMA_IWARP}="1"
+DRIVERS=="nes", ENV{ID_RDMA_IWARP}="1"
# Hardware that supports RoCE
DRIVERS=="be2net", ENV{ID_RDMA_ROCE}="1"
diff --git kernel-boot/rdma-hw-modules.rules kernel-boot/rdma-hw-modules.rules
index bee416dbe719..da4bbe363ac4 100644
--- kernel-boot/rdma-hw-modules.rules
+++ kernel-boot/rdma-hw-modules.rules
@@ -33,5 +33,6 @@ ENV{ID_NET_DRIVER}=="enic", RUN{builtin}+="kmod load usnic_verbs"
# ipathverbs
# mthca
# vmw_pvrdma
+# nes
LABEL="rdma_hw_modules_end"
diff --git libibverbs/verbs.h libibverbs/verbs.h
index 7b8d431024b9..d873f6d07327 100644
--- libibverbs/verbs.h
+++ libibverbs/verbs.h
@@ -2165,6 +2165,7 @@ extern const struct verbs_device_ops verbs_provider_ipathverbs;
extern const struct verbs_device_ops verbs_provider_mlx4;
extern const struct verbs_device_ops verbs_provider_mlx5;
extern const struct verbs_device_ops verbs_provider_mthca;
+extern const struct verbs_device_ops verbs_provider_nes;
extern const struct verbs_device_ops verbs_provider_ocrdma;
extern const struct verbs_device_ops verbs_provider_qedr;
extern const struct verbs_device_ops verbs_provider_rxe;
diff --git providers/nes/CMakeLists.txt providers/nes/CMakeLists.txt
new file mode 100644
index 000000000000..0c7fa8fad09f
--- /dev/null
+++ providers/nes/CMakeLists.txt
@@ -0,0 +1,4 @@
+rdma_provider(nes
+ nes_umain.c
+ nes_uverbs.c
+)
diff --git providers/nes/nes-abi.h providers/nes/nes-abi.h
new file mode 100644
index 000000000000..0a531230b062
--- /dev/null
+++ providers/nes/nes-abi.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2006 - 2010 Intel Corporation. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * gpl-2.0.txt in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef nes_ABI_H
+#define nes_ABI_H
+
+#include <infiniband/kern-abi.h>
+#include <rdma/nes-abi.h>
+#include <kernel-abi/nes-abi.h>
+
+DECLARE_DRV_CMD(nes_ualloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
+ empty, nes_alloc_pd_resp);
+DECLARE_DRV_CMD(nes_ucreate_cq, IB_USER_VERBS_CMD_CREATE_CQ,
+ nes_create_cq_req, nes_create_cq_resp);
+DECLARE_DRV_CMD(nes_ucreate_qp, IB_USER_VERBS_CMD_CREATE_QP,
+ nes_create_qp_req, nes_create_qp_resp);
+DECLARE_DRV_CMD(nes_get_context, IB_USER_VERBS_CMD_GET_CONTEXT,
+ nes_alloc_ucontext_req, nes_alloc_ucontext_resp);
+DECLARE_DRV_CMD(nes_ureg_mr, IB_USER_VERBS_CMD_REG_MR,
+ nes_mem_reg_req, empty);
+
+#endif /* nes_ABI_H */
diff --git providers/nes/nes_umain.c providers/nes/nes_umain.c
new file mode 100644
index 000000000000..07aa7ddd112a
--- /dev/null
+++ providers/nes/nes_umain.c
@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2006 - 2010 Intel Corporation. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * gpl-2.0.txt in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <pthread.h>
+
+#include "nes_umain.h"
+#include "nes-abi.h"
+
+unsigned int nes_debug_level = 0;
+long int page_size;
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#ifndef PCI_VENDOR_ID_NETEFFECT
+#define PCI_VENDOR_ID_NETEFFECT 0x1678
+#endif
+
+#define HCA(v, d) VERBS_PCI_MATCH(PCI_VENDOR_ID_##v, d, NULL)
+static const struct verbs_match_ent hca_table[] = {
+ VERBS_DRIVER_ID(RDMA_DRIVER_NES),
+ HCA(NETEFFECT, 0x0100),
+ HCA(NETEFFECT, 0x0110),
+ {},
+};
+
+static const struct verbs_context_ops nes_uctx_ops = {
+ .query_device = nes_uquery_device,
+ .query_port = nes_uquery_port,
+ .alloc_pd = nes_ualloc_pd,
+ .dealloc_pd = nes_ufree_pd,
+ .reg_mr = nes_ureg_mr,
+ .dereg_mr = nes_udereg_mr,
+ .create_cq = nes_ucreate_cq,
+ .poll_cq = nes_upoll_cq,
+ .req_notify_cq = nes_uarm_cq,
+ .cq_event = nes_cq_event,
+ .resize_cq = nes_uresize_cq,
+ .destroy_cq = nes_udestroy_cq,
+ .create_qp = nes_ucreate_qp,
+ .query_qp = nes_uquery_qp,
+ .modify_qp = nes_umodify_qp,
+ .destroy_qp = nes_udestroy_qp,
+ .post_send = nes_upost_send,
+ .post_recv = nes_upost_recv,
+ .create_ah = nes_ucreate_ah,
+ .destroy_ah = nes_udestroy_ah,
+ .attach_mcast = nes_uattach_mcast,
+ .detach_mcast = nes_udetach_mcast,
+ .async_event = nes_async_event
+};
+
+static const struct verbs_context_ops nes_uctx_no_db_ops = {
+ .poll_cq = nes_upoll_cq_no_db_read,
+};
+
+
+/**
+ * nes_ualloc_context
+ */
+static struct verbs_context *nes_ualloc_context(struct ibv_device *ibdev,
+ int cmd_fd,
+ void *private_data)
+{
+ struct ibv_pd *ibv_pd;
+ struct nes_uvcontext *nesvctx;
+ struct nes_get_context cmd;
+ struct nes_get_context_resp resp;
+ char value[16];
+ uint32_t nes_drv_opt = 0;
+
+ page_size = sysconf(_SC_PAGESIZE);
+
+ nesvctx = verbs_init_and_alloc_context(ibdev, cmd_fd, nesvctx, ibv_ctx,
+ RDMA_DRIVER_NES);
+ if (!nesvctx)
+ return NULL;
+
+ cmd.userspace_ver = NES_ABI_USERSPACE_VER;
+
+ if (ibv_cmd_get_context(&nesvctx->ibv_ctx, (struct ibv_get_context *)&cmd, sizeof cmd,
+ &resp.ibv_resp, sizeof(resp)))
+ goto err_free;
+
+ if (resp.kernel_ver != NES_ABI_KERNEL_VER) {
+ fprintf(stderr, PFX "%s: Invalid kernel driver version detected. Detected %d, should be %d\n",
+ __FUNCTION__, resp.kernel_ver, NES_ABI_KERNEL_VER);
+ goto err_free;
+ }
+
+ if (ibv_read_sysfs_file("/sys/module/iw_nes", "parameters/nes_drv_opt",
+ value, sizeof(value)) > 0) {
+ sscanf(value, "%d", &nes_drv_opt);
+ } else if (ibv_read_sysfs_file("/sys/module/iw_nes", "nes_drv_opt",
+ value, sizeof(value)) > 0) {
+ sscanf(value, "%d", &nes_drv_opt);
+ }
+
+ verbs_set_ops(&nesvctx->ibv_ctx, &nes_uctx_ops);
+ if (nes_drv_opt & NES_DRV_OPT_NO_DB_READ)
+ verbs_set_ops(&nesvctx->ibv_ctx, &nes_uctx_no_db_ops);
+
+ nesvctx->max_pds = resp.max_pds;
+ nesvctx->max_qps = resp.max_qps;
+ nesvctx->wq_size = resp.wq_size;
+ nesvctx->virtwq = resp.virtwq;
+ nesvctx->mcrqf = 0;
+
+ /* Get a doorbell region for the CQs */
+ ibv_pd = nes_ualloc_pd(&nesvctx->ibv_ctx.context);
+ if (!ibv_pd)
+ goto err_free;
+ ibv_pd->context = &nesvctx->ibv_ctx.context;
+ nesvctx->nesupd = to_nes_upd(ibv_pd);
+
+ return &nesvctx->ibv_ctx;
+
+err_free:
+ fprintf(stderr, PFX "%s: Failed to allocate context for device.\n", __FUNCTION__);
+ verbs_uninit_context(&nesvctx->ibv_ctx);
+ free(nesvctx);
+
+ return NULL;
+}
+
+
+/**
+ * nes_ufree_context
+ */
+static void nes_ufree_context(struct ibv_context *ibctx)
+{
+ struct nes_uvcontext *nesvctx = to_nes_uctx(ibctx);
+ nes_ufree_pd(&nesvctx->nesupd->ibv_pd);
+
+ verbs_uninit_context(&nesvctx->ibv_ctx);
+ free(nesvctx);
+}
+
+static void nes_uninit_device(struct verbs_device *verbs_device)
+{
+ struct nes_udevice *dev = to_nes_udev(&verbs_device->device);
+
+ free(dev);
+}
+
+static struct verbs_device *
+nes_device_alloc(struct verbs_sysfs_dev *sysfs_dev)
+{
+ struct nes_udevice *dev;
+ char value[16];
+
+ if (ibv_read_sysfs_file("/sys/module/iw_nes", "parameters/debug_level",
+ value, sizeof(value)) > 0) {
+ sscanf(value, "%u", &nes_debug_level);
+ } else if (ibv_read_sysfs_file("/sys/module/iw_nes", "debug_level",
+ value, sizeof(value)) > 0) {
+ sscanf(value, "%u", &nes_debug_level);
+ }
+
+ dev = calloc(1, sizeof(*dev));
+ if (!dev)
+ return NULL;
+
+ dev->page_size = sysconf(_SC_PAGESIZE);
+
+ nes_debug(NES_DBG_INIT, "libnes initialized\n");
+
+ return &dev->ibv_dev;
+}
+
+static const struct verbs_device_ops nes_udev_ops = {
+ .name = "nes",
+ .match_min_abi_version = 0,
+ .match_max_abi_version = INT_MAX,
+ .match_table = hca_table,
+ .alloc_device = nes_device_alloc,
+ .uninit_device = nes_uninit_device,
+ .alloc_context = nes_ualloc_context,
+ .free_context = nes_ufree_context,
+};
+PROVIDER_DRIVER(nes, nes_udev_ops);
diff --git providers/nes/nes_umain.h providers/nes/nes_umain.h
new file mode 100644
index 000000000000..1070ce4295a5
--- /dev/null
+++ providers/nes/nes_umain.h
@@ -0,0 +1,383 @@
+/*
+ * Copyright (c) 2006 - 2010 Intel Corporation. All rights reserved.
+ * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * gpl-2.0.txt in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef nes_umain_H
+#define nes_umain_H
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <endian.h>
+#include <util/compiler.h>
+
+#include <infiniband/driver.h>
+#include <util/udma_barrier.h>
+
+#define PFX "libnes: "
+
+#define NES_QP_MMAP 1
+#define NES_QP_VMAP 2
+
+#define NES_DRV_OPT_NO_INLINE_DATA 0x00000080
+#define NES_DRV_OPT_NO_DB_READ 0x00001000
+
+#define NES_DEBUG
+/* debug levels */
+/* must match kernel */
+#define NES_DBG_HW 0x00000001
+#define NES_DBG_INIT 0x00000002
+#define NES_DBG_ISR 0x00000004
+#define NES_DBG_PHY 0x00000008
+#define NES_DBG_NETDEV 0x00000010
+#define NES_DBG_CM 0x00000020
+#define NES_DBG_CM1 0x00000040
+#define NES_DBG_NIC_RX 0x00000080
+#define NES_DBG_NIC_TX 0x00000100
+#define NES_DBG_CQP 0x00000200
+#define NES_DBG_MMAP 0x00000400
+#define NES_DBG_MR 0x00000800
+#define NES_DBG_PD 0x00001000
+#define NES_DBG_CQ 0x00002000
+#define NES_DBG_QP 0x00004000
+#define NES_DBG_MOD_QP 0x00008000
+#define NES_DBG_AEQ 0x00010000
+#define NES_DBG_IW_RX 0x00020000
+#define NES_DBG_IW_TX 0x00040000
+#define NES_DBG_SHUTDOWN 0x00080000
+#define NES_DBG_UD 0x00100000
+#define NES_DBG_RSVD1 0x10000000
+#define NES_DBG_RSVD2 0x20000000
+#define NES_DBG_RSVD3 0x40000000
+#define NES_DBG_RSVD4 0x80000000
+#define NES_DBG_ALL 0xffffffff
+
+extern unsigned int nes_debug_level;
+#ifdef NES_DEBUG
+#define nes_debug(level, fmt, args...) \
+ if (level & nes_debug_level) \
+ fprintf(stderr, PFX "%s[%u]: " fmt, __FUNCTION__, __LINE__, ##args)
+#else
+#define nes_debug(level, fmt, args...)
+#endif
+
+enum nes_cqe_opcode_bits {
+ NES_CQE_STAG_VALID = (1<<6),
+ NES_CQE_ERROR = (1<<7),
+ NES_CQE_SQ = (1<<8),
+ NES_CQE_SE = (1<<9),
+ NES_CQE_PSH = (1<<29),
+ NES_CQE_FIN = (1<<30),
+ NES_CQE_VALID = (1<<31),
+};
+
+enum nes_cqe_word_idx {
+ NES_CQE_PAYLOAD_LENGTH_IDX = 0,
+ NES_CQE_COMP_COMP_CTX_LOW_IDX = 2,
+ NES_CQE_COMP_COMP_CTX_HIGH_IDX = 3,
+ NES_CQE_INV_STAG_IDX = 4,
+ NES_CQE_QP_ID_IDX = 5,
+ NES_CQE_ERROR_CODE_IDX = 6,
+ NES_CQE_OPCODE_IDX = 7,
+};
+
+enum nes_cqe_allocate_bits {
+ NES_CQE_ALLOC_INC_SELECT = (1<<28),
+ NES_CQE_ALLOC_NOTIFY_NEXT = (1<<29),
+ NES_CQE_ALLOC_NOTIFY_SE = (1<<30),
+ NES_CQE_ALLOC_RESET = (1<<31),
+};
+
+enum nes_iwarp_sq_wqe_word_idx {
+ NES_IWARP_SQ_WQE_MISC_IDX = 0,
+ NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX = 1,
+ NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX = 2,
+ NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX = 3,
+ NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX = 4,
+ NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX = 5,
+ NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX = 7,
+ NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX = 8,
+ NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX = 9,
+ NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX = 10,
+ NES_IWARP_SQ_WQE_RDMA_STAG_IDX = 11,
+ NES_IWARP_SQ_WQE_IMM_DATA_START_IDX = 12,
+ NES_IWARP_SQ_WQE_FRAG0_LOW_IDX = 16,
+ NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX = 17,
+ NES_IWARP_SQ_WQE_LENGTH0_IDX = 18,
+ NES_IWARP_SQ_WQE_STAG0_IDX = 19,
+ NES_IWARP_SQ_WQE_FRAG1_LOW_IDX = 20,
+ NES_IWARP_SQ_WQE_FRAG1_HIGH_IDX = 21,
+ NES_IWARP_SQ_WQE_LENGTH1_IDX = 22,
+ NES_IWARP_SQ_WQE_STAG1_IDX = 23,
+ NES_IWARP_SQ_WQE_FRAG2_LOW_IDX = 24,
+ NES_IWARP_SQ_WQE_FRAG2_HIGH_IDX = 25,
+ NES_IWARP_SQ_WQE_LENGTH2_IDX = 26,
+ NES_IWARP_SQ_WQE_STAG2_IDX = 27,
+ NES_IWARP_SQ_WQE_FRAG3_LOW_IDX = 28,
+ NES_IWARP_SQ_WQE_FRAG3_HIGH_IDX = 29,
+ NES_IWARP_SQ_WQE_LENGTH3_IDX = 30,
+ NES_IWARP_SQ_WQE_STAG3_IDX = 31,
+};
+
+enum nes_iwarp_rq_wqe_word_idx {
+ NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX = 1,
+ NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX = 2,
+ NES_IWARP_RQ_WQE_COMP_CTX_HIGH_IDX = 3,
+ NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX = 4,
+ NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX = 5,
+ NES_IWARP_RQ_WQE_FRAG0_LOW_IDX = 8,
+ NES_IWARP_RQ_WQE_FRAG0_HIGH_IDX = 9,
+ NES_IWARP_RQ_WQE_LENGTH0_IDX = 10,
+ NES_IWARP_RQ_WQE_STAG0_IDX = 11,
+ NES_IWARP_RQ_WQE_FRAG1_LOW_IDX = 12,
+ NES_IWARP_RQ_WQE_FRAG1_HIGH_IDX = 13,
+ NES_IWARP_RQ_WQE_LENGTH1_IDX = 14,
+ NES_IWARP_RQ_WQE_STAG1_IDX = 15,
+ NES_IWARP_RQ_WQE_FRAG2_LOW_IDX = 16,
+ NES_IWARP_RQ_WQE_FRAG2_HIGH_IDX = 17,
+ NES_IWARP_RQ_WQE_LENGTH2_IDX = 18,
+ NES_IWARP_RQ_WQE_STAG2_IDX = 19,
+ NES_IWARP_RQ_WQE_FRAG3_LOW_IDX = 20,
+ NES_IWARP_RQ_WQE_FRAG3_HIGH_IDX = 21,
+ NES_IWARP_RQ_WQE_LENGTH3_IDX = 22,
+ NES_IWARP_RQ_WQE_STAG3_IDX = 23,
+};
+
+enum nes_iwarp_sq_opcodes {
+ NES_IWARP_SQ_WQE_STREAMING = (1<<23),
+ NES_IWARP_SQ_WQE_IMM_DATA = (1<<28),
+ NES_IWARP_SQ_WQE_READ_FENCE = (1<<29),
+ NES_IWARP_SQ_WQE_LOCAL_FENCE = (1<<30),
+ NES_IWARP_SQ_WQE_SIGNALED_COMPL = (1<<31),
+};
+
+enum nes_iwarp_sq_wqe_bits {
+ NES_IWARP_SQ_OP_RDMAW = 0,
+ NES_IWARP_SQ_OP_RDMAR = 1,
+ NES_IWARP_SQ_OP_SEND = 3,
+ NES_IWARP_SQ_OP_SENDINV = 4,
+ NES_IWARP_SQ_OP_SENDSE = 5,
+ NES_IWARP_SQ_OP_SENDSEINV = 6,
+ NES_IWARP_SQ_OP_BIND = 8,
+ NES_IWARP_SQ_OP_FAST_REG = 9,
+ NES_IWARP_SQ_OP_LOCINV = 10,
+ NES_IWARP_SQ_OP_RDMAR_LOCINV = 11,
+ NES_IWARP_SQ_OP_NOP = 12,
+};
+
+enum nes_nic_cqe_word_idx {
+ NES_NIC_CQE_ACCQP_ID_IDX = 0,
+ NES_NIC_CQE_TAG_PKT_TYPE_IDX = 2,
+ NES_NIC_CQE_MISC_IDX = 3,
+};
+
+#define NES_NIC_CQE_ERRV_SHIFT 16
+enum nes_nic_ev_bits {
+ NES_NIC_ERRV_BITS_MODE = (1<<0),
+ NES_NIC_ERRV_BITS_IPV4_CSUM_ERR = (1<<1),
+ NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR = (1<<2),
+ NES_NIC_ERRV_BITS_WQE_OVERRUN = (1<<3),
+ NES_NIC_ERRV_BITS_IPH_ERR = (1<<4),
+};
+
+enum nes_nic_cqe_bits {
+ NES_NIC_CQE_ERRV_MASK = (0xff<<NES_NIC_CQE_ERRV_SHIFT),
+ NES_NIC_CQE_SQ = (1<<24),
+ NES_NIC_CQE_ACCQP_PORT = (1<<28),
+ NES_NIC_CQE_ACCQP_VALID = (1<<29),
+ NES_NIC_CQE_TAG_VALID = (1<<30),
+ NES_NIC_CQE_VALID = (1<<31),
+};
+struct nes_hw_nic_cqe {
+ uint32_t cqe_words[4];
+};
+
+enum nes_iwarp_cqe_major_code {
+ NES_IWARP_CQE_MAJOR_FLUSH = 1,
+ NES_IWARP_CQE_MAJOR_DRV = 0x8000
+};
+
+enum nes_iwarp_cqe_minor_code {
+ NES_IWARP_CQE_MINOR_FLUSH = 1
+};
+
+struct nes_hw_qp_wqe {
+ uint32_t wqe_words[32];
+};
+
+struct nes_hw_cqe {
+ uint32_t cqe_words[8];
+};
+
+struct nes_user_doorbell {
+ uint32_t wqe_alloc;
+ uint32_t reserved[3];
+ uint32_t cqe_alloc;
+};
+
+struct nes_udevice {
+ struct verbs_device ibv_dev;
+ int page_size;
+};
+
+struct nes_upd {
+ struct ibv_pd ibv_pd;
+ struct nes_user_doorbell volatile *udoorbell;
+ uint32_t pd_id;
+ uint32_t db_index;
+};
+
+struct nes_uvcontext {
+ struct verbs_context ibv_ctx;
+ struct nes_upd *nesupd;
+ uint32_t max_pds; /* maximum pds allowed for this user process */
+ uint32_t max_qps; /* maximum qps allowed for this user process */
+ uint32_t wq_size; /* size of the WQs (sq+rq) allocated to the mmaped area */
+ uint32_t mcrqf;
+ uint8_t virtwq ; /* flag if to use virt wqs or not */
+ uint8_t reserved[3];
+};
+
+struct nes_uqp;
+
+struct nes_ucq {
+ struct ibv_cq ibv_cq;
+ struct nes_hw_cqe volatile *cqes;
+ struct verbs_mr vmr;
+ pthread_spinlock_t lock;
+ uint32_t cq_id;
+ uint16_t size;
+ uint16_t head;
+ uint16_t polled_completions;
+ uint8_t is_armed;
+ uint8_t skip_arm;
+ int arm_sol;
+ int skip_sol;
+ int comp_vector;
+ struct nes_uqp *udqp;
+};
+
+struct nes_uqp {
+ struct ibv_qp ibv_qp;
+ struct nes_hw_qp_wqe volatile *sq_vbase;
+ struct nes_hw_qp_wqe volatile *rq_vbase;
+ uint32_t qp_id;
+ struct nes_ucq *send_cq;
+ struct nes_ucq *recv_cq;
+ struct verbs_mr vmr;
+ uint32_t nes_drv_opt;
+ pthread_spinlock_t lock;
+ uint16_t sq_db_index;
+ uint16_t sq_head;
+ uint16_t sq_tail;
+ uint16_t sq_size;
+ uint16_t sq_sig_all;
+ uint16_t rq_db_index;
+ uint16_t rq_head;
+ uint16_t rq_tail;
+ uint16_t rq_size;
+ uint16_t rdma0_msg;
+ uint16_t mapping;
+ uint16_t qperr;
+ uint16_t rsvd;
+ uint32_t pending_rcvs;
+ struct ibv_recv_wr *pend_rx_wr;
+ int nes_ud_sksq_fd;
+ void *sksq_shared_ctxt;
+ uint64_t send_wr_id[512]; /* IMA send wr_id ring content */
+ uint64_t recv_wr_id[512]; /* IMA receive wr_id ring content */
+};
+
+#define to_nes_uxxx(xxx, type) \
+ container_of(ib##xxx, struct nes_u##type, ibv_##xxx)
+
+static inline struct nes_udevice *to_nes_udev(struct ibv_device *ibdev)
+{
+ return container_of(ibdev, struct nes_udevice, ibv_dev.device);
+}
+
+static inline struct nes_uvcontext *to_nes_uctx(struct ibv_context *ibctx)
+{
+ return container_of(ibctx, struct nes_uvcontext, ibv_ctx.context);
+}
+
+static inline struct nes_upd *to_nes_upd(struct ibv_pd *ibpd)
+{
+ return to_nes_uxxx(pd, pd);
+}
+
+static inline struct nes_ucq *to_nes_ucq(struct ibv_cq *ibcq)
+{
+ return to_nes_uxxx(cq, cq);
+}
+
+static inline struct nes_uqp *to_nes_uqp(struct ibv_qp *ibqp)
+{
+ return to_nes_uxxx(qp, qp);
+}
+
+
+/* nes_uverbs.c */
+int nes_uquery_device(struct ibv_context *, struct ibv_device_attr *);
+int nes_uquery_port(struct ibv_context *, uint8_t, struct ibv_port_attr *);
+struct ibv_pd *nes_ualloc_pd(struct ibv_context *);
+int nes_ufree_pd(struct ibv_pd *);
+struct ibv_mr *nes_ureg_mr(struct ibv_pd *pd, void *addr, size_t length,
+ uint64_t hca_va, int access);
+int nes_udereg_mr(struct verbs_mr *vmr);
+struct ibv_cq *nes_ucreate_cq(struct ibv_context *, int, struct ibv_comp_channel *, int);
+int nes_uresize_cq(struct ibv_cq *, int);
+int nes_udestroy_cq(struct ibv_cq *);
+int nes_upoll_cq(struct ibv_cq *, int, struct ibv_wc *);
+int nes_upoll_cq_no_db_read(struct ibv_cq *, int, struct ibv_wc *);
+int nes_uarm_cq(struct ibv_cq *, int);
+void nes_cq_event(struct ibv_cq *);
+struct ibv_srq *nes_ucreate_srq(struct ibv_pd *, struct ibv_srq_init_attr *);
+int nes_umodify_srq(struct ibv_srq *, struct ibv_srq_attr *, int);
+int nes_udestroy_srq(struct ibv_srq *);
+int nes_upost_srq_recv(struct ibv_srq *, struct ibv_recv_wr *, struct ibv_recv_wr **);
+struct ibv_qp *nes_ucreate_qp(struct ibv_pd *, struct ibv_qp_init_attr *);
+int nes_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+ int, struct ibv_qp_init_attr *init_attr);
+int nes_umodify_qp(struct ibv_qp *, struct ibv_qp_attr *, int);
+int nes_udestroy_qp(struct ibv_qp *);
+int nes_upost_send(struct ibv_qp *, struct ibv_send_wr *, struct ibv_send_wr **);
+int nes_upost_recv(struct ibv_qp *, struct ibv_recv_wr *, struct ibv_recv_wr **);
+struct ibv_ah *nes_ucreate_ah(struct ibv_pd *, struct ibv_ah_attr *);
+int nes_udestroy_ah(struct ibv_ah *);
+int nes_uattach_mcast(struct ibv_qp *, const union ibv_gid *, uint16_t);
+int nes_udetach_mcast(struct ibv_qp *, const union ibv_gid *, uint16_t);
+void nes_async_event(struct ibv_context *context,
+ struct ibv_async_event *event);
+
+extern long int page_size;
+
+#endif /* nes_umain_H */
diff --git providers/nes/nes_uverbs.c providers/nes/nes_uverbs.c
new file mode 100644
index 000000000000..2b78468b4135
--- /dev/null
+++ providers/nes/nes_uverbs.c
@@ -0,0 +1,1535 @@
+/*
+ * Copyright (c) 2006 - 2010 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * gpl-2.0.txt in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <config.h>
+
+#include <endian.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <signal.h>
+#include <errno.h>
+#include <pthread.h>
+#include <malloc.h>
+#include <sys/mman.h>
+#include <linux/if_ether.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "nes_umain.h"
+#include "nes-abi.h"
+
+#define STATIC static
+#define INLINE inline
+
+#define NES_WC_WITH_VLAN 1 << 3
+#define NES_UD_RX_BATCH_SZ 64
+#define NES_UD_MAX_SG_LIST_SZ 1
+
+struct nes_ud_send_wr {
+ uint32_t wr_cnt;
+ uint32_t qpn;
+ uint32_t flags;
+ uint32_t resv[1];
+ struct ibv_sge sg_list[64];
+};
+
+struct nes_ud_recv_wr {
+ uint32_t wr_cnt;
+ uint32_t qpn;
+ uint32_t resv[2];
+ struct ibv_sge sg_list[64];
+};
+
+/**
+ * nes_uquery_device
+ */
+int nes_uquery_device(struct ibv_context *context, struct ibv_device_attr *attr)
+{
+ struct ibv_query_device cmd;
+ uint64_t nes_fw_ver;
+ int ret;
+ unsigned int minor, major;
+
+ ret = ibv_cmd_query_device(context, attr, &nes_fw_ver,
+ &cmd, sizeof cmd);
+ if (ret)
+ return ret;
+
+ major = (nes_fw_ver >> 16) & 0xffff;
+ minor = nes_fw_ver & 0xffff;
+
+ snprintf(attr->fw_ver, sizeof attr->fw_ver,
+ "%d.%d", major, minor);
+
+ return 0;
+}
+
+
+/**
+ * nes_uquery_port
+ */
+int nes_uquery_port(struct ibv_context *context, uint8_t port,
+ struct ibv_port_attr *attr)
+{
+ struct ibv_query_port cmd;
+
+ return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
+}
+
+
+/**
+ * nes_ualloc_pd
+ */
+struct ibv_pd *nes_ualloc_pd(struct ibv_context *context)
+{
+ struct ibv_alloc_pd cmd;
+ struct nes_ualloc_pd_resp resp;
+ struct nes_upd *nesupd;
+
+ nesupd = malloc(sizeof *nesupd);
+ if (!nesupd)
+ return NULL;
+
+ if (ibv_cmd_alloc_pd(context, &nesupd->ibv_pd, &cmd, sizeof cmd,
+ &resp.ibv_resp, sizeof resp)) {
+ free(nesupd);
+ return NULL;
+ }
+ nesupd->pd_id = resp.pd_id;
+ nesupd->db_index = resp.mmap_db_index;
+
+ nesupd->udoorbell = mmap(NULL, page_size, PROT_WRITE | PROT_READ, MAP_SHARED,
+ context->cmd_fd, nesupd->db_index * page_size);
+
+ if (nesupd->udoorbell == MAP_FAILED) {
+ free(nesupd);
+ return NULL;
+ }
+
+ return &nesupd->ibv_pd;
+}
+
+
+/**
+ * nes_ufree_pd
+ */
+int nes_ufree_pd(struct ibv_pd *pd)
+{
+ int ret;
+ struct nes_upd *nesupd;
+
+ nesupd = to_nes_upd(pd);
+
+ ret = ibv_cmd_dealloc_pd(pd);
+ if (ret)
+ return ret;
+
+ munmap((void *)nesupd->udoorbell, page_size);
+ free(nesupd);
+
+ return 0;
+}
+
+
+/**
+ * nes_ureg_mr
+ */
+struct ibv_mr *nes_ureg_mr(struct ibv_pd *pd, void *addr, size_t length,
+ uint64_t hca_va, int access)
+{
+ struct verbs_mr *vmr;
+ struct nes_ureg_mr cmd;
+ struct ib_uverbs_reg_mr_resp resp;
+
+ vmr = malloc(sizeof(*vmr));
+ if (!vmr)
+ return NULL;
+
+ cmd.reg_type = IWNES_MEMREG_TYPE_MEM;
+ if (ibv_cmd_reg_mr(pd, addr, length, hca_va, access, vmr, &cmd.ibv_cmd,
+ sizeof(cmd), &resp, sizeof(resp))) {
+ free(vmr);
+
+ return NULL;
+ }
+
+ return &vmr->ibv_mr;
+}
+
+
+/**
+ * nes_udereg_mr
+ */
+int nes_udereg_mr(struct verbs_mr *vmr)
+{
+ int ret;
+
+ ret = ibv_cmd_dereg_mr(vmr);
+ if (ret)
+ return ret;
+
+ free(vmr);
+ return 0;
+}
+
+/**
+ * nes_ucreate_cq
+ */
+struct ibv_cq *nes_ucreate_cq(struct ibv_context *context, int cqe,
+ struct ibv_comp_channel *channel, int comp_vector)
+{
+ struct nes_ucq *nesucq;
+ struct nes_ureg_mr reg_mr_cmd;
+ struct ib_uverbs_reg_mr_resp reg_mr_resp;
+ struct nes_ucreate_cq cmd;
+ struct nes_ucreate_cq_resp resp;
+ int ret;
+ struct nes_uvcontext *nesvctx = to_nes_uctx(context);
+
+ nesucq = malloc(sizeof *nesucq);
+ if (!nesucq) {
+ return NULL;
+ }
+ memset(nesucq, 0, sizeof(*nesucq));
+
+ if (pthread_spin_init(&nesucq->lock, PTHREAD_PROCESS_PRIVATE)) {
+ free(nesucq);
+ return NULL;
+ }
+
+ if (cqe < 4) /* a reasonable minimum */
+ cqe = 4;
+ nesucq->size = cqe + 1;
+ nesucq->comp_vector = comp_vector;
+
+ nesucq->cqes = memalign(page_size, nesucq->size*sizeof(struct nes_hw_cqe));
+ if (!nesucq->cqes)
+ goto err;
+
+ /* Register the memory for the CQ */
+ reg_mr_cmd.reg_type = IWNES_MEMREG_TYPE_CQ;
+
+ ret = ibv_cmd_reg_mr(&nesvctx->nesupd->ibv_pd, (void *)nesucq->cqes,
+ (nesucq->size*sizeof(struct nes_hw_cqe)),
+ (uintptr_t)nesucq->cqes, IBV_ACCESS_LOCAL_WRITE,
+ &nesucq->vmr, &reg_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd),
+ &reg_mr_resp, sizeof(reg_mr_resp));
+ if (ret) {
+ /* fprintf(stderr, "ibv_cmd_reg_mr failed (ret = %d).\n", ret); */
+ free((struct nes_hw_cqe *)nesucq->cqes);
+ goto err;
+ }
+
+ /* Create the CQ */
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.user_cq_buffer = (__u64)((uintptr_t)nesucq->cqes);
+ cmd.mcrqf = nesvctx->mcrqf;
+
+ ret = ibv_cmd_create_cq(context, nesucq->size-1, channel, comp_vector,
+ &nesucq->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
+ &resp.ibv_resp, sizeof resp);
+ if (ret)
+ goto err;
+
+ nesucq->cq_id = (uint16_t)resp.cq_id;
+
+ /* Zero out the CQ */
+ memset((struct nes_hw_cqe *)nesucq->cqes, 0, nesucq->size*sizeof(struct nes_hw_cqe));
+
+ return &nesucq->ibv_cq;
+
+err:
+ /* fprintf(stderr, PFX "%s: Error Creating CQ.\n", __FUNCTION__); */
+ pthread_spin_destroy(&nesucq->lock);
+ free(nesucq);
+
+ return NULL;
+}
+
+
+/**
+ * nes_uresize_cq
+ */
+int nes_uresize_cq(struct ibv_cq *cq, int cqe)
+{
+ /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+ return -ENOSYS;
+}
+
+/**
+ * nes_udestroy_cq
+ */
+int nes_udestroy_cq(struct ibv_cq *cq)
+{
+ struct nes_ucq *nesucq = to_nes_ucq(cq);
+ int ret;
+
+ ret = ibv_cmd_destroy_cq(cq);
+ if (ret)
+ return ret;
+
+ ret = ibv_cmd_dereg_mr(&nesucq->vmr);
+ if (ret)
+ fprintf(stderr, PFX "%s: Failed to deregister CQ Memory Region.\n", __FUNCTION__);
+
+ /* Free CQ the memory */
+ free((struct nes_hw_cqe *)nesucq->cqes);
+ pthread_spin_destroy(&nesucq->lock);
+ free(nesucq);
+
+ return 0;
+}
+
+#define NES_CQ_BUF_OV_ERR 0x3
+
+static inline
+int nes_ima_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry)
+{
+ struct nes_ucq *nesucq = to_nes_ucq(cq);
+ struct nes_uvcontext *nesvctx = to_nes_uctx(cq->context);
+ uint32_t cqe_misc;
+ int cqe_count = 0;
+ uint32_t head;
+ uint32_t cq_size;
+
+ volatile struct nes_hw_nic_cqe *cqe = NULL;
+ volatile struct nes_hw_nic_cqe *cqes;
+
+ struct nes_uqp *nesuqp = nesucq->udqp;
+ uint32_t vlan_tag = 0;
+
+ cqes = (volatile struct nes_hw_nic_cqe *)nesucq->cqes;
+ head = nesucq->head;
+ cq_size = nesucq->size;
+
+ if (!nesuqp || !nesvctx)
+ exit(0);
+ if (nesuqp->ibv_qp.state == IBV_QPS_ERR) {
+ while (cqe_count < num_entries) {
+ memset(entry, 0, sizeof *entry);
+
+ if (nesuqp->recv_cq == nesucq) {
+ if (nesuqp->rq_tail != nesuqp->rq_head) {
+ /* Working on a RQ Completion*/
+ entry->wr_id =
+ nesuqp->recv_wr_id[nesuqp->rq_tail];
+ if (++nesuqp->rq_tail >= nesuqp->rq_size)
+ nesuqp->rq_tail = 0;
+ } else
+ return cqe_count;
+ } else
+ if (nesuqp->send_cq == nesucq) {
+ if (nesuqp->sq_tail != nesuqp->sq_head) {
+ entry->wr_id =
+ nesuqp->send_wr_id[nesuqp->sq_tail];
+ /* Working on a SQ Completion*/
+ if (++nesuqp->sq_tail >= nesuqp->sq_size)
+ nesuqp->sq_tail = 0;
+ } else
+ return cqe_count;
+ }
+ entry->status = IBV_WC_WR_FLUSH_ERR;
+ entry++;
+ cqe_count++;
+ }
+ return cqe_count;
+ }
+
+ while (cqe_count < num_entries) {
+ const enum ibv_wc_opcode INVAL_OP = -1;
+
+ entry->opcode = INVAL_OP;
+ cqe = &cqes[head];
+ cqe_misc =
+ le32toh(cqe->cqe_words[NES_NIC_CQE_MISC_IDX]);
+ if (cqe_misc & NES_NIC_CQE_VALID) {
+ memset(entry, 0, sizeof *entry);
+ entry->opcode = INVAL_OP;
+ cqe->cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
+ entry->status = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >>
+ NES_NIC_CQE_ERRV_SHIFT;
+ entry->qp_num = nesuqp->qp_id;
+ entry->src_qp = nesuqp->qp_id;
+ if (cqe_misc & NES_NIC_CQE_SQ) {
+ entry->opcode = IBV_WC_SEND;
+
+ entry->wr_id =
+ nesuqp->send_wr_id[nesuqp->sq_tail];
+
+ /* Working on a SQ Completion*/
+ if (++nesuqp->sq_tail >= nesuqp->sq_size)
+ nesuqp->sq_tail = 0;
+ } else {
+ /* no CRC counting at all - all packets
+ go to higher layer as they are received -
+ the fastest path */
+
+ entry->byte_len = cqe_misc & 0xffff;
+ entry->opcode = IBV_WC_RECV;
+
+ entry->wr_id =
+ nesuqp->recv_wr_id[nesuqp->rq_tail];
+ if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
+ vlan_tag = le32toh(
+ cqe->cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
+ >> 16;
+ entry->sl = (vlan_tag >> 12) & 0x0f;
+ entry->pkey_index = vlan_tag & 0x0fff;
+ entry->wc_flags |= NES_WC_WITH_VLAN;
+ }
+
+
+ /* Working on a RQ Completion*/
+ if (++nesuqp->rq_tail >= nesuqp->rq_size)
+ nesuqp->rq_tail = 0;
+ if (entry->status == NES_CQ_BUF_OV_ERR)
+ entry->status = IBV_WC_LOC_LEN_ERR;
+ }
+
+ if (++head >= cq_size)
+ head = 0;
+
+ if (entry->opcode != INVAL_OP) {
+ /* it is possible that no entry will be
+ available */
+ cqe_count++;
+ entry++;
+ }
+
+ nesvctx->nesupd->udoorbell->cqe_alloc =
+ htole32(nesucq->cq_id | (1 << 16));
+ } else {
+ break;
+ }
+ }
+ nesucq->head = head;
+ return cqe_count;
+}
+
+/**
+ * nes_upoll_cq
+ */
+int nes_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry)
+{
+ uint64_t wrid;
+ struct nes_ucq *nesucq;
+ struct nes_uvcontext *nesvctx = NULL;
+ struct nes_uqp *nesuqp;
+ int cqe_count=0;
+ uint32_t head;
+ uint32_t cq_size;
+ uint32_t wqe_index;
+ uint32_t wq_tail = 0;
+ struct nes_hw_cqe cqe;
+ uint64_t u64temp;
+ int move_cq_head = 1;
+ uint32_t err_code;
+
+ nesucq = to_nes_ucq(cq);
+ nesvctx = to_nes_uctx(cq->context);
+
+ if (nesucq->cq_id < 64)
+ return nes_ima_upoll_cq(cq, num_entries, entry);
+
+ pthread_spin_lock(&nesucq->lock);
+
+ head = nesucq->head;
+ cq_size = nesucq->size;
+
+ while (cqe_count<num_entries) {
+ if ((le32toh(nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) == 0)
+ break;
+
+ /* Make sure we read CQ entry contents *after* we've checked the valid bit. */
+ udma_from_device_barrier();
+
+ cqe = (volatile struct nes_hw_cqe)nesucq->cqes[head];
+
+ /* parse CQE, get completion context from WQE (either rq or sq */
+ wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]) & 511;
+ u64temp = ((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]))) |
+ (((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32);
+
+ if (likely(u64temp)) {
+ nesuqp = (struct nes_uqp *)(uintptr_t)(u64temp & (~1023));
+ memset(entry, 0, sizeof *entry);
+ if (likely(le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]) == 0)) {
+ entry->status = IBV_WC_SUCCESS;
+ } else {
+ err_code = le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]);
+ if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16)) {
+ entry->status = err_code & 0x0000ffff;
+ } else {
+ entry->status = IBV_WC_WR_FLUSH_ERR;
+ if (le32toh(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) {
+ if (wqe_index == 0 && nesuqp->rdma0_msg) {
+ nesuqp->sq_tail = (wqe_index+1)&(nesuqp->sq_size - 1);
+ move_cq_head = 0;
+ wq_tail = nesuqp->sq_tail;
+ nesuqp->rdma0_msg = 0;
+ goto nes_upoll_cq_update;
+ }
+ }
+ }
+ }
+ entry->qp_num = nesuqp->qp_id;
+ entry->src_qp = nesuqp->qp_id;
+ nesuqp->rdma0_msg = 0;
+
+ if (le32toh(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) {
+ /* Working on a SQ Completion*/
+ wrid = ((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX])) |
+ (((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
+ entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]);
+
+ switch (le32toh(nesuqp->sq_vbase[wqe_index].
+ wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) {
+ case NES_IWARP_SQ_OP_RDMAW:
+ /* fprintf(stderr, PFX "%s: Operation = RDMA WRITE.\n",
+ __FUNCTION__ ); */
+ entry->opcode = IBV_WC_RDMA_WRITE;
+ break;
+ case NES_IWARP_SQ_OP_RDMAR:
+ /* fprintf(stderr, PFX "%s: Operation = RDMA READ.\n",
+ __FUNCTION__ ); */
+ entry->opcode = IBV_WC_RDMA_READ;
+ entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index].
+ wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]);
+ break;
+ case NES_IWARP_SQ_OP_SENDINV:
+ case NES_IWARP_SQ_OP_SENDSEINV:
+ case NES_IWARP_SQ_OP_SEND:
+ case NES_IWARP_SQ_OP_SENDSE:
+ /* fprintf(stderr, PFX "%s: Operation = Send.\n",
+ __FUNCTION__ ); */
+ entry->opcode = IBV_WC_SEND;
+ break;
+ }
+
+ nesuqp->sq_tail = (wqe_index+1)&(nesuqp->sq_size - 1);
+ if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->sq_tail != nesuqp->sq_head)) {
+ move_cq_head = 0;
+ wq_tail = nesuqp->sq_tail;
+ }
+ } else {
+ /* Working on a RQ Completion*/
+ entry->byte_len = le32toh(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]);
+ wrid = ((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX])) |
+ (((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
+ entry->opcode = IBV_WC_RECV;
+
+ nesuqp->rq_tail = (wqe_index+1)&(nesuqp->rq_size - 1);
+ if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->rq_tail != nesuqp->rq_head)) {
+ move_cq_head = 0;
+ wq_tail = nesuqp->rq_tail;
+ }
+ }
+
+ entry->wr_id = wrid;
+ entry++;
+ cqe_count++;
+ }
+nes_upoll_cq_update:
+ if (move_cq_head) {
+ nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
+ if (++head >= cq_size)
+ head = 0;
+ nesucq->polled_completions++;
+
+ if ((nesucq->polled_completions > (cq_size/2)) ||
+ (nesucq->polled_completions == 255)) {
+ if (nesvctx == NULL)
+ nesvctx = to_nes_uctx(cq->context);
+ nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id |
+ (nesucq->polled_completions << 16));
+ nesucq->polled_completions = 0;
+ }
+ } else {
+ /* Update the wqe index and set status to flush */
+ wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
+ wqe_index = (wqe_index & (~511)) | wq_tail;
+ nesucq->cqes[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] =
+ htole32(wqe_index);
+ nesucq->cqes[head].cqe_words[NES_CQE_ERROR_CODE_IDX] =
+ htole32((NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH);
+ move_cq_head = 1; /* ready for next pass */
+ }
+ }
+
+ if (nesucq->polled_completions) {
+ if (nesvctx == NULL)
+ nesvctx = to_nes_uctx(cq->context);
+ nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id |
+ (nesucq->polled_completions << 16));
+ nesucq->polled_completions = 0;
+ }
+ nesucq->head = head;
+
+ pthread_spin_unlock(&nesucq->lock);
+
+ return cqe_count;
+}
+
+
+/**
+ * nes_upoll_cq_no_db_read
+ */
+int nes_upoll_cq_no_db_read(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry)
+{
+ uint64_t wrid;
+ struct nes_ucq *nesucq;
+ struct nes_uvcontext *nesvctx = NULL;
+ struct nes_uqp *nesuqp;
+ int cqe_count=0;
+ uint32_t head;
+ uint32_t cq_size;
+ uint32_t wqe_index;
+ uint32_t wq_tail = 0;
+ struct nes_hw_cqe cqe;
+ uint64_t u64temp;
+ int move_cq_head = 1;
+ uint32_t err_code;
+
+ nesucq = to_nes_ucq(cq);
+ nesvctx = to_nes_uctx(cq->context);
+
+ if (nesucq->cq_id < 64)
+ return nes_ima_upoll_cq(cq, num_entries, entry);
+
+ pthread_spin_lock(&nesucq->lock);
+
+ head = nesucq->head;
+ cq_size = nesucq->size;
+
+ while (cqe_count<num_entries) {
+ if ((le32toh(nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) == 0)
+ break;
+
+ /* Make sure we read CQ entry contents *after* we've checked the valid bit. */
+ udma_from_device_barrier();
+
+ cqe = (volatile struct nes_hw_cqe)nesucq->cqes[head];
+
+ /* parse CQE, get completion context from WQE (either rq or sq */
+ wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]) & 511;
+ u64temp = ((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]))) |
+ (((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32);
+
+ if (likely(u64temp)) {
+ nesuqp = (struct nes_uqp *)(uintptr_t)(u64temp & (~1023));
+ memset(entry, 0, sizeof *entry);
+ if (likely(le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]) == 0)) {
+ entry->status = IBV_WC_SUCCESS;
+ } else {
+ err_code = le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]);
+ if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16))
+ entry->status = err_code & 0x0000ffff;
+ else
+ entry->status = IBV_WC_WR_FLUSH_ERR;
+ }
+ entry->qp_num = nesuqp->qp_id;
+ entry->src_qp = nesuqp->qp_id;
+
+ if (le32toh(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) {
+ /* Working on a SQ Completion*/
+ wrid = ((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX])) |
+ (((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
+ entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]);
+
+ switch (le32toh(nesuqp->sq_vbase[wqe_index].
+ wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) {
+ case NES_IWARP_SQ_OP_RDMAW:
+ /* fprintf(stderr, PFX "%s: Operation = RDMA WRITE.\n",
+ __FUNCTION__ ); */
+ entry->opcode = IBV_WC_RDMA_WRITE;
+ break;
+ case NES_IWARP_SQ_OP_RDMAR:
+ /* fprintf(stderr, PFX "%s: Operation = RDMA READ.\n",
+ __FUNCTION__ ); */
+ entry->opcode = IBV_WC_RDMA_READ;
+ entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index].
+ wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]);
+ break;
+ case NES_IWARP_SQ_OP_SENDINV:
+ case NES_IWARP_SQ_OP_SENDSEINV:
+ case NES_IWARP_SQ_OP_SEND:
+ case NES_IWARP_SQ_OP_SENDSE:
+ /* fprintf(stderr, PFX "%s: Operation = Send.\n",
+ __FUNCTION__ ); */
+ entry->opcode = IBV_WC_SEND;
+ break;
+ }
+
+ nesuqp->sq_tail = (wqe_index+1)&(nesuqp->sq_size - 1);
+ if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->sq_tail != nesuqp->sq_head)) {
+ move_cq_head = 0;
+ wq_tail = nesuqp->sq_tail;
+ }
+ } else {
+ /* Working on a RQ Completion*/
+ entry->byte_len = le32toh(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]);
+ wrid = ((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX])) |
+ (((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
+ entry->opcode = IBV_WC_RECV;
+
+ nesuqp->rq_tail = (wqe_index+1)&(nesuqp->rq_size - 1);
+ if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->rq_tail != nesuqp->rq_head)) {
+ move_cq_head = 0;
+ wq_tail = nesuqp->rq_tail;
+ }
+ }
+
+ entry->wr_id = wrid;
+ entry++;
+ cqe_count++;
+ }
+
+ if (move_cq_head) {
+ nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
+ if (++head >= cq_size)
+ head = 0;
+ nesucq->polled_completions++;
+
+ if ((nesucq->polled_completions > (cq_size/2)) ||
+ (nesucq->polled_completions == 255)) {
+ if (nesvctx == NULL)
+ nesvctx = to_nes_uctx(cq->context);
+ nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id |
+ (nesucq->polled_completions << 16));
+ nesucq->polled_completions = 0;
+ }
+ } else {
+ /* Update the wqe index and set status to flush */
+ wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
+ wqe_index = (wqe_index & (~511)) | wq_tail;
+ nesucq->cqes[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] =
+ htole32(wqe_index);
+ nesucq->cqes[head].cqe_words[NES_CQE_ERROR_CODE_IDX] =
+ htole32((NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH);
+ move_cq_head = 1; /* ready for next pass */
+ }
+ }
+
+ if (nesucq->polled_completions) {
+ if (nesvctx == NULL)
+ nesvctx = to_nes_uctx(cq->context);
+ nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id |
+ (nesucq->polled_completions << 16));
+ nesucq->polled_completions = 0;
+ }
+ nesucq->head = head;
+
+ pthread_spin_unlock(&nesucq->lock);
+
+ return cqe_count;
+}
+
+/**
+ * nes_arm_cq
+ */
+static void nes_arm_cq(struct nes_ucq *nesucq, struct nes_uvcontext *nesvctx, int sol)
+{
+ uint32_t cq_arm;
+
+ cq_arm = nesucq->cq_id;
+
+ if (sol)
+ cq_arm |= NES_CQE_ALLOC_NOTIFY_SE;
+ else
+ cq_arm |= NES_CQE_ALLOC_NOTIFY_NEXT;
+
+ nesvctx->nesupd->udoorbell->cqe_alloc = htole32(cq_arm);
+ nesucq->is_armed = 1;
+ nesucq->arm_sol = sol;
+ nesucq->skip_arm = 0;
+ nesucq->skip_sol = 1;
+}
+
+/**
+ * nes_uarm_cq
+ */
+int nes_uarm_cq(struct ibv_cq *cq, int solicited)
+{
+ struct nes_ucq *nesucq;
+ struct nes_uvcontext *nesvctx;
+
+ nesucq = to_nes_ucq(cq);
+ nesvctx = to_nes_uctx(cq->context);
+
+ pthread_spin_lock(&nesucq->lock);
+
+ if (nesucq->is_armed) {
+ /* don't arm again unless... */
+ if ((nesucq->arm_sol) && (!solicited)) {
+ /* solicited changed from notify SE to notify next */
+ nes_arm_cq(nesucq, nesvctx, solicited);
+ } else {
+ nesucq->skip_arm = 1;
+ nesucq->skip_sol &= solicited;
+ }
+ } else {
+ nes_arm_cq(nesucq, nesvctx, solicited);
+ }
+
+ pthread_spin_unlock(&nesucq->lock);
+
+ return 0;
+}
+
+
+/**
+ * nes_cq_event
+ */
+void nes_cq_event(struct ibv_cq *cq)
+{
+ struct nes_ucq *nesucq;
+
+ nesucq = to_nes_ucq(cq);
+
+ pthread_spin_lock(&nesucq->lock);
+
+ if (nesucq->skip_arm) {
+ struct nes_uvcontext *nesvctx;
+ nesvctx = to_nes_uctx(cq->context);
+ nes_arm_cq(nesucq, nesvctx, nesucq->skip_sol);
+ } else {
+ nesucq->is_armed = 0;
+ }
+
+ pthread_spin_unlock(&nesucq->lock);
+}
+
+
+/**
+ * nes_ucreate_srq
+ */
+struct ibv_srq *nes_ucreate_srq(struct ibv_pd *pd, struct ibv_srq_init_attr *attr)
+{
+ /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+ return (void *)-ENOSYS;
+}
+
+
+/**
+ * nes_umodify_srq
+ */
+int nes_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, int attr_mask)
+{
+ /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+ return -ENOSYS;
+}
+
+
+/**
+ * nes_udestroy_srq
+ */
+int nes_udestroy_srq(struct ibv_srq *srq)
+{
+ /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+ return -ENOSYS;
+}
+
+
+/**
+ * nes_upost_srq_recv
+ */
+int nes_upost_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr,
+ struct ibv_recv_wr **bad_wr)
+{
+ /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+ return -ENOSYS;
+}
+
+
+/**
+ * nes_mmapped_qp
+ * will not invoke registration of memory reqion and will allow
+ * the kernel module to allocate big chunk of contigous memory
+ * for sq and rq... returns 1 if succeeds, 0 if fails..
+ */
+static int nes_mmapped_qp(struct nes_uqp *nesuqp, struct ibv_pd *pd, struct ibv_qp_init_attr *attr,
+ struct nes_ucreate_qp_resp *resp)
+{
+
+ unsigned long mmap_offset;
+ struct nes_ucreate_qp cmd;
+ struct nes_uvcontext *nesvctx = to_nes_uctx(pd->context);
+ int ret;
+
+ memset (&cmd, 0, sizeof(cmd) );
+ cmd.user_qp_buffer = (__u64) ((uintptr_t) nesuqp);
+
+ /* fprintf(stderr, PFX "%s entering==>\n",__FUNCTION__); */
+ ret = ibv_cmd_create_qp(pd, &nesuqp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd,
+ &resp->ibv_resp, sizeof (struct nes_ucreate_qp_resp) );
+ if (ret)
+ return 0;
+ nesuqp->send_cq = to_nes_ucq(attr->send_cq);
+ nesuqp->recv_cq = to_nes_ucq(attr->recv_cq);
+ nesuqp->sq_db_index = resp->mmap_sq_db_index;
+ nesuqp->rq_db_index = resp->mmap_rq_db_index;
+ nesuqp->sq_size = resp->actual_sq_size;
+ nesuqp->rq_size = resp->actual_rq_size;
+
+ /* Map the SQ/RQ buffers */
+ mmap_offset = nesvctx->max_pds*page_size;
+ mmap_offset += (((sizeof(struct nes_hw_qp_wqe) * nesvctx->wq_size) + page_size-1) &
+ (~(page_size-1)))*nesuqp->sq_db_index;
+
+ nesuqp->sq_vbase = mmap(NULL, (nesuqp->sq_size+nesuqp->rq_size) *
+ sizeof(struct nes_hw_qp_wqe), PROT_WRITE | PROT_READ,
+ MAP_SHARED, pd->context->cmd_fd, mmap_offset);
+
+
+ if (nesuqp->sq_vbase == MAP_FAILED) {
+ return 0;
+ }
+ nesuqp->rq_vbase = (struct nes_hw_qp_wqe *)(((char *)nesuqp->sq_vbase) +
+ (nesuqp->sq_size*sizeof(struct nes_hw_qp_wqe)));
+ *((unsigned int *)nesuqp->sq_vbase) = 0;
+ nesuqp->mapping = NES_QP_MMAP;
+
+ return 1;
+}
+
+
+/**
+ * nes_vmapped_qp
+ * invoke registration of memory reqion. This method is used
+ * when kernel can not allocate qp memory (contigous physical).
+ *
+ * returns 1 if succeeds, 0 if fails..
+ */
+static int nes_vmapped_qp(struct nes_uqp *nesuqp, struct ibv_pd *pd, struct ibv_qp_init_attr *attr,
+ struct nes_ucreate_qp_resp *resp, int sqdepth, int rqdepth)
+{
+ struct nes_ucreate_qp cmd;
+ struct nes_ureg_mr reg_mr_cmd;
+ struct ib_uverbs_reg_mr_resp reg_mr_resp;
+ int totalqpsize;
+ int ret;
+
+ // fprintf(stderr, PFX "%s\n", __FUNCTION__);
+ totalqpsize = (sqdepth + rqdepth) * sizeof (struct nes_hw_qp_wqe) ;
+ nesuqp->sq_vbase = memalign(page_size, totalqpsize);
+ if (!nesuqp->sq_vbase) {
+ // fprintf(stderr, PFX "CREATE_QP could not allocate mem of size %d\n", totalqpsize);
+ return 0;
+ }
+ nesuqp->rq_vbase = (struct nes_hw_qp_wqe *) (((char *) nesuqp->sq_vbase) +
+ (nesuqp->sq_size * sizeof(struct nes_hw_qp_wqe)));
+
+ reg_mr_cmd.reg_type = IWNES_MEMREG_TYPE_QP;
+
+ //fprintf(stderr, PFX "qp_rq_vbase = %p qp_sq_vbase=%p reg_mr = %p\n",
+ // nesuqp->rq_vbase, nesuqp->sq_vbase, &nesuqp->mr);
+
+ ret = ibv_cmd_reg_mr(pd, (void *)nesuqp->sq_vbase,totalqpsize,
+ (uintptr_t)nesuqp->sq_vbase,
+ IBV_ACCESS_LOCAL_WRITE, &nesuqp->vmr,
+ &reg_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd),
+ &reg_mr_resp, sizeof(reg_mr_resp));
+ if (ret) {
+ // fprintf(stderr, PFX "%s ibv_cmd_reg_mr failed (ret = %d).\n", __FUNCTION__, ret);
+ free((void *) nesuqp->sq_vbase);
+ return 0;
+ }
+ // So now the memory has been registered..
+ memset (&cmd, 0, sizeof(cmd) );
+ cmd.user_wqe_buffers = (__u64) ((uintptr_t) nesuqp->sq_vbase);
+ cmd.user_qp_buffer = (__u64) ((uintptr_t) nesuqp);
+ ret = ibv_cmd_create_qp(pd, &nesuqp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd,
+ &resp->ibv_resp, sizeof (struct nes_ucreate_qp_resp) );
+ if (ret) {
+ ibv_cmd_dereg_mr(&nesuqp->vmr);
+ free((void *)nesuqp->sq_vbase);
+ return 0;
+ }
+ *((unsigned int *)nesuqp->rq_vbase) = 0;
+ nesuqp->send_cq = to_nes_ucq(attr->send_cq);
+ nesuqp->recv_cq = to_nes_ucq(attr->recv_cq);
+ nesuqp->sq_db_index = resp->mmap_sq_db_index;
+ nesuqp->rq_db_index = resp->mmap_rq_db_index;
+ nesuqp->sq_size = resp->actual_sq_size;
+ nesuqp->rq_size = resp->actual_rq_size;
+ nesuqp->mapping = NES_QP_VMAP;
+ return 1;
+}
+
+
+/**
+ * nes_qp_get_qdepth
+ * This routine will return the size of qdepth to be set for one
+ * of the qp (sq or rq)
+ */
+static int nes_qp_get_qdepth(uint32_t qdepth, uint32_t maxsges)
+{
+ int retdepth;
+
+ /* Do sanity check on the parameters */
+ /* Should the following be 510 or 511 */
+ if ((qdepth > 510) || (maxsges > 4) )
+ return 0;
+
+ /* Do we need to do the following of */
+ /* we can just return the actual value.. needed for alignment */
+ if (qdepth < 32)
+ retdepth = 32;
+ else if (qdepth < 128)
+ retdepth = 128;
+ else retdepth = 512;
+
+ return retdepth;
+}
+
+
+/**
+ * nes_ucreate_qp
+ */
+struct ibv_qp *nes_ucreate_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
+{
+ struct nes_ucreate_qp_resp resp;
+ struct nes_uvcontext *nesvctx = to_nes_uctx(pd->context);
+ struct nes_uqp *nesuqp;
+ int sqdepth, rqdepth;
+ int status = 1;
+
+ /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+
+ /* Sanity check QP size before proceeding */
+ sqdepth = nes_qp_get_qdepth(attr->cap.max_send_wr, attr->cap.max_send_sge);
+ if (!sqdepth) {
+ fprintf(stderr, PFX "%s Bad sq attr parameters max_send_wr=%d max_send_sge=%d\n",
+ __FUNCTION__, attr->cap.max_send_wr,attr->cap.max_send_sge);
+ return NULL;
+ }
+
+ rqdepth = nes_qp_get_qdepth(attr->cap.max_recv_wr, attr->cap.max_recv_sge);
+ if (!rqdepth) {
+ fprintf(stderr, PFX "%s Bad rq attr parameters max_recv_wr=%d max_recv_sge=%d\n",
+ __FUNCTION__, attr->cap.max_recv_wr,attr->cap.max_recv_sge);
+ return NULL;
+ }
+
+ nesuqp = memalign(1024, sizeof(*nesuqp));
+ if (!nesuqp)
+ return NULL;
+ memset(nesuqp, 0, sizeof(*nesuqp));
+
+ if (pthread_spin_init(&nesuqp->lock, PTHREAD_PROCESS_PRIVATE)) {
+ free(nesuqp);
+ return NULL;
+ }
+
+ /* Initially setting it up so we will know how much memory to allocate for mapping */
+ /* also setting it up in attr.. If we do not want to modify the attr struct, we */
+ /* can save the original values and restore them before return. */
+ nesuqp->sq_size = attr->cap.max_send_wr = sqdepth;
+ nesuqp->rq_size = attr->cap.max_recv_wr = rqdepth;
+
+ nesuqp->sq_sig_all = attr->sq_sig_all;
+ if (nesvctx->virtwq) {
+ status = nes_vmapped_qp(nesuqp,pd, attr,&resp,sqdepth,rqdepth);
+ }else {
+ status = nes_mmapped_qp(nesuqp,pd,attr, &resp);
+ }
+
+ if (!status) {
+ pthread_spin_destroy(&nesuqp->lock);
+ free(nesuqp);
+ return NULL;
+ }
+
+
+ /* The following are the common parameters no matter how the */
+ /* sq and rq memory was mapped.. */
+
+ /* Account for LSMM, in theory, could get overrun if app preposts to SQ */
+ nesuqp->sq_head = 1;
+ nesuqp->sq_tail = 1;
+ nesuqp->qp_id = resp.qp_id;
+ nesuqp->nes_drv_opt = resp.nes_drv_opt;
+ nesuqp->ibv_qp.qp_num = resp.qp_id;
+ nesuqp->rdma0_msg = 1;
+
+ return &nesuqp->ibv_qp;
+}
+
+
+/**
+ * nes_uquery_qp
+ */
+int nes_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+ int attr_mask, struct ibv_qp_init_attr *init_attr)
+{
+ struct ibv_query_qp cmd;
+
+ /* fprintf(stderr, PFX "nes_uquery_qp: calling ibv_cmd_query_qp\n"); */
+
+ return ibv_cmd_query_qp(qp, attr, attr_mask, init_attr, &cmd, sizeof(cmd));
+}
+
+
+/**
+ * nes_umodify_qp
+ */
+int nes_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
+{
+ struct ibv_modify_qp cmd = {};
+ return ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof cmd);
+}
+
+
+/**
+ * nes_clean_cq
+ */
+static void nes_clean_cq(struct nes_uqp *nesuqp, struct nes_ucq *nesucq)
+{
+ uint32_t cq_head;
+ uint32_t lo;
+ uint32_t hi;
+ uint64_t u64temp;
+
+ pthread_spin_lock(&nesucq->lock);
+
+ cq_head = nesucq->head;
+ while (le32toh(nesucq->cqes[cq_head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
+ udma_from_device_barrier();
+ lo = le32toh(nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
+ hi = le32toh(nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]);
+ u64temp = (((uint64_t)hi) << 32) | ((uint64_t)lo);
+ u64temp &= (~1023);
+ if (u64temp == (uint64_t)(uintptr_t)nesuqp) {
+ /* Zero the context value so cqe will be ignored */
+ nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = 0;
+ nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX] = 0;
+ }
+
+ if (++cq_head >= nesucq->size)
+ cq_head = 0;
+ }
+
+ pthread_spin_unlock(&nesucq->lock);
+}
+
+
+/**
+ * nes_udestroy_qp
+ */
+int nes_udestroy_qp(struct ibv_qp *qp)
+{
+ struct nes_uqp *nesuqp = to_nes_uqp(qp);
+ int ret = 0;
+
+ // fprintf(stderr, PFX "%s addr&mr= %p \n", __FUNCTION__, &nesuqp->mr );
+
+ if (nesuqp->mapping == NES_QP_VMAP) {
+ ret = ibv_cmd_dereg_mr(&nesuqp->vmr);
+ if (ret)
+ fprintf(stderr, PFX "%s dereg_mr FAILED\n", __FUNCTION__);
+ free((void *)nesuqp->sq_vbase);
+ }
+
+ if (nesuqp->mapping == NES_QP_MMAP) {
+ munmap((void *)nesuqp->sq_vbase, (nesuqp->sq_size+nesuqp->rq_size) *
+ sizeof(struct nes_hw_qp_wqe));
+ }
+
+ ret = ibv_cmd_destroy_qp(qp);
+ if (ret) {
+ fprintf(stderr, PFX "%s FAILED\n", __FUNCTION__);
+ return ret;
+ }
+
+ pthread_spin_destroy(&nesuqp->lock);
+
+ /* Clean any pending completions from the cq(s) */
+ if (nesuqp->send_cq)
+ nes_clean_cq(nesuqp, nesuqp->send_cq);
+
+ if ((nesuqp->recv_cq) && (nesuqp->recv_cq != nesuqp->send_cq))
+ nes_clean_cq(nesuqp, nesuqp->recv_cq);
+ free(nesuqp);
+
+ return 0;
+}
+
+/**
+ * nes_upost_send
+ */
+int nes_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
+ struct ibv_send_wr **bad_wr)
+{
+ uint64_t u64temp;
+ struct nes_uqp *nesuqp = to_nes_uqp(ib_qp);
+ struct nes_upd *nesupd = to_nes_upd(ib_qp->pd);
+ struct nes_hw_qp_wqe volatile *wqe;
+ uint32_t head;
+ uint32_t qsize = nesuqp->sq_size;
+ uint32_t counter;
+ uint32_t err = 0;
+ uint32_t wqe_count = 0;
+ uint32_t outstanding_wqes;
+ uint32_t total_payload_length = 0;
+ int sge_index;
+
+ pthread_spin_lock(&nesuqp->lock);
+ udma_to_device_barrier();
+
+ head = nesuqp->sq_head;
+ while (ib_wr) {
+ if (unlikely(nesuqp->qperr)) {
+ err = -EINVAL;
+ break;
+ }
+
+ /* Check for SQ overflow */
+ outstanding_wqes = head + (2 * qsize) - nesuqp->sq_tail;
+ outstanding_wqes &= qsize - 1;
+ if (unlikely(outstanding_wqes == (qsize - 1))) {
+ err = -EINVAL;
+ break;
+ }
+ if (unlikely(ib_wr->num_sge > 4)) {
+ err = -EINVAL;
+ break;
+ }
+
+ wqe = (struct nes_hw_qp_wqe *)&nesuqp->sq_vbase[head];
+ /* fprintf(stderr, PFX "%s: QP%u: processing sq wqe at %p, head = %u.\n",
+ __FUNCTION__, nesuqp->qp_id, wqe, head); */
+ u64temp = (uint64_t) ib_wr->wr_id;
+ wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX] = htole32((uint32_t)u64temp);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX] = htole32((uint32_t)(u64temp>>32));
+ u64temp = (uint64_t)((uintptr_t)nesuqp);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX] = htole32((uint32_t)u64temp);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX] = htole32((uint32_t)(u64temp>>32));
+ udma_ordering_write_barrier();
+ wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX] |= htole32(head);
+
+ switch (ib_wr->opcode) {
+ case IBV_WR_SEND:
+ case IBV_WR_SEND_WITH_IMM:
+ /* fprintf(stderr, PFX "%s: QP%u: processing sq wqe%u. Opcode = %s\n",
+ __FUNCTION__, nesuqp->qp_id, head, "Send"); */
+ if (ib_wr->send_flags & IBV_SEND_SOLICITED) {
+ wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_SENDSE);
+ } else {
+ wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_SEND);
+ }
+
+ if (ib_wr->send_flags & IBV_SEND_FENCE) {
+ wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_READ_FENCE);
+ }
+
+ /* if (ib_wr->send_flags & IBV_SEND_INLINE) {
+ fprintf(stderr, PFX "%s: Send SEND_INLINE, length=%d\n",
+ __FUNCTION__, ib_wr->sg_list[0].length);
+ } */
+ if ((ib_wr->send_flags & IBV_SEND_INLINE) && (ib_wr->sg_list[0].length <= 64) &&
+ ((nesuqp->nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
+ (ib_wr->num_sge == 1)) {
+ memcpy((void *)&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
+ (void *)(intptr_t)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(ib_wr->sg_list[0].length);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_IMM_DATA);
+ } else {
+ total_payload_length = 0;
+ for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
+ wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] =
+ htole32((uint32_t)ib_wr->sg_list[sge_index].addr);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] =
+ htole32((uint32_t)(ib_wr->sg_list[sge_index].addr>>32));
+ wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX+(sge_index*4)] =
+ htole32(ib_wr->sg_list[sge_index].length);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX+(sge_index*4)] =
+ htole32(ib_wr->sg_list[sge_index].lkey);
+ total_payload_length += ib_wr->sg_list[sge_index].length;
+ }
+ wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] =
+ htole32(total_payload_length);
+ }
+
+ break;
+ case IBV_WR_RDMA_WRITE:
+ case IBV_WR_RDMA_WRITE_WITH_IMM:
+ /* fprintf(stderr, PFX "%s:QP%u: processing sq wqe%u. Opcode = %s\n",
+ __FUNCTION__, nesuqp->qp_id, head, "Write"); */
+ wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_RDMAW);
+
+ if (ib_wr->send_flags & IBV_SEND_FENCE) {
+ wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_READ_FENCE);
+ }
+ wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = htole32(ib_wr->wr.rdma.rkey);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = htole32(
+ (uint32_t)ib_wr->wr.rdma.remote_addr);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = htole32(
+ (uint32_t)(ib_wr->wr.rdma.remote_addr>>32));
+
+ /* if (ib_wr->send_flags & IBV_SEND_INLINE) {
+ fprintf(stderr, PFX "%s: Write SEND_INLINE, length=%d\n",
+ __FUNCTION__, ib_wr->sg_list[0].length);
+ } */
+ if ((ib_wr->send_flags & IBV_SEND_INLINE) && (ib_wr->sg_list[0].length <= 64) &&
+ ((nesuqp->nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
+ (ib_wr->num_sge == 1)) {
+ memcpy((void *)&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
+ (void *)(intptr_t)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(ib_wr->sg_list[0].length);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_IMM_DATA);
+ } else {
+ total_payload_length = 0;
+ for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
+ wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] = htole32(
+ (uint32_t)ib_wr->sg_list[sge_index].addr);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] = htole32(
+ (uint32_t)(ib_wr->sg_list[sge_index].addr>>32));
+ wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX+(sge_index*4)] = htole32(
+ ib_wr->sg_list[sge_index].length);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX+(sge_index*4)] = htole32(
+ ib_wr->sg_list[sge_index].lkey);
+ total_payload_length += ib_wr->sg_list[sge_index].length;
+ }
+ wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(total_payload_length);
+ }
+ wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] =
+ wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX];
+ break;
+ case IBV_WR_RDMA_READ:
+ /* fprintf(stderr, PFX "%s:QP%u:processing sq wqe%u. Opcode = %s\n",
+ __FUNCTION__, nesuqp->qp_id, head, "Read"); */
+ /* IWarp only supports 1 sge for RDMA reads */
+ if (ib_wr->num_sge > 1) {
+ err = -EINVAL;
+ break;
+ }
+ wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_RDMAR);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = htole32((uint32_t)ib_wr->wr.rdma.remote_addr);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = htole32((uint32_t)(ib_wr->wr.rdma.remote_addr>>32));
+ wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = htole32(ib_wr->wr.rdma.rkey);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = htole32(ib_wr->sg_list->length);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX] = htole32((uint32_t)ib_wr->sg_list->addr);
+ wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX] = htole32((uint32_t)(ib_wr->sg_list->addr>>32));
+ wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = htole32(ib_wr->sg_list->lkey);
+ break;
+ default:
+ /* error */
+ err = -EINVAL;
+ break;
+ }
+
+ if ((ib_wr->send_flags & IBV_SEND_SIGNALED) || nesuqp->sq_sig_all) {
+ /* fprintf(stderr, PFX "%s:sq wqe%u is signalled\n", __FUNCTION__, head); */
+ wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_SIGNALED_COMPL);
+ }
+ ib_wr = ib_wr->next;
+ head++;
+ wqe_count++;
+ if (head >= qsize)
+ head = 0;
+ }
+
+ nesuqp->sq_head = head;
+ udma_to_device_barrier();
+ while (wqe_count) {
+ counter = (wqe_count<(uint32_t)255) ? wqe_count : 255;
+ wqe_count -= counter;
+ nesupd->udoorbell->wqe_alloc = htole32((counter<<24) | 0x00800000 | nesuqp->qp_id);
+ }
+
+ if (err)
+ *bad_wr = ib_wr;
+
+ pthread_spin_unlock(&nesuqp->lock);
+
+ return err;
+}
+
+/**
+ * nes_upost_recv
+ */
+int nes_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr,
+ struct ibv_recv_wr **bad_wr)
+{
+ uint64_t u64temp;
+ struct nes_uqp *nesuqp = to_nes_uqp(ib_qp);
+ struct nes_upd *nesupd = to_nes_upd(ib_qp->pd);
+ struct nes_hw_qp_wqe *wqe;
+ uint32_t head;
+ uint32_t qsize = nesuqp->rq_size;
+ uint32_t counter;
+ uint32_t err = 0;
+ uint32_t wqe_count = 0;
+ uint32_t outstanding_wqes;
+ uint32_t total_payload_length;
+ int sge_index;
+
+ if (unlikely(ib_wr->num_sge > 4)) {
+ *bad_wr = ib_wr;
+ return -EINVAL;
+ }
+
+ pthread_spin_lock(&nesuqp->lock);
+ udma_to_device_barrier();
+
+ head = nesuqp->rq_head;
+ while (ib_wr) {
+ if (unlikely(nesuqp->qperr)) {
+ err = -EINVAL;
+ break;
+ }
+
+ /* Check for RQ overflow */
+ outstanding_wqes = head + (2 * qsize) - nesuqp->rq_tail;
+ outstanding_wqes &= qsize - 1;
+ if (unlikely(outstanding_wqes == (qsize - 1))) {
+ err = -EINVAL;
+ break;
+ }
+
+ wqe = (struct nes_hw_qp_wqe *)&nesuqp->rq_vbase[head];
+ u64temp = ib_wr->wr_id;
+ wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX] =
+ htole32((uint32_t)u64temp);
+ wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX] =
+ htole32((uint32_t)(u64temp >> 32));
+ u64temp = (uint64_t)((uintptr_t)nesuqp);
+ wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX] =
+ htole32((uint32_t)u64temp);
+ wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_HIGH_IDX] =
+ htole32((uint32_t)(u64temp >> 32));
+ udma_ordering_write_barrier();
+ wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX] |= htole32(head);
+
+ total_payload_length = 0;
+ for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
+ wqe->wqe_words[NES_IWARP_RQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] =
+ htole32((uint32_t)ib_wr->sg_list[sge_index].addr);
+ wqe->wqe_words[NES_IWARP_RQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] =
+ htole32((uint32_t)(ib_wr->sg_list[sge_index].addr>>32));
+ wqe->wqe_words[NES_IWARP_RQ_WQE_LENGTH0_IDX+(sge_index*4)] =
+ htole32(ib_wr->sg_list[sge_index].length);
+ wqe->wqe_words[NES_IWARP_RQ_WQE_STAG0_IDX+(sge_index*4)] =
+ htole32(ib_wr->sg_list[sge_index].lkey);
+ total_payload_length += ib_wr->sg_list[sge_index].length;
+ }
+ wqe->wqe_words[NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(total_payload_length);
+
+ ib_wr = ib_wr->next;
+ head++;
+ wqe_count++;
+ if (head >= qsize)
+ head = 0;
+ }
+
+ nesuqp->rq_head = head;
+ udma_to_device_barrier();
+ while (wqe_count) {
+ counter = (wqe_count<(uint32_t)255) ? wqe_count : 255;
+ wqe_count -= counter;
+ nesupd->udoorbell->wqe_alloc = htole32((counter << 24) | nesuqp->qp_id);
+ }
+
+ if (err)
+ *bad_wr = ib_wr;
+
+ pthread_spin_unlock(&nesuqp->lock);
+
+ return err;
+}
+
+
+/**
+ * nes_ucreate_ah
+ */
+struct ibv_ah *nes_ucreate_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
+{
+ /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+ return (void *)-ENOSYS;
+}
+
+
+/**
+ * nes_udestroy_ah
+ */
+int nes_udestroy_ah(struct ibv_ah *ah)
+{
+ /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
+ return -ENOSYS;
+}
+
+
+/**
+ * nes_uattach_mcast
+ */
+int nes_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
+{
+ int ret = 0;
+ ret = ibv_cmd_attach_mcast(qp, gid, lid);
+ nes_debug(NES_DBG_UD, "%s ret=%d\n", __func__, ret);
+ return ret;
+}
+
+
+/**
+ * nes_udetach_mcast
+ */
+int nes_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
+{
+ int ret = 0;
+ ret = ibv_cmd_detach_mcast(qp, gid, lid);
+ nes_debug(NES_DBG_UD, "%s ret=%d\n", __func__, ret);
+ return ret;
+}
+
+/**
+ * nes_async_event
+ */
+void nes_async_event(struct ibv_context *context,
+ struct ibv_async_event *event)
+{
+ struct nes_uqp *nesuqp;
+
+ switch (event->event_type) {
+ case IBV_EVENT_QP_FATAL:
+ case IBV_EVENT_QP_ACCESS_ERR:
+ /* Do not let application queue anything else to the qp */
+ nesuqp = to_nes_uqp(event->element.qp);
+ nesuqp->qperr = 1;
+ break;
+
+ default:
+ break;
+ }
+}
diff --git redhat/rdma-core.spec redhat/rdma-core.spec
index 45e5bc10493b..e0b143364991 100644
--- redhat/rdma-core.spec
+++ redhat/rdma-core.spec
@@ -144,6 +144,8 @@ Provides: libmlx5 = %{version}-%{release}
Obsoletes: libmlx5 < %{version}-%{release}
Provides: libmthca = %{version}-%{release}
Obsoletes: libmthca < %{version}-%{release}
+Provides: libnes = %{version}-%{release}
+Obsoletes: libnes < %{version}-%{release}
Provides: libocrdma = %{version}-%{release}
Obsoletes: libocrdma < %{version}-%{release}
Provides: librxe = %{version}-%{release}
@@ -167,6 +169,7 @@ Device-specific plug-in ibverbs userspace drivers are included:
- libmlx4: Mellanox ConnectX-3 InfiniBand HCA
- libmlx5: Mellanox Connect-IB/X-4+ InfiniBand HCA
- libmthca: Mellanox InfiniBand HCA
+- libnes: NetEffect RNIC
- libocrdma: Emulex OneConnect RDMA/RoCE Device
- libqedr: QLogic QL4xxx RoCE HCA
- librxe: A software implementation of the RoCE protocol
diff --git suse/rdma-core.spec suse/rdma-core.spec
index 529968b4005a..a32d8f9cb966 100644
--- suse/rdma-core.spec
+++ suse/rdma-core.spec
@@ -190,6 +190,7 @@ Obsoletes: libipathverbs-rdmav2 < %{version}-%{release}
Obsoletes: libmlx4-rdmav2 < %{version}-%{release}
Obsoletes: libmlx5-rdmav2 < %{version}-%{release}
Obsoletes: libmthca-rdmav2 < %{version}-%{release}
+Obsoletes: libnes-rdmav2 < %{version}-%{release}
Obsoletes: libocrdma-rdmav2 < %{version}-%{release}
Obsoletes: librxe-rdmav2 < %{version}-%{release}
%if 0%{?dma_coherent}
@@ -219,6 +220,7 @@ Device-specific plug-in ibverbs userspace drivers are included:
- libmlx4: Mellanox ConnectX-3 InfiniBand HCA
- libmlx5: Mellanox Connect-IB/X-4+ InfiniBand HCA
- libmthca: Mellanox InfiniBand HCA
+- libnes: NetEffect RNIC
- libocrdma: Emulex OneConnect RDMA/RoCE Device
- libqedr: QLogic QL4xxx RoCE HCA
- librxe: A software implementation of the RoCE protocol
Reference in New Issue View Git Blame Copy Permalink