rdma-core-nes.patch

   1 From 4daf5c91c1296683924cb9668c3d879da072756b Mon Sep 17 00:00:00 2001
   2 From: Jason Gunthorpe <jgg@mellanox.com>
   3 Date: Thu, 24 Oct 2019 12:50:22 -0300
   4 Subject: [PATCH] libnes: Remove libnes from rdma-core
   5
   6 Remove the userspace provider for nes after removing it from kernel.
   7
   8 Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
   9 ---
  10  CMakeLists.txt                     |    1 -
  11  MAINTAINERS                        |    5 -
  12  README.md                          |    1 -
  13  debian/control                     |    4 -
  14  debian/copyright                   |    5 -
  15  kernel-boot/rdma-description.rules |    1 -
  16  kernel-boot/rdma-hw-modules.rules  |    1 -
  17  libibverbs/verbs.h                 |    1 -
  18  providers/nes/CMakeLists.txt       |    4 -
  19  providers/nes/nes-abi.h            |   52 -
  20  providers/nes/nes_umain.c          |  220 ----
  21  providers/nes/nes_umain.h          |  383 -------
  22  providers/nes/nes_uverbs.c         | 1535 ----------------------------
  23  redhat/rdma-core.spec              |    3 -
  24  suse/rdma-core.spec                |    2 -
  25  15 files changed, 2218 deletions(-)
  26  delete mode 100644 providers/nes/CMakeLists.txt
  27  delete mode 100644 providers/nes/nes-abi.h
  28  delete mode 100644 providers/nes/nes_umain.c
  29  delete mode 100644 providers/nes/nes_umain.h
  30  delete mode 100644 providers/nes/nes_uverbs.c
  31
  32 diff --git a/CMakeLists.txt b/CMakeLists.txt
  33 index 85485ba00..85eb25936 100644
  34 --- a/CMakeLists.txt
  35 +++ b/CMakeLists.txt
  36 @@ -625,7 +625,6 @@ add_subdirectory(providers/mlx4/man)
  37  add_subdirectory(providers/mlx5)
  38  add_subdirectory(providers/mlx5/man)
  39  add_subdirectory(providers/mthca)
  40 -add_subdirectory(providers/nes) # NO SPARSE
  41  add_subdirectory(providers/ocrdma)
  42  add_subdirectory(providers/qedr)
  43  add_subdirectory(providers/vmw_pvrdma)
  44 diff --git a/debian/copyright b/debian/copyright
  45 index db4951993..6f86d1c86 100644
  46 --- a/debian/copyright
  47 +++ b/debian/copyright
  48 @@ -197,11 +197,6 @@ Copyright: 2004-2005, Topspin Communications.
  49             2005, Mellanox Technologies Ltd.
  50  License: BSD-MIT or GPL-2
  51
  52 -Files: providers/nes/*
  53 -Copyright: 2006-2010, Intel Corporation.
  54 -           2006, Open Grid Computing, Inc.
  55 -License: BSD-MIT or GPL-2
  56 -
  57  Files: providers/ocrdma/*
  58  Copyright: 2008-2013, Emulex.
  59  License: BSD-2-clause or GPL-2
  60 diff --git a/kernel-boot/rdma-description.rules b/kernel-boot/rdma-description.rules
  61 index 4ea59ba19..48a7cede9 100644
  62 --- a/kernel-boot/rdma-description.rules
  63 +++ b/kernel-boot/rdma-description.rules
  64 @@ -24,7 +24,6 @@ DRIVERS=="hfi1", ENV{ID_RDMA_OPA}="1"
  65  # Hardware that supports iWarp
  66  DRIVERS=="cxgb4", ENV{ID_RDMA_IWARP}="1"
  67  DRIVERS=="i40e", ENV{ID_RDMA_IWARP}="1"
  68 -DRIVERS=="nes", ENV{ID_RDMA_IWARP}="1"
  69
  70  # Hardware that supports RoCE
  71  DRIVERS=="be2net", ENV{ID_RDMA_ROCE}="1"
  72 diff --git a/kernel-boot/rdma-hw-modules.rules b/kernel-boot/rdma-hw-modules.rules
  73 index da4bbe363..bee416dbe 100644
  74 --- a/kernel-boot/rdma-hw-modules.rules
  75 +++ b/kernel-boot/rdma-hw-modules.rules
  76 @@ -33,7 +33,6 @@ ENV{ID_NET_DRIVER}=="enic", RUN{builtin}+="kmod load usnic_verbs"
  77  # ipathverbs
  78  # mthca
  79  # vmw_pvrdma
  80 -# nes
  81
  82  LABEL="rdma_hw_modules_net_end"
  83
  84 diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
  85 index 12a33a99a..13e7c63e7 100644
  86 --- a/libibverbs/verbs.h
  87 +++ b/libibverbs/verbs.h
  88 @@ -2153,7 +2153,6 @@ extern const struct verbs_device_ops verbs_provider_ipathverbs;
  89  extern const struct verbs_device_ops verbs_provider_mlx4;
  90  extern const struct verbs_device_ops verbs_provider_mlx5;
  91  extern const struct verbs_device_ops verbs_provider_mthca;
  92 -extern const struct verbs_device_ops verbs_provider_nes;
  93  extern const struct verbs_device_ops verbs_provider_ocrdma;
  94  extern const struct verbs_device_ops verbs_provider_qedr;
  95  extern const struct verbs_device_ops verbs_provider_rxe;
  96 diff --git a/providers/nes/CMakeLists.txt b/providers/nes/CMakeLists.txt
  97 deleted file mode 100644
  98 index 0c7fa8fad..000000000
  99 --- a/providers/nes/CMakeLists.txt
 100 +++ /dev/null
 101 @@ -1,4 +0,0 @@
 102 -rdma_provider(nes
 103 -  nes_umain.c
 104 -  nes_uverbs.c
 105 -)
 106 diff --git a/providers/nes/nes-abi.h b/providers/nes/nes-abi.h
 107 deleted file mode 100644
 108 index 0a531230b..000000000
 109 --- a/providers/nes/nes-abi.h
 110 +++ /dev/null
 111 @@ -1,52 +0,0 @@
 112 -/*
 113 - * Copyright (c) 2006 - 2010 Intel Corporation.  All rights reserved.
 114 - * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
 115 - *
 116 - * This software is available to you under a choice of one of two
 117 - * licenses.  You may choose to be licensed under the terms of the GNU
 118 - * General Public License (GPL) Version 2, available from the file
 119 - * gpl-2.0.txt in the main directory of this source tree, or the
 120 - * OpenIB.org BSD license below:
 121 - *
 122 - *     Redistribution and use in source and binary forms, with or
 123 - *     without modification, are permitted provided that the following
 124 - *     conditions are met:
 125 - *
 126 - *      - Redistributions of source code must retain the above
 127 - *        copyright notice, this list of conditions and the following
 128 - *        disclaimer.
 129 - *
 130 - *      - Redistributions in binary form must reproduce the above
 131 - *        copyright notice, this list of conditions and the following
 132 - *        disclaimer in the documentation and/or other materials
 133 - *        provided with the distribution.
 134 - *
 135 - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 136 - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 137 - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 138 - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 139 - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 140 - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 141 - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 142 - * SOFTWARE.
 143 - */
 144 -
 145 -#ifndef nes_ABI_H
 146 -#define nes_ABI_H
 147 -
 148 -#include <infiniband/kern-abi.h>
 149 -#include <rdma/nes-abi.h>
 150 -#include <kernel-abi/nes-abi.h>
 151 -
 152 -DECLARE_DRV_CMD(nes_ualloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
 153 -               empty, nes_alloc_pd_resp);
 154 -DECLARE_DRV_CMD(nes_ucreate_cq, IB_USER_VERBS_CMD_CREATE_CQ,
 155 -               nes_create_cq_req, nes_create_cq_resp);
 156 -DECLARE_DRV_CMD(nes_ucreate_qp, IB_USER_VERBS_CMD_CREATE_QP,
 157 -               nes_create_qp_req, nes_create_qp_resp);
 158 -DECLARE_DRV_CMD(nes_get_context, IB_USER_VERBS_CMD_GET_CONTEXT,
 159 -               nes_alloc_ucontext_req, nes_alloc_ucontext_resp);
 160 -DECLARE_DRV_CMD(nes_ureg_mr, IB_USER_VERBS_CMD_REG_MR,
 161 -               nes_mem_reg_req, empty);
 162 -
 163 -#endif                 /* nes_ABI_H */
 164 diff --git a/providers/nes/nes_umain.c b/providers/nes/nes_umain.c
 165 deleted file mode 100644
 166 index 07aa7ddd1..000000000
 167 --- a/providers/nes/nes_umain.c
 168 +++ /dev/null
 169 @@ -1,220 +0,0 @@
 170 -/*
 171 - * Copyright (c) 2006 - 2010 Intel Corporation.  All rights reserved.
 172 - * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
 173 - *
 174 - * This software is available to you under a choice of one of two
 175 - * licenses.  You may choose to be licensed under the terms of the GNU
 176 - * General Public License (GPL) Version 2, available from the file
 177 - * gpl-2.0.txt in the main directory of this source tree, or the
 178 - * OpenIB.org BSD license below:
 179 - *
 180 - *     Redistribution and use in source and binary forms, with or
 181 - *     without modification, are permitted provided that the following
 182 - *     conditions are met:
 183 - *
 184 - *      - Redistributions of source code must retain the above
 185 - *        copyright notice, this list of conditions and the following
 186 - *        disclaimer.
 187 - *
 188 - *      - Redistributions in binary form must reproduce the above
 189 - *        copyright notice, this list of conditions and the following
 190 - *        disclaimer in the documentation and/or other materials
 191 - *        provided with the distribution.
 192 - *
 193 - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 194 - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 195 - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 196 - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 197 - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 198 - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 199 - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 200 - * SOFTWARE.
 201 - */
 202 -
 203 -#include <config.h>
 204 -
 205 -#include <stdio.h>
 206 -#include <stdlib.h>
 207 -#include <string.h>
 208 -#include <unistd.h>
 209 -#include <errno.h>
 210 -#include <sys/mman.h>
 211 -#include <pthread.h>
 212 -
 213 -#include "nes_umain.h"
 214 -#include "nes-abi.h"
 215 -
 216 -unsigned int nes_debug_level = 0;
 217 -long int page_size;
 218 -
 219 -#include <sys/types.h>
 220 -#include <sys/stat.h>
 221 -#include <fcntl.h>
 222 -
 223 -#ifndef PCI_VENDOR_ID_NETEFFECT
 224 -#define PCI_VENDOR_ID_NETEFFECT                0x1678
 225 -#endif
 226 -
 227 -#define HCA(v, d) VERBS_PCI_MATCH(PCI_VENDOR_ID_##v, d, NULL)
 228 -static const struct verbs_match_ent hca_table[] = {
 229 -       VERBS_DRIVER_ID(RDMA_DRIVER_NES),
 230 -       HCA(NETEFFECT, 0x0100),
 231 -       HCA(NETEFFECT, 0x0110),
 232 -       {},
 233 -};
 234 -
 235 -static const struct verbs_context_ops nes_uctx_ops = {
 236 -       .query_device = nes_uquery_device,
 237 -       .query_port = nes_uquery_port,
 238 -       .alloc_pd = nes_ualloc_pd,
 239 -       .dealloc_pd = nes_ufree_pd,
 240 -       .reg_mr = nes_ureg_mr,
 241 -       .dereg_mr = nes_udereg_mr,
 242 -       .create_cq = nes_ucreate_cq,
 243 -       .poll_cq = nes_upoll_cq,
 244 -       .req_notify_cq = nes_uarm_cq,
 245 -       .cq_event = nes_cq_event,
 246 -       .resize_cq = nes_uresize_cq,
 247 -       .destroy_cq = nes_udestroy_cq,
 248 -       .create_qp = nes_ucreate_qp,
 249 -       .query_qp = nes_uquery_qp,
 250 -       .modify_qp = nes_umodify_qp,
 251 -       .destroy_qp = nes_udestroy_qp,
 252 -       .post_send = nes_upost_send,
 253 -       .post_recv = nes_upost_recv,
 254 -       .create_ah = nes_ucreate_ah,
 255 -       .destroy_ah = nes_udestroy_ah,
 256 -       .attach_mcast = nes_uattach_mcast,
 257 -       .detach_mcast = nes_udetach_mcast,
 258 -       .async_event = nes_async_event
 259 -};
 260 -
 261 -static const struct verbs_context_ops nes_uctx_no_db_ops = {
 262 -       .poll_cq = nes_upoll_cq_no_db_read,
 263 -};
 264 -
 265 -
 266 -/**
 267 - * nes_ualloc_context
 268 - */
 269 -static struct verbs_context *nes_ualloc_context(struct ibv_device *ibdev,
 270 -                                               int cmd_fd,
 271 -                                               void *private_data)
 272 -{
 273 -       struct ibv_pd *ibv_pd;
 274 -       struct nes_uvcontext *nesvctx;
 275 -       struct nes_get_context cmd;
 276 -       struct nes_get_context_resp resp;
 277 -       char value[16];
 278 -       uint32_t nes_drv_opt = 0;
 279 -
 280 -       page_size = sysconf(_SC_PAGESIZE);
 281 -
 282 -       nesvctx = verbs_init_and_alloc_context(ibdev, cmd_fd, nesvctx, ibv_ctx,
 283 -                                              RDMA_DRIVER_NES);
 284 -       if (!nesvctx)
 285 -               return NULL;
 286 -
 287 -       cmd.userspace_ver = NES_ABI_USERSPACE_VER;
 288 -
 289 -       if (ibv_cmd_get_context(&nesvctx->ibv_ctx, (struct ibv_get_context *)&cmd, sizeof cmd,
 290 -                               &resp.ibv_resp, sizeof(resp)))
 291 -               goto err_free;
 292 -
 293 -       if (resp.kernel_ver != NES_ABI_KERNEL_VER) {
 294 -               fprintf(stderr, PFX "%s: Invalid kernel driver version detected. Detected %d, should be %d\n",
 295 -                       __FUNCTION__, resp.kernel_ver, NES_ABI_KERNEL_VER);
 296 -               goto err_free;
 297 -       }
 298 -
 299 -       if (ibv_read_sysfs_file("/sys/module/iw_nes", "parameters/nes_drv_opt",
 300 -                       value, sizeof(value)) > 0) {
 301 -               sscanf(value, "%d", &nes_drv_opt);
 302 -       } else if (ibv_read_sysfs_file("/sys/module/iw_nes", "nes_drv_opt",
 303 -                               value, sizeof(value)) > 0) {
 304 -                       sscanf(value, "%d", &nes_drv_opt);
 305 -       }
 306 -
 307 -       verbs_set_ops(&nesvctx->ibv_ctx, &nes_uctx_ops);
 308 -       if (nes_drv_opt & NES_DRV_OPT_NO_DB_READ)
 309 -               verbs_set_ops(&nesvctx->ibv_ctx, &nes_uctx_no_db_ops);
 310 -
 311 -       nesvctx->max_pds = resp.max_pds;
 312 -       nesvctx->max_qps = resp.max_qps;
 313 -       nesvctx->wq_size = resp.wq_size;
 314 -       nesvctx->virtwq = resp.virtwq;
 315 -       nesvctx->mcrqf = 0;
 316 -
 317 -       /* Get a doorbell region for the CQs */
 318 -       ibv_pd = nes_ualloc_pd(&nesvctx->ibv_ctx.context);
 319 -       if (!ibv_pd)
 320 -               goto err_free;
 321 -       ibv_pd->context = &nesvctx->ibv_ctx.context;
 322 -       nesvctx->nesupd = to_nes_upd(ibv_pd);
 323 -
 324 -       return &nesvctx->ibv_ctx;
 325 -
 326 -err_free:
 327 -       fprintf(stderr, PFX "%s: Failed to allocate context for device.\n", __FUNCTION__);
 328 -       verbs_uninit_context(&nesvctx->ibv_ctx);
 329 -       free(nesvctx);
 330 -
 331 -       return NULL;
 332 -}
 333 -
 334 -
 335 -/**
 336 - * nes_ufree_context
 337 - */
 338 -static void nes_ufree_context(struct ibv_context *ibctx)
 339 -{
 340 -       struct nes_uvcontext *nesvctx = to_nes_uctx(ibctx);
 341 -       nes_ufree_pd(&nesvctx->nesupd->ibv_pd);
 342 -
 343 -       verbs_uninit_context(&nesvctx->ibv_ctx);
 344 -       free(nesvctx);
 345 -}
 346 -
 347 -static void nes_uninit_device(struct verbs_device *verbs_device)
 348 -{
 349 -       struct nes_udevice *dev = to_nes_udev(&verbs_device->device);
 350 -
 351 -       free(dev);
 352 -}
 353 -
 354 -static struct verbs_device *
 355 -nes_device_alloc(struct verbs_sysfs_dev *sysfs_dev)
 356 -{
 357 -       struct nes_udevice *dev;
 358 -       char value[16];
 359 -
 360 -       if (ibv_read_sysfs_file("/sys/module/iw_nes", "parameters/debug_level",
 361 -                       value, sizeof(value)) > 0) {
 362 -               sscanf(value, "%u", &nes_debug_level);
 363 -       } else if (ibv_read_sysfs_file("/sys/module/iw_nes", "debug_level",
 364 -                               value, sizeof(value)) > 0) {
 365 -                       sscanf(value, "%u", &nes_debug_level);
 366 -       }
 367 -
 368 -       dev = calloc(1, sizeof(*dev));
 369 -       if (!dev)
 370 -               return NULL;
 371 -
 372 -       dev->page_size = sysconf(_SC_PAGESIZE);
 373 -
 374 -       nes_debug(NES_DBG_INIT, "libnes initialized\n");
 375 -
 376 -       return &dev->ibv_dev;
 377 -}
 378 -
 379 -static const struct verbs_device_ops nes_udev_ops = {
 380 -       .name = "nes",
 381 -       .match_min_abi_version = 0,
 382 -       .match_max_abi_version = INT_MAX,
 383 -       .match_table = hca_table,
 384 -       .alloc_device = nes_device_alloc,
 385 -       .uninit_device = nes_uninit_device,
 386 -       .alloc_context = nes_ualloc_context,
 387 -       .free_context = nes_ufree_context,
 388 -};
 389 -PROVIDER_DRIVER(nes, nes_udev_ops);
 390 diff --git a/providers/nes/nes_umain.h b/providers/nes/nes_umain.h
 391 deleted file mode 100644
 392 index 1070ce429..000000000
 393 --- a/providers/nes/nes_umain.h
 394 +++ /dev/null
 395 @@ -1,383 +0,0 @@
 396 -/*
 397 - * Copyright (c) 2006 - 2010 Intel Corporation.  All rights reserved.
 398 - * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved.
 399 - *
 400 - * This software is available to you under a choice of one of two
 401 - * licenses.  You may choose to be licensed under the terms of the GNU
 402 - * General Public License (GPL) Version 2, available from the file
 403 - * gpl-2.0.txt in the main directory of this source tree, or the
 404 - * OpenIB.org BSD license below:
 405 - *
 406 - *     Redistribution and use in source and binary forms, with or
 407 - *     without modification, are permitted provided that the following
 408 - *     conditions are met:
 409 - *
 410 - *      - Redistributions of source code must retain the above
 411 - *        copyright notice, this list of conditions and the following
 412 - *        disclaimer.
 413 - *
 414 - *      - Redistributions in binary form must reproduce the above
 415 - *        copyright notice, this list of conditions and the following
 416 - *        disclaimer in the documentation and/or other materials
 417 - *        provided with the distribution.
 418 - *
 419 - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 420 - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 421 - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 422 - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 423 - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 424 - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 425 - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 426 - * SOFTWARE.
 427 - */
 428 -
 429 -#ifndef nes_umain_H
 430 -#define nes_umain_H
 431 -
 432 -#include <inttypes.h>
 433 -#include <stddef.h>
 434 -#include <endian.h>
 435 -#include <util/compiler.h>
 436 -
 437 -#include <infiniband/driver.h>
 438 -#include <util/udma_barrier.h>
 439 -
 440 -#define PFX    "libnes: "
 441 -
 442 -#define  NES_QP_MMAP           1
 443 -#define  NES_QP_VMAP           2
 444 -
 445 -#define NES_DRV_OPT_NO_INLINE_DATA     0x00000080
 446 -#define NES_DRV_OPT_NO_DB_READ         0x00001000
 447 -
 448 -#define NES_DEBUG
 449 -/* debug levels */
 450 -/* must match kernel */
 451 -#define NES_DBG_HW          0x00000001
 452 -#define NES_DBG_INIT        0x00000002
 453 -#define NES_DBG_ISR         0x00000004
 454 -#define NES_DBG_PHY         0x00000008
 455 -#define NES_DBG_NETDEV      0x00000010
 456 -#define NES_DBG_CM          0x00000020
 457 -#define NES_DBG_CM1         0x00000040
 458 -#define NES_DBG_NIC_RX      0x00000080
 459 -#define NES_DBG_NIC_TX      0x00000100
 460 -#define NES_DBG_CQP         0x00000200
 461 -#define NES_DBG_MMAP        0x00000400
 462 -#define NES_DBG_MR          0x00000800
 463 -#define NES_DBG_PD          0x00001000
 464 -#define NES_DBG_CQ          0x00002000
 465 -#define NES_DBG_QP          0x00004000
 466 -#define NES_DBG_MOD_QP      0x00008000
 467 -#define NES_DBG_AEQ         0x00010000
 468 -#define NES_DBG_IW_RX       0x00020000
 469 -#define NES_DBG_IW_TX       0x00040000
 470 -#define NES_DBG_SHUTDOWN    0x00080000
 471 -#define NES_DBG_UD          0x00100000
 472 -#define NES_DBG_RSVD1       0x10000000
 473 -#define NES_DBG_RSVD2       0x20000000
 474 -#define NES_DBG_RSVD3       0x40000000
 475 -#define NES_DBG_RSVD4       0x80000000
 476 -#define NES_DBG_ALL         0xffffffff
 477 -
 478 -extern unsigned int nes_debug_level;
 479 -#ifdef NES_DEBUG
 480 -#define nes_debug(level, fmt, args...) \
 481 -       if (level & nes_debug_level) \
 482 -               fprintf(stderr, PFX "%s[%u]: " fmt, __FUNCTION__, __LINE__, ##args)
 483 -#else
 484 -#define nes_debug(level, fmt, args...)
 485 -#endif
 486 -
 487 -enum nes_cqe_opcode_bits {
 488 -       NES_CQE_STAG_VALID = (1<<6),
 489 -       NES_CQE_ERROR = (1<<7),
 490 -       NES_CQE_SQ = (1<<8),
 491 -       NES_CQE_SE = (1<<9),
 492 -       NES_CQE_PSH = (1<<29),
 493 -       NES_CQE_FIN = (1<<30),
 494 -       NES_CQE_VALID = (1<<31),
 495 -};
 496 -
 497 -enum nes_cqe_word_idx {
 498 -       NES_CQE_PAYLOAD_LENGTH_IDX = 0,
 499 -       NES_CQE_COMP_COMP_CTX_LOW_IDX = 2,
 500 -       NES_CQE_COMP_COMP_CTX_HIGH_IDX = 3,
 501 -       NES_CQE_INV_STAG_IDX = 4,
 502 -       NES_CQE_QP_ID_IDX = 5,
 503 -       NES_CQE_ERROR_CODE_IDX = 6,
 504 -       NES_CQE_OPCODE_IDX = 7,
 505 -};
 506 -
 507 -enum nes_cqe_allocate_bits {
 508 -       NES_CQE_ALLOC_INC_SELECT = (1<<28),
 509 -       NES_CQE_ALLOC_NOTIFY_NEXT = (1<<29),
 510 -       NES_CQE_ALLOC_NOTIFY_SE = (1<<30),
 511 -       NES_CQE_ALLOC_RESET = (1<<31),
 512 -};
 513 -
 514 -enum nes_iwarp_sq_wqe_word_idx {
 515 -       NES_IWARP_SQ_WQE_MISC_IDX = 0,
 516 -       NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX = 1,
 517 -       NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX = 2,
 518 -       NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX = 3,
 519 -       NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX = 4,
 520 -       NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX = 5,
 521 -       NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX = 7,
 522 -       NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX = 8,
 523 -       NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX = 9,
 524 -       NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX = 10,
 525 -       NES_IWARP_SQ_WQE_RDMA_STAG_IDX = 11,
 526 -       NES_IWARP_SQ_WQE_IMM_DATA_START_IDX = 12,
 527 -       NES_IWARP_SQ_WQE_FRAG0_LOW_IDX = 16,
 528 -       NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX = 17,
 529 -       NES_IWARP_SQ_WQE_LENGTH0_IDX = 18,
 530 -       NES_IWARP_SQ_WQE_STAG0_IDX = 19,
 531 -       NES_IWARP_SQ_WQE_FRAG1_LOW_IDX = 20,
 532 -       NES_IWARP_SQ_WQE_FRAG1_HIGH_IDX = 21,
 533 -       NES_IWARP_SQ_WQE_LENGTH1_IDX = 22,
 534 -       NES_IWARP_SQ_WQE_STAG1_IDX = 23,
 535 -       NES_IWARP_SQ_WQE_FRAG2_LOW_IDX = 24,
 536 -       NES_IWARP_SQ_WQE_FRAG2_HIGH_IDX = 25,
 537 -       NES_IWARP_SQ_WQE_LENGTH2_IDX = 26,
 538 -       NES_IWARP_SQ_WQE_STAG2_IDX = 27,
 539 -       NES_IWARP_SQ_WQE_FRAG3_LOW_IDX = 28,
 540 -       NES_IWARP_SQ_WQE_FRAG3_HIGH_IDX = 29,
 541 -       NES_IWARP_SQ_WQE_LENGTH3_IDX = 30,
 542 -       NES_IWARP_SQ_WQE_STAG3_IDX = 31,
 543 -};
 544 -
 545 -enum nes_iwarp_rq_wqe_word_idx {
 546 -       NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX = 1,
 547 -       NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX = 2,
 548 -       NES_IWARP_RQ_WQE_COMP_CTX_HIGH_IDX = 3,
 549 -       NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX = 4,
 550 -       NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX = 5,
 551 -       NES_IWARP_RQ_WQE_FRAG0_LOW_IDX = 8,
 552 -       NES_IWARP_RQ_WQE_FRAG0_HIGH_IDX = 9,
 553 -       NES_IWARP_RQ_WQE_LENGTH0_IDX = 10,
 554 -       NES_IWARP_RQ_WQE_STAG0_IDX = 11,
 555 -       NES_IWARP_RQ_WQE_FRAG1_LOW_IDX = 12,
 556 -       NES_IWARP_RQ_WQE_FRAG1_HIGH_IDX = 13,
 557 -       NES_IWARP_RQ_WQE_LENGTH1_IDX = 14,
 558 -       NES_IWARP_RQ_WQE_STAG1_IDX = 15,
 559 -       NES_IWARP_RQ_WQE_FRAG2_LOW_IDX = 16,
 560 -       NES_IWARP_RQ_WQE_FRAG2_HIGH_IDX = 17,
 561 -       NES_IWARP_RQ_WQE_LENGTH2_IDX = 18,
 562 -       NES_IWARP_RQ_WQE_STAG2_IDX = 19,
 563 -       NES_IWARP_RQ_WQE_FRAG3_LOW_IDX = 20,
 564 -       NES_IWARP_RQ_WQE_FRAG3_HIGH_IDX = 21,
 565 -       NES_IWARP_RQ_WQE_LENGTH3_IDX = 22,
 566 -       NES_IWARP_RQ_WQE_STAG3_IDX = 23,
 567 -};
 568 -
 569 -enum nes_iwarp_sq_opcodes {
 570 -       NES_IWARP_SQ_WQE_STREAMING = (1<<23),
 571 -       NES_IWARP_SQ_WQE_IMM_DATA = (1<<28),
 572 -       NES_IWARP_SQ_WQE_READ_FENCE = (1<<29),
 573 -       NES_IWARP_SQ_WQE_LOCAL_FENCE = (1<<30),
 574 -       NES_IWARP_SQ_WQE_SIGNALED_COMPL = (1<<31),
 575 -};
 576 -
 577 -enum nes_iwarp_sq_wqe_bits {
 578 -       NES_IWARP_SQ_OP_RDMAW = 0,
 579 -       NES_IWARP_SQ_OP_RDMAR = 1,
 580 -       NES_IWARP_SQ_OP_SEND = 3,
 581 -       NES_IWARP_SQ_OP_SENDINV = 4,
 582 -       NES_IWARP_SQ_OP_SENDSE = 5,
 583 -       NES_IWARP_SQ_OP_SENDSEINV = 6,
 584 -       NES_IWARP_SQ_OP_BIND = 8,
 585 -       NES_IWARP_SQ_OP_FAST_REG = 9,
 586 -       NES_IWARP_SQ_OP_LOCINV = 10,
 587 -       NES_IWARP_SQ_OP_RDMAR_LOCINV = 11,
 588 -       NES_IWARP_SQ_OP_NOP = 12,
 589 -};
 590 -
 591 -enum nes_nic_cqe_word_idx {
 592 -       NES_NIC_CQE_ACCQP_ID_IDX = 0,
 593 -       NES_NIC_CQE_TAG_PKT_TYPE_IDX = 2,
 594 -       NES_NIC_CQE_MISC_IDX = 3,
 595 -};
 596 -
 597 -#define NES_NIC_CQE_ERRV_SHIFT 16
 598 -enum nes_nic_ev_bits {
 599 -       NES_NIC_ERRV_BITS_MODE = (1<<0),
 600 -       NES_NIC_ERRV_BITS_IPV4_CSUM_ERR = (1<<1),
 601 -       NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR = (1<<2),
 602 -       NES_NIC_ERRV_BITS_WQE_OVERRUN = (1<<3),
 603 -       NES_NIC_ERRV_BITS_IPH_ERR = (1<<4),
 604 -};
 605 -
 606 -enum nes_nic_cqe_bits {
 607 -       NES_NIC_CQE_ERRV_MASK = (0xff<<NES_NIC_CQE_ERRV_SHIFT),
 608 -       NES_NIC_CQE_SQ = (1<<24),
 609 -       NES_NIC_CQE_ACCQP_PORT = (1<<28),
 610 -       NES_NIC_CQE_ACCQP_VALID = (1<<29),
 611 -       NES_NIC_CQE_TAG_VALID = (1<<30),
 612 -       NES_NIC_CQE_VALID = (1<<31),
 613 -};
 614 -struct nes_hw_nic_cqe {
 615 -       uint32_t cqe_words[4];
 616 -};
 617 -
 618 -enum nes_iwarp_cqe_major_code {
 619 -       NES_IWARP_CQE_MAJOR_FLUSH = 1,
 620 -       NES_IWARP_CQE_MAJOR_DRV = 0x8000
 621 -};
 622 -
 623 -enum nes_iwarp_cqe_minor_code {
 624 -       NES_IWARP_CQE_MINOR_FLUSH = 1
 625 -};
 626 -
 627 -struct nes_hw_qp_wqe {
 628 -       uint32_t wqe_words[32];
 629 -};
 630 -
 631 -struct nes_hw_cqe {
 632 -       uint32_t cqe_words[8];
 633 -};
 634 -
 635 -struct nes_user_doorbell {
 636 -       uint32_t wqe_alloc;
 637 -       uint32_t reserved[3];
 638 -       uint32_t cqe_alloc;
 639 -};
 640 -
 641 -struct nes_udevice {
 642 -       struct verbs_device ibv_dev;
 643 -       int page_size;
 644 -};
 645 -
 646 -struct nes_upd {
 647 -       struct ibv_pd ibv_pd;
 648 -       struct nes_user_doorbell volatile *udoorbell;
 649 -       uint32_t pd_id;
 650 -       uint32_t db_index;
 651 -};
 652 -
 653 -struct nes_uvcontext {
 654 -       struct verbs_context ibv_ctx;
 655 -       struct nes_upd *nesupd;
 656 -       uint32_t max_pds; /* maximum pds allowed for this user process */
 657 -       uint32_t max_qps; /* maximum qps allowed for this user process */
 658 -       uint32_t wq_size; /* size of the WQs (sq+rq) allocated to the mmaped area */
 659 -       uint32_t mcrqf;
 660 -       uint8_t virtwq ; /*  flag if to use virt wqs or not */
 661 -       uint8_t reserved[3];
 662 -};
 663 -
 664 -struct nes_uqp;
 665 -
 666 -struct nes_ucq {
 667 -       struct ibv_cq ibv_cq;
 668 -       struct nes_hw_cqe volatile *cqes;
 669 -       struct verbs_mr vmr;
 670 -       pthread_spinlock_t lock;
 671 -       uint32_t cq_id;
 672 -       uint16_t size;
 673 -       uint16_t head;
 674 -       uint16_t polled_completions;
 675 -       uint8_t is_armed;
 676 -       uint8_t skip_arm;
 677 -       int arm_sol;
 678 -       int skip_sol;
 679 -       int comp_vector;
 680 -       struct nes_uqp *udqp;
 681 -};
 682 -
 683 -struct nes_uqp {
 684 -       struct ibv_qp ibv_qp;
 685 -       struct nes_hw_qp_wqe volatile *sq_vbase;
 686 -       struct nes_hw_qp_wqe volatile *rq_vbase;
 687 -       uint32_t qp_id;
 688 -       struct nes_ucq *send_cq;
 689 -       struct nes_ucq *recv_cq;
 690 -       struct  verbs_mr vmr;
 691 -       uint32_t nes_drv_opt;
 692 -       pthread_spinlock_t lock;
 693 -       uint16_t sq_db_index;
 694 -       uint16_t sq_head;
 695 -       uint16_t sq_tail;
 696 -       uint16_t sq_size;
 697 -       uint16_t sq_sig_all;
 698 -       uint16_t rq_db_index;
 699 -       uint16_t rq_head;
 700 -       uint16_t rq_tail;
 701 -       uint16_t rq_size;
 702 -       uint16_t rdma0_msg;
 703 -       uint16_t mapping;
 704 -       uint16_t qperr;
 705 -       uint16_t rsvd;
 706 -       uint32_t pending_rcvs;
 707 -       struct ibv_recv_wr *pend_rx_wr;
 708 -       int nes_ud_sksq_fd;
 709 -       void *sksq_shared_ctxt;
 710 -       uint64_t send_wr_id[512]; /* IMA send wr_id ring content */
 711 -       uint64_t recv_wr_id[512]; /* IMA receive wr_id ring content */
 712 -};
 713 -
 714 -#define to_nes_uxxx(xxx, type)                                                 \
 715 -       container_of(ib##xxx, struct nes_u##type, ibv_##xxx)
 716 -
 717 -static inline struct nes_udevice *to_nes_udev(struct ibv_device *ibdev)
 718 -{
 719 -       return container_of(ibdev, struct nes_udevice, ibv_dev.device);
 720 -}
 721 -
 722 -static inline struct nes_uvcontext *to_nes_uctx(struct ibv_context *ibctx)
 723 -{
 724 -       return container_of(ibctx, struct nes_uvcontext, ibv_ctx.context);
 725 -}
 726 -
 727 -static inline struct nes_upd *to_nes_upd(struct ibv_pd *ibpd)
 728 -{
 729 -       return to_nes_uxxx(pd, pd);
 730 -}
 731 -
 732 -static inline struct nes_ucq *to_nes_ucq(struct ibv_cq *ibcq)
 733 -{
 734 -       return to_nes_uxxx(cq, cq);
 735 -}
 736 -
 737 -static inline struct nes_uqp *to_nes_uqp(struct ibv_qp *ibqp)
 738 -{
 739 -       return to_nes_uxxx(qp, qp);
 740 -}
 741 -
 742 -
 743 -/* nes_uverbs.c */
 744 -int nes_uquery_device(struct ibv_context *, struct ibv_device_attr *);
 745 -int nes_uquery_port(struct ibv_context *, uint8_t, struct ibv_port_attr *);
 746 -struct ibv_pd *nes_ualloc_pd(struct ibv_context *);
 747 -int nes_ufree_pd(struct ibv_pd *);
 748 -struct ibv_mr *nes_ureg_mr(struct ibv_pd *pd, void *addr, size_t length,
 749 -                          uint64_t hca_va, int access);
 750 -int nes_udereg_mr(struct verbs_mr *vmr);
 751 -struct ibv_cq *nes_ucreate_cq(struct ibv_context *, int, struct ibv_comp_channel *, int);
 752 -int nes_uresize_cq(struct ibv_cq *, int);
 753 -int nes_udestroy_cq(struct ibv_cq *);
 754 -int nes_upoll_cq(struct ibv_cq *, int, struct ibv_wc *);
 755 -int nes_upoll_cq_no_db_read(struct ibv_cq *, int, struct ibv_wc *);
 756 -int nes_uarm_cq(struct ibv_cq *, int);
 757 -void nes_cq_event(struct ibv_cq *);
 758 -struct ibv_srq *nes_ucreate_srq(struct ibv_pd *, struct ibv_srq_init_attr *);
 759 -int nes_umodify_srq(struct ibv_srq *, struct ibv_srq_attr *, int);
 760 -int nes_udestroy_srq(struct ibv_srq *);
 761 -int nes_upost_srq_recv(struct ibv_srq *, struct ibv_recv_wr *, struct ibv_recv_wr **);
 762 -struct ibv_qp *nes_ucreate_qp(struct ibv_pd *, struct ibv_qp_init_attr *);
 763 -int nes_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 764 -                 int, struct ibv_qp_init_attr *init_attr);
 765 -int nes_umodify_qp(struct ibv_qp *, struct ibv_qp_attr *, int);
 766 -int nes_udestroy_qp(struct ibv_qp *);
 767 -int nes_upost_send(struct ibv_qp *, struct ibv_send_wr *, struct ibv_send_wr **);
 768 -int nes_upost_recv(struct ibv_qp *, struct ibv_recv_wr *, struct ibv_recv_wr **);
 769 -struct ibv_ah *nes_ucreate_ah(struct ibv_pd *, struct ibv_ah_attr *);
 770 -int nes_udestroy_ah(struct ibv_ah *);
 771 -int nes_uattach_mcast(struct ibv_qp *, const union ibv_gid *, uint16_t);
 772 -int nes_udetach_mcast(struct ibv_qp *, const union ibv_gid *, uint16_t);
 773 -void nes_async_event(struct ibv_context *context,
 774 -                    struct ibv_async_event *event);
 775 -
 776 -extern long int page_size;
 777 -
 778 -#endif                         /* nes_umain_H */
 779 diff --git a/providers/nes/nes_uverbs.c b/providers/nes/nes_uverbs.c
 780 deleted file mode 100644
 781 index 2b78468b4..000000000
 782 --- a/providers/nes/nes_uverbs.c
 783 +++ /dev/null
 784 @@ -1,1535 +0,0 @@
 785 -/*
 786 - * Copyright (c) 2006 - 2010 Intel Corporation.  All rights reserved.
 787 - *
 788 - * This software is available to you under a choice of one of two
 789 - * licenses.  You may choose to be licensed under the terms of the GNU
 790 - * General Public License (GPL) Version 2, available from the file
 791 - * gpl-2.0.txt in the main directory of this source tree, or the
 792 - * OpenIB.org BSD license below:
 793 - *
 794 - *     Redistribution and use in source and binary forms, with or
 795 - *     without modification, are permitted provided that the following
 796 - *     conditions are met:
 797 - *
 798 - *      - Redistributions of source code must retain the above
 799 - *        copyright notice, this list of conditions and the following
 800 - *        disclaimer.
 801 - *
 802 - *      - Redistributions in binary form must reproduce the above
 803 - *        copyright notice, this list of conditions and the following
 804 - *        disclaimer in the documentation and/or other materials
 805 - *        provided with the distribution.
 806 - *
 807 - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 808 - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 809 - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 810 - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 811 - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 812 - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 813 - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 814 - * SOFTWARE.
 815 - */
 816 -
 817 -#include <config.h>
 818 -
 819 -#include <endian.h>
 820 -#include <stdlib.h>
 821 -#include <stdio.h>
 822 -#include <string.h>
 823 -#include <unistd.h>
 824 -#include <signal.h>
 825 -#include <errno.h>
 826 -#include <pthread.h>
 827 -#include <malloc.h>
 828 -#include <sys/mman.h>
 829 -#include <linux/if_ether.h>
 830 -#include <sys/stat.h>
 831 -#include <fcntl.h>
 832 -
 833 -#include "nes_umain.h"
 834 -#include "nes-abi.h"
 835 -
 836 -#define STATIC static
 837 -#define INLINE inline
 838 -
 839 -#define NES_WC_WITH_VLAN   1 << 3
 840 -#define NES_UD_RX_BATCH_SZ 64
 841 -#define NES_UD_MAX_SG_LIST_SZ 1
 842 -
 843 -struct nes_ud_send_wr {
 844 -       uint32_t               wr_cnt;
 845 -       uint32_t               qpn;
 846 -       uint32_t               flags;
 847 -       uint32_t               resv[1];
 848 -       struct ibv_sge         sg_list[64];
 849 -};
 850 -
 851 -struct nes_ud_recv_wr {
 852 -       uint32_t               wr_cnt;
 853 -       uint32_t               qpn;
 854 -       uint32_t               resv[2];
 855 -       struct ibv_sge         sg_list[64];
 856 -};
 857 -
 858 -/**
 859 - * nes_uquery_device
 860 - */
 861 -int nes_uquery_device(struct ibv_context *context, struct ibv_device_attr *attr)
 862 -{
 863 -       struct ibv_query_device cmd;
 864 -       uint64_t nes_fw_ver;
 865 -       int ret;
 866 -       unsigned int minor, major;
 867 -
 868 -       ret = ibv_cmd_query_device(context, attr, &nes_fw_ver,
 869 -                                       &cmd, sizeof cmd);
 870 -       if (ret)
 871 -               return ret;
 872 -
 873 -       major = (nes_fw_ver >> 16) & 0xffff;
 874 -       minor = nes_fw_ver & 0xffff;
 875 -
 876 -       snprintf(attr->fw_ver, sizeof attr->fw_ver,
 877 -               "%d.%d", major, minor);
 878 -
 879 -       return 0;
 880 -}
 881 -
 882 -
 883 -/**
 884 - * nes_uquery_port
 885 - */
 886 -int nes_uquery_port(struct ibv_context *context, uint8_t port,
 887 -               struct ibv_port_attr *attr)
 888 -{
 889 -       struct ibv_query_port cmd;
 890 -
 891 -       return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
 892 -}
 893 -
 894 -
 895 -/**
 896 - * nes_ualloc_pd
 897 - */
 898 -struct ibv_pd *nes_ualloc_pd(struct ibv_context *context)
 899 -{
 900 -       struct ibv_alloc_pd cmd;
 901 -       struct nes_ualloc_pd_resp resp;
 902 -       struct nes_upd *nesupd;
 903 -
 904 -       nesupd = malloc(sizeof *nesupd);
 905 -       if (!nesupd)
 906 -               return NULL;
 907 -
 908 -       if (ibv_cmd_alloc_pd(context, &nesupd->ibv_pd, &cmd, sizeof cmd,
 909 -                       &resp.ibv_resp, sizeof resp)) {
 910 -               free(nesupd);
 911 -               return NULL;
 912 -       }
 913 -       nesupd->pd_id = resp.pd_id;
 914 -       nesupd->db_index = resp.mmap_db_index;
 915 -
 916 -       nesupd->udoorbell = mmap(NULL, page_size, PROT_WRITE | PROT_READ, MAP_SHARED,
 917 -                       context->cmd_fd, nesupd->db_index * page_size);
 918 -
 919 -       if (nesupd->udoorbell == MAP_FAILED) {
 920 -               free(nesupd);
 921 -               return NULL;
 922 -       }
 923 -
 924 -       return &nesupd->ibv_pd;
 925 -}
 926 -
 927 -
 928 -/**
 929 - * nes_ufree_pd
 930 - */
 931 -int nes_ufree_pd(struct ibv_pd *pd)
 932 -{
 933 -       int ret;
 934 -       struct nes_upd *nesupd;
 935 -
 936 -       nesupd = to_nes_upd(pd);
 937 -
 938 -       ret = ibv_cmd_dealloc_pd(pd);
 939 -       if (ret)
 940 -               return ret;
 941 -
 942 -       munmap((void *)nesupd->udoorbell, page_size);
 943 -       free(nesupd);
 944 -
 945 -       return 0;
 946 -}
 947 -
 948 -
 949 -/**
 950 - * nes_ureg_mr
 951 - */
 952 -struct ibv_mr *nes_ureg_mr(struct ibv_pd *pd, void *addr, size_t length,
 953 -                          uint64_t hca_va, int access)
 954 -{
 955 -       struct verbs_mr *vmr;
 956 -       struct nes_ureg_mr cmd;
 957 -       struct ib_uverbs_reg_mr_resp resp;
 958 -
 959 -       vmr = malloc(sizeof(*vmr));
 960 -       if (!vmr)
 961 -               return NULL;
 962 -
 963 -       cmd.reg_type = IWNES_MEMREG_TYPE_MEM;
 964 -       if (ibv_cmd_reg_mr(pd, addr, length, hca_va, access, vmr, &cmd.ibv_cmd,
 965 -                          sizeof(cmd), &resp, sizeof(resp))) {
 966 -               free(vmr);
 967 -
 968 -               return NULL;
 969 -       }
 970 -
 971 -       return &vmr->ibv_mr;
 972 -}
 973 -
 974 -
 975 -/**
 976 - * nes_udereg_mr
 977 - */
 978 -int nes_udereg_mr(struct verbs_mr *vmr)
 979 -{
 980 -       int ret;
 981 -
 982 -       ret = ibv_cmd_dereg_mr(vmr);
 983 -       if (ret)
 984 -               return ret;
 985 -
 986 -       free(vmr);
 987 -       return 0;
 988 -}
 989 -
 990 -/**
 991 - * nes_ucreate_cq
 992 - */
 993 -struct ibv_cq *nes_ucreate_cq(struct ibv_context *context, int cqe,
 994 -               struct ibv_comp_channel *channel, int comp_vector)
 995 -{
 996 -       struct nes_ucq *nesucq;
 997 -       struct nes_ureg_mr reg_mr_cmd;
 998 -       struct ib_uverbs_reg_mr_resp reg_mr_resp;
 999 -       struct nes_ucreate_cq cmd;
1000 -       struct nes_ucreate_cq_resp resp;
1001 -       int ret;
1002 -       struct nes_uvcontext *nesvctx = to_nes_uctx(context);
1003 -
1004 -       nesucq = malloc(sizeof *nesucq);
1005 -       if (!nesucq) {
1006 -               return NULL;
1007 -       }
1008 -       memset(nesucq, 0, sizeof(*nesucq));
1009 -
1010 -       if (pthread_spin_init(&nesucq->lock, PTHREAD_PROCESS_PRIVATE)) {
1011 -               free(nesucq);
1012 -               return NULL;
1013 -       }
1014 -
1015 -       if (cqe < 4)    /* a reasonable minimum */
1016 -               cqe = 4;
1017 -       nesucq->size = cqe + 1;
1018 -       nesucq->comp_vector = comp_vector;
1019 -
1020 -       nesucq->cqes = memalign(page_size, nesucq->size*sizeof(struct nes_hw_cqe));
1021 -       if (!nesucq->cqes)
1022 -               goto err;
1023 -
1024 -       /* Register the memory for the CQ */
1025 -       reg_mr_cmd.reg_type = IWNES_MEMREG_TYPE_CQ;
1026 -
1027 -       ret = ibv_cmd_reg_mr(&nesvctx->nesupd->ibv_pd, (void *)nesucq->cqes,
1028 -                       (nesucq->size*sizeof(struct nes_hw_cqe)),
1029 -                       (uintptr_t)nesucq->cqes, IBV_ACCESS_LOCAL_WRITE,
1030 -                       &nesucq->vmr, &reg_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd),
1031 -                       &reg_mr_resp, sizeof(reg_mr_resp));
1032 -       if (ret) {
1033 -               /* fprintf(stderr, "ibv_cmd_reg_mr failed (ret = %d).\n", ret); */
1034 -               free((struct nes_hw_cqe *)nesucq->cqes);
1035 -               goto err;
1036 -       }
1037 -
1038 -       /* Create the CQ */
1039 -       memset(&cmd, 0, sizeof(cmd));
1040 -       cmd.user_cq_buffer = (__u64)((uintptr_t)nesucq->cqes);
1041 -       cmd.mcrqf = nesvctx->mcrqf;
1042 -
1043 -       ret = ibv_cmd_create_cq(context, nesucq->size-1, channel, comp_vector,
1044 -                       &nesucq->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
1045 -                       &resp.ibv_resp, sizeof resp);
1046 -       if (ret)
1047 -               goto err;
1048 -
1049 -       nesucq->cq_id = (uint16_t)resp.cq_id;
1050 -
1051 -       /* Zero out the CQ */
1052 -       memset((struct nes_hw_cqe *)nesucq->cqes, 0, nesucq->size*sizeof(struct nes_hw_cqe));
1053 -
1054 -       return &nesucq->ibv_cq;
1055 -
1056 -err:
1057 -       /* fprintf(stderr, PFX "%s: Error Creating CQ.\n", __FUNCTION__); */
1058 -       pthread_spin_destroy(&nesucq->lock);
1059 -       free(nesucq);
1060 -
1061 -       return NULL;
1062 -}
1063 -
1064 -
1065 -/**
1066 - * nes_uresize_cq
1067 - */
1068 -int nes_uresize_cq(struct ibv_cq *cq, int cqe)
1069 -{
1070 -       /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
1071 -       return -ENOSYS;
1072 -}
1073 -
1074 -/**
1075 - * nes_udestroy_cq
1076 - */
1077 -int nes_udestroy_cq(struct ibv_cq *cq)
1078 -{
1079 -       struct nes_ucq *nesucq = to_nes_ucq(cq);
1080 -       int ret;
1081 -
1082 -       ret = ibv_cmd_destroy_cq(cq);
1083 -       if (ret)
1084 -               return ret;
1085 -
1086 -       ret = ibv_cmd_dereg_mr(&nesucq->vmr);
1087 -       if (ret)
1088 -               fprintf(stderr, PFX "%s: Failed to deregister CQ Memory Region.\n", __FUNCTION__);
1089 -
1090 -       /* Free CQ the memory */
1091 -       free((struct nes_hw_cqe *)nesucq->cqes);
1092 -       pthread_spin_destroy(&nesucq->lock);
1093 -       free(nesucq);
1094 -
1095 -       return 0;
1096 -}
1097 -
1098 -#define  NES_CQ_BUF_OV_ERR 0x3
1099 -
1100 -static inline
1101 -int nes_ima_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry)
1102 -{
1103 -       struct nes_ucq *nesucq = to_nes_ucq(cq);
1104 -       struct nes_uvcontext *nesvctx = to_nes_uctx(cq->context);
1105 -       uint32_t cqe_misc;
1106 -       int cqe_count = 0;
1107 -       uint32_t head;
1108 -       uint32_t cq_size;
1109 -
1110 -       volatile struct nes_hw_nic_cqe *cqe = NULL;
1111 -       volatile struct nes_hw_nic_cqe *cqes;
1112 -
1113 -       struct nes_uqp *nesuqp = nesucq->udqp;
1114 -       uint32_t vlan_tag = 0;
1115 -
1116 -       cqes = (volatile struct nes_hw_nic_cqe *)nesucq->cqes;
1117 -       head = nesucq->head;
1118 -       cq_size = nesucq->size;
1119 -
1120 -       if (!nesuqp || !nesvctx)
1121 -               exit(0);
1122 -       if (nesuqp->ibv_qp.state == IBV_QPS_ERR) {
1123 -               while (cqe_count < num_entries) {
1124 -                       memset(entry, 0, sizeof *entry);
1125 -
1126 -               if (nesuqp->recv_cq == nesucq) {
1127 -                       if (nesuqp->rq_tail != nesuqp->rq_head) {
1128 -                               /* Working on a RQ Completion*/
1129 -                               entry->wr_id =
1130 -                                       nesuqp->recv_wr_id[nesuqp->rq_tail];
1131 -                               if (++nesuqp->rq_tail >= nesuqp->rq_size)
1132 -                                       nesuqp->rq_tail = 0;
1133 -                       } else
1134 -                               return cqe_count;
1135 -               } else
1136 -               if (nesuqp->send_cq == nesucq) {
1137 -                       if (nesuqp->sq_tail != nesuqp->sq_head) {
1138 -                               entry->wr_id =
1139 -                                       nesuqp->send_wr_id[nesuqp->sq_tail];
1140 -                               /* Working on a SQ Completion*/
1141 -                               if (++nesuqp->sq_tail >= nesuqp->sq_size)
1142 -                                       nesuqp->sq_tail = 0;
1143 -                       } else
1144 -                               return cqe_count;
1145 -               }
1146 -               entry->status = IBV_WC_WR_FLUSH_ERR;
1147 -               entry++;
1148 -               cqe_count++;
1149 -               }
1150 -               return cqe_count;
1151 -       }
1152 -
1153 -       while (cqe_count < num_entries) {
1154 -               const enum ibv_wc_opcode INVAL_OP = -1;
1155 -
1156 -               entry->opcode = INVAL_OP;
1157 -               cqe = &cqes[head];
1158 -               cqe_misc =
1159 -                       le32toh(cqe->cqe_words[NES_NIC_CQE_MISC_IDX]);
1160 -               if (cqe_misc & NES_NIC_CQE_VALID) {
1161 -                       memset(entry, 0, sizeof *entry);
1162 -                       entry->opcode = INVAL_OP;
1163 -                       cqe->cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
1164 -                       entry->status = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >>
1165 -                                               NES_NIC_CQE_ERRV_SHIFT;
1166 -                       entry->qp_num = nesuqp->qp_id;
1167 -                       entry->src_qp = nesuqp->qp_id;
1168 -                       if (cqe_misc & NES_NIC_CQE_SQ) {
1169 -                               entry->opcode = IBV_WC_SEND;
1170 -
1171 -                               entry->wr_id =
1172 -                                       nesuqp->send_wr_id[nesuqp->sq_tail];
1173 -
1174 -                               /* Working on a SQ Completion*/
1175 -                               if (++nesuqp->sq_tail >= nesuqp->sq_size)
1176 -                                       nesuqp->sq_tail = 0;
1177 -                       } else {
1178 -                               /* no CRC counting at all - all packets
1179 -                               go to higher layer as they are received -
1180 -                               the fastest path */
1181 -
1182 -                               entry->byte_len = cqe_misc & 0xffff;
1183 -                               entry->opcode = IBV_WC_RECV;
1184 -
1185 -                               entry->wr_id =
1186 -                                       nesuqp->recv_wr_id[nesuqp->rq_tail];
1187 -                               if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
1188 -                                       vlan_tag = le32toh(
1189 -                               cqe->cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
1190 -                                                                       >> 16;
1191 -                                       entry->sl = (vlan_tag >> 12) & 0x0f;
1192 -                                       entry->pkey_index = vlan_tag & 0x0fff;
1193 -                                       entry->wc_flags |= NES_WC_WITH_VLAN;
1194 -                               }
1195 -
1196 -
1197 -                               /* Working on a RQ Completion*/
1198 -                               if (++nesuqp->rq_tail >= nesuqp->rq_size)
1199 -                                       nesuqp->rq_tail = 0;
1200 -                               if (entry->status == NES_CQ_BUF_OV_ERR)
1201 -                                       entry->status = IBV_WC_LOC_LEN_ERR;
1202 -                       }
1203 -
1204 -                       if (++head >= cq_size)
1205 -                               head = 0;
1206 -
1207 -                       if (entry->opcode != INVAL_OP) {
1208 -                               /* it is possible that no entry will be
1209 -                                 available */
1210 -                               cqe_count++;
1211 -                               entry++;
1212 -                       }
1213 -
1214 -                       nesvctx->nesupd->udoorbell->cqe_alloc =
1215 -                               htole32(nesucq->cq_id | (1 << 16));
1216 -               } else {
1217 -                       break;
1218 -               }
1219 -       }
1220 -       nesucq->head = head;
1221 -       return cqe_count;
1222 -}
1223 -
1224 -/**
1225 - * nes_upoll_cq
1226 - */
1227 -int nes_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry)
1228 -{
1229 -       uint64_t wrid;
1230 -       struct nes_ucq *nesucq;
1231 -       struct nes_uvcontext *nesvctx = NULL;
1232 -       struct nes_uqp *nesuqp;
1233 -       int cqe_count=0;
1234 -       uint32_t head;
1235 -       uint32_t cq_size;
1236 -       uint32_t wqe_index;
1237 -       uint32_t wq_tail = 0;
1238 -       struct nes_hw_cqe cqe;
1239 -       uint64_t u64temp;
1240 -       int move_cq_head = 1;
1241 -       uint32_t err_code;
1242 -
1243 -       nesucq = to_nes_ucq(cq);
1244 -       nesvctx = to_nes_uctx(cq->context);
1245 -
1246 -       if (nesucq->cq_id < 64)
1247 -               return nes_ima_upoll_cq(cq, num_entries, entry);
1248 -
1249 -       pthread_spin_lock(&nesucq->lock);
1250 -
1251 -       head = nesucq->head;
1252 -       cq_size = nesucq->size;
1253 -
1254 -       while (cqe_count<num_entries) {
1255 -               if ((le32toh(nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) == 0)
1256 -                       break;
1257 -
1258 -               /* Make sure we read CQ entry contents *after* we've checked the valid bit. */
1259 -               udma_from_device_barrier();
1260 -
1261 -               cqe = (volatile struct nes_hw_cqe)nesucq->cqes[head];
1262 -
1263 -               /* parse CQE, get completion context from WQE (either rq or sq */
1264 -               wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]) & 511;
1265 -               u64temp = ((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]))) |
1266 -                               (((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32);
1267 -
1268 -               if (likely(u64temp)) {
1269 -                       nesuqp = (struct nes_uqp *)(uintptr_t)(u64temp & (~1023));
1270 -                       memset(entry, 0, sizeof *entry);
1271 -                       if (likely(le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]) == 0)) {
1272 -                               entry->status = IBV_WC_SUCCESS;
1273 -                       } else {
1274 -                               err_code = le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]);
1275 -                               if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16)) {
1276 -                                       entry->status = err_code & 0x0000ffff;
1277 -                               } else {
1278 -                                       entry->status = IBV_WC_WR_FLUSH_ERR;
1279 -                                       if (le32toh(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) {
1280 -                                               if (wqe_index == 0 && nesuqp->rdma0_msg) {
1281 -                                                       nesuqp->sq_tail = (wqe_index+1)&(nesuqp->sq_size - 1);
1282 -                                                       move_cq_head = 0;
1283 -                                                       wq_tail = nesuqp->sq_tail;
1284 -                                                       nesuqp->rdma0_msg = 0;
1285 -                                                       goto nes_upoll_cq_update;
1286 -                                               }
1287 -                                       }
1288 -                               }
1289 -                       }
1290 -                       entry->qp_num = nesuqp->qp_id;
1291 -                       entry->src_qp = nesuqp->qp_id;
1292 -                       nesuqp->rdma0_msg = 0;
1293 -
1294 -                       if (le32toh(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) {
1295 -                               /* Working on a SQ Completion*/
1296 -                               wrid = ((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX])) |
1297 -                                       (((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
1298 -                               entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]);
1299 -
1300 -                               switch (le32toh(nesuqp->sq_vbase[wqe_index].
1301 -                                               wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) {
1302 -                                       case NES_IWARP_SQ_OP_RDMAW:
1303 -                                               /* fprintf(stderr, PFX "%s: Operation = RDMA WRITE.\n",
1304 -                                                               __FUNCTION__ ); */
1305 -                                               entry->opcode = IBV_WC_RDMA_WRITE;
1306 -                                               break;
1307 -                                       case NES_IWARP_SQ_OP_RDMAR:
1308 -                                               /* fprintf(stderr, PFX "%s: Operation = RDMA READ.\n",
1309 -                                                               __FUNCTION__ ); */
1310 -                                               entry->opcode = IBV_WC_RDMA_READ;
1311 -                                               entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index].
1312 -                                                               wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]);
1313 -                                               break;
1314 -                                       case NES_IWARP_SQ_OP_SENDINV:
1315 -                                       case NES_IWARP_SQ_OP_SENDSEINV:
1316 -                                       case NES_IWARP_SQ_OP_SEND:
1317 -                                       case NES_IWARP_SQ_OP_SENDSE:
1318 -                                               /* fprintf(stderr, PFX "%s: Operation = Send.\n",
1319 -                                                               __FUNCTION__ ); */
1320 -                                               entry->opcode = IBV_WC_SEND;
1321 -                                               break;
1322 -                               }
1323 -
1324 -                               nesuqp->sq_tail = (wqe_index+1)&(nesuqp->sq_size - 1);
1325 -                               if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->sq_tail != nesuqp->sq_head)) {
1326 -                                       move_cq_head = 0;
1327 -                                       wq_tail = nesuqp->sq_tail;
1328 -                               }
1329 -                       } else {
1330 -                               /* Working on a RQ Completion*/
1331 -                               entry->byte_len = le32toh(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]);
1332 -                               wrid = ((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX])) |
1333 -                                       (((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
1334 -                               entry->opcode = IBV_WC_RECV;
1335 -
1336 -                               nesuqp->rq_tail = (wqe_index+1)&(nesuqp->rq_size - 1);
1337 -                               if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->rq_tail != nesuqp->rq_head)) {
1338 -                                       move_cq_head = 0;
1339 -                                       wq_tail = nesuqp->rq_tail;
1340 -                               }
1341 -                       }
1342 -
1343 -                       entry->wr_id = wrid;
1344 -                       entry++;
1345 -                       cqe_count++;
1346 -               }
1347 -nes_upoll_cq_update:
1348 -               if (move_cq_head) {
1349 -                       nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
1350 -                       if (++head >= cq_size)
1351 -                               head = 0;
1352 -                       nesucq->polled_completions++;
1353 -
1354 -                       if ((nesucq->polled_completions > (cq_size/2)) ||
1355 -                                       (nesucq->polled_completions == 255)) {
1356 -                               if (nesvctx == NULL)
1357 -                                       nesvctx = to_nes_uctx(cq->context);
1358 -                               nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id |
1359 -                                               (nesucq->polled_completions << 16));
1360 -                               nesucq->polled_completions = 0;
1361 -                       }
1362 -               } else {
1363 -                       /* Update the wqe index and set status to flush */
1364 -                       wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
1365 -                       wqe_index = (wqe_index & (~511)) | wq_tail;
1366 -                       nesucq->cqes[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] =
1367 -                               htole32(wqe_index);
1368 -                       nesucq->cqes[head].cqe_words[NES_CQE_ERROR_CODE_IDX] =
1369 -                               htole32((NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH);
1370 -                       move_cq_head = 1; /* ready for next pass */
1371 -               }
1372 -       }
1373 -
1374 -       if (nesucq->polled_completions) {
1375 -               if (nesvctx == NULL)
1376 -                       nesvctx = to_nes_uctx(cq->context);
1377 -               nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id |
1378 -                               (nesucq->polled_completions << 16));
1379 -               nesucq->polled_completions = 0;
1380 -       }
1381 -       nesucq->head = head;
1382 -
1383 -       pthread_spin_unlock(&nesucq->lock);
1384 -
1385 -       return cqe_count;
1386 -}
1387 -
1388 -
1389 -/**
1390 - * nes_upoll_cq_no_db_read
1391 - */
1392 -int nes_upoll_cq_no_db_read(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry)
1393 -{
1394 -       uint64_t wrid;
1395 -       struct nes_ucq *nesucq;
1396 -       struct nes_uvcontext *nesvctx = NULL;
1397 -       struct nes_uqp *nesuqp;
1398 -       int cqe_count=0;
1399 -       uint32_t head;
1400 -       uint32_t cq_size;
1401 -       uint32_t wqe_index;
1402 -       uint32_t wq_tail = 0;
1403 -       struct nes_hw_cqe cqe;
1404 -       uint64_t u64temp;
1405 -       int move_cq_head = 1;
1406 -       uint32_t err_code;
1407 -
1408 -       nesucq = to_nes_ucq(cq);
1409 -       nesvctx = to_nes_uctx(cq->context);
1410 -
1411 -       if (nesucq->cq_id < 64)
1412 -               return nes_ima_upoll_cq(cq, num_entries, entry);
1413 -
1414 -       pthread_spin_lock(&nesucq->lock);
1415 -
1416 -       head = nesucq->head;
1417 -       cq_size = nesucq->size;
1418 -
1419 -       while (cqe_count<num_entries) {
1420 -               if ((le32toh(nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) == 0)
1421 -                       break;
1422 -
1423 -               /* Make sure we read CQ entry contents *after* we've checked the valid bit. */
1424 -               udma_from_device_barrier();
1425 -
1426 -               cqe = (volatile struct nes_hw_cqe)nesucq->cqes[head];
1427 -
1428 -               /* parse CQE, get completion context from WQE (either rq or sq */
1429 -               wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]) & 511;
1430 -               u64temp = ((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]))) |
1431 -                               (((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32);
1432 -
1433 -               if (likely(u64temp)) {
1434 -                       nesuqp = (struct nes_uqp *)(uintptr_t)(u64temp & (~1023));
1435 -                       memset(entry, 0, sizeof *entry);
1436 -                       if (likely(le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]) == 0)) {
1437 -                               entry->status = IBV_WC_SUCCESS;
1438 -                       } else {
1439 -                               err_code = le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]);
1440 -                               if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16))
1441 -                                       entry->status = err_code & 0x0000ffff;
1442 -                               else
1443 -                                       entry->status = IBV_WC_WR_FLUSH_ERR;
1444 -                       }
1445 -                       entry->qp_num = nesuqp->qp_id;
1446 -                       entry->src_qp = nesuqp->qp_id;
1447 -
1448 -                       if (le32toh(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) {
1449 -                               /* Working on a SQ Completion*/
1450 -                               wrid = ((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX])) |
1451 -                                       (((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
1452 -                               entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]);
1453 -
1454 -                               switch (le32toh(nesuqp->sq_vbase[wqe_index].
1455 -                                               wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) {
1456 -                                       case NES_IWARP_SQ_OP_RDMAW:
1457 -                                               /* fprintf(stderr, PFX "%s: Operation = RDMA WRITE.\n",
1458 -                                                               __FUNCTION__ ); */
1459 -                                               entry->opcode = IBV_WC_RDMA_WRITE;
1460 -                                               break;
1461 -                                       case NES_IWARP_SQ_OP_RDMAR:
1462 -                                               /* fprintf(stderr, PFX "%s: Operation = RDMA READ.\n",
1463 -                                                               __FUNCTION__ ); */
1464 -                                               entry->opcode = IBV_WC_RDMA_READ;
1465 -                                               entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index].
1466 -                                                               wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]);
1467 -                                               break;
1468 -                                       case NES_IWARP_SQ_OP_SENDINV:
1469 -                                       case NES_IWARP_SQ_OP_SENDSEINV:
1470 -                                       case NES_IWARP_SQ_OP_SEND:
1471 -                                       case NES_IWARP_SQ_OP_SENDSE:
1472 -                                               /* fprintf(stderr, PFX "%s: Operation = Send.\n",
1473 -                                                               __FUNCTION__ ); */
1474 -                                               entry->opcode = IBV_WC_SEND;
1475 -                                               break;
1476 -                               }
1477 -
1478 -                               nesuqp->sq_tail = (wqe_index+1)&(nesuqp->sq_size - 1);
1479 -                               if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->sq_tail != nesuqp->sq_head)) {
1480 -                                       move_cq_head = 0;
1481 -                                       wq_tail = nesuqp->sq_tail;
1482 -                               }
1483 -                       } else {
1484 -                               /* Working on a RQ Completion*/
1485 -                               entry->byte_len = le32toh(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]);
1486 -                               wrid = ((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX])) |
1487 -                                       (((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32);
1488 -                               entry->opcode = IBV_WC_RECV;
1489 -
1490 -                               nesuqp->rq_tail = (wqe_index+1)&(nesuqp->rq_size - 1);
1491 -                               if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->rq_tail != nesuqp->rq_head)) {
1492 -                                       move_cq_head = 0;
1493 -                                       wq_tail = nesuqp->rq_tail;
1494 -                               }
1495 -                       }
1496 -
1497 -                       entry->wr_id = wrid;
1498 -                       entry++;
1499 -                       cqe_count++;
1500 -               }
1501 -
1502 -               if (move_cq_head) {
1503 -                       nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
1504 -                       if (++head >= cq_size)
1505 -                               head = 0;
1506 -                       nesucq->polled_completions++;
1507 -
1508 -                       if ((nesucq->polled_completions > (cq_size/2)) ||
1509 -                                       (nesucq->polled_completions == 255)) {
1510 -                               if (nesvctx == NULL)
1511 -                                       nesvctx = to_nes_uctx(cq->context);
1512 -                               nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id |
1513 -                                               (nesucq->polled_completions << 16));
1514 -                               nesucq->polled_completions = 0;
1515 -                       }
1516 -               } else {
1517 -                       /* Update the wqe index and set status to flush */
1518 -                       wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
1519 -                       wqe_index = (wqe_index & (~511)) | wq_tail;
1520 -                       nesucq->cqes[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] =
1521 -                               htole32(wqe_index);
1522 -                       nesucq->cqes[head].cqe_words[NES_CQE_ERROR_CODE_IDX] =
1523 -                               htole32((NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH);
1524 -                       move_cq_head = 1; /* ready for next pass */
1525 -               }
1526 -       }
1527 -
1528 -       if (nesucq->polled_completions) {
1529 -               if (nesvctx == NULL)
1530 -                       nesvctx = to_nes_uctx(cq->context);
1531 -               nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id |
1532 -                               (nesucq->polled_completions << 16));
1533 -               nesucq->polled_completions = 0;
1534 -       }
1535 -       nesucq->head = head;
1536 -
1537 -       pthread_spin_unlock(&nesucq->lock);
1538 -
1539 -       return cqe_count;
1540 -}
1541 -
1542 -/**
1543 - * nes_arm_cq
1544 - */
1545 -static void nes_arm_cq(struct nes_ucq *nesucq, struct nes_uvcontext *nesvctx, int sol)
1546 -{
1547 -       uint32_t cq_arm;
1548 -
1549 -       cq_arm = nesucq->cq_id;
1550 -
1551 -       if (sol)
1552 -               cq_arm |= NES_CQE_ALLOC_NOTIFY_SE;
1553 -       else
1554 -               cq_arm |= NES_CQE_ALLOC_NOTIFY_NEXT;
1555 -
1556 -       nesvctx->nesupd->udoorbell->cqe_alloc = htole32(cq_arm);
1557 -       nesucq->is_armed = 1;
1558 -       nesucq->arm_sol = sol;
1559 -       nesucq->skip_arm = 0;
1560 -       nesucq->skip_sol = 1;
1561 -}
1562 -
1563 -/**
1564 - * nes_uarm_cq
1565 - */
1566 -int nes_uarm_cq(struct ibv_cq *cq, int solicited)
1567 -{
1568 -       struct nes_ucq *nesucq;
1569 -       struct nes_uvcontext *nesvctx;
1570 -
1571 -       nesucq = to_nes_ucq(cq);
1572 -       nesvctx = to_nes_uctx(cq->context);
1573 -
1574 -       pthread_spin_lock(&nesucq->lock);
1575 -
1576 -       if (nesucq->is_armed) {
1577 -       /* don't arm again unless... */
1578 -               if ((nesucq->arm_sol) && (!solicited)) {
1579 -                       /* solicited changed from notify SE to notify next */
1580 -                       nes_arm_cq(nesucq, nesvctx, solicited);
1581 -               } else {
1582 -                       nesucq->skip_arm = 1;
1583 -                       nesucq->skip_sol &= solicited;
1584 -               }
1585 -       } else {
1586 -               nes_arm_cq(nesucq, nesvctx, solicited);
1587 -       }
1588 -
1589 -       pthread_spin_unlock(&nesucq->lock);
1590 -
1591 -       return 0;
1592 -}
1593 -
1594 -
1595 -/**
1596 - * nes_cq_event
1597 - */
1598 -void nes_cq_event(struct ibv_cq *cq)
1599 -{
1600 -       struct nes_ucq *nesucq;
1601 -
1602 -       nesucq = to_nes_ucq(cq);
1603 -
1604 -       pthread_spin_lock(&nesucq->lock);
1605 -
1606 -       if (nesucq->skip_arm) {
1607 -               struct nes_uvcontext *nesvctx;
1608 -               nesvctx = to_nes_uctx(cq->context);
1609 -               nes_arm_cq(nesucq, nesvctx, nesucq->skip_sol);
1610 -       } else {
1611 -               nesucq->is_armed = 0;
1612 -       }
1613 -
1614 -       pthread_spin_unlock(&nesucq->lock);
1615 -}
1616 -
1617 -
1618 -/**
1619 - * nes_ucreate_srq
1620 - */
1621 -struct ibv_srq *nes_ucreate_srq(struct ibv_pd *pd, struct ibv_srq_init_attr *attr)
1622 -{
1623 -       /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
1624 -       return (void *)-ENOSYS;
1625 -}
1626 -
1627 -
1628 -/**
1629 - * nes_umodify_srq
1630 - */
1631 -int nes_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, int attr_mask)
1632 -{
1633 -       /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
1634 -       return -ENOSYS;
1635 -}
1636 -
1637 -
1638 -/**
1639 - * nes_udestroy_srq
1640 - */
1641 -int nes_udestroy_srq(struct ibv_srq *srq)
1642 -{
1643 -       /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
1644 -       return -ENOSYS;
1645 -}
1646 -
1647 -
1648 -/**
1649 - * nes_upost_srq_recv
1650 - */
1651 -int nes_upost_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr,
1652 -               struct ibv_recv_wr **bad_wr)
1653 -{
1654 -       /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
1655 -       return -ENOSYS;
1656 -}
1657 -
1658 -
1659 -/**
1660 - * nes_mmapped_qp
1661 - * will not invoke registration of memory reqion and will allow
1662 - * the kernel module to allocate big chunk of contigous memory
1663 - * for sq and rq... returns 1 if succeeds, 0 if fails..
1664 - */
1665 -static int nes_mmapped_qp(struct nes_uqp *nesuqp, struct ibv_pd *pd, struct ibv_qp_init_attr *attr,
1666 -               struct nes_ucreate_qp_resp *resp)
1667 -{
1668 -
1669 -       unsigned long mmap_offset;
1670 -       struct nes_ucreate_qp cmd;
1671 -       struct nes_uvcontext *nesvctx = to_nes_uctx(pd->context);
1672 -       int ret;
1673 -
1674 -       memset (&cmd, 0, sizeof(cmd) );
1675 -       cmd.user_qp_buffer = (__u64) ((uintptr_t) nesuqp);
1676 -
1677 -       /* fprintf(stderr, PFX "%s entering==>\n",__FUNCTION__); */
1678 -       ret = ibv_cmd_create_qp(pd, &nesuqp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd,
1679 -               &resp->ibv_resp, sizeof (struct nes_ucreate_qp_resp) );
1680 -       if (ret)
1681 -               return 0;
1682 -       nesuqp->send_cq = to_nes_ucq(attr->send_cq);
1683 -       nesuqp->recv_cq = to_nes_ucq(attr->recv_cq);
1684 -       nesuqp->sq_db_index = resp->mmap_sq_db_index;
1685 -       nesuqp->rq_db_index = resp->mmap_rq_db_index;
1686 -       nesuqp->sq_size = resp->actual_sq_size;
1687 -       nesuqp->rq_size = resp->actual_rq_size;
1688 -
1689 -       /* Map the SQ/RQ buffers */
1690 -       mmap_offset = nesvctx->max_pds*page_size;
1691 -       mmap_offset += (((sizeof(struct nes_hw_qp_wqe) * nesvctx->wq_size) + page_size-1) &
1692 -                       (~(page_size-1)))*nesuqp->sq_db_index;
1693 -
1694 -       nesuqp->sq_vbase = mmap(NULL, (nesuqp->sq_size+nesuqp->rq_size) *
1695 -                       sizeof(struct nes_hw_qp_wqe), PROT_WRITE | PROT_READ,
1696 -                       MAP_SHARED, pd->context->cmd_fd, mmap_offset);
1697 -
1698 -
1699 -       if (nesuqp->sq_vbase == MAP_FAILED) {
1700 -               return 0;
1701 -       }
1702 -       nesuqp->rq_vbase = (struct nes_hw_qp_wqe *)(((char *)nesuqp->sq_vbase) +
1703 -                       (nesuqp->sq_size*sizeof(struct nes_hw_qp_wqe)));
1704 -       *((unsigned int *)nesuqp->sq_vbase) = 0;
1705 -       nesuqp->mapping = NES_QP_MMAP;
1706 -
1707 -       return 1;
1708 -}
1709 -
1710 -
1711 -/**
1712 - * nes_vmapped_qp
1713 - * invoke registration of memory reqion. This method is used
1714 - * when kernel can not allocate qp memory (contigous physical).
1715 - *
1716 - * returns 1 if succeeds, 0 if fails..
1717 - */
1718 -static int nes_vmapped_qp(struct nes_uqp *nesuqp, struct ibv_pd *pd, struct ibv_qp_init_attr *attr,
1719 -                         struct nes_ucreate_qp_resp *resp, int sqdepth, int rqdepth)
1720 -{
1721 -       struct nes_ucreate_qp cmd;
1722 -       struct nes_ureg_mr reg_mr_cmd;
1723 -       struct ib_uverbs_reg_mr_resp reg_mr_resp;
1724 -       int totalqpsize;
1725 -       int ret;
1726 -
1727 -       // fprintf(stderr, PFX "%s\n", __FUNCTION__);
1728 -       totalqpsize = (sqdepth + rqdepth) * sizeof (struct nes_hw_qp_wqe) ;
1729 -       nesuqp->sq_vbase = memalign(page_size, totalqpsize);
1730 -       if (!nesuqp->sq_vbase) {
1731 -       //      fprintf(stderr, PFX "CREATE_QP could not allocate mem of size %d\n", totalqpsize);
1732 -               return 0;
1733 -       }
1734 -       nesuqp->rq_vbase = (struct nes_hw_qp_wqe *) (((char *) nesuqp->sq_vbase) +
1735 -                          (nesuqp->sq_size * sizeof(struct nes_hw_qp_wqe)));
1736 -
1737 -       reg_mr_cmd.reg_type = IWNES_MEMREG_TYPE_QP;
1738 -
1739 -       //fprintf(stderr, PFX "qp_rq_vbase = %p qp_sq_vbase=%p reg_mr = %p\n",
1740 -       //              nesuqp->rq_vbase, nesuqp->sq_vbase, &nesuqp->mr);
1741 -
1742 -        ret = ibv_cmd_reg_mr(pd, (void *)nesuqp->sq_vbase,totalqpsize,
1743 -                            (uintptr_t)nesuqp->sq_vbase,
1744 -                            IBV_ACCESS_LOCAL_WRITE, &nesuqp->vmr,
1745 -                            &reg_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd),
1746 -                            &reg_mr_resp, sizeof(reg_mr_resp));
1747 -        if (ret) {
1748 -                // fprintf(stderr, PFX "%s ibv_cmd_reg_mr failed (ret = %d).\n", __FUNCTION__, ret);
1749 -               free((void *) nesuqp->sq_vbase);
1750 -               return 0;
1751 -        }
1752 -       // So now the memory has been registered..
1753 -       memset (&cmd, 0, sizeof(cmd) );
1754 -       cmd.user_wqe_buffers = (__u64) ((uintptr_t) nesuqp->sq_vbase);
1755 -       cmd.user_qp_buffer = (__u64) ((uintptr_t) nesuqp);
1756 -       ret = ibv_cmd_create_qp(pd, &nesuqp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd,
1757 -                               &resp->ibv_resp, sizeof (struct nes_ucreate_qp_resp) );
1758 -       if (ret) {
1759 -               ibv_cmd_dereg_mr(&nesuqp->vmr);
1760 -               free((void *)nesuqp->sq_vbase);
1761 -               return 0;
1762 -       }
1763 -       *((unsigned int *)nesuqp->rq_vbase) = 0;
1764 -       nesuqp->send_cq = to_nes_ucq(attr->send_cq);
1765 -       nesuqp->recv_cq = to_nes_ucq(attr->recv_cq);
1766 -       nesuqp->sq_db_index = resp->mmap_sq_db_index;
1767 -       nesuqp->rq_db_index = resp->mmap_rq_db_index;
1768 -       nesuqp->sq_size = resp->actual_sq_size;
1769 -       nesuqp->rq_size = resp->actual_rq_size;
1770 -       nesuqp->mapping = NES_QP_VMAP;
1771 -       return 1;
1772 -}
1773 -
1774 -
1775 -/**
1776 - * nes_qp_get_qdepth
1777 - * This routine will return the size of qdepth to be set for one
1778 - * of the qp (sq or rq)
1779 - */
1780 -static int nes_qp_get_qdepth(uint32_t qdepth, uint32_t maxsges)
1781 -{
1782 -       int     retdepth;
1783 -
1784 -       /* Do sanity check on the parameters */
1785 -       /* Should the following be 510 or 511 */
1786 -       if ((qdepth > 510) || (maxsges > 4) )
1787 -               return 0;
1788 -
1789 -       /* Do we need to do the following of */
1790 -       /* we can just return the actual value.. needed for alignment */
1791 -       if (qdepth < 32)
1792 -               retdepth = 32;
1793 -       else if (qdepth < 128)
1794 -               retdepth = 128;
1795 -       else retdepth = 512;
1796 -
1797 -       return retdepth;
1798 -}
1799 -
1800 -
1801 -/**
1802 - * nes_ucreate_qp
1803 - */
1804 -struct ibv_qp *nes_ucreate_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
1805 -{
1806 -       struct nes_ucreate_qp_resp resp;
1807 -       struct nes_uvcontext *nesvctx = to_nes_uctx(pd->context);
1808 -       struct nes_uqp *nesuqp;
1809 -       int     sqdepth, rqdepth;
1810 -       int      status = 1;
1811 -
1812 -       /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
1813 -
1814 -       /* Sanity check QP size before proceeding */
1815 -       sqdepth = nes_qp_get_qdepth(attr->cap.max_send_wr, attr->cap.max_send_sge);
1816 -       if (!sqdepth) {
1817 -               fprintf(stderr, PFX "%s Bad sq attr parameters max_send_wr=%d max_send_sge=%d\n",
1818 -                       __FUNCTION__, attr->cap.max_send_wr,attr->cap.max_send_sge);
1819 -               return NULL;
1820 -       }
1821 -
1822 -       rqdepth = nes_qp_get_qdepth(attr->cap.max_recv_wr, attr->cap.max_recv_sge);
1823 -       if (!rqdepth) {
1824 -               fprintf(stderr, PFX "%s Bad rq attr parameters max_recv_wr=%d max_recv_sge=%d\n",
1825 -                       __FUNCTION__, attr->cap.max_recv_wr,attr->cap.max_recv_sge);
1826 -               return NULL;
1827 -       }
1828 -
1829 -       nesuqp = memalign(1024, sizeof(*nesuqp));
1830 -       if (!nesuqp)
1831 -               return NULL;
1832 -       memset(nesuqp, 0, sizeof(*nesuqp));
1833 -
1834 -       if (pthread_spin_init(&nesuqp->lock, PTHREAD_PROCESS_PRIVATE)) {
1835 -               free(nesuqp);
1836 -               return NULL;
1837 -       }
1838 -
1839 -       /* Initially setting it up so we will know how much memory to allocate for mapping */
1840 -       /* also setting it up in attr.. If we do not want to modify the attr struct, we */
1841 -       /* can save the original values and restore them before return. */
1842 -       nesuqp->sq_size = attr->cap.max_send_wr = sqdepth;
1843 -       nesuqp->rq_size = attr->cap.max_recv_wr = rqdepth;
1844 -
1845 -       nesuqp->sq_sig_all = attr->sq_sig_all;
1846 -       if (nesvctx->virtwq) {
1847 -               status = nes_vmapped_qp(nesuqp,pd, attr,&resp,sqdepth,rqdepth);
1848 -       }else {
1849 -               status = nes_mmapped_qp(nesuqp,pd,attr, &resp);
1850 -       }
1851 -
1852 -       if (!status) {
1853 -               pthread_spin_destroy(&nesuqp->lock);
1854 -               free(nesuqp);
1855 -               return NULL;
1856 -       }
1857 -
1858 -
1859 -       /* The following are the common parameters no matter how the */
1860 -       /* sq and rq memory was mapped.. */
1861 -
1862 -       /* Account for LSMM, in theory, could get overrun if app preposts to SQ */
1863 -       nesuqp->sq_head = 1;
1864 -       nesuqp->sq_tail = 1;
1865 -       nesuqp->qp_id = resp.qp_id;
1866 -       nesuqp->nes_drv_opt = resp.nes_drv_opt;
1867 -       nesuqp->ibv_qp.qp_num = resp.qp_id;
1868 -       nesuqp->rdma0_msg = 1;
1869 -
1870 -       return &nesuqp->ibv_qp;
1871 -}
1872 -
1873 -
1874 -/**
1875 - * nes_uquery_qp
1876 - */
1877 -int nes_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
1878 -                 int attr_mask, struct ibv_qp_init_attr *init_attr)
1879 -{
1880 -       struct ibv_query_qp cmd;
1881 -
1882 -       /* fprintf(stderr, PFX "nes_uquery_qp: calling ibv_cmd_query_qp\n"); */
1883 -
1884 -       return ibv_cmd_query_qp(qp, attr, attr_mask, init_attr, &cmd, sizeof(cmd));
1885 -}
1886 -
1887 -
1888 -/**
1889 - * nes_umodify_qp
1890 - */
1891 -int nes_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
1892 -{
1893 -       struct ibv_modify_qp cmd = {};
1894 -       return ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof cmd);
1895 -}
1896 -
1897 -
1898 -/**
1899 - * nes_clean_cq
1900 - */
1901 -static void nes_clean_cq(struct nes_uqp *nesuqp, struct nes_ucq *nesucq)
1902 -{
1903 -       uint32_t cq_head;
1904 -       uint32_t lo;
1905 -       uint32_t hi;
1906 -       uint64_t u64temp;
1907 -
1908 -       pthread_spin_lock(&nesucq->lock);
1909 -
1910 -       cq_head = nesucq->head;
1911 -       while (le32toh(nesucq->cqes[cq_head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
1912 -               udma_from_device_barrier();
1913 -               lo = le32toh(nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]);
1914 -               hi = le32toh(nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]);
1915 -               u64temp = (((uint64_t)hi) << 32) | ((uint64_t)lo);
1916 -               u64temp &= (~1023);
1917 -               if (u64temp == (uint64_t)(uintptr_t)nesuqp) {
1918 -                       /* Zero the context value so cqe will be ignored */
1919 -                       nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = 0;
1920 -                       nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX] = 0;
1921 -               }
1922 -
1923 -               if (++cq_head >= nesucq->size)
1924 -                       cq_head = 0;
1925 -       }
1926 -
1927 -       pthread_spin_unlock(&nesucq->lock);
1928 -}
1929 -
1930 -
1931 -/**
1932 - * nes_udestroy_qp
1933 - */
1934 -int nes_udestroy_qp(struct ibv_qp *qp)
1935 -{
1936 -       struct nes_uqp *nesuqp = to_nes_uqp(qp);
1937 -       int ret = 0;
1938 -
1939 -       // fprintf(stderr, PFX "%s addr&mr= %p  \n", __FUNCTION__, &nesuqp->mr );
1940 -
1941 -       if (nesuqp->mapping == NES_QP_VMAP) {
1942 -               ret = ibv_cmd_dereg_mr(&nesuqp->vmr);
1943 -               if (ret)
1944 -                       fprintf(stderr, PFX "%s dereg_mr FAILED\n", __FUNCTION__);
1945 -               free((void *)nesuqp->sq_vbase);
1946 -       }
1947 -
1948 -       if (nesuqp->mapping == NES_QP_MMAP) {
1949 -               munmap((void *)nesuqp->sq_vbase, (nesuqp->sq_size+nesuqp->rq_size) *
1950 -                       sizeof(struct nes_hw_qp_wqe));
1951 -       }
1952 -
1953 -       ret = ibv_cmd_destroy_qp(qp);
1954 -       if (ret) {
1955 -               fprintf(stderr, PFX "%s FAILED\n", __FUNCTION__);
1956 -               return ret;
1957 -       }
1958 -
1959 -       pthread_spin_destroy(&nesuqp->lock);
1960 -
1961 -       /* Clean any pending completions from the cq(s) */
1962 -       if (nesuqp->send_cq)
1963 -               nes_clean_cq(nesuqp, nesuqp->send_cq);
1964 -
1965 -       if ((nesuqp->recv_cq) && (nesuqp->recv_cq != nesuqp->send_cq))
1966 -               nes_clean_cq(nesuqp, nesuqp->recv_cq);
1967 -       free(nesuqp);
1968 -
1969 -       return 0;
1970 -}
1971 -
1972 -/**
1973 - * nes_upost_send
1974 - */
1975 -int nes_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
1976 -               struct ibv_send_wr **bad_wr)
1977 -{
1978 -       uint64_t u64temp;
1979 -       struct nes_uqp *nesuqp = to_nes_uqp(ib_qp);
1980 -       struct nes_upd *nesupd = to_nes_upd(ib_qp->pd);
1981 -       struct nes_hw_qp_wqe volatile *wqe;
1982 -       uint32_t head;
1983 -       uint32_t qsize = nesuqp->sq_size;
1984 -       uint32_t counter;
1985 -       uint32_t err = 0;
1986 -       uint32_t wqe_count = 0;
1987 -       uint32_t outstanding_wqes;
1988 -       uint32_t total_payload_length = 0;
1989 -       int sge_index;
1990 -
1991 -       pthread_spin_lock(&nesuqp->lock);
1992 -       udma_to_device_barrier();
1993 -
1994 -       head = nesuqp->sq_head;
1995 -       while (ib_wr) {
1996 -               if (unlikely(nesuqp->qperr)) {
1997 -                       err = -EINVAL;
1998 -                       break;
1999 -               }
2000 -
2001 -               /* Check for SQ overflow */
2002 -               outstanding_wqes = head + (2 * qsize) - nesuqp->sq_tail;
2003 -               outstanding_wqes &= qsize - 1;
2004 -               if (unlikely(outstanding_wqes == (qsize - 1))) {
2005 -                       err = -EINVAL;
2006 -                       break;
2007 -               }
2008 -               if (unlikely(ib_wr->num_sge > 4)) {
2009 -                       err = -EINVAL;
2010 -                       break;
2011 -               }
2012 -
2013 -               wqe = (struct nes_hw_qp_wqe *)&nesuqp->sq_vbase[head];
2014 -               /* fprintf(stderr, PFX "%s: QP%u: processing sq wqe at %p, head = %u.\n",
2015 -                               __FUNCTION__, nesuqp->qp_id, wqe, head);  */
2016 -               u64temp = (uint64_t) ib_wr->wr_id;
2017 -               wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX] = htole32((uint32_t)u64temp);
2018 -               wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX] = htole32((uint32_t)(u64temp>>32));
2019 -               u64temp = (uint64_t)((uintptr_t)nesuqp);
2020 -               wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX] = htole32((uint32_t)u64temp);
2021 -               wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX] = htole32((uint32_t)(u64temp>>32));
2022 -               udma_ordering_write_barrier();
2023 -               wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX] |= htole32(head);
2024 -
2025 -               switch (ib_wr->opcode) {
2026 -               case IBV_WR_SEND:
2027 -               case IBV_WR_SEND_WITH_IMM:
2028 -                       /* fprintf(stderr, PFX "%s: QP%u: processing sq wqe%u. Opcode = %s\n",
2029 -                                       __FUNCTION__, nesuqp->qp_id, head, "Send"); */
2030 -                       if (ib_wr->send_flags & IBV_SEND_SOLICITED) {
2031 -                               wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_SENDSE);
2032 -                       } else {
2033 -                               wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_SEND);
2034 -                       }
2035 -
2036 -                       if (ib_wr->send_flags & IBV_SEND_FENCE) {
2037 -                               wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_READ_FENCE);
2038 -                       }
2039 -
2040 -                       /* if (ib_wr->send_flags & IBV_SEND_INLINE) {
2041 -                               fprintf(stderr, PFX "%s: Send SEND_INLINE, length=%d\n",
2042 -                                               __FUNCTION__, ib_wr->sg_list[0].length);
2043 -                       } */
2044 -                       if ((ib_wr->send_flags & IBV_SEND_INLINE) && (ib_wr->sg_list[0].length <= 64) &&
2045 -                               ((nesuqp->nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
2046 -                               (ib_wr->num_sge == 1)) {
2047 -                               memcpy((void *)&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
2048 -                                               (void *)(intptr_t)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
2049 -                               wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(ib_wr->sg_list[0].length);
2050 -                               wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_IMM_DATA);
2051 -                       } else {
2052 -                               total_payload_length = 0;
2053 -                               for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
2054 -                                       wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] =
2055 -                                                       htole32((uint32_t)ib_wr->sg_list[sge_index].addr);
2056 -                                       wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] =
2057 -                                                       htole32((uint32_t)(ib_wr->sg_list[sge_index].addr>>32));
2058 -                                       wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX+(sge_index*4)] =
2059 -                                                       htole32(ib_wr->sg_list[sge_index].length);
2060 -                                       wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX+(sge_index*4)] =
2061 -                                                       htole32(ib_wr->sg_list[sge_index].lkey);
2062 -                                       total_payload_length += ib_wr->sg_list[sge_index].length;
2063 -                               }
2064 -                               wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] =
2065 -                                               htole32(total_payload_length);
2066 -                       }
2067 -
2068 -                       break;
2069 -               case IBV_WR_RDMA_WRITE:
2070 -               case IBV_WR_RDMA_WRITE_WITH_IMM:
2071 -                       /* fprintf(stderr, PFX "%s:QP%u: processing sq wqe%u. Opcode = %s\n",
2072 -                                       __FUNCTION__, nesuqp->qp_id, head, "Write"); */
2073 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_RDMAW);
2074 -
2075 -                       if (ib_wr->send_flags & IBV_SEND_FENCE) {
2076 -                               wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_READ_FENCE);
2077 -                       }
2078 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = htole32(ib_wr->wr.rdma.rkey);
2079 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = htole32(
2080 -                                       (uint32_t)ib_wr->wr.rdma.remote_addr);
2081 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = htole32(
2082 -                                       (uint32_t)(ib_wr->wr.rdma.remote_addr>>32));
2083 -
2084 -                       /* if (ib_wr->send_flags & IBV_SEND_INLINE) {
2085 -                               fprintf(stderr, PFX "%s: Write SEND_INLINE, length=%d\n",
2086 -                                               __FUNCTION__, ib_wr->sg_list[0].length);
2087 -                       } */
2088 -                       if ((ib_wr->send_flags & IBV_SEND_INLINE) && (ib_wr->sg_list[0].length <= 64) &&
2089 -                               ((nesuqp->nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) &&
2090 -                               (ib_wr->num_sge == 1)) {
2091 -                               memcpy((void *)&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX],
2092 -                                               (void *)(intptr_t)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length);
2093 -                               wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(ib_wr->sg_list[0].length);
2094 -                               wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_IMM_DATA);
2095 -                       } else {
2096 -                               total_payload_length = 0;
2097 -                               for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
2098 -                                       wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] = htole32(
2099 -                                                       (uint32_t)ib_wr->sg_list[sge_index].addr);
2100 -                                       wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] = htole32(
2101 -                                                       (uint32_t)(ib_wr->sg_list[sge_index].addr>>32));
2102 -                                       wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX+(sge_index*4)] = htole32(
2103 -                                                       ib_wr->sg_list[sge_index].length);
2104 -                                       wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX+(sge_index*4)] = htole32(
2105 -                                                       ib_wr->sg_list[sge_index].lkey);
2106 -                                       total_payload_length += ib_wr->sg_list[sge_index].length;
2107 -                               }
2108 -                               wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(total_payload_length);
2109 -                       }
2110 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] =
2111 -                                       wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX];
2112 -                       break;
2113 -               case IBV_WR_RDMA_READ:
2114 -                       /* fprintf(stderr, PFX "%s:QP%u:processing sq wqe%u. Opcode = %s\n",
2115 -                                       __FUNCTION__, nesuqp->qp_id, head, "Read"); */
2116 -                       /* IWarp only supports 1 sge for RDMA reads */
2117 -                       if (ib_wr->num_sge > 1) {
2118 -                               err = -EINVAL;
2119 -                               break;
2120 -                       }
2121 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_RDMAR);
2122 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = htole32((uint32_t)ib_wr->wr.rdma.remote_addr);
2123 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = htole32((uint32_t)(ib_wr->wr.rdma.remote_addr>>32));
2124 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = htole32(ib_wr->wr.rdma.rkey);
2125 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = htole32(ib_wr->sg_list->length);
2126 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX] = htole32((uint32_t)ib_wr->sg_list->addr);
2127 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX] = htole32((uint32_t)(ib_wr->sg_list->addr>>32));
2128 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = htole32(ib_wr->sg_list->lkey);
2129 -                       break;
2130 -               default:
2131 -                       /* error */
2132 -                       err = -EINVAL;
2133 -                       break;
2134 -               }
2135 -
2136 -                       if ((ib_wr->send_flags & IBV_SEND_SIGNALED) || nesuqp->sq_sig_all) {
2137 -                       /* fprintf(stderr, PFX "%s:sq wqe%u is signalled\n", __FUNCTION__, head); */
2138 -                       wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_SIGNALED_COMPL);
2139 -               }
2140 -               ib_wr = ib_wr->next;
2141 -               head++;
2142 -               wqe_count++;
2143 -               if (head >= qsize)
2144 -                       head = 0;
2145 -       }
2146 -
2147 -       nesuqp->sq_head = head;
2148 -       udma_to_device_barrier();
2149 -       while (wqe_count) {
2150 -               counter = (wqe_count<(uint32_t)255) ? wqe_count : 255;
2151 -               wqe_count -= counter;
2152 -               nesupd->udoorbell->wqe_alloc =  htole32((counter<<24) | 0x00800000 | nesuqp->qp_id);
2153 -       }
2154 -
2155 -       if (err)
2156 -               *bad_wr = ib_wr;
2157 -
2158 -       pthread_spin_unlock(&nesuqp->lock);
2159 -
2160 -       return err;
2161 -}
2162 -
2163 -/**
2164 - * nes_upost_recv
2165 - */
2166 -int nes_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr,
2167 -               struct ibv_recv_wr **bad_wr)
2168 -{
2169 -       uint64_t u64temp;
2170 -       struct nes_uqp *nesuqp = to_nes_uqp(ib_qp);
2171 -       struct nes_upd *nesupd = to_nes_upd(ib_qp->pd);
2172 -       struct nes_hw_qp_wqe *wqe;
2173 -       uint32_t head;
2174 -       uint32_t qsize = nesuqp->rq_size;
2175 -       uint32_t counter;
2176 -       uint32_t err = 0;
2177 -       uint32_t wqe_count = 0;
2178 -       uint32_t outstanding_wqes;
2179 -       uint32_t total_payload_length;
2180 -       int sge_index;
2181 -
2182 -       if (unlikely(ib_wr->num_sge > 4)) {
2183 -               *bad_wr = ib_wr;
2184 -               return -EINVAL;
2185 -       }
2186 -
2187 -       pthread_spin_lock(&nesuqp->lock);
2188 -       udma_to_device_barrier();
2189 -
2190 -       head = nesuqp->rq_head;
2191 -       while (ib_wr) {
2192 -               if (unlikely(nesuqp->qperr)) {
2193 -                       err = -EINVAL;
2194 -                       break;
2195 -               }
2196 -
2197 -               /* Check for RQ overflow */
2198 -               outstanding_wqes = head + (2 * qsize) - nesuqp->rq_tail;
2199 -               outstanding_wqes &= qsize - 1;
2200 -               if (unlikely(outstanding_wqes == (qsize - 1))) {
2201 -                       err = -EINVAL;
2202 -                       break;
2203 -               }
2204 -
2205 -               wqe = (struct nes_hw_qp_wqe *)&nesuqp->rq_vbase[head];
2206 -               u64temp = ib_wr->wr_id;
2207 -               wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX] =
2208 -                               htole32((uint32_t)u64temp);
2209 -               wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX] =
2210 -                               htole32((uint32_t)(u64temp >> 32));
2211 -               u64temp = (uint64_t)((uintptr_t)nesuqp);
2212 -               wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX] =
2213 -                               htole32((uint32_t)u64temp);
2214 -               wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_HIGH_IDX] =
2215 -                               htole32((uint32_t)(u64temp >> 32));
2216 -               udma_ordering_write_barrier();
2217 -               wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX] |= htole32(head);
2218 -
2219 -               total_payload_length = 0;
2220 -               for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) {
2221 -                       wqe->wqe_words[NES_IWARP_RQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] =
2222 -                                       htole32((uint32_t)ib_wr->sg_list[sge_index].addr);
2223 -                       wqe->wqe_words[NES_IWARP_RQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] =
2224 -                                       htole32((uint32_t)(ib_wr->sg_list[sge_index].addr>>32));
2225 -                       wqe->wqe_words[NES_IWARP_RQ_WQE_LENGTH0_IDX+(sge_index*4)] =
2226 -                                       htole32(ib_wr->sg_list[sge_index].length);
2227 -                       wqe->wqe_words[NES_IWARP_RQ_WQE_STAG0_IDX+(sge_index*4)] =
2228 -                                       htole32(ib_wr->sg_list[sge_index].lkey);
2229 -                       total_payload_length += ib_wr->sg_list[sge_index].length;
2230 -               }
2231 -               wqe->wqe_words[NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(total_payload_length);
2232 -
2233 -               ib_wr = ib_wr->next;
2234 -               head++;
2235 -               wqe_count++;
2236 -               if (head >= qsize)
2237 -                       head = 0;
2238 -       }
2239 -
2240 -       nesuqp->rq_head = head;
2241 -       udma_to_device_barrier();
2242 -       while (wqe_count) {
2243 -               counter = (wqe_count<(uint32_t)255) ? wqe_count : 255;
2244 -               wqe_count -= counter;
2245 -               nesupd->udoorbell->wqe_alloc = htole32((counter << 24) | nesuqp->qp_id);
2246 -       }
2247 -
2248 -       if (err)
2249 -               *bad_wr = ib_wr;
2250 -
2251 -       pthread_spin_unlock(&nesuqp->lock);
2252 -
2253 -       return err;
2254 -}
2255 -
2256 -
2257 -/**
2258 - * nes_ucreate_ah
2259 - */
2260 -struct ibv_ah *nes_ucreate_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
2261 -{
2262 -       /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
2263 -       return (void *)-ENOSYS;
2264 -}
2265 -
2266 -
2267 -/**
2268 - * nes_udestroy_ah
2269 - */
2270 -int nes_udestroy_ah(struct ibv_ah *ah)
2271 -{
2272 -       /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */
2273 -       return -ENOSYS;
2274 -}
2275 -
2276 -
2277 -/**
2278 - * nes_uattach_mcast
2279 - */
2280 -int nes_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
2281 -{
2282 -       int ret = 0;
2283 -       ret =  ibv_cmd_attach_mcast(qp, gid, lid);
2284 -       nes_debug(NES_DBG_UD, "%s ret=%d\n", __func__, ret);
2285 -       return ret;
2286 -}
2287 -
2288 -
2289 -/**
2290 - * nes_udetach_mcast
2291 - */
2292 -int nes_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
2293 -{
2294 -       int ret = 0;
2295 -       ret = ibv_cmd_detach_mcast(qp, gid, lid);
2296 -       nes_debug(NES_DBG_UD, "%s ret=%d\n", __func__, ret);
2297 -       return ret;
2298 -}
2299 -
2300 -/**
2301 - * nes_async_event
2302 - */
2303 -void nes_async_event(struct ibv_context *context,
2304 -                    struct ibv_async_event *event)
2305 -{
2306 -       struct nes_uqp *nesuqp;
2307 -
2308 -       switch (event->event_type) {
2309 -       case IBV_EVENT_QP_FATAL:
2310 -       case IBV_EVENT_QP_ACCESS_ERR:
2311 -               /* Do not let application queue anything else to the qp */
2312 -               nesuqp = to_nes_uqp(event->element.qp);
2313 -               nesuqp->qperr = 1;
2314 -               break;
2315 -
2316 -       default:
2317 -               break;
2318 -       }
2319 -}