]>
Commit | Line | Data |
---|---|---|
59dabf6f JB |
1 | From 4daf5c91c1296683924cb9668c3d879da072756b Mon Sep 17 00:00:00 2001 |
2 | From: Jason Gunthorpe <jgg@mellanox.com> | |
3 | Date: Thu, 24 Oct 2019 12:50:22 -0300 | |
4 | Subject: [PATCH] libnes: Remove libnes from rdma-core | |
5 | ||
6 | Remove the userspace provider for nes after removing it from kernel. | |
7 | ||
8 | Signed-off-by: Jason Gunthorpe <jgg@mellanox.com> | |
9 | --- | |
10 | CMakeLists.txt | 1 - | |
11 | MAINTAINERS | 5 - | |
12 | README.md | 1 - | |
13 | debian/control | 4 - | |
14 | debian/copyright | 5 - | |
15 | kernel-boot/rdma-description.rules | 1 - | |
16 | kernel-boot/rdma-hw-modules.rules | 1 - | |
17 | libibverbs/verbs.h | 1 - | |
18 | providers/nes/CMakeLists.txt | 4 - | |
19 | providers/nes/nes-abi.h | 52 - | |
20 | providers/nes/nes_umain.c | 220 ---- | |
21 | providers/nes/nes_umain.h | 383 ------- | |
22 | providers/nes/nes_uverbs.c | 1535 ---------------------------- | |
23 | redhat/rdma-core.spec | 3 - | |
24 | suse/rdma-core.spec | 2 - | |
25 | 15 files changed, 2218 deletions(-) | |
26 | delete mode 100644 providers/nes/CMakeLists.txt | |
27 | delete mode 100644 providers/nes/nes-abi.h | |
28 | delete mode 100644 providers/nes/nes_umain.c | |
29 | delete mode 100644 providers/nes/nes_umain.h | |
30 | delete mode 100644 providers/nes/nes_uverbs.c | |
31 | ||
32 | diff --git a/CMakeLists.txt b/CMakeLists.txt | |
33 | index 85485ba00..85eb25936 100644 | |
34 | --- a/CMakeLists.txt | |
35 | +++ b/CMakeLists.txt | |
36 | @@ -625,7 +625,6 @@ add_subdirectory(providers/mlx4/man) | |
37 | add_subdirectory(providers/mlx5) | |
38 | add_subdirectory(providers/mlx5/man) | |
39 | add_subdirectory(providers/mthca) | |
40 | -add_subdirectory(providers/nes) # NO SPARSE | |
41 | add_subdirectory(providers/ocrdma) | |
42 | add_subdirectory(providers/qedr) | |
43 | add_subdirectory(providers/vmw_pvrdma) | |
44 | diff --git a/debian/copyright b/debian/copyright | |
45 | index db4951993..6f86d1c86 100644 | |
46 | --- a/debian/copyright | |
47 | +++ b/debian/copyright | |
48 | @@ -197,11 +197,6 @@ Copyright: 2004-2005, Topspin Communications. | |
49 | 2005, Mellanox Technologies Ltd. | |
50 | License: BSD-MIT or GPL-2 | |
51 | ||
52 | -Files: providers/nes/* | |
53 | -Copyright: 2006-2010, Intel Corporation. | |
54 | - 2006, Open Grid Computing, Inc. | |
55 | -License: BSD-MIT or GPL-2 | |
56 | - | |
57 | Files: providers/ocrdma/* | |
58 | Copyright: 2008-2013, Emulex. | |
59 | License: BSD-2-clause or GPL-2 | |
60 | diff --git a/kernel-boot/rdma-description.rules b/kernel-boot/rdma-description.rules | |
61 | index 4ea59ba19..48a7cede9 100644 | |
62 | --- a/kernel-boot/rdma-description.rules | |
63 | +++ b/kernel-boot/rdma-description.rules | |
64 | @@ -24,7 +24,6 @@ DRIVERS=="hfi1", ENV{ID_RDMA_OPA}="1" | |
65 | # Hardware that supports iWarp | |
66 | DRIVERS=="cxgb4", ENV{ID_RDMA_IWARP}="1" | |
67 | DRIVERS=="i40e", ENV{ID_RDMA_IWARP}="1" | |
68 | -DRIVERS=="nes", ENV{ID_RDMA_IWARP}="1" | |
69 | ||
70 | # Hardware that supports RoCE | |
71 | DRIVERS=="be2net", ENV{ID_RDMA_ROCE}="1" | |
72 | diff --git a/kernel-boot/rdma-hw-modules.rules b/kernel-boot/rdma-hw-modules.rules | |
73 | index da4bbe363..bee416dbe 100644 | |
74 | --- a/kernel-boot/rdma-hw-modules.rules | |
75 | +++ b/kernel-boot/rdma-hw-modules.rules | |
76 | @@ -33,6 +33,5 @@ ENV{ID_NET_DRIVER}=="enic", RUN{builtin}+="kmod load usnic_verbs" | |
77 | # ipathverbs | |
78 | # mthca | |
79 | # vmw_pvrdma | |
80 | -# nes | |
81 | ||
82 | LABEL="rdma_hw_modules_end" | |
83 | diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h | |
84 | index 12a33a99a..13e7c63e7 100644 | |
85 | --- a/libibverbs/verbs.h | |
86 | +++ b/libibverbs/verbs.h | |
87 | @@ -2153,7 +2153,6 @@ extern const struct verbs_device_ops verbs_provider_ipathverbs; | |
88 | extern const struct verbs_device_ops verbs_provider_mlx4; | |
89 | extern const struct verbs_device_ops verbs_provider_mlx5; | |
90 | extern const struct verbs_device_ops verbs_provider_mthca; | |
91 | -extern const struct verbs_device_ops verbs_provider_nes; | |
92 | extern const struct verbs_device_ops verbs_provider_ocrdma; | |
93 | extern const struct verbs_device_ops verbs_provider_qedr; | |
94 | extern const struct verbs_device_ops verbs_provider_rxe; | |
95 | diff --git a/providers/nes/CMakeLists.txt b/providers/nes/CMakeLists.txt | |
96 | deleted file mode 100644 | |
97 | index 0c7fa8fad..000000000 | |
98 | --- a/providers/nes/CMakeLists.txt | |
99 | +++ /dev/null | |
100 | @@ -1,4 +0,0 @@ | |
101 | -rdma_provider(nes | |
102 | - nes_umain.c | |
103 | - nes_uverbs.c | |
104 | -) | |
105 | diff --git a/providers/nes/nes-abi.h b/providers/nes/nes-abi.h | |
106 | deleted file mode 100644 | |
107 | index 0a531230b..000000000 | |
108 | --- a/providers/nes/nes-abi.h | |
109 | +++ /dev/null | |
110 | @@ -1,52 +0,0 @@ | |
111 | -/* | |
112 | - * Copyright (c) 2006 - 2010 Intel Corporation. All rights reserved. | |
113 | - * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved. | |
114 | - * | |
115 | - * This software is available to you under a choice of one of two | |
116 | - * licenses. You may choose to be licensed under the terms of the GNU | |
117 | - * General Public License (GPL) Version 2, available from the file | |
118 | - * gpl-2.0.txt in the main directory of this source tree, or the | |
119 | - * OpenIB.org BSD license below: | |
120 | - * | |
121 | - * Redistribution and use in source and binary forms, with or | |
122 | - * without modification, are permitted provided that the following | |
123 | - * conditions are met: | |
124 | - * | |
125 | - * - Redistributions of source code must retain the above | |
126 | - * copyright notice, this list of conditions and the following | |
127 | - * disclaimer. | |
128 | - * | |
129 | - * - Redistributions in binary form must reproduce the above | |
130 | - * copyright notice, this list of conditions and the following | |
131 | - * disclaimer in the documentation and/or other materials | |
132 | - * provided with the distribution. | |
133 | - * | |
134 | - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
135 | - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
136 | - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
137 | - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
138 | - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
139 | - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
140 | - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
141 | - * SOFTWARE. | |
142 | - */ | |
143 | - | |
144 | -#ifndef nes_ABI_H | |
145 | -#define nes_ABI_H | |
146 | - | |
147 | -#include <infiniband/kern-abi.h> | |
148 | -#include <rdma/nes-abi.h> | |
149 | -#include <kernel-abi/nes-abi.h> | |
150 | - | |
151 | -DECLARE_DRV_CMD(nes_ualloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, | |
152 | - empty, nes_alloc_pd_resp); | |
153 | -DECLARE_DRV_CMD(nes_ucreate_cq, IB_USER_VERBS_CMD_CREATE_CQ, | |
154 | - nes_create_cq_req, nes_create_cq_resp); | |
155 | -DECLARE_DRV_CMD(nes_ucreate_qp, IB_USER_VERBS_CMD_CREATE_QP, | |
156 | - nes_create_qp_req, nes_create_qp_resp); | |
157 | -DECLARE_DRV_CMD(nes_get_context, IB_USER_VERBS_CMD_GET_CONTEXT, | |
158 | - nes_alloc_ucontext_req, nes_alloc_ucontext_resp); | |
159 | -DECLARE_DRV_CMD(nes_ureg_mr, IB_USER_VERBS_CMD_REG_MR, | |
160 | - nes_mem_reg_req, empty); | |
161 | - | |
162 | -#endif /* nes_ABI_H */ | |
163 | diff --git a/providers/nes/nes_umain.c b/providers/nes/nes_umain.c | |
164 | deleted file mode 100644 | |
165 | index 07aa7ddd1..000000000 | |
166 | --- a/providers/nes/nes_umain.c | |
167 | +++ /dev/null | |
168 | @@ -1,220 +0,0 @@ | |
169 | -/* | |
170 | - * Copyright (c) 2006 - 2010 Intel Corporation. All rights reserved. | |
171 | - * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved. | |
172 | - * | |
173 | - * This software is available to you under a choice of one of two | |
174 | - * licenses. You may choose to be licensed under the terms of the GNU | |
175 | - * General Public License (GPL) Version 2, available from the file | |
176 | - * gpl-2.0.txt in the main directory of this source tree, or the | |
177 | - * OpenIB.org BSD license below: | |
178 | - * | |
179 | - * Redistribution and use in source and binary forms, with or | |
180 | - * without modification, are permitted provided that the following | |
181 | - * conditions are met: | |
182 | - * | |
183 | - * - Redistributions of source code must retain the above | |
184 | - * copyright notice, this list of conditions and the following | |
185 | - * disclaimer. | |
186 | - * | |
187 | - * - Redistributions in binary form must reproduce the above | |
188 | - * copyright notice, this list of conditions and the following | |
189 | - * disclaimer in the documentation and/or other materials | |
190 | - * provided with the distribution. | |
191 | - * | |
192 | - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
193 | - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
194 | - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
195 | - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
196 | - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
197 | - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
198 | - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
199 | - * SOFTWARE. | |
200 | - */ | |
201 | - | |
202 | -#include <config.h> | |
203 | - | |
204 | -#include <stdio.h> | |
205 | -#include <stdlib.h> | |
206 | -#include <string.h> | |
207 | -#include <unistd.h> | |
208 | -#include <errno.h> | |
209 | -#include <sys/mman.h> | |
210 | -#include <pthread.h> | |
211 | - | |
212 | -#include "nes_umain.h" | |
213 | -#include "nes-abi.h" | |
214 | - | |
215 | -unsigned int nes_debug_level = 0; | |
216 | -long int page_size; | |
217 | - | |
218 | -#include <sys/types.h> | |
219 | -#include <sys/stat.h> | |
220 | -#include <fcntl.h> | |
221 | - | |
222 | -#ifndef PCI_VENDOR_ID_NETEFFECT | |
223 | -#define PCI_VENDOR_ID_NETEFFECT 0x1678 | |
224 | -#endif | |
225 | - | |
226 | -#define HCA(v, d) VERBS_PCI_MATCH(PCI_VENDOR_ID_##v, d, NULL) | |
227 | -static const struct verbs_match_ent hca_table[] = { | |
228 | - VERBS_DRIVER_ID(RDMA_DRIVER_NES), | |
229 | - HCA(NETEFFECT, 0x0100), | |
230 | - HCA(NETEFFECT, 0x0110), | |
231 | - {}, | |
232 | -}; | |
233 | - | |
234 | -static const struct verbs_context_ops nes_uctx_ops = { | |
235 | - .query_device = nes_uquery_device, | |
236 | - .query_port = nes_uquery_port, | |
237 | - .alloc_pd = nes_ualloc_pd, | |
238 | - .dealloc_pd = nes_ufree_pd, | |
239 | - .reg_mr = nes_ureg_mr, | |
240 | - .dereg_mr = nes_udereg_mr, | |
241 | - .create_cq = nes_ucreate_cq, | |
242 | - .poll_cq = nes_upoll_cq, | |
243 | - .req_notify_cq = nes_uarm_cq, | |
244 | - .cq_event = nes_cq_event, | |
245 | - .resize_cq = nes_uresize_cq, | |
246 | - .destroy_cq = nes_udestroy_cq, | |
247 | - .create_qp = nes_ucreate_qp, | |
248 | - .query_qp = nes_uquery_qp, | |
249 | - .modify_qp = nes_umodify_qp, | |
250 | - .destroy_qp = nes_udestroy_qp, | |
251 | - .post_send = nes_upost_send, | |
252 | - .post_recv = nes_upost_recv, | |
253 | - .create_ah = nes_ucreate_ah, | |
254 | - .destroy_ah = nes_udestroy_ah, | |
255 | - .attach_mcast = nes_uattach_mcast, | |
256 | - .detach_mcast = nes_udetach_mcast, | |
257 | - .async_event = nes_async_event | |
258 | -}; | |
259 | - | |
260 | -static const struct verbs_context_ops nes_uctx_no_db_ops = { | |
261 | - .poll_cq = nes_upoll_cq_no_db_read, | |
262 | -}; | |
263 | - | |
264 | - | |
265 | -/** | |
266 | - * nes_ualloc_context | |
267 | - */ | |
268 | -static struct verbs_context *nes_ualloc_context(struct ibv_device *ibdev, | |
269 | - int cmd_fd, | |
270 | - void *private_data) | |
271 | -{ | |
272 | - struct ibv_pd *ibv_pd; | |
273 | - struct nes_uvcontext *nesvctx; | |
274 | - struct nes_get_context cmd; | |
275 | - struct nes_get_context_resp resp; | |
276 | - char value[16]; | |
277 | - uint32_t nes_drv_opt = 0; | |
278 | - | |
279 | - page_size = sysconf(_SC_PAGESIZE); | |
280 | - | |
281 | - nesvctx = verbs_init_and_alloc_context(ibdev, cmd_fd, nesvctx, ibv_ctx, | |
282 | - RDMA_DRIVER_NES); | |
283 | - if (!nesvctx) | |
284 | - return NULL; | |
285 | - | |
286 | - cmd.userspace_ver = NES_ABI_USERSPACE_VER; | |
287 | - | |
288 | - if (ibv_cmd_get_context(&nesvctx->ibv_ctx, (struct ibv_get_context *)&cmd, sizeof cmd, | |
289 | - &resp.ibv_resp, sizeof(resp))) | |
290 | - goto err_free; | |
291 | - | |
292 | - if (resp.kernel_ver != NES_ABI_KERNEL_VER) { | |
293 | - fprintf(stderr, PFX "%s: Invalid kernel driver version detected. Detected %d, should be %d\n", | |
294 | - __FUNCTION__, resp.kernel_ver, NES_ABI_KERNEL_VER); | |
295 | - goto err_free; | |
296 | - } | |
297 | - | |
298 | - if (ibv_read_sysfs_file("/sys/module/iw_nes", "parameters/nes_drv_opt", | |
299 | - value, sizeof(value)) > 0) { | |
300 | - sscanf(value, "%d", &nes_drv_opt); | |
301 | - } else if (ibv_read_sysfs_file("/sys/module/iw_nes", "nes_drv_opt", | |
302 | - value, sizeof(value)) > 0) { | |
303 | - sscanf(value, "%d", &nes_drv_opt); | |
304 | - } | |
305 | - | |
306 | - verbs_set_ops(&nesvctx->ibv_ctx, &nes_uctx_ops); | |
307 | - if (nes_drv_opt & NES_DRV_OPT_NO_DB_READ) | |
308 | - verbs_set_ops(&nesvctx->ibv_ctx, &nes_uctx_no_db_ops); | |
309 | - | |
310 | - nesvctx->max_pds = resp.max_pds; | |
311 | - nesvctx->max_qps = resp.max_qps; | |
312 | - nesvctx->wq_size = resp.wq_size; | |
313 | - nesvctx->virtwq = resp.virtwq; | |
314 | - nesvctx->mcrqf = 0; | |
315 | - | |
316 | - /* Get a doorbell region for the CQs */ | |
317 | - ibv_pd = nes_ualloc_pd(&nesvctx->ibv_ctx.context); | |
318 | - if (!ibv_pd) | |
319 | - goto err_free; | |
320 | - ibv_pd->context = &nesvctx->ibv_ctx.context; | |
321 | - nesvctx->nesupd = to_nes_upd(ibv_pd); | |
322 | - | |
323 | - return &nesvctx->ibv_ctx; | |
324 | - | |
325 | -err_free: | |
326 | - fprintf(stderr, PFX "%s: Failed to allocate context for device.\n", __FUNCTION__); | |
327 | - verbs_uninit_context(&nesvctx->ibv_ctx); | |
328 | - free(nesvctx); | |
329 | - | |
330 | - return NULL; | |
331 | -} | |
332 | - | |
333 | - | |
334 | -/** | |
335 | - * nes_ufree_context | |
336 | - */ | |
337 | -static void nes_ufree_context(struct ibv_context *ibctx) | |
338 | -{ | |
339 | - struct nes_uvcontext *nesvctx = to_nes_uctx(ibctx); | |
340 | - nes_ufree_pd(&nesvctx->nesupd->ibv_pd); | |
341 | - | |
342 | - verbs_uninit_context(&nesvctx->ibv_ctx); | |
343 | - free(nesvctx); | |
344 | -} | |
345 | - | |
346 | -static void nes_uninit_device(struct verbs_device *verbs_device) | |
347 | -{ | |
348 | - struct nes_udevice *dev = to_nes_udev(&verbs_device->device); | |
349 | - | |
350 | - free(dev); | |
351 | -} | |
352 | - | |
353 | -static struct verbs_device * | |
354 | -nes_device_alloc(struct verbs_sysfs_dev *sysfs_dev) | |
355 | -{ | |
356 | - struct nes_udevice *dev; | |
357 | - char value[16]; | |
358 | - | |
359 | - if (ibv_read_sysfs_file("/sys/module/iw_nes", "parameters/debug_level", | |
360 | - value, sizeof(value)) > 0) { | |
361 | - sscanf(value, "%u", &nes_debug_level); | |
362 | - } else if (ibv_read_sysfs_file("/sys/module/iw_nes", "debug_level", | |
363 | - value, sizeof(value)) > 0) { | |
364 | - sscanf(value, "%u", &nes_debug_level); | |
365 | - } | |
366 | - | |
367 | - dev = calloc(1, sizeof(*dev)); | |
368 | - if (!dev) | |
369 | - return NULL; | |
370 | - | |
371 | - dev->page_size = sysconf(_SC_PAGESIZE); | |
372 | - | |
373 | - nes_debug(NES_DBG_INIT, "libnes initialized\n"); | |
374 | - | |
375 | - return &dev->ibv_dev; | |
376 | -} | |
377 | - | |
378 | -static const struct verbs_device_ops nes_udev_ops = { | |
379 | - .name = "nes", | |
380 | - .match_min_abi_version = 0, | |
381 | - .match_max_abi_version = INT_MAX, | |
382 | - .match_table = hca_table, | |
383 | - .alloc_device = nes_device_alloc, | |
384 | - .uninit_device = nes_uninit_device, | |
385 | - .alloc_context = nes_ualloc_context, | |
386 | - .free_context = nes_ufree_context, | |
387 | -}; | |
388 | -PROVIDER_DRIVER(nes, nes_udev_ops); | |
389 | diff --git a/providers/nes/nes_umain.h b/providers/nes/nes_umain.h | |
390 | deleted file mode 100644 | |
391 | index 1070ce429..000000000 | |
392 | --- a/providers/nes/nes_umain.h | |
393 | +++ /dev/null | |
394 | @@ -1,383 +0,0 @@ | |
395 | -/* | |
396 | - * Copyright (c) 2006 - 2010 Intel Corporation. All rights reserved. | |
397 | - * Copyright (c) 2006 Open Grid Computing, Inc. All rights reserved. | |
398 | - * | |
399 | - * This software is available to you under a choice of one of two | |
400 | - * licenses. You may choose to be licensed under the terms of the GNU | |
401 | - * General Public License (GPL) Version 2, available from the file | |
402 | - * gpl-2.0.txt in the main directory of this source tree, or the | |
403 | - * OpenIB.org BSD license below: | |
404 | - * | |
405 | - * Redistribution and use in source and binary forms, with or | |
406 | - * without modification, are permitted provided that the following | |
407 | - * conditions are met: | |
408 | - * | |
409 | - * - Redistributions of source code must retain the above | |
410 | - * copyright notice, this list of conditions and the following | |
411 | - * disclaimer. | |
412 | - * | |
413 | - * - Redistributions in binary form must reproduce the above | |
414 | - * copyright notice, this list of conditions and the following | |
415 | - * disclaimer in the documentation and/or other materials | |
416 | - * provided with the distribution. | |
417 | - * | |
418 | - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
419 | - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
420 | - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
421 | - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
422 | - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
423 | - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
424 | - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
425 | - * SOFTWARE. | |
426 | - */ | |
427 | - | |
428 | -#ifndef nes_umain_H | |
429 | -#define nes_umain_H | |
430 | - | |
431 | -#include <inttypes.h> | |
432 | -#include <stddef.h> | |
433 | -#include <endian.h> | |
434 | -#include <util/compiler.h> | |
435 | - | |
436 | -#include <infiniband/driver.h> | |
437 | -#include <util/udma_barrier.h> | |
438 | - | |
439 | -#define PFX "libnes: " | |
440 | - | |
441 | -#define NES_QP_MMAP 1 | |
442 | -#define NES_QP_VMAP 2 | |
443 | - | |
444 | -#define NES_DRV_OPT_NO_INLINE_DATA 0x00000080 | |
445 | -#define NES_DRV_OPT_NO_DB_READ 0x00001000 | |
446 | - | |
447 | -#define NES_DEBUG | |
448 | -/* debug levels */ | |
449 | -/* must match kernel */ | |
450 | -#define NES_DBG_HW 0x00000001 | |
451 | -#define NES_DBG_INIT 0x00000002 | |
452 | -#define NES_DBG_ISR 0x00000004 | |
453 | -#define NES_DBG_PHY 0x00000008 | |
454 | -#define NES_DBG_NETDEV 0x00000010 | |
455 | -#define NES_DBG_CM 0x00000020 | |
456 | -#define NES_DBG_CM1 0x00000040 | |
457 | -#define NES_DBG_NIC_RX 0x00000080 | |
458 | -#define NES_DBG_NIC_TX 0x00000100 | |
459 | -#define NES_DBG_CQP 0x00000200 | |
460 | -#define NES_DBG_MMAP 0x00000400 | |
461 | -#define NES_DBG_MR 0x00000800 | |
462 | -#define NES_DBG_PD 0x00001000 | |
463 | -#define NES_DBG_CQ 0x00002000 | |
464 | -#define NES_DBG_QP 0x00004000 | |
465 | -#define NES_DBG_MOD_QP 0x00008000 | |
466 | -#define NES_DBG_AEQ 0x00010000 | |
467 | -#define NES_DBG_IW_RX 0x00020000 | |
468 | -#define NES_DBG_IW_TX 0x00040000 | |
469 | -#define NES_DBG_SHUTDOWN 0x00080000 | |
470 | -#define NES_DBG_UD 0x00100000 | |
471 | -#define NES_DBG_RSVD1 0x10000000 | |
472 | -#define NES_DBG_RSVD2 0x20000000 | |
473 | -#define NES_DBG_RSVD3 0x40000000 | |
474 | -#define NES_DBG_RSVD4 0x80000000 | |
475 | -#define NES_DBG_ALL 0xffffffff | |
476 | - | |
477 | -extern unsigned int nes_debug_level; | |
478 | -#ifdef NES_DEBUG | |
479 | -#define nes_debug(level, fmt, args...) \ | |
480 | - if (level & nes_debug_level) \ | |
481 | - fprintf(stderr, PFX "%s[%u]: " fmt, __FUNCTION__, __LINE__, ##args) | |
482 | -#else | |
483 | -#define nes_debug(level, fmt, args...) | |
484 | -#endif | |
485 | - | |
486 | -enum nes_cqe_opcode_bits { | |
487 | - NES_CQE_STAG_VALID = (1<<6), | |
488 | - NES_CQE_ERROR = (1<<7), | |
489 | - NES_CQE_SQ = (1<<8), | |
490 | - NES_CQE_SE = (1<<9), | |
491 | - NES_CQE_PSH = (1<<29), | |
492 | - NES_CQE_FIN = (1<<30), | |
493 | - NES_CQE_VALID = (1<<31), | |
494 | -}; | |
495 | - | |
496 | -enum nes_cqe_word_idx { | |
497 | - NES_CQE_PAYLOAD_LENGTH_IDX = 0, | |
498 | - NES_CQE_COMP_COMP_CTX_LOW_IDX = 2, | |
499 | - NES_CQE_COMP_COMP_CTX_HIGH_IDX = 3, | |
500 | - NES_CQE_INV_STAG_IDX = 4, | |
501 | - NES_CQE_QP_ID_IDX = 5, | |
502 | - NES_CQE_ERROR_CODE_IDX = 6, | |
503 | - NES_CQE_OPCODE_IDX = 7, | |
504 | -}; | |
505 | - | |
506 | -enum nes_cqe_allocate_bits { | |
507 | - NES_CQE_ALLOC_INC_SELECT = (1<<28), | |
508 | - NES_CQE_ALLOC_NOTIFY_NEXT = (1<<29), | |
509 | - NES_CQE_ALLOC_NOTIFY_SE = (1<<30), | |
510 | - NES_CQE_ALLOC_RESET = (1<<31), | |
511 | -}; | |
512 | - | |
513 | -enum nes_iwarp_sq_wqe_word_idx { | |
514 | - NES_IWARP_SQ_WQE_MISC_IDX = 0, | |
515 | - NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX = 1, | |
516 | - NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX = 2, | |
517 | - NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX = 3, | |
518 | - NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX = 4, | |
519 | - NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX = 5, | |
520 | - NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX = 7, | |
521 | - NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX = 8, | |
522 | - NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX = 9, | |
523 | - NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX = 10, | |
524 | - NES_IWARP_SQ_WQE_RDMA_STAG_IDX = 11, | |
525 | - NES_IWARP_SQ_WQE_IMM_DATA_START_IDX = 12, | |
526 | - NES_IWARP_SQ_WQE_FRAG0_LOW_IDX = 16, | |
527 | - NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX = 17, | |
528 | - NES_IWARP_SQ_WQE_LENGTH0_IDX = 18, | |
529 | - NES_IWARP_SQ_WQE_STAG0_IDX = 19, | |
530 | - NES_IWARP_SQ_WQE_FRAG1_LOW_IDX = 20, | |
531 | - NES_IWARP_SQ_WQE_FRAG1_HIGH_IDX = 21, | |
532 | - NES_IWARP_SQ_WQE_LENGTH1_IDX = 22, | |
533 | - NES_IWARP_SQ_WQE_STAG1_IDX = 23, | |
534 | - NES_IWARP_SQ_WQE_FRAG2_LOW_IDX = 24, | |
535 | - NES_IWARP_SQ_WQE_FRAG2_HIGH_IDX = 25, | |
536 | - NES_IWARP_SQ_WQE_LENGTH2_IDX = 26, | |
537 | - NES_IWARP_SQ_WQE_STAG2_IDX = 27, | |
538 | - NES_IWARP_SQ_WQE_FRAG3_LOW_IDX = 28, | |
539 | - NES_IWARP_SQ_WQE_FRAG3_HIGH_IDX = 29, | |
540 | - NES_IWARP_SQ_WQE_LENGTH3_IDX = 30, | |
541 | - NES_IWARP_SQ_WQE_STAG3_IDX = 31, | |
542 | -}; | |
543 | - | |
544 | -enum nes_iwarp_rq_wqe_word_idx { | |
545 | - NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX = 1, | |
546 | - NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX = 2, | |
547 | - NES_IWARP_RQ_WQE_COMP_CTX_HIGH_IDX = 3, | |
548 | - NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX = 4, | |
549 | - NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX = 5, | |
550 | - NES_IWARP_RQ_WQE_FRAG0_LOW_IDX = 8, | |
551 | - NES_IWARP_RQ_WQE_FRAG0_HIGH_IDX = 9, | |
552 | - NES_IWARP_RQ_WQE_LENGTH0_IDX = 10, | |
553 | - NES_IWARP_RQ_WQE_STAG0_IDX = 11, | |
554 | - NES_IWARP_RQ_WQE_FRAG1_LOW_IDX = 12, | |
555 | - NES_IWARP_RQ_WQE_FRAG1_HIGH_IDX = 13, | |
556 | - NES_IWARP_RQ_WQE_LENGTH1_IDX = 14, | |
557 | - NES_IWARP_RQ_WQE_STAG1_IDX = 15, | |
558 | - NES_IWARP_RQ_WQE_FRAG2_LOW_IDX = 16, | |
559 | - NES_IWARP_RQ_WQE_FRAG2_HIGH_IDX = 17, | |
560 | - NES_IWARP_RQ_WQE_LENGTH2_IDX = 18, | |
561 | - NES_IWARP_RQ_WQE_STAG2_IDX = 19, | |
562 | - NES_IWARP_RQ_WQE_FRAG3_LOW_IDX = 20, | |
563 | - NES_IWARP_RQ_WQE_FRAG3_HIGH_IDX = 21, | |
564 | - NES_IWARP_RQ_WQE_LENGTH3_IDX = 22, | |
565 | - NES_IWARP_RQ_WQE_STAG3_IDX = 23, | |
566 | -}; | |
567 | - | |
568 | -enum nes_iwarp_sq_opcodes { | |
569 | - NES_IWARP_SQ_WQE_STREAMING = (1<<23), | |
570 | - NES_IWARP_SQ_WQE_IMM_DATA = (1<<28), | |
571 | - NES_IWARP_SQ_WQE_READ_FENCE = (1<<29), | |
572 | - NES_IWARP_SQ_WQE_LOCAL_FENCE = (1<<30), | |
573 | - NES_IWARP_SQ_WQE_SIGNALED_COMPL = (1<<31), | |
574 | -}; | |
575 | - | |
576 | -enum nes_iwarp_sq_wqe_bits { | |
577 | - NES_IWARP_SQ_OP_RDMAW = 0, | |
578 | - NES_IWARP_SQ_OP_RDMAR = 1, | |
579 | - NES_IWARP_SQ_OP_SEND = 3, | |
580 | - NES_IWARP_SQ_OP_SENDINV = 4, | |
581 | - NES_IWARP_SQ_OP_SENDSE = 5, | |
582 | - NES_IWARP_SQ_OP_SENDSEINV = 6, | |
583 | - NES_IWARP_SQ_OP_BIND = 8, | |
584 | - NES_IWARP_SQ_OP_FAST_REG = 9, | |
585 | - NES_IWARP_SQ_OP_LOCINV = 10, | |
586 | - NES_IWARP_SQ_OP_RDMAR_LOCINV = 11, | |
587 | - NES_IWARP_SQ_OP_NOP = 12, | |
588 | -}; | |
589 | - | |
590 | -enum nes_nic_cqe_word_idx { | |
591 | - NES_NIC_CQE_ACCQP_ID_IDX = 0, | |
592 | - NES_NIC_CQE_TAG_PKT_TYPE_IDX = 2, | |
593 | - NES_NIC_CQE_MISC_IDX = 3, | |
594 | -}; | |
595 | - | |
596 | -#define NES_NIC_CQE_ERRV_SHIFT 16 | |
597 | -enum nes_nic_ev_bits { | |
598 | - NES_NIC_ERRV_BITS_MODE = (1<<0), | |
599 | - NES_NIC_ERRV_BITS_IPV4_CSUM_ERR = (1<<1), | |
600 | - NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR = (1<<2), | |
601 | - NES_NIC_ERRV_BITS_WQE_OVERRUN = (1<<3), | |
602 | - NES_NIC_ERRV_BITS_IPH_ERR = (1<<4), | |
603 | -}; | |
604 | - | |
605 | -enum nes_nic_cqe_bits { | |
606 | - NES_NIC_CQE_ERRV_MASK = (0xff<<NES_NIC_CQE_ERRV_SHIFT), | |
607 | - NES_NIC_CQE_SQ = (1<<24), | |
608 | - NES_NIC_CQE_ACCQP_PORT = (1<<28), | |
609 | - NES_NIC_CQE_ACCQP_VALID = (1<<29), | |
610 | - NES_NIC_CQE_TAG_VALID = (1<<30), | |
611 | - NES_NIC_CQE_VALID = (1<<31), | |
612 | -}; | |
613 | -struct nes_hw_nic_cqe { | |
614 | - uint32_t cqe_words[4]; | |
615 | -}; | |
616 | - | |
617 | -enum nes_iwarp_cqe_major_code { | |
618 | - NES_IWARP_CQE_MAJOR_FLUSH = 1, | |
619 | - NES_IWARP_CQE_MAJOR_DRV = 0x8000 | |
620 | -}; | |
621 | - | |
622 | -enum nes_iwarp_cqe_minor_code { | |
623 | - NES_IWARP_CQE_MINOR_FLUSH = 1 | |
624 | -}; | |
625 | - | |
626 | -struct nes_hw_qp_wqe { | |
627 | - uint32_t wqe_words[32]; | |
628 | -}; | |
629 | - | |
630 | -struct nes_hw_cqe { | |
631 | - uint32_t cqe_words[8]; | |
632 | -}; | |
633 | - | |
634 | -struct nes_user_doorbell { | |
635 | - uint32_t wqe_alloc; | |
636 | - uint32_t reserved[3]; | |
637 | - uint32_t cqe_alloc; | |
638 | -}; | |
639 | - | |
640 | -struct nes_udevice { | |
641 | - struct verbs_device ibv_dev; | |
642 | - int page_size; | |
643 | -}; | |
644 | - | |
645 | -struct nes_upd { | |
646 | - struct ibv_pd ibv_pd; | |
647 | - struct nes_user_doorbell volatile *udoorbell; | |
648 | - uint32_t pd_id; | |
649 | - uint32_t db_index; | |
650 | -}; | |
651 | - | |
652 | -struct nes_uvcontext { | |
653 | - struct verbs_context ibv_ctx; | |
654 | - struct nes_upd *nesupd; | |
655 | - uint32_t max_pds; /* maximum pds allowed for this user process */ | |
656 | - uint32_t max_qps; /* maximum qps allowed for this user process */ | |
657 | - uint32_t wq_size; /* size of the WQs (sq+rq) allocated to the mmaped area */ | |
658 | - uint32_t mcrqf; | |
659 | - uint8_t virtwq ; /* flag if to use virt wqs or not */ | |
660 | - uint8_t reserved[3]; | |
661 | -}; | |
662 | - | |
663 | -struct nes_uqp; | |
664 | - | |
665 | -struct nes_ucq { | |
666 | - struct ibv_cq ibv_cq; | |
667 | - struct nes_hw_cqe volatile *cqes; | |
668 | - struct verbs_mr vmr; | |
669 | - pthread_spinlock_t lock; | |
670 | - uint32_t cq_id; | |
671 | - uint16_t size; | |
672 | - uint16_t head; | |
673 | - uint16_t polled_completions; | |
674 | - uint8_t is_armed; | |
675 | - uint8_t skip_arm; | |
676 | - int arm_sol; | |
677 | - int skip_sol; | |
678 | - int comp_vector; | |
679 | - struct nes_uqp *udqp; | |
680 | -}; | |
681 | - | |
682 | -struct nes_uqp { | |
683 | - struct ibv_qp ibv_qp; | |
684 | - struct nes_hw_qp_wqe volatile *sq_vbase; | |
685 | - struct nes_hw_qp_wqe volatile *rq_vbase; | |
686 | - uint32_t qp_id; | |
687 | - struct nes_ucq *send_cq; | |
688 | - struct nes_ucq *recv_cq; | |
689 | - struct verbs_mr vmr; | |
690 | - uint32_t nes_drv_opt; | |
691 | - pthread_spinlock_t lock; | |
692 | - uint16_t sq_db_index; | |
693 | - uint16_t sq_head; | |
694 | - uint16_t sq_tail; | |
695 | - uint16_t sq_size; | |
696 | - uint16_t sq_sig_all; | |
697 | - uint16_t rq_db_index; | |
698 | - uint16_t rq_head; | |
699 | - uint16_t rq_tail; | |
700 | - uint16_t rq_size; | |
701 | - uint16_t rdma0_msg; | |
702 | - uint16_t mapping; | |
703 | - uint16_t qperr; | |
704 | - uint16_t rsvd; | |
705 | - uint32_t pending_rcvs; | |
706 | - struct ibv_recv_wr *pend_rx_wr; | |
707 | - int nes_ud_sksq_fd; | |
708 | - void *sksq_shared_ctxt; | |
709 | - uint64_t send_wr_id[512]; /* IMA send wr_id ring content */ | |
710 | - uint64_t recv_wr_id[512]; /* IMA receive wr_id ring content */ | |
711 | -}; | |
712 | - | |
713 | -#define to_nes_uxxx(xxx, type) \ | |
714 | - container_of(ib##xxx, struct nes_u##type, ibv_##xxx) | |
715 | - | |
716 | -static inline struct nes_udevice *to_nes_udev(struct ibv_device *ibdev) | |
717 | -{ | |
718 | - return container_of(ibdev, struct nes_udevice, ibv_dev.device); | |
719 | -} | |
720 | - | |
721 | -static inline struct nes_uvcontext *to_nes_uctx(struct ibv_context *ibctx) | |
722 | -{ | |
723 | - return container_of(ibctx, struct nes_uvcontext, ibv_ctx.context); | |
724 | -} | |
725 | - | |
726 | -static inline struct nes_upd *to_nes_upd(struct ibv_pd *ibpd) | |
727 | -{ | |
728 | - return to_nes_uxxx(pd, pd); | |
729 | -} | |
730 | - | |
731 | -static inline struct nes_ucq *to_nes_ucq(struct ibv_cq *ibcq) | |
732 | -{ | |
733 | - return to_nes_uxxx(cq, cq); | |
734 | -} | |
735 | - | |
736 | -static inline struct nes_uqp *to_nes_uqp(struct ibv_qp *ibqp) | |
737 | -{ | |
738 | - return to_nes_uxxx(qp, qp); | |
739 | -} | |
740 | - | |
741 | - | |
742 | -/* nes_uverbs.c */ | |
743 | -int nes_uquery_device(struct ibv_context *, struct ibv_device_attr *); | |
744 | -int nes_uquery_port(struct ibv_context *, uint8_t, struct ibv_port_attr *); | |
745 | -struct ibv_pd *nes_ualloc_pd(struct ibv_context *); | |
746 | -int nes_ufree_pd(struct ibv_pd *); | |
747 | -struct ibv_mr *nes_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, | |
748 | - uint64_t hca_va, int access); | |
749 | -int nes_udereg_mr(struct verbs_mr *vmr); | |
750 | -struct ibv_cq *nes_ucreate_cq(struct ibv_context *, int, struct ibv_comp_channel *, int); | |
751 | -int nes_uresize_cq(struct ibv_cq *, int); | |
752 | -int nes_udestroy_cq(struct ibv_cq *); | |
753 | -int nes_upoll_cq(struct ibv_cq *, int, struct ibv_wc *); | |
754 | -int nes_upoll_cq_no_db_read(struct ibv_cq *, int, struct ibv_wc *); | |
755 | -int nes_uarm_cq(struct ibv_cq *, int); | |
756 | -void nes_cq_event(struct ibv_cq *); | |
757 | -struct ibv_srq *nes_ucreate_srq(struct ibv_pd *, struct ibv_srq_init_attr *); | |
758 | -int nes_umodify_srq(struct ibv_srq *, struct ibv_srq_attr *, int); | |
759 | -int nes_udestroy_srq(struct ibv_srq *); | |
760 | -int nes_upost_srq_recv(struct ibv_srq *, struct ibv_recv_wr *, struct ibv_recv_wr **); | |
761 | -struct ibv_qp *nes_ucreate_qp(struct ibv_pd *, struct ibv_qp_init_attr *); | |
762 | -int nes_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, | |
763 | - int, struct ibv_qp_init_attr *init_attr); | |
764 | -int nes_umodify_qp(struct ibv_qp *, struct ibv_qp_attr *, int); | |
765 | -int nes_udestroy_qp(struct ibv_qp *); | |
766 | -int nes_upost_send(struct ibv_qp *, struct ibv_send_wr *, struct ibv_send_wr **); | |
767 | -int nes_upost_recv(struct ibv_qp *, struct ibv_recv_wr *, struct ibv_recv_wr **); | |
768 | -struct ibv_ah *nes_ucreate_ah(struct ibv_pd *, struct ibv_ah_attr *); | |
769 | -int nes_udestroy_ah(struct ibv_ah *); | |
770 | -int nes_uattach_mcast(struct ibv_qp *, const union ibv_gid *, uint16_t); | |
771 | -int nes_udetach_mcast(struct ibv_qp *, const union ibv_gid *, uint16_t); | |
772 | -void nes_async_event(struct ibv_context *context, | |
773 | - struct ibv_async_event *event); | |
774 | - | |
775 | -extern long int page_size; | |
776 | - | |
777 | -#endif /* nes_umain_H */ | |
778 | diff --git a/providers/nes/nes_uverbs.c b/providers/nes/nes_uverbs.c | |
779 | deleted file mode 100644 | |
780 | index 2b78468b4..000000000 | |
781 | --- a/providers/nes/nes_uverbs.c | |
782 | +++ /dev/null | |
783 | @@ -1,1535 +0,0 @@ | |
784 | -/* | |
785 | - * Copyright (c) 2006 - 2010 Intel Corporation. All rights reserved. | |
786 | - * | |
787 | - * This software is available to you under a choice of one of two | |
788 | - * licenses. You may choose to be licensed under the terms of the GNU | |
789 | - * General Public License (GPL) Version 2, available from the file | |
790 | - * gpl-2.0.txt in the main directory of this source tree, or the | |
791 | - * OpenIB.org BSD license below: | |
792 | - * | |
793 | - * Redistribution and use in source and binary forms, with or | |
794 | - * without modification, are permitted provided that the following | |
795 | - * conditions are met: | |
796 | - * | |
797 | - * - Redistributions of source code must retain the above | |
798 | - * copyright notice, this list of conditions and the following | |
799 | - * disclaimer. | |
800 | - * | |
801 | - * - Redistributions in binary form must reproduce the above | |
802 | - * copyright notice, this list of conditions and the following | |
803 | - * disclaimer in the documentation and/or other materials | |
804 | - * provided with the distribution. | |
805 | - * | |
806 | - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
807 | - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
808 | - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
809 | - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
810 | - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
811 | - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
812 | - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
813 | - * SOFTWARE. | |
814 | - */ | |
815 | - | |
816 | -#include <config.h> | |
817 | - | |
818 | -#include <endian.h> | |
819 | -#include <stdlib.h> | |
820 | -#include <stdio.h> | |
821 | -#include <string.h> | |
822 | -#include <unistd.h> | |
823 | -#include <signal.h> | |
824 | -#include <errno.h> | |
825 | -#include <pthread.h> | |
826 | -#include <malloc.h> | |
827 | -#include <sys/mman.h> | |
828 | -#include <linux/if_ether.h> | |
829 | -#include <sys/stat.h> | |
830 | -#include <fcntl.h> | |
831 | - | |
832 | -#include "nes_umain.h" | |
833 | -#include "nes-abi.h" | |
834 | - | |
835 | -#define STATIC static | |
836 | -#define INLINE inline | |
837 | - | |
838 | -#define NES_WC_WITH_VLAN 1 << 3 | |
839 | -#define NES_UD_RX_BATCH_SZ 64 | |
840 | -#define NES_UD_MAX_SG_LIST_SZ 1 | |
841 | - | |
842 | -struct nes_ud_send_wr { | |
843 | - uint32_t wr_cnt; | |
844 | - uint32_t qpn; | |
845 | - uint32_t flags; | |
846 | - uint32_t resv[1]; | |
847 | - struct ibv_sge sg_list[64]; | |
848 | -}; | |
849 | - | |
850 | -struct nes_ud_recv_wr { | |
851 | - uint32_t wr_cnt; | |
852 | - uint32_t qpn; | |
853 | - uint32_t resv[2]; | |
854 | - struct ibv_sge sg_list[64]; | |
855 | -}; | |
856 | - | |
857 | -/** | |
858 | - * nes_uquery_device | |
859 | - */ | |
860 | -int nes_uquery_device(struct ibv_context *context, struct ibv_device_attr *attr) | |
861 | -{ | |
862 | - struct ibv_query_device cmd; | |
863 | - uint64_t nes_fw_ver; | |
864 | - int ret; | |
865 | - unsigned int minor, major; | |
866 | - | |
867 | - ret = ibv_cmd_query_device(context, attr, &nes_fw_ver, | |
868 | - &cmd, sizeof cmd); | |
869 | - if (ret) | |
870 | - return ret; | |
871 | - | |
872 | - major = (nes_fw_ver >> 16) & 0xffff; | |
873 | - minor = nes_fw_ver & 0xffff; | |
874 | - | |
875 | - snprintf(attr->fw_ver, sizeof attr->fw_ver, | |
876 | - "%d.%d", major, minor); | |
877 | - | |
878 | - return 0; | |
879 | -} | |
880 | - | |
881 | - | |
882 | -/** | |
883 | - * nes_uquery_port | |
884 | - */ | |
885 | -int nes_uquery_port(struct ibv_context *context, uint8_t port, | |
886 | - struct ibv_port_attr *attr) | |
887 | -{ | |
888 | - struct ibv_query_port cmd; | |
889 | - | |
890 | - return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd); | |
891 | -} | |
892 | - | |
893 | - | |
894 | -/** | |
895 | - * nes_ualloc_pd | |
896 | - */ | |
897 | -struct ibv_pd *nes_ualloc_pd(struct ibv_context *context) | |
898 | -{ | |
899 | - struct ibv_alloc_pd cmd; | |
900 | - struct nes_ualloc_pd_resp resp; | |
901 | - struct nes_upd *nesupd; | |
902 | - | |
903 | - nesupd = malloc(sizeof *nesupd); | |
904 | - if (!nesupd) | |
905 | - return NULL; | |
906 | - | |
907 | - if (ibv_cmd_alloc_pd(context, &nesupd->ibv_pd, &cmd, sizeof cmd, | |
908 | - &resp.ibv_resp, sizeof resp)) { | |
909 | - free(nesupd); | |
910 | - return NULL; | |
911 | - } | |
912 | - nesupd->pd_id = resp.pd_id; | |
913 | - nesupd->db_index = resp.mmap_db_index; | |
914 | - | |
915 | - nesupd->udoorbell = mmap(NULL, page_size, PROT_WRITE | PROT_READ, MAP_SHARED, | |
916 | - context->cmd_fd, nesupd->db_index * page_size); | |
917 | - | |
918 | - if (nesupd->udoorbell == MAP_FAILED) { | |
919 | - free(nesupd); | |
920 | - return NULL; | |
921 | - } | |
922 | - | |
923 | - return &nesupd->ibv_pd; | |
924 | -} | |
925 | - | |
926 | - | |
927 | -/** | |
928 | - * nes_ufree_pd | |
929 | - */ | |
930 | -int nes_ufree_pd(struct ibv_pd *pd) | |
931 | -{ | |
932 | - int ret; | |
933 | - struct nes_upd *nesupd; | |
934 | - | |
935 | - nesupd = to_nes_upd(pd); | |
936 | - | |
937 | - ret = ibv_cmd_dealloc_pd(pd); | |
938 | - if (ret) | |
939 | - return ret; | |
940 | - | |
941 | - munmap((void *)nesupd->udoorbell, page_size); | |
942 | - free(nesupd); | |
943 | - | |
944 | - return 0; | |
945 | -} | |
946 | - | |
947 | - | |
948 | -/** | |
949 | - * nes_ureg_mr | |
950 | - */ | |
951 | -struct ibv_mr *nes_ureg_mr(struct ibv_pd *pd, void *addr, size_t length, | |
952 | - uint64_t hca_va, int access) | |
953 | -{ | |
954 | - struct verbs_mr *vmr; | |
955 | - struct nes_ureg_mr cmd; | |
956 | - struct ib_uverbs_reg_mr_resp resp; | |
957 | - | |
958 | - vmr = malloc(sizeof(*vmr)); | |
959 | - if (!vmr) | |
960 | - return NULL; | |
961 | - | |
962 | - cmd.reg_type = IWNES_MEMREG_TYPE_MEM; | |
963 | - if (ibv_cmd_reg_mr(pd, addr, length, hca_va, access, vmr, &cmd.ibv_cmd, | |
964 | - sizeof(cmd), &resp, sizeof(resp))) { | |
965 | - free(vmr); | |
966 | - | |
967 | - return NULL; | |
968 | - } | |
969 | - | |
970 | - return &vmr->ibv_mr; | |
971 | -} | |
972 | - | |
973 | - | |
974 | -/** | |
975 | - * nes_udereg_mr | |
976 | - */ | |
977 | -int nes_udereg_mr(struct verbs_mr *vmr) | |
978 | -{ | |
979 | - int ret; | |
980 | - | |
981 | - ret = ibv_cmd_dereg_mr(vmr); | |
982 | - if (ret) | |
983 | - return ret; | |
984 | - | |
985 | - free(vmr); | |
986 | - return 0; | |
987 | -} | |
988 | - | |
989 | -/** | |
990 | - * nes_ucreate_cq | |
991 | - */ | |
992 | -struct ibv_cq *nes_ucreate_cq(struct ibv_context *context, int cqe, | |
993 | - struct ibv_comp_channel *channel, int comp_vector) | |
994 | -{ | |
995 | - struct nes_ucq *nesucq; | |
996 | - struct nes_ureg_mr reg_mr_cmd; | |
997 | - struct ib_uverbs_reg_mr_resp reg_mr_resp; | |
998 | - struct nes_ucreate_cq cmd; | |
999 | - struct nes_ucreate_cq_resp resp; | |
1000 | - int ret; | |
1001 | - struct nes_uvcontext *nesvctx = to_nes_uctx(context); | |
1002 | - | |
1003 | - nesucq = malloc(sizeof *nesucq); | |
1004 | - if (!nesucq) { | |
1005 | - return NULL; | |
1006 | - } | |
1007 | - memset(nesucq, 0, sizeof(*nesucq)); | |
1008 | - | |
1009 | - if (pthread_spin_init(&nesucq->lock, PTHREAD_PROCESS_PRIVATE)) { | |
1010 | - free(nesucq); | |
1011 | - return NULL; | |
1012 | - } | |
1013 | - | |
1014 | - if (cqe < 4) /* a reasonable minimum */ | |
1015 | - cqe = 4; | |
1016 | - nesucq->size = cqe + 1; | |
1017 | - nesucq->comp_vector = comp_vector; | |
1018 | - | |
1019 | - nesucq->cqes = memalign(page_size, nesucq->size*sizeof(struct nes_hw_cqe)); | |
1020 | - if (!nesucq->cqes) | |
1021 | - goto err; | |
1022 | - | |
1023 | - /* Register the memory for the CQ */ | |
1024 | - reg_mr_cmd.reg_type = IWNES_MEMREG_TYPE_CQ; | |
1025 | - | |
1026 | - ret = ibv_cmd_reg_mr(&nesvctx->nesupd->ibv_pd, (void *)nesucq->cqes, | |
1027 | - (nesucq->size*sizeof(struct nes_hw_cqe)), | |
1028 | - (uintptr_t)nesucq->cqes, IBV_ACCESS_LOCAL_WRITE, | |
1029 | - &nesucq->vmr, ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), | |
1030 | - ®_mr_resp, sizeof(reg_mr_resp)); | |
1031 | - if (ret) { | |
1032 | - /* fprintf(stderr, "ibv_cmd_reg_mr failed (ret = %d).\n", ret); */ | |
1033 | - free((struct nes_hw_cqe *)nesucq->cqes); | |
1034 | - goto err; | |
1035 | - } | |
1036 | - | |
1037 | - /* Create the CQ */ | |
1038 | - memset(&cmd, 0, sizeof(cmd)); | |
1039 | - cmd.user_cq_buffer = (__u64)((uintptr_t)nesucq->cqes); | |
1040 | - cmd.mcrqf = nesvctx->mcrqf; | |
1041 | - | |
1042 | - ret = ibv_cmd_create_cq(context, nesucq->size-1, channel, comp_vector, | |
1043 | - &nesucq->ibv_cq, &cmd.ibv_cmd, sizeof cmd, | |
1044 | - &resp.ibv_resp, sizeof resp); | |
1045 | - if (ret) | |
1046 | - goto err; | |
1047 | - | |
1048 | - nesucq->cq_id = (uint16_t)resp.cq_id; | |
1049 | - | |
1050 | - /* Zero out the CQ */ | |
1051 | - memset((struct nes_hw_cqe *)nesucq->cqes, 0, nesucq->size*sizeof(struct nes_hw_cqe)); | |
1052 | - | |
1053 | - return &nesucq->ibv_cq; | |
1054 | - | |
1055 | -err: | |
1056 | - /* fprintf(stderr, PFX "%s: Error Creating CQ.\n", __FUNCTION__); */ | |
1057 | - pthread_spin_destroy(&nesucq->lock); | |
1058 | - free(nesucq); | |
1059 | - | |
1060 | - return NULL; | |
1061 | -} | |
1062 | - | |
1063 | - | |
1064 | -/** | |
1065 | - * nes_uresize_cq | |
1066 | - */ | |
1067 | -int nes_uresize_cq(struct ibv_cq *cq, int cqe) | |
1068 | -{ | |
1069 | - /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */ | |
1070 | - return -ENOSYS; | |
1071 | -} | |
1072 | - | |
1073 | -/** | |
1074 | - * nes_udestroy_cq | |
1075 | - */ | |
1076 | -int nes_udestroy_cq(struct ibv_cq *cq) | |
1077 | -{ | |
1078 | - struct nes_ucq *nesucq = to_nes_ucq(cq); | |
1079 | - int ret; | |
1080 | - | |
1081 | - ret = ibv_cmd_destroy_cq(cq); | |
1082 | - if (ret) | |
1083 | - return ret; | |
1084 | - | |
1085 | - ret = ibv_cmd_dereg_mr(&nesucq->vmr); | |
1086 | - if (ret) | |
1087 | - fprintf(stderr, PFX "%s: Failed to deregister CQ Memory Region.\n", __FUNCTION__); | |
1088 | - | |
1089 | - /* Free CQ the memory */ | |
1090 | - free((struct nes_hw_cqe *)nesucq->cqes); | |
1091 | - pthread_spin_destroy(&nesucq->lock); | |
1092 | - free(nesucq); | |
1093 | - | |
1094 | - return 0; | |
1095 | -} | |
1096 | - | |
1097 | -#define NES_CQ_BUF_OV_ERR 0x3 | |
1098 | - | |
1099 | -static inline | |
1100 | -int nes_ima_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry) | |
1101 | -{ | |
1102 | - struct nes_ucq *nesucq = to_nes_ucq(cq); | |
1103 | - struct nes_uvcontext *nesvctx = to_nes_uctx(cq->context); | |
1104 | - uint32_t cqe_misc; | |
1105 | - int cqe_count = 0; | |
1106 | - uint32_t head; | |
1107 | - uint32_t cq_size; | |
1108 | - | |
1109 | - volatile struct nes_hw_nic_cqe *cqe = NULL; | |
1110 | - volatile struct nes_hw_nic_cqe *cqes; | |
1111 | - | |
1112 | - struct nes_uqp *nesuqp = nesucq->udqp; | |
1113 | - uint32_t vlan_tag = 0; | |
1114 | - | |
1115 | - cqes = (volatile struct nes_hw_nic_cqe *)nesucq->cqes; | |
1116 | - head = nesucq->head; | |
1117 | - cq_size = nesucq->size; | |
1118 | - | |
1119 | - if (!nesuqp || !nesvctx) | |
1120 | - exit(0); | |
1121 | - if (nesuqp->ibv_qp.state == IBV_QPS_ERR) { | |
1122 | - while (cqe_count < num_entries) { | |
1123 | - memset(entry, 0, sizeof *entry); | |
1124 | - | |
1125 | - if (nesuqp->recv_cq == nesucq) { | |
1126 | - if (nesuqp->rq_tail != nesuqp->rq_head) { | |
1127 | - /* Working on a RQ Completion*/ | |
1128 | - entry->wr_id = | |
1129 | - nesuqp->recv_wr_id[nesuqp->rq_tail]; | |
1130 | - if (++nesuqp->rq_tail >= nesuqp->rq_size) | |
1131 | - nesuqp->rq_tail = 0; | |
1132 | - } else | |
1133 | - return cqe_count; | |
1134 | - } else | |
1135 | - if (nesuqp->send_cq == nesucq) { | |
1136 | - if (nesuqp->sq_tail != nesuqp->sq_head) { | |
1137 | - entry->wr_id = | |
1138 | - nesuqp->send_wr_id[nesuqp->sq_tail]; | |
1139 | - /* Working on a SQ Completion*/ | |
1140 | - if (++nesuqp->sq_tail >= nesuqp->sq_size) | |
1141 | - nesuqp->sq_tail = 0; | |
1142 | - } else | |
1143 | - return cqe_count; | |
1144 | - } | |
1145 | - entry->status = IBV_WC_WR_FLUSH_ERR; | |
1146 | - entry++; | |
1147 | - cqe_count++; | |
1148 | - } | |
1149 | - return cqe_count; | |
1150 | - } | |
1151 | - | |
1152 | - while (cqe_count < num_entries) { | |
1153 | - const enum ibv_wc_opcode INVAL_OP = -1; | |
1154 | - | |
1155 | - entry->opcode = INVAL_OP; | |
1156 | - cqe = &cqes[head]; | |
1157 | - cqe_misc = | |
1158 | - le32toh(cqe->cqe_words[NES_NIC_CQE_MISC_IDX]); | |
1159 | - if (cqe_misc & NES_NIC_CQE_VALID) { | |
1160 | - memset(entry, 0, sizeof *entry); | |
1161 | - entry->opcode = INVAL_OP; | |
1162 | - cqe->cqe_words[NES_NIC_CQE_MISC_IDX] = 0; | |
1163 | - entry->status = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> | |
1164 | - NES_NIC_CQE_ERRV_SHIFT; | |
1165 | - entry->qp_num = nesuqp->qp_id; | |
1166 | - entry->src_qp = nesuqp->qp_id; | |
1167 | - if (cqe_misc & NES_NIC_CQE_SQ) { | |
1168 | - entry->opcode = IBV_WC_SEND; | |
1169 | - | |
1170 | - entry->wr_id = | |
1171 | - nesuqp->send_wr_id[nesuqp->sq_tail]; | |
1172 | - | |
1173 | - /* Working on a SQ Completion*/ | |
1174 | - if (++nesuqp->sq_tail >= nesuqp->sq_size) | |
1175 | - nesuqp->sq_tail = 0; | |
1176 | - } else { | |
1177 | - /* no CRC counting at all - all packets | |
1178 | - go to higher layer as they are received - | |
1179 | - the fastest path */ | |
1180 | - | |
1181 | - entry->byte_len = cqe_misc & 0xffff; | |
1182 | - entry->opcode = IBV_WC_RECV; | |
1183 | - | |
1184 | - entry->wr_id = | |
1185 | - nesuqp->recv_wr_id[nesuqp->rq_tail]; | |
1186 | - if (cqe_misc & NES_NIC_CQE_TAG_VALID) { | |
1187 | - vlan_tag = le32toh( | |
1188 | - cqe->cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]) | |
1189 | - >> 16; | |
1190 | - entry->sl = (vlan_tag >> 12) & 0x0f; | |
1191 | - entry->pkey_index = vlan_tag & 0x0fff; | |
1192 | - entry->wc_flags |= NES_WC_WITH_VLAN; | |
1193 | - } | |
1194 | - | |
1195 | - | |
1196 | - /* Working on a RQ Completion*/ | |
1197 | - if (++nesuqp->rq_tail >= nesuqp->rq_size) | |
1198 | - nesuqp->rq_tail = 0; | |
1199 | - if (entry->status == NES_CQ_BUF_OV_ERR) | |
1200 | - entry->status = IBV_WC_LOC_LEN_ERR; | |
1201 | - } | |
1202 | - | |
1203 | - if (++head >= cq_size) | |
1204 | - head = 0; | |
1205 | - | |
1206 | - if (entry->opcode != INVAL_OP) { | |
1207 | - /* it is possible that no entry will be | |
1208 | - available */ | |
1209 | - cqe_count++; | |
1210 | - entry++; | |
1211 | - } | |
1212 | - | |
1213 | - nesvctx->nesupd->udoorbell->cqe_alloc = | |
1214 | - htole32(nesucq->cq_id | (1 << 16)); | |
1215 | - } else { | |
1216 | - break; | |
1217 | - } | |
1218 | - } | |
1219 | - nesucq->head = head; | |
1220 | - return cqe_count; | |
1221 | -} | |
1222 | - | |
1223 | -/** | |
1224 | - * nes_upoll_cq | |
1225 | - */ | |
1226 | -int nes_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry) | |
1227 | -{ | |
1228 | - uint64_t wrid; | |
1229 | - struct nes_ucq *nesucq; | |
1230 | - struct nes_uvcontext *nesvctx = NULL; | |
1231 | - struct nes_uqp *nesuqp; | |
1232 | - int cqe_count=0; | |
1233 | - uint32_t head; | |
1234 | - uint32_t cq_size; | |
1235 | - uint32_t wqe_index; | |
1236 | - uint32_t wq_tail = 0; | |
1237 | - struct nes_hw_cqe cqe; | |
1238 | - uint64_t u64temp; | |
1239 | - int move_cq_head = 1; | |
1240 | - uint32_t err_code; | |
1241 | - | |
1242 | - nesucq = to_nes_ucq(cq); | |
1243 | - nesvctx = to_nes_uctx(cq->context); | |
1244 | - | |
1245 | - if (nesucq->cq_id < 64) | |
1246 | - return nes_ima_upoll_cq(cq, num_entries, entry); | |
1247 | - | |
1248 | - pthread_spin_lock(&nesucq->lock); | |
1249 | - | |
1250 | - head = nesucq->head; | |
1251 | - cq_size = nesucq->size; | |
1252 | - | |
1253 | - while (cqe_count<num_entries) { | |
1254 | - if ((le32toh(nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) == 0) | |
1255 | - break; | |
1256 | - | |
1257 | - /* Make sure we read CQ entry contents *after* we've checked the valid bit. */ | |
1258 | - udma_from_device_barrier(); | |
1259 | - | |
1260 | - cqe = (volatile struct nes_hw_cqe)nesucq->cqes[head]; | |
1261 | - | |
1262 | - /* parse CQE, get completion context from WQE (either rq or sq */ | |
1263 | - wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]) & 511; | |
1264 | - u64temp = ((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]))) | | |
1265 | - (((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32); | |
1266 | - | |
1267 | - if (likely(u64temp)) { | |
1268 | - nesuqp = (struct nes_uqp *)(uintptr_t)(u64temp & (~1023)); | |
1269 | - memset(entry, 0, sizeof *entry); | |
1270 | - if (likely(le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]) == 0)) { | |
1271 | - entry->status = IBV_WC_SUCCESS; | |
1272 | - } else { | |
1273 | - err_code = le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]); | |
1274 | - if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16)) { | |
1275 | - entry->status = err_code & 0x0000ffff; | |
1276 | - } else { | |
1277 | - entry->status = IBV_WC_WR_FLUSH_ERR; | |
1278 | - if (le32toh(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) { | |
1279 | - if (wqe_index == 0 && nesuqp->rdma0_msg) { | |
1280 | - nesuqp->sq_tail = (wqe_index+1)&(nesuqp->sq_size - 1); | |
1281 | - move_cq_head = 0; | |
1282 | - wq_tail = nesuqp->sq_tail; | |
1283 | - nesuqp->rdma0_msg = 0; | |
1284 | - goto nes_upoll_cq_update; | |
1285 | - } | |
1286 | - } | |
1287 | - } | |
1288 | - } | |
1289 | - entry->qp_num = nesuqp->qp_id; | |
1290 | - entry->src_qp = nesuqp->qp_id; | |
1291 | - nesuqp->rdma0_msg = 0; | |
1292 | - | |
1293 | - if (le32toh(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) { | |
1294 | - /* Working on a SQ Completion*/ | |
1295 | - wrid = ((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX])) | | |
1296 | - (((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); | |
1297 | - entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]); | |
1298 | - | |
1299 | - switch (le32toh(nesuqp->sq_vbase[wqe_index]. | |
1300 | - wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) { | |
1301 | - case NES_IWARP_SQ_OP_RDMAW: | |
1302 | - /* fprintf(stderr, PFX "%s: Operation = RDMA WRITE.\n", | |
1303 | - __FUNCTION__ ); */ | |
1304 | - entry->opcode = IBV_WC_RDMA_WRITE; | |
1305 | - break; | |
1306 | - case NES_IWARP_SQ_OP_RDMAR: | |
1307 | - /* fprintf(stderr, PFX "%s: Operation = RDMA READ.\n", | |
1308 | - __FUNCTION__ ); */ | |
1309 | - entry->opcode = IBV_WC_RDMA_READ; | |
1310 | - entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index]. | |
1311 | - wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]); | |
1312 | - break; | |
1313 | - case NES_IWARP_SQ_OP_SENDINV: | |
1314 | - case NES_IWARP_SQ_OP_SENDSEINV: | |
1315 | - case NES_IWARP_SQ_OP_SEND: | |
1316 | - case NES_IWARP_SQ_OP_SENDSE: | |
1317 | - /* fprintf(stderr, PFX "%s: Operation = Send.\n", | |
1318 | - __FUNCTION__ ); */ | |
1319 | - entry->opcode = IBV_WC_SEND; | |
1320 | - break; | |
1321 | - } | |
1322 | - | |
1323 | - nesuqp->sq_tail = (wqe_index+1)&(nesuqp->sq_size - 1); | |
1324 | - if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->sq_tail != nesuqp->sq_head)) { | |
1325 | - move_cq_head = 0; | |
1326 | - wq_tail = nesuqp->sq_tail; | |
1327 | - } | |
1328 | - } else { | |
1329 | - /* Working on a RQ Completion*/ | |
1330 | - entry->byte_len = le32toh(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]); | |
1331 | - wrid = ((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX])) | | |
1332 | - (((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); | |
1333 | - entry->opcode = IBV_WC_RECV; | |
1334 | - | |
1335 | - nesuqp->rq_tail = (wqe_index+1)&(nesuqp->rq_size - 1); | |
1336 | - if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->rq_tail != nesuqp->rq_head)) { | |
1337 | - move_cq_head = 0; | |
1338 | - wq_tail = nesuqp->rq_tail; | |
1339 | - } | |
1340 | - } | |
1341 | - | |
1342 | - entry->wr_id = wrid; | |
1343 | - entry++; | |
1344 | - cqe_count++; | |
1345 | - } | |
1346 | -nes_upoll_cq_update: | |
1347 | - if (move_cq_head) { | |
1348 | - nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; | |
1349 | - if (++head >= cq_size) | |
1350 | - head = 0; | |
1351 | - nesucq->polled_completions++; | |
1352 | - | |
1353 | - if ((nesucq->polled_completions > (cq_size/2)) || | |
1354 | - (nesucq->polled_completions == 255)) { | |
1355 | - if (nesvctx == NULL) | |
1356 | - nesvctx = to_nes_uctx(cq->context); | |
1357 | - nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id | | |
1358 | - (nesucq->polled_completions << 16)); | |
1359 | - nesucq->polled_completions = 0; | |
1360 | - } | |
1361 | - } else { | |
1362 | - /* Update the wqe index and set status to flush */ | |
1363 | - wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); | |
1364 | - wqe_index = (wqe_index & (~511)) | wq_tail; | |
1365 | - nesucq->cqes[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = | |
1366 | - htole32(wqe_index); | |
1367 | - nesucq->cqes[head].cqe_words[NES_CQE_ERROR_CODE_IDX] = | |
1368 | - htole32((NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH); | |
1369 | - move_cq_head = 1; /* ready for next pass */ | |
1370 | - } | |
1371 | - } | |
1372 | - | |
1373 | - if (nesucq->polled_completions) { | |
1374 | - if (nesvctx == NULL) | |
1375 | - nesvctx = to_nes_uctx(cq->context); | |
1376 | - nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id | | |
1377 | - (nesucq->polled_completions << 16)); | |
1378 | - nesucq->polled_completions = 0; | |
1379 | - } | |
1380 | - nesucq->head = head; | |
1381 | - | |
1382 | - pthread_spin_unlock(&nesucq->lock); | |
1383 | - | |
1384 | - return cqe_count; | |
1385 | -} | |
1386 | - | |
1387 | - | |
1388 | -/** | |
1389 | - * nes_upoll_cq_no_db_read | |
1390 | - */ | |
1391 | -int nes_upoll_cq_no_db_read(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry) | |
1392 | -{ | |
1393 | - uint64_t wrid; | |
1394 | - struct nes_ucq *nesucq; | |
1395 | - struct nes_uvcontext *nesvctx = NULL; | |
1396 | - struct nes_uqp *nesuqp; | |
1397 | - int cqe_count=0; | |
1398 | - uint32_t head; | |
1399 | - uint32_t cq_size; | |
1400 | - uint32_t wqe_index; | |
1401 | - uint32_t wq_tail = 0; | |
1402 | - struct nes_hw_cqe cqe; | |
1403 | - uint64_t u64temp; | |
1404 | - int move_cq_head = 1; | |
1405 | - uint32_t err_code; | |
1406 | - | |
1407 | - nesucq = to_nes_ucq(cq); | |
1408 | - nesvctx = to_nes_uctx(cq->context); | |
1409 | - | |
1410 | - if (nesucq->cq_id < 64) | |
1411 | - return nes_ima_upoll_cq(cq, num_entries, entry); | |
1412 | - | |
1413 | - pthread_spin_lock(&nesucq->lock); | |
1414 | - | |
1415 | - head = nesucq->head; | |
1416 | - cq_size = nesucq->size; | |
1417 | - | |
1418 | - while (cqe_count<num_entries) { | |
1419 | - if ((le32toh(nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) == 0) | |
1420 | - break; | |
1421 | - | |
1422 | - /* Make sure we read CQ entry contents *after* we've checked the valid bit. */ | |
1423 | - udma_from_device_barrier(); | |
1424 | - | |
1425 | - cqe = (volatile struct nes_hw_cqe)nesucq->cqes[head]; | |
1426 | - | |
1427 | - /* parse CQE, get completion context from WQE (either rq or sq */ | |
1428 | - wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]) & 511; | |
1429 | - u64temp = ((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]))) | | |
1430 | - (((uint64_t) (le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32); | |
1431 | - | |
1432 | - if (likely(u64temp)) { | |
1433 | - nesuqp = (struct nes_uqp *)(uintptr_t)(u64temp & (~1023)); | |
1434 | - memset(entry, 0, sizeof *entry); | |
1435 | - if (likely(le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]) == 0)) { | |
1436 | - entry->status = IBV_WC_SUCCESS; | |
1437 | - } else { | |
1438 | - err_code = le32toh(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]); | |
1439 | - if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16)) | |
1440 | - entry->status = err_code & 0x0000ffff; | |
1441 | - else | |
1442 | - entry->status = IBV_WC_WR_FLUSH_ERR; | |
1443 | - } | |
1444 | - entry->qp_num = nesuqp->qp_id; | |
1445 | - entry->src_qp = nesuqp->qp_id; | |
1446 | - | |
1447 | - if (le32toh(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) { | |
1448 | - /* Working on a SQ Completion*/ | |
1449 | - wrid = ((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX])) | | |
1450 | - (((uint64_t) le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); | |
1451 | - entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index].wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]); | |
1452 | - | |
1453 | - switch (le32toh(nesuqp->sq_vbase[wqe_index]. | |
1454 | - wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) { | |
1455 | - case NES_IWARP_SQ_OP_RDMAW: | |
1456 | - /* fprintf(stderr, PFX "%s: Operation = RDMA WRITE.\n", | |
1457 | - __FUNCTION__ ); */ | |
1458 | - entry->opcode = IBV_WC_RDMA_WRITE; | |
1459 | - break; | |
1460 | - case NES_IWARP_SQ_OP_RDMAR: | |
1461 | - /* fprintf(stderr, PFX "%s: Operation = RDMA READ.\n", | |
1462 | - __FUNCTION__ ); */ | |
1463 | - entry->opcode = IBV_WC_RDMA_READ; | |
1464 | - entry->byte_len = le32toh(nesuqp->sq_vbase[wqe_index]. | |
1465 | - wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]); | |
1466 | - break; | |
1467 | - case NES_IWARP_SQ_OP_SENDINV: | |
1468 | - case NES_IWARP_SQ_OP_SENDSEINV: | |
1469 | - case NES_IWARP_SQ_OP_SEND: | |
1470 | - case NES_IWARP_SQ_OP_SENDSE: | |
1471 | - /* fprintf(stderr, PFX "%s: Operation = Send.\n", | |
1472 | - __FUNCTION__ ); */ | |
1473 | - entry->opcode = IBV_WC_SEND; | |
1474 | - break; | |
1475 | - } | |
1476 | - | |
1477 | - nesuqp->sq_tail = (wqe_index+1)&(nesuqp->sq_size - 1); | |
1478 | - if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->sq_tail != nesuqp->sq_head)) { | |
1479 | - move_cq_head = 0; | |
1480 | - wq_tail = nesuqp->sq_tail; | |
1481 | - } | |
1482 | - } else { | |
1483 | - /* Working on a RQ Completion*/ | |
1484 | - entry->byte_len = le32toh(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]); | |
1485 | - wrid = ((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX])) | | |
1486 | - (((uint64_t) le32toh(nesuqp->rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); | |
1487 | - entry->opcode = IBV_WC_RECV; | |
1488 | - | |
1489 | - nesuqp->rq_tail = (wqe_index+1)&(nesuqp->rq_size - 1); | |
1490 | - if ((entry->status != IBV_WC_SUCCESS) && (nesuqp->rq_tail != nesuqp->rq_head)) { | |
1491 | - move_cq_head = 0; | |
1492 | - wq_tail = nesuqp->rq_tail; | |
1493 | - } | |
1494 | - } | |
1495 | - | |
1496 | - entry->wr_id = wrid; | |
1497 | - entry++; | |
1498 | - cqe_count++; | |
1499 | - } | |
1500 | - | |
1501 | - if (move_cq_head) { | |
1502 | - nesucq->cqes[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; | |
1503 | - if (++head >= cq_size) | |
1504 | - head = 0; | |
1505 | - nesucq->polled_completions++; | |
1506 | - | |
1507 | - if ((nesucq->polled_completions > (cq_size/2)) || | |
1508 | - (nesucq->polled_completions == 255)) { | |
1509 | - if (nesvctx == NULL) | |
1510 | - nesvctx = to_nes_uctx(cq->context); | |
1511 | - nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id | | |
1512 | - (nesucq->polled_completions << 16)); | |
1513 | - nesucq->polled_completions = 0; | |
1514 | - } | |
1515 | - } else { | |
1516 | - /* Update the wqe index and set status to flush */ | |
1517 | - wqe_index = le32toh(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); | |
1518 | - wqe_index = (wqe_index & (~511)) | wq_tail; | |
1519 | - nesucq->cqes[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = | |
1520 | - htole32(wqe_index); | |
1521 | - nesucq->cqes[head].cqe_words[NES_CQE_ERROR_CODE_IDX] = | |
1522 | - htole32((NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH); | |
1523 | - move_cq_head = 1; /* ready for next pass */ | |
1524 | - } | |
1525 | - } | |
1526 | - | |
1527 | - if (nesucq->polled_completions) { | |
1528 | - if (nesvctx == NULL) | |
1529 | - nesvctx = to_nes_uctx(cq->context); | |
1530 | - nesvctx->nesupd->udoorbell->cqe_alloc = htole32(nesucq->cq_id | | |
1531 | - (nesucq->polled_completions << 16)); | |
1532 | - nesucq->polled_completions = 0; | |
1533 | - } | |
1534 | - nesucq->head = head; | |
1535 | - | |
1536 | - pthread_spin_unlock(&nesucq->lock); | |
1537 | - | |
1538 | - return cqe_count; | |
1539 | -} | |
1540 | - | |
1541 | -/** | |
1542 | - * nes_arm_cq | |
1543 | - */ | |
1544 | -static void nes_arm_cq(struct nes_ucq *nesucq, struct nes_uvcontext *nesvctx, int sol) | |
1545 | -{ | |
1546 | - uint32_t cq_arm; | |
1547 | - | |
1548 | - cq_arm = nesucq->cq_id; | |
1549 | - | |
1550 | - if (sol) | |
1551 | - cq_arm |= NES_CQE_ALLOC_NOTIFY_SE; | |
1552 | - else | |
1553 | - cq_arm |= NES_CQE_ALLOC_NOTIFY_NEXT; | |
1554 | - | |
1555 | - nesvctx->nesupd->udoorbell->cqe_alloc = htole32(cq_arm); | |
1556 | - nesucq->is_armed = 1; | |
1557 | - nesucq->arm_sol = sol; | |
1558 | - nesucq->skip_arm = 0; | |
1559 | - nesucq->skip_sol = 1; | |
1560 | -} | |
1561 | - | |
1562 | -/** | |
1563 | - * nes_uarm_cq | |
1564 | - */ | |
1565 | -int nes_uarm_cq(struct ibv_cq *cq, int solicited) | |
1566 | -{ | |
1567 | - struct nes_ucq *nesucq; | |
1568 | - struct nes_uvcontext *nesvctx; | |
1569 | - | |
1570 | - nesucq = to_nes_ucq(cq); | |
1571 | - nesvctx = to_nes_uctx(cq->context); | |
1572 | - | |
1573 | - pthread_spin_lock(&nesucq->lock); | |
1574 | - | |
1575 | - if (nesucq->is_armed) { | |
1576 | - /* don't arm again unless... */ | |
1577 | - if ((nesucq->arm_sol) && (!solicited)) { | |
1578 | - /* solicited changed from notify SE to notify next */ | |
1579 | - nes_arm_cq(nesucq, nesvctx, solicited); | |
1580 | - } else { | |
1581 | - nesucq->skip_arm = 1; | |
1582 | - nesucq->skip_sol &= solicited; | |
1583 | - } | |
1584 | - } else { | |
1585 | - nes_arm_cq(nesucq, nesvctx, solicited); | |
1586 | - } | |
1587 | - | |
1588 | - pthread_spin_unlock(&nesucq->lock); | |
1589 | - | |
1590 | - return 0; | |
1591 | -} | |
1592 | - | |
1593 | - | |
1594 | -/** | |
1595 | - * nes_cq_event | |
1596 | - */ | |
1597 | -void nes_cq_event(struct ibv_cq *cq) | |
1598 | -{ | |
1599 | - struct nes_ucq *nesucq; | |
1600 | - | |
1601 | - nesucq = to_nes_ucq(cq); | |
1602 | - | |
1603 | - pthread_spin_lock(&nesucq->lock); | |
1604 | - | |
1605 | - if (nesucq->skip_arm) { | |
1606 | - struct nes_uvcontext *nesvctx; | |
1607 | - nesvctx = to_nes_uctx(cq->context); | |
1608 | - nes_arm_cq(nesucq, nesvctx, nesucq->skip_sol); | |
1609 | - } else { | |
1610 | - nesucq->is_armed = 0; | |
1611 | - } | |
1612 | - | |
1613 | - pthread_spin_unlock(&nesucq->lock); | |
1614 | -} | |
1615 | - | |
1616 | - | |
1617 | -/** | |
1618 | - * nes_ucreate_srq | |
1619 | - */ | |
1620 | -struct ibv_srq *nes_ucreate_srq(struct ibv_pd *pd, struct ibv_srq_init_attr *attr) | |
1621 | -{ | |
1622 | - /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */ | |
1623 | - return (void *)-ENOSYS; | |
1624 | -} | |
1625 | - | |
1626 | - | |
1627 | -/** | |
1628 | - * nes_umodify_srq | |
1629 | - */ | |
1630 | -int nes_umodify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, int attr_mask) | |
1631 | -{ | |
1632 | - /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */ | |
1633 | - return -ENOSYS; | |
1634 | -} | |
1635 | - | |
1636 | - | |
1637 | -/** | |
1638 | - * nes_udestroy_srq | |
1639 | - */ | |
1640 | -int nes_udestroy_srq(struct ibv_srq *srq) | |
1641 | -{ | |
1642 | - /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */ | |
1643 | - return -ENOSYS; | |
1644 | -} | |
1645 | - | |
1646 | - | |
1647 | -/** | |
1648 | - * nes_upost_srq_recv | |
1649 | - */ | |
1650 | -int nes_upost_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr, | |
1651 | - struct ibv_recv_wr **bad_wr) | |
1652 | -{ | |
1653 | - /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */ | |
1654 | - return -ENOSYS; | |
1655 | -} | |
1656 | - | |
1657 | - | |
1658 | -/** | |
1659 | - * nes_mmapped_qp | |
1660 | - * will not invoke registration of memory reqion and will allow | |
1661 | - * the kernel module to allocate big chunk of contigous memory | |
1662 | - * for sq and rq... returns 1 if succeeds, 0 if fails.. | |
1663 | - */ | |
1664 | -static int nes_mmapped_qp(struct nes_uqp *nesuqp, struct ibv_pd *pd, struct ibv_qp_init_attr *attr, | |
1665 | - struct nes_ucreate_qp_resp *resp) | |
1666 | -{ | |
1667 | - | |
1668 | - unsigned long mmap_offset; | |
1669 | - struct nes_ucreate_qp cmd; | |
1670 | - struct nes_uvcontext *nesvctx = to_nes_uctx(pd->context); | |
1671 | - int ret; | |
1672 | - | |
1673 | - memset (&cmd, 0, sizeof(cmd) ); | |
1674 | - cmd.user_qp_buffer = (__u64) ((uintptr_t) nesuqp); | |
1675 | - | |
1676 | - /* fprintf(stderr, PFX "%s entering==>\n",__FUNCTION__); */ | |
1677 | - ret = ibv_cmd_create_qp(pd, &nesuqp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd, | |
1678 | - &resp->ibv_resp, sizeof (struct nes_ucreate_qp_resp) ); | |
1679 | - if (ret) | |
1680 | - return 0; | |
1681 | - nesuqp->send_cq = to_nes_ucq(attr->send_cq); | |
1682 | - nesuqp->recv_cq = to_nes_ucq(attr->recv_cq); | |
1683 | - nesuqp->sq_db_index = resp->mmap_sq_db_index; | |
1684 | - nesuqp->rq_db_index = resp->mmap_rq_db_index; | |
1685 | - nesuqp->sq_size = resp->actual_sq_size; | |
1686 | - nesuqp->rq_size = resp->actual_rq_size; | |
1687 | - | |
1688 | - /* Map the SQ/RQ buffers */ | |
1689 | - mmap_offset = nesvctx->max_pds*page_size; | |
1690 | - mmap_offset += (((sizeof(struct nes_hw_qp_wqe) * nesvctx->wq_size) + page_size-1) & | |
1691 | - (~(page_size-1)))*nesuqp->sq_db_index; | |
1692 | - | |
1693 | - nesuqp->sq_vbase = mmap(NULL, (nesuqp->sq_size+nesuqp->rq_size) * | |
1694 | - sizeof(struct nes_hw_qp_wqe), PROT_WRITE | PROT_READ, | |
1695 | - MAP_SHARED, pd->context->cmd_fd, mmap_offset); | |
1696 | - | |
1697 | - | |
1698 | - if (nesuqp->sq_vbase == MAP_FAILED) { | |
1699 | - return 0; | |
1700 | - } | |
1701 | - nesuqp->rq_vbase = (struct nes_hw_qp_wqe *)(((char *)nesuqp->sq_vbase) + | |
1702 | - (nesuqp->sq_size*sizeof(struct nes_hw_qp_wqe))); | |
1703 | - *((unsigned int *)nesuqp->sq_vbase) = 0; | |
1704 | - nesuqp->mapping = NES_QP_MMAP; | |
1705 | - | |
1706 | - return 1; | |
1707 | -} | |
1708 | - | |
1709 | - | |
1710 | -/** | |
1711 | - * nes_vmapped_qp | |
1712 | - * invoke registration of memory reqion. This method is used | |
1713 | - * when kernel can not allocate qp memory (contigous physical). | |
1714 | - * | |
1715 | - * returns 1 if succeeds, 0 if fails.. | |
1716 | - */ | |
1717 | -static int nes_vmapped_qp(struct nes_uqp *nesuqp, struct ibv_pd *pd, struct ibv_qp_init_attr *attr, | |
1718 | - struct nes_ucreate_qp_resp *resp, int sqdepth, int rqdepth) | |
1719 | -{ | |
1720 | - struct nes_ucreate_qp cmd; | |
1721 | - struct nes_ureg_mr reg_mr_cmd; | |
1722 | - struct ib_uverbs_reg_mr_resp reg_mr_resp; | |
1723 | - int totalqpsize; | |
1724 | - int ret; | |
1725 | - | |
1726 | - // fprintf(stderr, PFX "%s\n", __FUNCTION__); | |
1727 | - totalqpsize = (sqdepth + rqdepth) * sizeof (struct nes_hw_qp_wqe) ; | |
1728 | - nesuqp->sq_vbase = memalign(page_size, totalqpsize); | |
1729 | - if (!nesuqp->sq_vbase) { | |
1730 | - // fprintf(stderr, PFX "CREATE_QP could not allocate mem of size %d\n", totalqpsize); | |
1731 | - return 0; | |
1732 | - } | |
1733 | - nesuqp->rq_vbase = (struct nes_hw_qp_wqe *) (((char *) nesuqp->sq_vbase) + | |
1734 | - (nesuqp->sq_size * sizeof(struct nes_hw_qp_wqe))); | |
1735 | - | |
1736 | - reg_mr_cmd.reg_type = IWNES_MEMREG_TYPE_QP; | |
1737 | - | |
1738 | - //fprintf(stderr, PFX "qp_rq_vbase = %p qp_sq_vbase=%p reg_mr = %p\n", | |
1739 | - // nesuqp->rq_vbase, nesuqp->sq_vbase, &nesuqp->mr); | |
1740 | - | |
1741 | - ret = ibv_cmd_reg_mr(pd, (void *)nesuqp->sq_vbase,totalqpsize, | |
1742 | - (uintptr_t)nesuqp->sq_vbase, | |
1743 | - IBV_ACCESS_LOCAL_WRITE, &nesuqp->vmr, | |
1744 | - ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), | |
1745 | - ®_mr_resp, sizeof(reg_mr_resp)); | |
1746 | - if (ret) { | |
1747 | - // fprintf(stderr, PFX "%s ibv_cmd_reg_mr failed (ret = %d).\n", __FUNCTION__, ret); | |
1748 | - free((void *) nesuqp->sq_vbase); | |
1749 | - return 0; | |
1750 | - } | |
1751 | - // So now the memory has been registered.. | |
1752 | - memset (&cmd, 0, sizeof(cmd) ); | |
1753 | - cmd.user_wqe_buffers = (__u64) ((uintptr_t) nesuqp->sq_vbase); | |
1754 | - cmd.user_qp_buffer = (__u64) ((uintptr_t) nesuqp); | |
1755 | - ret = ibv_cmd_create_qp(pd, &nesuqp->ibv_qp, attr, &cmd.ibv_cmd, sizeof cmd, | |
1756 | - &resp->ibv_resp, sizeof (struct nes_ucreate_qp_resp) ); | |
1757 | - if (ret) { | |
1758 | - ibv_cmd_dereg_mr(&nesuqp->vmr); | |
1759 | - free((void *)nesuqp->sq_vbase); | |
1760 | - return 0; | |
1761 | - } | |
1762 | - *((unsigned int *)nesuqp->rq_vbase) = 0; | |
1763 | - nesuqp->send_cq = to_nes_ucq(attr->send_cq); | |
1764 | - nesuqp->recv_cq = to_nes_ucq(attr->recv_cq); | |
1765 | - nesuqp->sq_db_index = resp->mmap_sq_db_index; | |
1766 | - nesuqp->rq_db_index = resp->mmap_rq_db_index; | |
1767 | - nesuqp->sq_size = resp->actual_sq_size; | |
1768 | - nesuqp->rq_size = resp->actual_rq_size; | |
1769 | - nesuqp->mapping = NES_QP_VMAP; | |
1770 | - return 1; | |
1771 | -} | |
1772 | - | |
1773 | - | |
1774 | -/** | |
1775 | - * nes_qp_get_qdepth | |
1776 | - * This routine will return the size of qdepth to be set for one | |
1777 | - * of the qp (sq or rq) | |
1778 | - */ | |
1779 | -static int nes_qp_get_qdepth(uint32_t qdepth, uint32_t maxsges) | |
1780 | -{ | |
1781 | - int retdepth; | |
1782 | - | |
1783 | - /* Do sanity check on the parameters */ | |
1784 | - /* Should the following be 510 or 511 */ | |
1785 | - if ((qdepth > 510) || (maxsges > 4) ) | |
1786 | - return 0; | |
1787 | - | |
1788 | - /* Do we need to do the following of */ | |
1789 | - /* we can just return the actual value.. needed for alignment */ | |
1790 | - if (qdepth < 32) | |
1791 | - retdepth = 32; | |
1792 | - else if (qdepth < 128) | |
1793 | - retdepth = 128; | |
1794 | - else retdepth = 512; | |
1795 | - | |
1796 | - return retdepth; | |
1797 | -} | |
1798 | - | |
1799 | - | |
1800 | -/** | |
1801 | - * nes_ucreate_qp | |
1802 | - */ | |
1803 | -struct ibv_qp *nes_ucreate_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr) | |
1804 | -{ | |
1805 | - struct nes_ucreate_qp_resp resp; | |
1806 | - struct nes_uvcontext *nesvctx = to_nes_uctx(pd->context); | |
1807 | - struct nes_uqp *nesuqp; | |
1808 | - int sqdepth, rqdepth; | |
1809 | - int status = 1; | |
1810 | - | |
1811 | - /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */ | |
1812 | - | |
1813 | - /* Sanity check QP size before proceeding */ | |
1814 | - sqdepth = nes_qp_get_qdepth(attr->cap.max_send_wr, attr->cap.max_send_sge); | |
1815 | - if (!sqdepth) { | |
1816 | - fprintf(stderr, PFX "%s Bad sq attr parameters max_send_wr=%d max_send_sge=%d\n", | |
1817 | - __FUNCTION__, attr->cap.max_send_wr,attr->cap.max_send_sge); | |
1818 | - return NULL; | |
1819 | - } | |
1820 | - | |
1821 | - rqdepth = nes_qp_get_qdepth(attr->cap.max_recv_wr, attr->cap.max_recv_sge); | |
1822 | - if (!rqdepth) { | |
1823 | - fprintf(stderr, PFX "%s Bad rq attr parameters max_recv_wr=%d max_recv_sge=%d\n", | |
1824 | - __FUNCTION__, attr->cap.max_recv_wr,attr->cap.max_recv_sge); | |
1825 | - return NULL; | |
1826 | - } | |
1827 | - | |
1828 | - nesuqp = memalign(1024, sizeof(*nesuqp)); | |
1829 | - if (!nesuqp) | |
1830 | - return NULL; | |
1831 | - memset(nesuqp, 0, sizeof(*nesuqp)); | |
1832 | - | |
1833 | - if (pthread_spin_init(&nesuqp->lock, PTHREAD_PROCESS_PRIVATE)) { | |
1834 | - free(nesuqp); | |
1835 | - return NULL; | |
1836 | - } | |
1837 | - | |
1838 | - /* Initially setting it up so we will know how much memory to allocate for mapping */ | |
1839 | - /* also setting it up in attr.. If we do not want to modify the attr struct, we */ | |
1840 | - /* can save the original values and restore them before return. */ | |
1841 | - nesuqp->sq_size = attr->cap.max_send_wr = sqdepth; | |
1842 | - nesuqp->rq_size = attr->cap.max_recv_wr = rqdepth; | |
1843 | - | |
1844 | - nesuqp->sq_sig_all = attr->sq_sig_all; | |
1845 | - if (nesvctx->virtwq) { | |
1846 | - status = nes_vmapped_qp(nesuqp,pd, attr,&resp,sqdepth,rqdepth); | |
1847 | - }else { | |
1848 | - status = nes_mmapped_qp(nesuqp,pd,attr, &resp); | |
1849 | - } | |
1850 | - | |
1851 | - if (!status) { | |
1852 | - pthread_spin_destroy(&nesuqp->lock); | |
1853 | - free(nesuqp); | |
1854 | - return NULL; | |
1855 | - } | |
1856 | - | |
1857 | - | |
1858 | - /* The following are the common parameters no matter how the */ | |
1859 | - /* sq and rq memory was mapped.. */ | |
1860 | - | |
1861 | - /* Account for LSMM, in theory, could get overrun if app preposts to SQ */ | |
1862 | - nesuqp->sq_head = 1; | |
1863 | - nesuqp->sq_tail = 1; | |
1864 | - nesuqp->qp_id = resp.qp_id; | |
1865 | - nesuqp->nes_drv_opt = resp.nes_drv_opt; | |
1866 | - nesuqp->ibv_qp.qp_num = resp.qp_id; | |
1867 | - nesuqp->rdma0_msg = 1; | |
1868 | - | |
1869 | - return &nesuqp->ibv_qp; | |
1870 | -} | |
1871 | - | |
1872 | - | |
1873 | -/** | |
1874 | - * nes_uquery_qp | |
1875 | - */ | |
1876 | -int nes_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, | |
1877 | - int attr_mask, struct ibv_qp_init_attr *init_attr) | |
1878 | -{ | |
1879 | - struct ibv_query_qp cmd; | |
1880 | - | |
1881 | - /* fprintf(stderr, PFX "nes_uquery_qp: calling ibv_cmd_query_qp\n"); */ | |
1882 | - | |
1883 | - return ibv_cmd_query_qp(qp, attr, attr_mask, init_attr, &cmd, sizeof(cmd)); | |
1884 | -} | |
1885 | - | |
1886 | - | |
1887 | -/** | |
1888 | - * nes_umodify_qp | |
1889 | - */ | |
1890 | -int nes_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) | |
1891 | -{ | |
1892 | - struct ibv_modify_qp cmd = {}; | |
1893 | - return ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof cmd); | |
1894 | -} | |
1895 | - | |
1896 | - | |
1897 | -/** | |
1898 | - * nes_clean_cq | |
1899 | - */ | |
1900 | -static void nes_clean_cq(struct nes_uqp *nesuqp, struct nes_ucq *nesucq) | |
1901 | -{ | |
1902 | - uint32_t cq_head; | |
1903 | - uint32_t lo; | |
1904 | - uint32_t hi; | |
1905 | - uint64_t u64temp; | |
1906 | - | |
1907 | - pthread_spin_lock(&nesucq->lock); | |
1908 | - | |
1909 | - cq_head = nesucq->head; | |
1910 | - while (le32toh(nesucq->cqes[cq_head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) { | |
1911 | - udma_from_device_barrier(); | |
1912 | - lo = le32toh(nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); | |
1913 | - hi = le32toh(nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]); | |
1914 | - u64temp = (((uint64_t)hi) << 32) | ((uint64_t)lo); | |
1915 | - u64temp &= (~1023); | |
1916 | - if (u64temp == (uint64_t)(uintptr_t)nesuqp) { | |
1917 | - /* Zero the context value so cqe will be ignored */ | |
1918 | - nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = 0; | |
1919 | - nesucq->cqes[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX] = 0; | |
1920 | - } | |
1921 | - | |
1922 | - if (++cq_head >= nesucq->size) | |
1923 | - cq_head = 0; | |
1924 | - } | |
1925 | - | |
1926 | - pthread_spin_unlock(&nesucq->lock); | |
1927 | -} | |
1928 | - | |
1929 | - | |
1930 | -/** | |
1931 | - * nes_udestroy_qp | |
1932 | - */ | |
1933 | -int nes_udestroy_qp(struct ibv_qp *qp) | |
1934 | -{ | |
1935 | - struct nes_uqp *nesuqp = to_nes_uqp(qp); | |
1936 | - int ret = 0; | |
1937 | - | |
1938 | - // fprintf(stderr, PFX "%s addr&mr= %p \n", __FUNCTION__, &nesuqp->mr ); | |
1939 | - | |
1940 | - if (nesuqp->mapping == NES_QP_VMAP) { | |
1941 | - ret = ibv_cmd_dereg_mr(&nesuqp->vmr); | |
1942 | - if (ret) | |
1943 | - fprintf(stderr, PFX "%s dereg_mr FAILED\n", __FUNCTION__); | |
1944 | - free((void *)nesuqp->sq_vbase); | |
1945 | - } | |
1946 | - | |
1947 | - if (nesuqp->mapping == NES_QP_MMAP) { | |
1948 | - munmap((void *)nesuqp->sq_vbase, (nesuqp->sq_size+nesuqp->rq_size) * | |
1949 | - sizeof(struct nes_hw_qp_wqe)); | |
1950 | - } | |
1951 | - | |
1952 | - ret = ibv_cmd_destroy_qp(qp); | |
1953 | - if (ret) { | |
1954 | - fprintf(stderr, PFX "%s FAILED\n", __FUNCTION__); | |
1955 | - return ret; | |
1956 | - } | |
1957 | - | |
1958 | - pthread_spin_destroy(&nesuqp->lock); | |
1959 | - | |
1960 | - /* Clean any pending completions from the cq(s) */ | |
1961 | - if (nesuqp->send_cq) | |
1962 | - nes_clean_cq(nesuqp, nesuqp->send_cq); | |
1963 | - | |
1964 | - if ((nesuqp->recv_cq) && (nesuqp->recv_cq != nesuqp->send_cq)) | |
1965 | - nes_clean_cq(nesuqp, nesuqp->recv_cq); | |
1966 | - free(nesuqp); | |
1967 | - | |
1968 | - return 0; | |
1969 | -} | |
1970 | - | |
1971 | -/** | |
1972 | - * nes_upost_send | |
1973 | - */ | |
1974 | -int nes_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, | |
1975 | - struct ibv_send_wr **bad_wr) | |
1976 | -{ | |
1977 | - uint64_t u64temp; | |
1978 | - struct nes_uqp *nesuqp = to_nes_uqp(ib_qp); | |
1979 | - struct nes_upd *nesupd = to_nes_upd(ib_qp->pd); | |
1980 | - struct nes_hw_qp_wqe volatile *wqe; | |
1981 | - uint32_t head; | |
1982 | - uint32_t qsize = nesuqp->sq_size; | |
1983 | - uint32_t counter; | |
1984 | - uint32_t err = 0; | |
1985 | - uint32_t wqe_count = 0; | |
1986 | - uint32_t outstanding_wqes; | |
1987 | - uint32_t total_payload_length = 0; | |
1988 | - int sge_index; | |
1989 | - | |
1990 | - pthread_spin_lock(&nesuqp->lock); | |
1991 | - udma_to_device_barrier(); | |
1992 | - | |
1993 | - head = nesuqp->sq_head; | |
1994 | - while (ib_wr) { | |
1995 | - if (unlikely(nesuqp->qperr)) { | |
1996 | - err = -EINVAL; | |
1997 | - break; | |
1998 | - } | |
1999 | - | |
2000 | - /* Check for SQ overflow */ | |
2001 | - outstanding_wqes = head + (2 * qsize) - nesuqp->sq_tail; | |
2002 | - outstanding_wqes &= qsize - 1; | |
2003 | - if (unlikely(outstanding_wqes == (qsize - 1))) { | |
2004 | - err = -EINVAL; | |
2005 | - break; | |
2006 | - } | |
2007 | - if (unlikely(ib_wr->num_sge > 4)) { | |
2008 | - err = -EINVAL; | |
2009 | - break; | |
2010 | - } | |
2011 | - | |
2012 | - wqe = (struct nes_hw_qp_wqe *)&nesuqp->sq_vbase[head]; | |
2013 | - /* fprintf(stderr, PFX "%s: QP%u: processing sq wqe at %p, head = %u.\n", | |
2014 | - __FUNCTION__, nesuqp->qp_id, wqe, head); */ | |
2015 | - u64temp = (uint64_t) ib_wr->wr_id; | |
2016 | - wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX] = htole32((uint32_t)u64temp); | |
2017 | - wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX] = htole32((uint32_t)(u64temp>>32)); | |
2018 | - u64temp = (uint64_t)((uintptr_t)nesuqp); | |
2019 | - wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX] = htole32((uint32_t)u64temp); | |
2020 | - wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX] = htole32((uint32_t)(u64temp>>32)); | |
2021 | - udma_ordering_write_barrier(); | |
2022 | - wqe->wqe_words[NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX] |= htole32(head); | |
2023 | - | |
2024 | - switch (ib_wr->opcode) { | |
2025 | - case IBV_WR_SEND: | |
2026 | - case IBV_WR_SEND_WITH_IMM: | |
2027 | - /* fprintf(stderr, PFX "%s: QP%u: processing sq wqe%u. Opcode = %s\n", | |
2028 | - __FUNCTION__, nesuqp->qp_id, head, "Send"); */ | |
2029 | - if (ib_wr->send_flags & IBV_SEND_SOLICITED) { | |
2030 | - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_SENDSE); | |
2031 | - } else { | |
2032 | - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_SEND); | |
2033 | - } | |
2034 | - | |
2035 | - if (ib_wr->send_flags & IBV_SEND_FENCE) { | |
2036 | - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_READ_FENCE); | |
2037 | - } | |
2038 | - | |
2039 | - /* if (ib_wr->send_flags & IBV_SEND_INLINE) { | |
2040 | - fprintf(stderr, PFX "%s: Send SEND_INLINE, length=%d\n", | |
2041 | - __FUNCTION__, ib_wr->sg_list[0].length); | |
2042 | - } */ | |
2043 | - if ((ib_wr->send_flags & IBV_SEND_INLINE) && (ib_wr->sg_list[0].length <= 64) && | |
2044 | - ((nesuqp->nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) && | |
2045 | - (ib_wr->num_sge == 1)) { | |
2046 | - memcpy((void *)&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX], | |
2047 | - (void *)(intptr_t)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length); | |
2048 | - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(ib_wr->sg_list[0].length); | |
2049 | - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_IMM_DATA); | |
2050 | - } else { | |
2051 | - total_payload_length = 0; | |
2052 | - for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) { | |
2053 | - wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] = | |
2054 | - htole32((uint32_t)ib_wr->sg_list[sge_index].addr); | |
2055 | - wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] = | |
2056 | - htole32((uint32_t)(ib_wr->sg_list[sge_index].addr>>32)); | |
2057 | - wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX+(sge_index*4)] = | |
2058 | - htole32(ib_wr->sg_list[sge_index].length); | |
2059 | - wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX+(sge_index*4)] = | |
2060 | - htole32(ib_wr->sg_list[sge_index].lkey); | |
2061 | - total_payload_length += ib_wr->sg_list[sge_index].length; | |
2062 | - } | |
2063 | - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = | |
2064 | - htole32(total_payload_length); | |
2065 | - } | |
2066 | - | |
2067 | - break; | |
2068 | - case IBV_WR_RDMA_WRITE: | |
2069 | - case IBV_WR_RDMA_WRITE_WITH_IMM: | |
2070 | - /* fprintf(stderr, PFX "%s:QP%u: processing sq wqe%u. Opcode = %s\n", | |
2071 | - __FUNCTION__, nesuqp->qp_id, head, "Write"); */ | |
2072 | - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_RDMAW); | |
2073 | - | |
2074 | - if (ib_wr->send_flags & IBV_SEND_FENCE) { | |
2075 | - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_READ_FENCE); | |
2076 | - } | |
2077 | - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = htole32(ib_wr->wr.rdma.rkey); | |
2078 | - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = htole32( | |
2079 | - (uint32_t)ib_wr->wr.rdma.remote_addr); | |
2080 | - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = htole32( | |
2081 | - (uint32_t)(ib_wr->wr.rdma.remote_addr>>32)); | |
2082 | - | |
2083 | - /* if (ib_wr->send_flags & IBV_SEND_INLINE) { | |
2084 | - fprintf(stderr, PFX "%s: Write SEND_INLINE, length=%d\n", | |
2085 | - __FUNCTION__, ib_wr->sg_list[0].length); | |
2086 | - } */ | |
2087 | - if ((ib_wr->send_flags & IBV_SEND_INLINE) && (ib_wr->sg_list[0].length <= 64) && | |
2088 | - ((nesuqp->nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) && | |
2089 | - (ib_wr->num_sge == 1)) { | |
2090 | - memcpy((void *)&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX], | |
2091 | - (void *)(intptr_t)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length); | |
2092 | - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(ib_wr->sg_list[0].length); | |
2093 | - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_IMM_DATA); | |
2094 | - } else { | |
2095 | - total_payload_length = 0; | |
2096 | - for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) { | |
2097 | - wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] = htole32( | |
2098 | - (uint32_t)ib_wr->sg_list[sge_index].addr); | |
2099 | - wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] = htole32( | |
2100 | - (uint32_t)(ib_wr->sg_list[sge_index].addr>>32)); | |
2101 | - wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX+(sge_index*4)] = htole32( | |
2102 | - ib_wr->sg_list[sge_index].length); | |
2103 | - wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX+(sge_index*4)] = htole32( | |
2104 | - ib_wr->sg_list[sge_index].lkey); | |
2105 | - total_payload_length += ib_wr->sg_list[sge_index].length; | |
2106 | - } | |
2107 | - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(total_payload_length); | |
2108 | - } | |
2109 | - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = | |
2110 | - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]; | |
2111 | - break; | |
2112 | - case IBV_WR_RDMA_READ: | |
2113 | - /* fprintf(stderr, PFX "%s:QP%u:processing sq wqe%u. Opcode = %s\n", | |
2114 | - __FUNCTION__, nesuqp->qp_id, head, "Read"); */ | |
2115 | - /* IWarp only supports 1 sge for RDMA reads */ | |
2116 | - if (ib_wr->num_sge > 1) { | |
2117 | - err = -EINVAL; | |
2118 | - break; | |
2119 | - } | |
2120 | - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = htole32(NES_IWARP_SQ_OP_RDMAR); | |
2121 | - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = htole32((uint32_t)ib_wr->wr.rdma.remote_addr); | |
2122 | - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = htole32((uint32_t)(ib_wr->wr.rdma.remote_addr>>32)); | |
2123 | - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = htole32(ib_wr->wr.rdma.rkey); | |
2124 | - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = htole32(ib_wr->sg_list->length); | |
2125 | - wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX] = htole32((uint32_t)ib_wr->sg_list->addr); | |
2126 | - wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX] = htole32((uint32_t)(ib_wr->sg_list->addr>>32)); | |
2127 | - wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = htole32(ib_wr->sg_list->lkey); | |
2128 | - break; | |
2129 | - default: | |
2130 | - /* error */ | |
2131 | - err = -EINVAL; | |
2132 | - break; | |
2133 | - } | |
2134 | - | |
2135 | - if ((ib_wr->send_flags & IBV_SEND_SIGNALED) || nesuqp->sq_sig_all) { | |
2136 | - /* fprintf(stderr, PFX "%s:sq wqe%u is signalled\n", __FUNCTION__, head); */ | |
2137 | - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] |= htole32(NES_IWARP_SQ_WQE_SIGNALED_COMPL); | |
2138 | - } | |
2139 | - ib_wr = ib_wr->next; | |
2140 | - head++; | |
2141 | - wqe_count++; | |
2142 | - if (head >= qsize) | |
2143 | - head = 0; | |
2144 | - } | |
2145 | - | |
2146 | - nesuqp->sq_head = head; | |
2147 | - udma_to_device_barrier(); | |
2148 | - while (wqe_count) { | |
2149 | - counter = (wqe_count<(uint32_t)255) ? wqe_count : 255; | |
2150 | - wqe_count -= counter; | |
2151 | - nesupd->udoorbell->wqe_alloc = htole32((counter<<24) | 0x00800000 | nesuqp->qp_id); | |
2152 | - } | |
2153 | - | |
2154 | - if (err) | |
2155 | - *bad_wr = ib_wr; | |
2156 | - | |
2157 | - pthread_spin_unlock(&nesuqp->lock); | |
2158 | - | |
2159 | - return err; | |
2160 | -} | |
2161 | - | |
2162 | -/** | |
2163 | - * nes_upost_recv | |
2164 | - */ | |
2165 | -int nes_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, | |
2166 | - struct ibv_recv_wr **bad_wr) | |
2167 | -{ | |
2168 | - uint64_t u64temp; | |
2169 | - struct nes_uqp *nesuqp = to_nes_uqp(ib_qp); | |
2170 | - struct nes_upd *nesupd = to_nes_upd(ib_qp->pd); | |
2171 | - struct nes_hw_qp_wqe *wqe; | |
2172 | - uint32_t head; | |
2173 | - uint32_t qsize = nesuqp->rq_size; | |
2174 | - uint32_t counter; | |
2175 | - uint32_t err = 0; | |
2176 | - uint32_t wqe_count = 0; | |
2177 | - uint32_t outstanding_wqes; | |
2178 | - uint32_t total_payload_length; | |
2179 | - int sge_index; | |
2180 | - | |
2181 | - if (unlikely(ib_wr->num_sge > 4)) { | |
2182 | - *bad_wr = ib_wr; | |
2183 | - return -EINVAL; | |
2184 | - } | |
2185 | - | |
2186 | - pthread_spin_lock(&nesuqp->lock); | |
2187 | - udma_to_device_barrier(); | |
2188 | - | |
2189 | - head = nesuqp->rq_head; | |
2190 | - while (ib_wr) { | |
2191 | - if (unlikely(nesuqp->qperr)) { | |
2192 | - err = -EINVAL; | |
2193 | - break; | |
2194 | - } | |
2195 | - | |
2196 | - /* Check for RQ overflow */ | |
2197 | - outstanding_wqes = head + (2 * qsize) - nesuqp->rq_tail; | |
2198 | - outstanding_wqes &= qsize - 1; | |
2199 | - if (unlikely(outstanding_wqes == (qsize - 1))) { | |
2200 | - err = -EINVAL; | |
2201 | - break; | |
2202 | - } | |
2203 | - | |
2204 | - wqe = (struct nes_hw_qp_wqe *)&nesuqp->rq_vbase[head]; | |
2205 | - u64temp = ib_wr->wr_id; | |
2206 | - wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX] = | |
2207 | - htole32((uint32_t)u64temp); | |
2208 | - wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX] = | |
2209 | - htole32((uint32_t)(u64temp >> 32)); | |
2210 | - u64temp = (uint64_t)((uintptr_t)nesuqp); | |
2211 | - wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX] = | |
2212 | - htole32((uint32_t)u64temp); | |
2213 | - wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_HIGH_IDX] = | |
2214 | - htole32((uint32_t)(u64temp >> 32)); | |
2215 | - udma_ordering_write_barrier(); | |
2216 | - wqe->wqe_words[NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX] |= htole32(head); | |
2217 | - | |
2218 | - total_payload_length = 0; | |
2219 | - for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) { | |
2220 | - wqe->wqe_words[NES_IWARP_RQ_WQE_FRAG0_LOW_IDX+(sge_index*4)] = | |
2221 | - htole32((uint32_t)ib_wr->sg_list[sge_index].addr); | |
2222 | - wqe->wqe_words[NES_IWARP_RQ_WQE_FRAG0_HIGH_IDX+(sge_index*4)] = | |
2223 | - htole32((uint32_t)(ib_wr->sg_list[sge_index].addr>>32)); | |
2224 | - wqe->wqe_words[NES_IWARP_RQ_WQE_LENGTH0_IDX+(sge_index*4)] = | |
2225 | - htole32(ib_wr->sg_list[sge_index].length); | |
2226 | - wqe->wqe_words[NES_IWARP_RQ_WQE_STAG0_IDX+(sge_index*4)] = | |
2227 | - htole32(ib_wr->sg_list[sge_index].lkey); | |
2228 | - total_payload_length += ib_wr->sg_list[sge_index].length; | |
2229 | - } | |
2230 | - wqe->wqe_words[NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX] = htole32(total_payload_length); | |
2231 | - | |
2232 | - ib_wr = ib_wr->next; | |
2233 | - head++; | |
2234 | - wqe_count++; | |
2235 | - if (head >= qsize) | |
2236 | - head = 0; | |
2237 | - } | |
2238 | - | |
2239 | - nesuqp->rq_head = head; | |
2240 | - udma_to_device_barrier(); | |
2241 | - while (wqe_count) { | |
2242 | - counter = (wqe_count<(uint32_t)255) ? wqe_count : 255; | |
2243 | - wqe_count -= counter; | |
2244 | - nesupd->udoorbell->wqe_alloc = htole32((counter << 24) | nesuqp->qp_id); | |
2245 | - } | |
2246 | - | |
2247 | - if (err) | |
2248 | - *bad_wr = ib_wr; | |
2249 | - | |
2250 | - pthread_spin_unlock(&nesuqp->lock); | |
2251 | - | |
2252 | - return err; | |
2253 | -} | |
2254 | - | |
2255 | - | |
2256 | -/** | |
2257 | - * nes_ucreate_ah | |
2258 | - */ | |
2259 | -struct ibv_ah *nes_ucreate_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr) | |
2260 | -{ | |
2261 | - /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */ | |
2262 | - return (void *)-ENOSYS; | |
2263 | -} | |
2264 | - | |
2265 | - | |
2266 | -/** | |
2267 | - * nes_udestroy_ah | |
2268 | - */ | |
2269 | -int nes_udestroy_ah(struct ibv_ah *ah) | |
2270 | -{ | |
2271 | - /* fprintf(stderr, PFX "%s\n", __FUNCTION__); */ | |
2272 | - return -ENOSYS; | |
2273 | -} | |
2274 | - | |
2275 | - | |
2276 | -/** | |
2277 | - * nes_uattach_mcast | |
2278 | - */ | |
2279 | -int nes_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) | |
2280 | -{ | |
2281 | - int ret = 0; | |
2282 | - ret = ibv_cmd_attach_mcast(qp, gid, lid); | |
2283 | - nes_debug(NES_DBG_UD, "%s ret=%d\n", __func__, ret); | |
2284 | - return ret; | |
2285 | -} | |
2286 | - | |
2287 | - | |
2288 | -/** | |
2289 | - * nes_udetach_mcast | |
2290 | - */ | |
2291 | -int nes_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid) | |
2292 | -{ | |
2293 | - int ret = 0; | |
2294 | - ret = ibv_cmd_detach_mcast(qp, gid, lid); | |
2295 | - nes_debug(NES_DBG_UD, "%s ret=%d\n", __func__, ret); | |
2296 | - return ret; | |
2297 | -} | |
2298 | - | |
2299 | -/** | |
2300 | - * nes_async_event | |
2301 | - */ | |
2302 | -void nes_async_event(struct ibv_context *context, | |
2303 | - struct ibv_async_event *event) | |
2304 | -{ | |
2305 | - struct nes_uqp *nesuqp; | |
2306 | - | |
2307 | - switch (event->event_type) { | |
2308 | - case IBV_EVENT_QP_FATAL: | |
2309 | - case IBV_EVENT_QP_ACCESS_ERR: | |
2310 | - /* Do not let application queue anything else to the qp */ | |
2311 | - nesuqp = to_nes_uqp(event->element.qp); | |
2312 | - nesuqp->qperr = 1; | |
2313 | - break; | |
2314 | - | |
2315 | - default: | |
2316 | - break; | |
2317 | - } | |
2318 | -} |