1d6b92ffaSHans Petter Selasky /*
2d6b92ffaSHans Petter Selasky * Copyright (c) 2006-2016 Chelsio, Inc. All rights reserved.
3d6b92ffaSHans Petter Selasky *
4d6b92ffaSHans Petter Selasky * This software is available to you under a choice of one of two
5d6b92ffaSHans Petter Selasky * licenses. You may choose to be licensed under the terms of the GNU
6d6b92ffaSHans Petter Selasky * General Public License (GPL) Version 2, available from the file
7d6b92ffaSHans Petter Selasky * COPYING in the main directory of this source tree, or the
8d6b92ffaSHans Petter Selasky * OpenIB.org BSD license below:
9d6b92ffaSHans Petter Selasky *
10d6b92ffaSHans Petter Selasky * Redistribution and use in source and binary forms, with or
11d6b92ffaSHans Petter Selasky * without modification, are permitted provided that the following
12d6b92ffaSHans Petter Selasky * conditions are met:
13d6b92ffaSHans Petter Selasky *
14d6b92ffaSHans Petter Selasky * - Redistributions of source code must retain the above
15d6b92ffaSHans Petter Selasky * copyright notice, this list of conditions and the following
16d6b92ffaSHans Petter Selasky * disclaimer.
17d6b92ffaSHans Petter Selasky *
18d6b92ffaSHans Petter Selasky * - Redistributions in binary form must reproduce the above
19d6b92ffaSHans Petter Selasky * copyright notice, this list of conditions and the following
20d6b92ffaSHans Petter Selasky * disclaimer in the documentation and/or other materials
21d6b92ffaSHans Petter Selasky * provided with the distribution.
22d6b92ffaSHans Petter Selasky *
23d6b92ffaSHans Petter Selasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24d6b92ffaSHans Petter Selasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25d6b92ffaSHans Petter Selasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26d6b92ffaSHans Petter Selasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27d6b92ffaSHans Petter Selasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28d6b92ffaSHans Petter Selasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29d6b92ffaSHans Petter Selasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30d6b92ffaSHans Petter Selasky * SOFTWARE.
31d6b92ffaSHans Petter Selasky */
32d6b92ffaSHans Petter Selasky #include <config.h>
33d6b92ffaSHans Petter Selasky
34d6b92ffaSHans Petter Selasky #include <stdio.h>
35d6b92ffaSHans Petter Selasky #include <stdlib.h>
36d6b92ffaSHans Petter Selasky #include <unistd.h>
37d6b92ffaSHans Petter Selasky #include <errno.h>
38d6b92ffaSHans Petter Selasky #include <sys/mman.h>
39d6b92ffaSHans Petter Selasky #include <pthread.h>
40d6b92ffaSHans Petter Selasky #include <string.h>
41d6b92ffaSHans Petter Selasky #include <signal.h>
425c2bacdeSNavdeep Parhar #include <stdbool.h>
43d6b92ffaSHans Petter Selasky
44d6b92ffaSHans Petter Selasky #include "libcxgb4.h"
45d6b92ffaSHans Petter Selasky #include "cxgb4-abi.h"
46d6b92ffaSHans Petter Selasky
47d6b92ffaSHans Petter Selasky #define PCI_VENDOR_ID_CHELSIO 0x1425
48d6b92ffaSHans Petter Selasky
49d6b92ffaSHans Petter Selasky /*
50d6b92ffaSHans Petter Selasky * Macros needed to support the PCI Device ID Table ...
51d6b92ffaSHans Petter Selasky */
52d6b92ffaSHans Petter Selasky #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
53d6b92ffaSHans Petter Selasky static struct { \
54d6b92ffaSHans Petter Selasky unsigned vendor; \
55d6b92ffaSHans Petter Selasky unsigned device; \
56d6b92ffaSHans Petter Selasky } hca_table[] = {
57d6b92ffaSHans Petter Selasky
58d6b92ffaSHans Petter Selasky #define CH_PCI_DEVICE_ID_FUNCTION \
59d6b92ffaSHans Petter Selasky 0x4
60d6b92ffaSHans Petter Selasky
61d6b92ffaSHans Petter Selasky #define CH_PCI_ID_TABLE_ENTRY(__DeviceID) \
62d6b92ffaSHans Petter Selasky { \
63d6b92ffaSHans Petter Selasky .vendor = PCI_VENDOR_ID_CHELSIO, \
64d6b92ffaSHans Petter Selasky .device = (__DeviceID), \
65d6b92ffaSHans Petter Selasky }
66d6b92ffaSHans Petter Selasky
67d6b92ffaSHans Petter Selasky #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
68d6b92ffaSHans Petter Selasky }
69d6b92ffaSHans Petter Selasky
70d6b92ffaSHans Petter Selasky #include "t4_chip_type.h"
71d6b92ffaSHans Petter Selasky #include "t4_pci_id_tbl.h"
72d6b92ffaSHans Petter Selasky
73d6b92ffaSHans Petter Selasky unsigned long c4iw_page_size;
74d6b92ffaSHans Petter Selasky unsigned long c4iw_page_shift;
75d6b92ffaSHans Petter Selasky unsigned long c4iw_page_mask;
76d6b92ffaSHans Petter Selasky int ma_wr;
77d6b92ffaSHans Petter Selasky int t5_en_wc = 1;
78d6b92ffaSHans Petter Selasky
79d6b92ffaSHans Petter Selasky static TAILQ_HEAD(,c4iw_dev) devices = TAILQ_HEAD_INITIALIZER(devices);
80d6b92ffaSHans Petter Selasky
81d6b92ffaSHans Petter Selasky static struct ibv_context_ops c4iw_ctx_ops = {
82d6b92ffaSHans Petter Selasky .query_device = c4iw_query_device,
83d6b92ffaSHans Petter Selasky .query_port = c4iw_query_port,
84d6b92ffaSHans Petter Selasky .alloc_pd = c4iw_alloc_pd,
85d6b92ffaSHans Petter Selasky .dealloc_pd = c4iw_free_pd,
86d6b92ffaSHans Petter Selasky .reg_mr = c4iw_reg_mr,
87d6b92ffaSHans Petter Selasky .dereg_mr = c4iw_dereg_mr,
88d6b92ffaSHans Petter Selasky .create_cq = c4iw_create_cq,
89d6b92ffaSHans Petter Selasky .resize_cq = c4iw_resize_cq,
90d6b92ffaSHans Petter Selasky .destroy_cq = c4iw_destroy_cq,
91d6b92ffaSHans Petter Selasky .create_srq = c4iw_create_srq,
92d6b92ffaSHans Petter Selasky .modify_srq = c4iw_modify_srq,
93d6b92ffaSHans Petter Selasky .destroy_srq = c4iw_destroy_srq,
94d6b92ffaSHans Petter Selasky .create_qp = c4iw_create_qp,
95d6b92ffaSHans Petter Selasky .modify_qp = c4iw_modify_qp,
96d6b92ffaSHans Petter Selasky .destroy_qp = c4iw_destroy_qp,
97d6b92ffaSHans Petter Selasky .query_qp = c4iw_query_qp,
98d6b92ffaSHans Petter Selasky .create_ah = c4iw_create_ah,
99d6b92ffaSHans Petter Selasky .destroy_ah = c4iw_destroy_ah,
100d6b92ffaSHans Petter Selasky .attach_mcast = c4iw_attach_mcast,
101d6b92ffaSHans Petter Selasky .detach_mcast = c4iw_detach_mcast,
102d6b92ffaSHans Petter Selasky .post_srq_recv = c4iw_post_srq_recv,
103d6b92ffaSHans Petter Selasky .req_notify_cq = c4iw_arm_cq,
104d6b92ffaSHans Petter Selasky };
105d6b92ffaSHans Petter Selasky
c4iw_alloc_context(struct ibv_device * ibdev,int cmd_fd)106d6b92ffaSHans Petter Selasky static struct ibv_context *c4iw_alloc_context(struct ibv_device *ibdev,
107d6b92ffaSHans Petter Selasky int cmd_fd)
108d6b92ffaSHans Petter Selasky {
109d6b92ffaSHans Petter Selasky struct c4iw_context *context;
110d6b92ffaSHans Petter Selasky struct ibv_get_context cmd;
111d6b92ffaSHans Petter Selasky struct c4iw_alloc_ucontext_resp resp;
112d6b92ffaSHans Petter Selasky struct c4iw_dev *rhp = to_c4iw_dev(ibdev);
113d6b92ffaSHans Petter Selasky struct ibv_query_device qcmd;
114d6b92ffaSHans Petter Selasky uint64_t raw_fw_ver;
115d6b92ffaSHans Petter Selasky struct ibv_device_attr attr;
116d6b92ffaSHans Petter Selasky
117d6b92ffaSHans Petter Selasky context = malloc(sizeof *context);
118d6b92ffaSHans Petter Selasky if (!context)
119d6b92ffaSHans Petter Selasky return NULL;
120d6b92ffaSHans Petter Selasky
121d6b92ffaSHans Petter Selasky memset(context, 0, sizeof *context);
122d6b92ffaSHans Petter Selasky context->ibv_ctx.cmd_fd = cmd_fd;
123d6b92ffaSHans Petter Selasky
124d6b92ffaSHans Petter Selasky resp.status_page_size = 0;
125d6b92ffaSHans Petter Selasky resp.reserved = 0;
126d6b92ffaSHans Petter Selasky if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
127d6b92ffaSHans Petter Selasky &resp.ibv_resp, sizeof resp))
128d6b92ffaSHans Petter Selasky goto err_free;
129d6b92ffaSHans Petter Selasky
130d6b92ffaSHans Petter Selasky if (resp.reserved)
131d6b92ffaSHans Petter Selasky PDBG("%s c4iw_alloc_ucontext_resp reserved field modified by kernel\n",
132d6b92ffaSHans Petter Selasky __FUNCTION__);
133d6b92ffaSHans Petter Selasky
134d6b92ffaSHans Petter Selasky context->status_page_size = resp.status_page_size;
135d6b92ffaSHans Petter Selasky if (resp.status_page_size) {
136d6b92ffaSHans Petter Selasky context->status_page = mmap(NULL, resp.status_page_size,
137d6b92ffaSHans Petter Selasky PROT_READ, MAP_SHARED, cmd_fd,
138d6b92ffaSHans Petter Selasky resp.status_page_key);
139d6b92ffaSHans Petter Selasky if (context->status_page == MAP_FAILED)
140d6b92ffaSHans Petter Selasky goto err_free;
141d6b92ffaSHans Petter Selasky }
142d6b92ffaSHans Petter Selasky
143d6b92ffaSHans Petter Selasky context->ibv_ctx.device = ibdev;
144d6b92ffaSHans Petter Selasky context->ibv_ctx.ops = c4iw_ctx_ops;
145d6b92ffaSHans Petter Selasky
146d6b92ffaSHans Petter Selasky switch (rhp->chip_version) {
147d6b92ffaSHans Petter Selasky case CHELSIO_T6:
148d6b92ffaSHans Petter Selasky PDBG("%s T6/T5/T4 device\n", __FUNCTION__);
149d6b92ffaSHans Petter Selasky case CHELSIO_T5:
150d6b92ffaSHans Petter Selasky PDBG("%s T5/T4 device\n", __FUNCTION__);
151d6b92ffaSHans Petter Selasky case CHELSIO_T4:
152d6b92ffaSHans Petter Selasky PDBG("%s T4 device\n", __FUNCTION__);
153d6b92ffaSHans Petter Selasky context->ibv_ctx.ops.async_event = c4iw_async_event;
154d6b92ffaSHans Petter Selasky context->ibv_ctx.ops.post_send = c4iw_post_send;
155d6b92ffaSHans Petter Selasky context->ibv_ctx.ops.post_recv = c4iw_post_receive;
156d6b92ffaSHans Petter Selasky context->ibv_ctx.ops.poll_cq = c4iw_poll_cq;
157d6b92ffaSHans Petter Selasky context->ibv_ctx.ops.req_notify_cq = c4iw_arm_cq;
158d6b92ffaSHans Petter Selasky break;
159d6b92ffaSHans Petter Selasky default:
160d6b92ffaSHans Petter Selasky PDBG("%s unknown hca type %d\n", __FUNCTION__,
161d6b92ffaSHans Petter Selasky rhp->chip_version);
162d6b92ffaSHans Petter Selasky goto err_unmap;
163d6b92ffaSHans Petter Selasky break;
164d6b92ffaSHans Petter Selasky }
165d6b92ffaSHans Petter Selasky
166d6b92ffaSHans Petter Selasky if (!rhp->mmid2ptr) {
167d6b92ffaSHans Petter Selasky int ret;
168d6b92ffaSHans Petter Selasky
169d6b92ffaSHans Petter Selasky ret = ibv_cmd_query_device(&context->ibv_ctx, &attr, &raw_fw_ver, &qcmd,
170d6b92ffaSHans Petter Selasky sizeof qcmd);
171d6b92ffaSHans Petter Selasky if (ret)
172d6b92ffaSHans Petter Selasky goto err_unmap;
173d6b92ffaSHans Petter Selasky rhp->max_mr = attr.max_mr;
174d6b92ffaSHans Petter Selasky rhp->mmid2ptr = calloc(attr.max_mr, sizeof(void *));
175d6b92ffaSHans Petter Selasky if (!rhp->mmid2ptr) {
176d6b92ffaSHans Petter Selasky goto err_unmap;
177d6b92ffaSHans Petter Selasky }
178d6b92ffaSHans Petter Selasky if (rhp->abi_version < 3) {
179d6b92ffaSHans Petter Selasky fprintf(stderr, "Warning: iw_cxgb4 driver is of older version"
180d6b92ffaSHans Petter Selasky " than libcxgb4:: %d\n", rhp->abi_version);
181d6b92ffaSHans Petter Selasky rhp->max_qp = T4_QID_BASE + attr.max_qp;
182d6b92ffaSHans Petter Selasky } else {
183d6b92ffaSHans Petter Selasky rhp->max_qp = context->status_page->qp_start +
184d6b92ffaSHans Petter Selasky context->status_page->qp_size;
185d6b92ffaSHans Petter Selasky }
186d6b92ffaSHans Petter Selasky rhp->qpid2ptr = calloc(rhp->max_qp, sizeof(void *));
187d6b92ffaSHans Petter Selasky if (!rhp->qpid2ptr) {
188d6b92ffaSHans Petter Selasky goto err_unmap;
189d6b92ffaSHans Petter Selasky }
190d6b92ffaSHans Petter Selasky if (rhp->abi_version < 3)
191d6b92ffaSHans Petter Selasky rhp->max_cq = T4_QID_BASE + attr.max_cq;
192d6b92ffaSHans Petter Selasky else
193d6b92ffaSHans Petter Selasky rhp->max_cq = context->status_page->cq_start +
194d6b92ffaSHans Petter Selasky context->status_page->cq_size;
195d6b92ffaSHans Petter Selasky rhp->cqid2ptr = calloc(rhp->max_cq, sizeof(void *));
196d6b92ffaSHans Petter Selasky if (!rhp->cqid2ptr)
197d6b92ffaSHans Petter Selasky goto err_unmap;
1985c2bacdeSNavdeep Parhar
1995c2bacdeSNavdeep Parhar /* Disable userspace WC if architecture/adapter does not
2005c2bacdeSNavdeep Parhar * support WC.
2015c2bacdeSNavdeep Parhar * Note: To forcefully disable WC in kernel driver use the
2025c2bacdeSNavdeep Parhar * loader tunable "hw.cxl.write_combine=0"
2035c2bacdeSNavdeep Parhar */
2045c2bacdeSNavdeep Parhar if (t5_en_wc && !context->status_page->wc_supported) {
2055c2bacdeSNavdeep Parhar t5_en_wc = 0;
2065c2bacdeSNavdeep Parhar }
207d6b92ffaSHans Petter Selasky }
208d6b92ffaSHans Petter Selasky
209d6b92ffaSHans Petter Selasky return &context->ibv_ctx;
210d6b92ffaSHans Petter Selasky
211d6b92ffaSHans Petter Selasky err_unmap:
212d6b92ffaSHans Petter Selasky munmap(context->status_page, context->status_page_size);
213d6b92ffaSHans Petter Selasky err_free:
214d6b92ffaSHans Petter Selasky if (rhp->cqid2ptr)
215d6b92ffaSHans Petter Selasky free(rhp->cqid2ptr);
216d6b92ffaSHans Petter Selasky if (rhp->qpid2ptr)
217d6b92ffaSHans Petter Selasky free(rhp->cqid2ptr);
218d6b92ffaSHans Petter Selasky if (rhp->mmid2ptr)
219d6b92ffaSHans Petter Selasky free(rhp->cqid2ptr);
220d6b92ffaSHans Petter Selasky free(context);
221d6b92ffaSHans Petter Selasky return NULL;
222d6b92ffaSHans Petter Selasky }
223d6b92ffaSHans Petter Selasky
c4iw_free_context(struct ibv_context * ibctx)224d6b92ffaSHans Petter Selasky static void c4iw_free_context(struct ibv_context *ibctx)
225d6b92ffaSHans Petter Selasky {
226d6b92ffaSHans Petter Selasky struct c4iw_context *context = to_c4iw_context(ibctx);
227d6b92ffaSHans Petter Selasky
228d6b92ffaSHans Petter Selasky if (context->status_page_size)
229d6b92ffaSHans Petter Selasky munmap(context->status_page, context->status_page_size);
230d6b92ffaSHans Petter Selasky free(context);
231d6b92ffaSHans Petter Selasky }
232d6b92ffaSHans Petter Selasky
233d6b92ffaSHans Petter Selasky static struct verbs_device_ops c4iw_dev_ops = {
234d6b92ffaSHans Petter Selasky .alloc_context = c4iw_alloc_context,
235d6b92ffaSHans Petter Selasky .free_context = c4iw_free_context
236d6b92ffaSHans Petter Selasky };
237d6b92ffaSHans Petter Selasky
238d6b92ffaSHans Petter Selasky #ifdef STALL_DETECTION
239d6b92ffaSHans Petter Selasky
240d6b92ffaSHans Petter Selasky int stall_to;
241d6b92ffaSHans Petter Selasky
dump_cq(struct c4iw_cq * chp)242d6b92ffaSHans Petter Selasky static void dump_cq(struct c4iw_cq *chp)
243d6b92ffaSHans Petter Selasky {
244d6b92ffaSHans Petter Selasky int i;
245d6b92ffaSHans Petter Selasky
246d6b92ffaSHans Petter Selasky fprintf(stderr,
247d6b92ffaSHans Petter Selasky "CQ: %p id %u queue %p cidx 0x%08x sw_queue %p sw_cidx %d sw_pidx %d sw_in_use %d depth %u error %u gen %d "
248d6b92ffaSHans Petter Selasky "cidx_inc %d bits_type_ts %016" PRIx64 " notempty %d\n", chp,
249d6b92ffaSHans Petter Selasky chp->cq.cqid, chp->cq.queue, chp->cq.cidx,
250d6b92ffaSHans Petter Selasky chp->cq.sw_queue, chp->cq.sw_cidx, chp->cq.sw_pidx, chp->cq.sw_in_use,
251d6b92ffaSHans Petter Selasky chp->cq.size, chp->cq.error, chp->cq.gen, chp->cq.cidx_inc, be64toh(chp->cq.bits_type_ts),
252d6b92ffaSHans Petter Selasky t4_cq_notempty(&chp->cq));
253d6b92ffaSHans Petter Selasky
254d6b92ffaSHans Petter Selasky for (i=0; i < chp->cq.size; i++) {
255d6b92ffaSHans Petter Selasky u64 *p = (u64 *)(chp->cq.queue + i);
256d6b92ffaSHans Petter Selasky
257d6b92ffaSHans Petter Selasky fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64, i, be64toh(p[0]), be64toh(p[1]));
258d6b92ffaSHans Petter Selasky if (i == chp->cq.cidx)
259d6b92ffaSHans Petter Selasky fprintf(stderr, " <-- cidx\n");
260d6b92ffaSHans Petter Selasky else
261d6b92ffaSHans Petter Selasky fprintf(stderr, "\n");
262d6b92ffaSHans Petter Selasky p+= 2;
263d6b92ffaSHans Petter Selasky fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64 "\n", i, be64toh(p[0]), be64toh(p[1]));
264d6b92ffaSHans Petter Selasky p+= 2;
265d6b92ffaSHans Petter Selasky fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64 "\n", i, be64toh(p[0]), be64toh(p[1]));
266d6b92ffaSHans Petter Selasky p+= 2;
267d6b92ffaSHans Petter Selasky fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64 "\n", i, be64toh(p[0]), be64toh(p[1]));
268d6b92ffaSHans Petter Selasky p+= 2;
269d6b92ffaSHans Petter Selasky }
270d6b92ffaSHans Petter Selasky }
271d6b92ffaSHans Petter Selasky
dump_qp(struct c4iw_qp * qhp)272d6b92ffaSHans Petter Selasky static void dump_qp(struct c4iw_qp *qhp)
273d6b92ffaSHans Petter Selasky {
274d6b92ffaSHans Petter Selasky int i;
275d6b92ffaSHans Petter Selasky int j;
276d6b92ffaSHans Petter Selasky struct t4_swsqe *swsqe;
277d6b92ffaSHans Petter Selasky struct t4_swrqe *swrqe;
278d6b92ffaSHans Petter Selasky u16 cidx, pidx;
279d6b92ffaSHans Petter Selasky u64 *p;
280d6b92ffaSHans Petter Selasky
281d6b92ffaSHans Petter Selasky fprintf(stderr,
282d6b92ffaSHans Petter Selasky "QP: %p id %u error %d flushed %d qid_mask 0x%x\n"
283d6b92ffaSHans Petter Selasky " SQ: id %u queue %p sw_queue %p cidx %u pidx %u in_use %u wq_pidx %u depth %u flags 0x%x flush_cidx %d\n"
284d6b92ffaSHans Petter Selasky " RQ: id %u queue %p sw_queue %p cidx %u pidx %u in_use %u depth %u\n",
285d6b92ffaSHans Petter Selasky qhp,
286d6b92ffaSHans Petter Selasky qhp->wq.sq.qid,
287d6b92ffaSHans Petter Selasky qhp->wq.error,
288d6b92ffaSHans Petter Selasky qhp->wq.flushed,
289d6b92ffaSHans Petter Selasky qhp->wq.qid_mask,
290d6b92ffaSHans Petter Selasky qhp->wq.sq.qid,
291d6b92ffaSHans Petter Selasky qhp->wq.sq.queue,
292d6b92ffaSHans Petter Selasky qhp->wq.sq.sw_sq,
293d6b92ffaSHans Petter Selasky qhp->wq.sq.cidx,
294d6b92ffaSHans Petter Selasky qhp->wq.sq.pidx,
295d6b92ffaSHans Petter Selasky qhp->wq.sq.in_use,
296d6b92ffaSHans Petter Selasky qhp->wq.sq.wq_pidx,
297d6b92ffaSHans Petter Selasky qhp->wq.sq.size,
298d6b92ffaSHans Petter Selasky qhp->wq.sq.flags,
299d6b92ffaSHans Petter Selasky qhp->wq.sq.flush_cidx,
300d6b92ffaSHans Petter Selasky qhp->wq.rq.qid,
301d6b92ffaSHans Petter Selasky qhp->wq.rq.queue,
302d6b92ffaSHans Petter Selasky qhp->wq.rq.sw_rq,
303d6b92ffaSHans Petter Selasky qhp->wq.rq.cidx,
304d6b92ffaSHans Petter Selasky qhp->wq.rq.pidx,
305d6b92ffaSHans Petter Selasky qhp->wq.rq.in_use,
306d6b92ffaSHans Petter Selasky qhp->wq.rq.size);
307d6b92ffaSHans Petter Selasky cidx = qhp->wq.sq.cidx;
308d6b92ffaSHans Petter Selasky pidx = qhp->wq.sq.pidx;
309d6b92ffaSHans Petter Selasky if (cidx != pidx)
310d6b92ffaSHans Petter Selasky fprintf(stderr, "SQ: \n");
311d6b92ffaSHans Petter Selasky while (cidx != pidx) {
312d6b92ffaSHans Petter Selasky swsqe = &qhp->wq.sq.sw_sq[cidx];
313d6b92ffaSHans Petter Selasky fprintf(stderr, "%04u: wr_id %016" PRIx64
314d6b92ffaSHans Petter Selasky " sq_wptr %08x read_len %u opcode 0x%x "
315d6b92ffaSHans Petter Selasky "complete %u signaled %u cqe %016" PRIx64 " %016" PRIx64 " %016" PRIx64 " %016" PRIx64 "\n",
316d6b92ffaSHans Petter Selasky cidx,
317d6b92ffaSHans Petter Selasky swsqe->wr_id,
318d6b92ffaSHans Petter Selasky swsqe->idx,
319d6b92ffaSHans Petter Selasky swsqe->read_len,
320d6b92ffaSHans Petter Selasky swsqe->opcode,
321d6b92ffaSHans Petter Selasky swsqe->complete,
322d6b92ffaSHans Petter Selasky swsqe->signaled,
323d6b92ffaSHans Petter Selasky htobe64(((uint64_t *)&swsqe->cqe)[0]),
324d6b92ffaSHans Petter Selasky htobe64(((uint64_t *)&swsqe->cqe)[1]),
325d6b92ffaSHans Petter Selasky htobe64(((uint64_t *)&swsqe->cqe)[2]),
326d6b92ffaSHans Petter Selasky htobe64(((uint64_t *)&swsqe->cqe)[3]));
327d6b92ffaSHans Petter Selasky if (++cidx == qhp->wq.sq.size)
328d6b92ffaSHans Petter Selasky cidx = 0;
329d6b92ffaSHans Petter Selasky }
330d6b92ffaSHans Petter Selasky
331d6b92ffaSHans Petter Selasky fprintf(stderr, "SQ WQ: \n");
332d6b92ffaSHans Petter Selasky p = (u64 *)qhp->wq.sq.queue;
333d6b92ffaSHans Petter Selasky for (i=0; i < qhp->wq.sq.size * T4_SQ_NUM_SLOTS; i++) {
334d6b92ffaSHans Petter Selasky for (j=0; j < T4_EQ_ENTRY_SIZE / 16; j++) {
335d6b92ffaSHans Petter Selasky fprintf(stderr, "%04u %016" PRIx64 " %016" PRIx64 " ",
336d6b92ffaSHans Petter Selasky i, be64toh(p[0]), be64toh(p[1]));
337d6b92ffaSHans Petter Selasky if (j == 0 && i == qhp->wq.sq.wq_pidx)
338d6b92ffaSHans Petter Selasky fprintf(stderr, " <-- pidx");
339d6b92ffaSHans Petter Selasky fprintf(stderr, "\n");
340d6b92ffaSHans Petter Selasky p += 2;
341d6b92ffaSHans Petter Selasky }
342d6b92ffaSHans Petter Selasky }
343d6b92ffaSHans Petter Selasky cidx = qhp->wq.rq.cidx;
344d6b92ffaSHans Petter Selasky pidx = qhp->wq.rq.pidx;
345d6b92ffaSHans Petter Selasky if (cidx != pidx)
346d6b92ffaSHans Petter Selasky fprintf(stderr, "RQ: \n");
347d6b92ffaSHans Petter Selasky while (cidx != pidx) {
348d6b92ffaSHans Petter Selasky swrqe = &qhp->wq.rq.sw_rq[cidx];
349d6b92ffaSHans Petter Selasky fprintf(stderr, "%04u: wr_id %016" PRIx64 "\n",
350d6b92ffaSHans Petter Selasky cidx,
351d6b92ffaSHans Petter Selasky swrqe->wr_id );
352d6b92ffaSHans Petter Selasky if (++cidx == qhp->wq.rq.size)
353d6b92ffaSHans Petter Selasky cidx = 0;
354d6b92ffaSHans Petter Selasky }
355d6b92ffaSHans Petter Selasky
356d6b92ffaSHans Petter Selasky fprintf(stderr, "RQ WQ: \n");
357d6b92ffaSHans Petter Selasky p = (u64 *)qhp->wq.rq.queue;
358d6b92ffaSHans Petter Selasky for (i=0; i < qhp->wq.rq.size * T4_RQ_NUM_SLOTS; i++) {
359d6b92ffaSHans Petter Selasky for (j=0; j < T4_EQ_ENTRY_SIZE / 16; j++) {
360d6b92ffaSHans Petter Selasky fprintf(stderr, "%04u %016" PRIx64 " %016" PRIx64 " ",
361d6b92ffaSHans Petter Selasky i, be64toh(p[0]), be64toh(p[1]));
362d6b92ffaSHans Petter Selasky if (j == 0 && i == qhp->wq.rq.pidx)
363d6b92ffaSHans Petter Selasky fprintf(stderr, " <-- pidx");
364d6b92ffaSHans Petter Selasky if (j == 0 && i == qhp->wq.rq.cidx)
365d6b92ffaSHans Petter Selasky fprintf(stderr, " <-- cidx");
366d6b92ffaSHans Petter Selasky fprintf(stderr, "\n");
367d6b92ffaSHans Petter Selasky p+=2;
368d6b92ffaSHans Petter Selasky }
369d6b92ffaSHans Petter Selasky }
370d6b92ffaSHans Petter Selasky }
371d6b92ffaSHans Petter Selasky
dump_state(void)372d6b92ffaSHans Petter Selasky void dump_state(void)
373d6b92ffaSHans Petter Selasky {
374d6b92ffaSHans Petter Selasky struct c4iw_dev *dev;
375d6b92ffaSHans Petter Selasky int i;
376d6b92ffaSHans Petter Selasky
377d6b92ffaSHans Petter Selasky fprintf(stderr, "STALL DETECTED:\n");
378d6b92ffaSHans Petter Selasky TAILQ_FOREACH(dev, &devices, list) {
379d6b92ffaSHans Petter Selasky //pthread_spin_lock(&dev->lock);
380d6b92ffaSHans Petter Selasky fprintf(stderr, "Device %s\n", dev->ibv_dev.name);
381d6b92ffaSHans Petter Selasky for (i=0; i < dev->max_cq; i++) {
382d6b92ffaSHans Petter Selasky if (dev->cqid2ptr[i]) {
383d6b92ffaSHans Petter Selasky struct c4iw_cq *chp = dev->cqid2ptr[i];
384d6b92ffaSHans Petter Selasky //pthread_spin_lock(&chp->lock);
385d6b92ffaSHans Petter Selasky dump_cq(chp);
386d6b92ffaSHans Petter Selasky //pthread_spin_unlock(&chp->lock);
387d6b92ffaSHans Petter Selasky }
388d6b92ffaSHans Petter Selasky }
389d6b92ffaSHans Petter Selasky for (i=0; i < dev->max_qp; i++) {
390d6b92ffaSHans Petter Selasky if (dev->qpid2ptr[i]) {
391d6b92ffaSHans Petter Selasky struct c4iw_qp *qhp = dev->qpid2ptr[i];
392d6b92ffaSHans Petter Selasky //pthread_spin_lock(&qhp->lock);
393d6b92ffaSHans Petter Selasky dump_qp(qhp);
394d6b92ffaSHans Petter Selasky //pthread_spin_unlock(&qhp->lock);
395d6b92ffaSHans Petter Selasky }
396d6b92ffaSHans Petter Selasky }
397d6b92ffaSHans Petter Selasky //pthread_spin_unlock(&dev->lock);
398d6b92ffaSHans Petter Selasky }
399d6b92ffaSHans Petter Selasky fprintf(stderr, "DUMP COMPLETE:\n");
400d6b92ffaSHans Petter Selasky fflush(stderr);
401d6b92ffaSHans Petter Selasky }
402d6b92ffaSHans Petter Selasky #endif /* end of STALL_DETECTION */
403d6b92ffaSHans Petter Selasky
404d6b92ffaSHans Petter Selasky /*
405d6b92ffaSHans Petter Selasky * c4iw_abi_version is used to store ABI for iw_cxgb4 so the user mode library
406d6b92ffaSHans Petter Selasky * can know if the driver supports the kernel mode db ringing.
407d6b92ffaSHans Petter Selasky */
408d6b92ffaSHans Petter Selasky int c4iw_abi_version = 1;
409d6b92ffaSHans Petter Selasky
cxgb4_driver_init(const char * uverbs_sys_path,int abi_version)410d6b92ffaSHans Petter Selasky static struct verbs_device *cxgb4_driver_init(const char *uverbs_sys_path,
411d6b92ffaSHans Petter Selasky int abi_version)
412d6b92ffaSHans Petter Selasky {
4135c2bacdeSNavdeep Parhar char devstr[IBV_SYSFS_PATH_MAX], ibdev[16], value[128], *cp;
4145c2bacdeSNavdeep Parhar char dev_str[IBV_SYSFS_PATH_MAX];
415d6b92ffaSHans Petter Selasky struct c4iw_dev *dev;
416d6b92ffaSHans Petter Selasky unsigned vendor, device, fw_maj, fw_min;
417d6b92ffaSHans Petter Selasky int i;
4185c2bacdeSNavdeep Parhar char devnum;
4195c2bacdeSNavdeep Parhar char ib_param[16];
420d6b92ffaSHans Petter Selasky
4215c2bacdeSNavdeep Parhar #ifndef __linux__
4225c2bacdeSNavdeep Parhar if (ibv_read_sysfs_file(uverbs_sys_path, "ibdev",
4235c2bacdeSNavdeep Parhar ibdev, sizeof ibdev) < 0)
4245c2bacdeSNavdeep Parhar return NULL;
4255c2bacdeSNavdeep Parhar
4265c2bacdeSNavdeep Parhar devnum = atoi(&ibdev[5]);
4275c2bacdeSNavdeep Parhar
4285c2bacdeSNavdeep Parhar if (ibdev[0] == 't' && ibdev[1] >= '4' && ibdev[1] <= '6' &&
4295c2bacdeSNavdeep Parhar strstr(&ibdev[2], "nex") && devnum >= 0) {
4305c2bacdeSNavdeep Parhar snprintf(dev_str, sizeof(dev_str), "/dev/t%cnex/%d", ibdev[1],
4315c2bacdeSNavdeep Parhar devnum);
4325c2bacdeSNavdeep Parhar } else
4335c2bacdeSNavdeep Parhar return NULL;
4345c2bacdeSNavdeep Parhar
4355c2bacdeSNavdeep Parhar if (ibv_read_sysfs_file(dev_str, "\%pnpinfo", value, sizeof value) < 0)
4365c2bacdeSNavdeep Parhar return NULL;
4375c2bacdeSNavdeep Parhar else {
4385c2bacdeSNavdeep Parhar if (strstr(value, "vendor=")) {
4395c2bacdeSNavdeep Parhar strncpy(ib_param, strstr(value, "vendor=") +
4405c2bacdeSNavdeep Parhar strlen("vendor="), 6);
4415c2bacdeSNavdeep Parhar sscanf(ib_param, "%i", &vendor);
4425c2bacdeSNavdeep Parhar }
4435c2bacdeSNavdeep Parhar
4445c2bacdeSNavdeep Parhar if (strstr(value, "device=")) {
4455c2bacdeSNavdeep Parhar strncpy(ib_param, strstr(value, "device=") +
4465c2bacdeSNavdeep Parhar strlen("device="), 6);
4475c2bacdeSNavdeep Parhar sscanf(ib_param, "%i", &device);
4485c2bacdeSNavdeep Parhar }
4495c2bacdeSNavdeep Parhar }
4505c2bacdeSNavdeep Parhar #else
451d6b92ffaSHans Petter Selasky if (ibv_read_sysfs_file(uverbs_sys_path, "device/vendor",
452d6b92ffaSHans Petter Selasky value, sizeof value) < 0)
453d6b92ffaSHans Petter Selasky return NULL;
454d6b92ffaSHans Petter Selasky sscanf(value, "%i", &vendor);
455d6b92ffaSHans Petter Selasky
456d6b92ffaSHans Petter Selasky if (ibv_read_sysfs_file(uverbs_sys_path, "device/device",
457d6b92ffaSHans Petter Selasky value, sizeof value) < 0)
458d6b92ffaSHans Petter Selasky return NULL;
459d6b92ffaSHans Petter Selasky sscanf(value, "%i", &device);
4605c2bacdeSNavdeep Parhar #endif
461d6b92ffaSHans Petter Selasky
462d6b92ffaSHans Petter Selasky for (i = 0; i < sizeof hca_table / sizeof hca_table[0]; ++i)
463d6b92ffaSHans Petter Selasky if (vendor == hca_table[i].vendor &&
464d6b92ffaSHans Petter Selasky device == hca_table[i].device)
465d6b92ffaSHans Petter Selasky goto found;
466d6b92ffaSHans Petter Selasky
467d6b92ffaSHans Petter Selasky return NULL;
468d6b92ffaSHans Petter Selasky
469d6b92ffaSHans Petter Selasky found:
470d6b92ffaSHans Petter Selasky c4iw_abi_version = abi_version;
471d6b92ffaSHans Petter Selasky
4725c2bacdeSNavdeep Parhar #ifndef __linux__
4735c2bacdeSNavdeep Parhar if (ibv_read_sysfs_file(dev_str, "firmware_version",
4745c2bacdeSNavdeep Parhar value, sizeof value) < 0)
4755c2bacdeSNavdeep Parhar return NULL;
4765c2bacdeSNavdeep Parhar #else
477d6b92ffaSHans Petter Selasky /*
478d6b92ffaSHans Petter Selasky * Verify that the firmware major number matches. Major number
479d6b92ffaSHans Petter Selasky * mismatches are fatal. Minor number mismatches are tolerated.
480d6b92ffaSHans Petter Selasky */
481d6b92ffaSHans Petter Selasky if (ibv_read_sysfs_file(uverbs_sys_path, "ibdev",
482d6b92ffaSHans Petter Selasky ibdev, sizeof ibdev) < 0)
483d6b92ffaSHans Petter Selasky return NULL;
484d6b92ffaSHans Petter Selasky
485d6b92ffaSHans Petter Selasky memset(devstr, 0, sizeof devstr);
486d6b92ffaSHans Petter Selasky snprintf(devstr, sizeof devstr, "%s/class/infiniband/%s",
487d6b92ffaSHans Petter Selasky ibv_get_sysfs_path(), ibdev);
488d6b92ffaSHans Petter Selasky if (ibv_read_sysfs_file(devstr, "fw_ver", value, sizeof value) < 0)
489d6b92ffaSHans Petter Selasky return NULL;
4905c2bacdeSNavdeep Parhar #endif
491d6b92ffaSHans Petter Selasky
492d6b92ffaSHans Petter Selasky cp = strtok(value+1, ".");
493d6b92ffaSHans Petter Selasky sscanf(cp, "%i", &fw_maj);
494d6b92ffaSHans Petter Selasky cp = strtok(NULL, ".");
495d6b92ffaSHans Petter Selasky sscanf(cp, "%i", &fw_min);
496d6b92ffaSHans Petter Selasky
497d6b92ffaSHans Petter Selasky if ((signed int)fw_maj < FW_MAJ) {
498d6b92ffaSHans Petter Selasky fprintf(stderr, "libcxgb4: Fatal firmware version mismatch. "
499d6b92ffaSHans Petter Selasky "Firmware major number is %u and libcxgb4 needs %u.\n",
500d6b92ffaSHans Petter Selasky fw_maj, FW_MAJ);
501d6b92ffaSHans Petter Selasky fflush(stderr);
502d6b92ffaSHans Petter Selasky return NULL;
503d6b92ffaSHans Petter Selasky }
504d6b92ffaSHans Petter Selasky
505d6b92ffaSHans Petter Selasky DBGLOG("libcxgb4");
506d6b92ffaSHans Petter Selasky
507d6b92ffaSHans Petter Selasky if ((signed int)fw_min < FW_MIN) {
508d6b92ffaSHans Petter Selasky PDBG("libcxgb4: non-fatal firmware version mismatch. "
509d6b92ffaSHans Petter Selasky "Firmware minor number is %u and libcxgb4 needs %u.\n",
510d6b92ffaSHans Petter Selasky fw_min, FW_MIN);
511d6b92ffaSHans Petter Selasky fflush(stderr);
512d6b92ffaSHans Petter Selasky }
513d6b92ffaSHans Petter Selasky
514d6b92ffaSHans Petter Selasky PDBG("%s found vendor %d device %d type %d\n",
515d6b92ffaSHans Petter Selasky __FUNCTION__, vendor, device, CHELSIO_CHIP_VERSION(hca_table[i].device >> 8));
516d6b92ffaSHans Petter Selasky
517d6b92ffaSHans Petter Selasky dev = calloc(1, sizeof *dev);
518d6b92ffaSHans Petter Selasky if (!dev) {
519d6b92ffaSHans Petter Selasky return NULL;
520d6b92ffaSHans Petter Selasky }
521d6b92ffaSHans Petter Selasky
522a687910fSSean Lim if (pthread_spin_init(&dev->lock, PTHREAD_PROCESS_PRIVATE))
523a687910fSSean Lim goto err;
524a687910fSSean Lim
525d6b92ffaSHans Petter Selasky dev->ibv_dev.ops = &c4iw_dev_ops;
526d6b92ffaSHans Petter Selasky dev->chip_version = CHELSIO_CHIP_VERSION(hca_table[i].device >> 8);
527d6b92ffaSHans Petter Selasky dev->abi_version = abi_version;
528d6b92ffaSHans Petter Selasky
529d6b92ffaSHans Petter Selasky PDBG("%s device claimed\n", __FUNCTION__);
530d6b92ffaSHans Petter Selasky TAILQ_INSERT_TAIL(&devices, dev, list);
531d6b92ffaSHans Petter Selasky #ifdef STALL_DETECTION
532d6b92ffaSHans Petter Selasky {
533d6b92ffaSHans Petter Selasky char *c = getenv("CXGB4_STALL_TIMEOUT");
534d6b92ffaSHans Petter Selasky if (c) {
535d6b92ffaSHans Petter Selasky stall_to = strtol(c, NULL, 0);
536d6b92ffaSHans Petter Selasky if (errno || stall_to < 0)
537d6b92ffaSHans Petter Selasky stall_to = 0;
538d6b92ffaSHans Petter Selasky }
539d6b92ffaSHans Petter Selasky }
540d6b92ffaSHans Petter Selasky #endif
541d6b92ffaSHans Petter Selasky {
542d6b92ffaSHans Petter Selasky char *c = getenv("CXGB4_MA_WR");
543d6b92ffaSHans Petter Selasky if (c) {
544d6b92ffaSHans Petter Selasky ma_wr = strtol(c, NULL, 0);
545d6b92ffaSHans Petter Selasky if (ma_wr != 1)
546d6b92ffaSHans Petter Selasky ma_wr = 0;
547d6b92ffaSHans Petter Selasky }
548d6b92ffaSHans Petter Selasky }
549d6b92ffaSHans Petter Selasky {
550d6b92ffaSHans Petter Selasky char *c = getenv("T5_ENABLE_WC");
551d6b92ffaSHans Petter Selasky if (c) {
552d6b92ffaSHans Petter Selasky t5_en_wc = strtol(c, NULL, 0);
553d6b92ffaSHans Petter Selasky if (t5_en_wc != 1)
554d6b92ffaSHans Petter Selasky t5_en_wc = 0;
555d6b92ffaSHans Petter Selasky }
556d6b92ffaSHans Petter Selasky }
557d6b92ffaSHans Petter Selasky
558d6b92ffaSHans Petter Selasky return &dev->ibv_dev;
559a687910fSSean Lim
560a687910fSSean Lim err:
561a687910fSSean Lim free(dev);
562a687910fSSean Lim
563a687910fSSean Lim return NULL;
564d6b92ffaSHans Petter Selasky }
565d6b92ffaSHans Petter Selasky
cxgb4_register_driver(void)566d6b92ffaSHans Petter Selasky static __attribute__((constructor)) void cxgb4_register_driver(void)
567d6b92ffaSHans Petter Selasky {
568d6b92ffaSHans Petter Selasky c4iw_page_size = sysconf(_SC_PAGESIZE);
569d6b92ffaSHans Petter Selasky c4iw_page_shift = long_log2(c4iw_page_size);
570d6b92ffaSHans Petter Selasky c4iw_page_mask = ~(c4iw_page_size - 1);
571d6b92ffaSHans Petter Selasky verbs_register_driver("cxgb4", cxgb4_driver_init);
572d6b92ffaSHans Petter Selasky }
573d6b92ffaSHans Petter Selasky
574d6b92ffaSHans Petter Selasky #ifdef STATS
575d6b92ffaSHans Petter Selasky void __attribute__ ((destructor)) cs_fini(void);
cs_fini(void)576d6b92ffaSHans Petter Selasky void __attribute__ ((destructor)) cs_fini(void)
577d6b92ffaSHans Petter Selasky {
578d6b92ffaSHans Petter Selasky syslog(LOG_NOTICE, "cxgb4 stats - sends %lu recv %lu read %lu "
579d6b92ffaSHans Petter Selasky "write %lu arm %lu cqe %lu mr %lu qp %lu cq %lu\n",
580d6b92ffaSHans Petter Selasky c4iw_stats.send, c4iw_stats.recv, c4iw_stats.read,
581d6b92ffaSHans Petter Selasky c4iw_stats.write, c4iw_stats.arm, c4iw_stats.cqe,
582d6b92ffaSHans Petter Selasky c4iw_stats.mr, c4iw_stats.qp, c4iw_stats.cq);
583d6b92ffaSHans Petter Selasky }
584d6b92ffaSHans Petter Selasky #endif
585