1d6b92ffaSHans Petter Selasky /*
2d6b92ffaSHans Petter Selasky * Copyright (c) 2005 Topspin Communications. All rights reserved.
3d6b92ffaSHans Petter Selasky * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
4d6b92ffaSHans Petter Selasky * Copyright (c) 2006, 2007 Cisco Systems. All rights reserved.
5d6b92ffaSHans Petter Selasky *
6d6b92ffaSHans Petter Selasky * This software is available to you under a choice of one of two
7d6b92ffaSHans Petter Selasky * licenses. You may choose to be licensed under the terms of the GNU
8d6b92ffaSHans Petter Selasky * General Public License (GPL) Version 2, available from the file
9d6b92ffaSHans Petter Selasky * COPYING in the main directory of this source tree, or the
10d6b92ffaSHans Petter Selasky * OpenIB.org BSD license below:
11d6b92ffaSHans Petter Selasky *
12d6b92ffaSHans Petter Selasky * Redistribution and use in source and binary forms, with or
13d6b92ffaSHans Petter Selasky * without modification, are permitted provided that the following
14d6b92ffaSHans Petter Selasky * conditions are met:
15d6b92ffaSHans Petter Selasky *
16d6b92ffaSHans Petter Selasky * - Redistributions of source code must retain the above
17d6b92ffaSHans Petter Selasky * copyright notice, this list of conditions and the following
18d6b92ffaSHans Petter Selasky * disclaimer.
19d6b92ffaSHans Petter Selasky *
20d6b92ffaSHans Petter Selasky * - Redistributions in binary form must reproduce the above
21d6b92ffaSHans Petter Selasky * copyright notice, this list of conditions and the following
22d6b92ffaSHans Petter Selasky * disclaimer in the documentation and/or other materials
23d6b92ffaSHans Petter Selasky * provided with the distribution.
24d6b92ffaSHans Petter Selasky *
25d6b92ffaSHans Petter Selasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26d6b92ffaSHans Petter Selasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27d6b92ffaSHans Petter Selasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28d6b92ffaSHans Petter Selasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29d6b92ffaSHans Petter Selasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30d6b92ffaSHans Petter Selasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31d6b92ffaSHans Petter Selasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32d6b92ffaSHans Petter Selasky * SOFTWARE.
33d6b92ffaSHans Petter Selasky */
34d6b92ffaSHans Petter Selasky
35d6b92ffaSHans Petter Selasky #include <config.h>
36d6b92ffaSHans Petter Selasky
37d6b92ffaSHans Petter Selasky #include <stdio.h>
38d6b92ffaSHans Petter Selasky #include <stdlib.h>
39d6b92ffaSHans Petter Selasky #include <pthread.h>
40d6b92ffaSHans Petter Selasky #include <string.h>
41d6b92ffaSHans Petter Selasky
42d6b92ffaSHans Petter Selasky #include <infiniband/opcode.h>
43d6b92ffaSHans Petter Selasky
44d6b92ffaSHans Petter Selasky #include "mlx4.h"
45d6b92ffaSHans Petter Selasky #include "doorbell.h"
46d6b92ffaSHans Petter Selasky
47d6b92ffaSHans Petter Selasky enum {
48d6b92ffaSHans Petter Selasky MLX4_CQ_DOORBELL = 0x20
49d6b92ffaSHans Petter Selasky };
50d6b92ffaSHans Petter Selasky
51d6b92ffaSHans Petter Selasky enum {
52d6b92ffaSHans Petter Selasky CQ_OK = 0,
53d6b92ffaSHans Petter Selasky CQ_EMPTY = -1,
54d6b92ffaSHans Petter Selasky CQ_POLL_ERR = -2
55d6b92ffaSHans Petter Selasky };
56d6b92ffaSHans Petter Selasky
57d6b92ffaSHans Petter Selasky #define MLX4_CQ_DB_REQ_NOT_SOL (1 << 24)
58d6b92ffaSHans Petter Selasky #define MLX4_CQ_DB_REQ_NOT (2 << 24)
59d6b92ffaSHans Petter Selasky
60d6b92ffaSHans Petter Selasky enum {
61d6b92ffaSHans Petter Selasky MLX4_CQE_VLAN_PRESENT_MASK = 1 << 29,
62d6b92ffaSHans Petter Selasky MLX4_CQE_QPN_MASK = 0xffffff,
63d6b92ffaSHans Petter Selasky };
64d6b92ffaSHans Petter Selasky
65d6b92ffaSHans Petter Selasky enum {
66d6b92ffaSHans Petter Selasky MLX4_CQE_OWNER_MASK = 0x80,
67d6b92ffaSHans Petter Selasky MLX4_CQE_IS_SEND_MASK = 0x40,
68d6b92ffaSHans Petter Selasky MLX4_CQE_OPCODE_MASK = 0x1f
69d6b92ffaSHans Petter Selasky };
70d6b92ffaSHans Petter Selasky
71d6b92ffaSHans Petter Selasky enum {
72d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01,
73d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02,
74d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04,
75d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_WR_FLUSH_ERR = 0x05,
76d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_MW_BIND_ERR = 0x06,
77d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_BAD_RESP_ERR = 0x10,
78d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11,
79d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12,
80d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13,
81d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_REMOTE_OP_ERR = 0x14,
82d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15,
83d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16,
84d6b92ffaSHans Petter Selasky MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22,
85d6b92ffaSHans Petter Selasky };
86d6b92ffaSHans Petter Selasky
87d6b92ffaSHans Petter Selasky struct mlx4_err_cqe {
88d6b92ffaSHans Petter Selasky uint32_t vlan_my_qpn;
89d6b92ffaSHans Petter Selasky uint32_t reserved1[5];
90d6b92ffaSHans Petter Selasky uint16_t wqe_index;
91d6b92ffaSHans Petter Selasky uint8_t vendor_err;
92d6b92ffaSHans Petter Selasky uint8_t syndrome;
93d6b92ffaSHans Petter Selasky uint8_t reserved2[3];
94d6b92ffaSHans Petter Selasky uint8_t owner_sr_opcode;
95d6b92ffaSHans Petter Selasky };
96d6b92ffaSHans Petter Selasky
get_cqe(struct mlx4_cq * cq,int entry)97d6b92ffaSHans Petter Selasky static struct mlx4_cqe *get_cqe(struct mlx4_cq *cq, int entry)
98d6b92ffaSHans Petter Selasky {
99d6b92ffaSHans Petter Selasky return cq->buf.buf + entry * cq->cqe_size;
100d6b92ffaSHans Petter Selasky }
101d6b92ffaSHans Petter Selasky
get_sw_cqe(struct mlx4_cq * cq,int n)102d6b92ffaSHans Petter Selasky static void *get_sw_cqe(struct mlx4_cq *cq, int n)
103d6b92ffaSHans Petter Selasky {
104d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibv_cq.cqe);
105d6b92ffaSHans Petter Selasky struct mlx4_cqe *tcqe = cq->cqe_size == 64 ? cqe + 1 : cqe;
106d6b92ffaSHans Petter Selasky
107d6b92ffaSHans Petter Selasky return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
108d6b92ffaSHans Petter Selasky !!(n & (cq->ibv_cq.cqe + 1))) ? NULL : cqe;
109d6b92ffaSHans Petter Selasky }
110d6b92ffaSHans Petter Selasky
next_cqe_sw(struct mlx4_cq * cq)111d6b92ffaSHans Petter Selasky static struct mlx4_cqe *next_cqe_sw(struct mlx4_cq *cq)
112d6b92ffaSHans Petter Selasky {
113d6b92ffaSHans Petter Selasky return get_sw_cqe(cq, cq->cons_index);
114d6b92ffaSHans Petter Selasky }
115d6b92ffaSHans Petter Selasky
mlx4_handle_error_cqe(struct mlx4_err_cqe * cqe)116d6b92ffaSHans Petter Selasky static enum ibv_wc_status mlx4_handle_error_cqe(struct mlx4_err_cqe *cqe)
117d6b92ffaSHans Petter Selasky {
118d6b92ffaSHans Petter Selasky if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR)
119d6b92ffaSHans Petter Selasky printf(PFX "local QP operation err "
120d6b92ffaSHans Petter Selasky "(QPN %06x, WQE index %x, vendor syndrome %02x, "
121d6b92ffaSHans Petter Selasky "opcode = %02x)\n",
122d6b92ffaSHans Petter Selasky htobe32(cqe->vlan_my_qpn), htobe32(cqe->wqe_index),
123d6b92ffaSHans Petter Selasky cqe->vendor_err,
124d6b92ffaSHans Petter Selasky cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
125d6b92ffaSHans Petter Selasky
126d6b92ffaSHans Petter Selasky switch (cqe->syndrome) {
127d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:
128d6b92ffaSHans Petter Selasky return IBV_WC_LOC_LEN_ERR;
129d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:
130d6b92ffaSHans Petter Selasky return IBV_WC_LOC_QP_OP_ERR;
131d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:
132d6b92ffaSHans Petter Selasky return IBV_WC_LOC_PROT_ERR;
133d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:
134d6b92ffaSHans Petter Selasky return IBV_WC_WR_FLUSH_ERR;
135d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_MW_BIND_ERR:
136d6b92ffaSHans Petter Selasky return IBV_WC_MW_BIND_ERR;
137d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_BAD_RESP_ERR:
138d6b92ffaSHans Petter Selasky return IBV_WC_BAD_RESP_ERR;
139d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:
140d6b92ffaSHans Petter Selasky return IBV_WC_LOC_ACCESS_ERR;
141d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
142d6b92ffaSHans Petter Selasky return IBV_WC_REM_INV_REQ_ERR;
143d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:
144d6b92ffaSHans Petter Selasky return IBV_WC_REM_ACCESS_ERR;
145d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:
146d6b92ffaSHans Petter Selasky return IBV_WC_REM_OP_ERR;
147d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
148d6b92ffaSHans Petter Selasky return IBV_WC_RETRY_EXC_ERR;
149d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
150d6b92ffaSHans Petter Selasky return IBV_WC_RNR_RETRY_EXC_ERR;
151d6b92ffaSHans Petter Selasky case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:
152d6b92ffaSHans Petter Selasky return IBV_WC_REM_ABORT_ERR;
153d6b92ffaSHans Petter Selasky default:
154d6b92ffaSHans Petter Selasky return IBV_WC_GENERAL_ERR;
155d6b92ffaSHans Petter Selasky }
156d6b92ffaSHans Petter Selasky }
157d6b92ffaSHans Petter Selasky
handle_good_req(struct ibv_wc * wc,struct mlx4_cqe * cqe)158d6b92ffaSHans Petter Selasky static inline void handle_good_req(struct ibv_wc *wc, struct mlx4_cqe *cqe)
159d6b92ffaSHans Petter Selasky {
160d6b92ffaSHans Petter Selasky wc->wc_flags = 0;
161d6b92ffaSHans Petter Selasky switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
162d6b92ffaSHans Petter Selasky case MLX4_OPCODE_RDMA_WRITE_IMM:
163d6b92ffaSHans Petter Selasky wc->wc_flags |= IBV_WC_WITH_IMM;
164d6b92ffaSHans Petter Selasky SWITCH_FALLTHROUGH;
165d6b92ffaSHans Petter Selasky case MLX4_OPCODE_RDMA_WRITE:
166d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_RDMA_WRITE;
167d6b92ffaSHans Petter Selasky break;
168d6b92ffaSHans Petter Selasky case MLX4_OPCODE_SEND_IMM:
169d6b92ffaSHans Petter Selasky wc->wc_flags |= IBV_WC_WITH_IMM;
170d6b92ffaSHans Petter Selasky SWITCH_FALLTHROUGH;
171d6b92ffaSHans Petter Selasky case MLX4_OPCODE_SEND:
172d6b92ffaSHans Petter Selasky case MLX4_OPCODE_SEND_INVAL:
173d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_SEND;
174d6b92ffaSHans Petter Selasky break;
175d6b92ffaSHans Petter Selasky case MLX4_OPCODE_RDMA_READ:
176d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_RDMA_READ;
177d6b92ffaSHans Petter Selasky wc->byte_len = be32toh(cqe->byte_cnt);
178d6b92ffaSHans Petter Selasky break;
179d6b92ffaSHans Petter Selasky case MLX4_OPCODE_ATOMIC_CS:
180d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_COMP_SWAP;
181d6b92ffaSHans Petter Selasky wc->byte_len = 8;
182d6b92ffaSHans Petter Selasky break;
183d6b92ffaSHans Petter Selasky case MLX4_OPCODE_ATOMIC_FA:
184d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_FETCH_ADD;
185d6b92ffaSHans Petter Selasky wc->byte_len = 8;
186d6b92ffaSHans Petter Selasky break;
187d6b92ffaSHans Petter Selasky case MLX4_OPCODE_LOCAL_INVAL:
188d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_LOCAL_INV;
189d6b92ffaSHans Petter Selasky break;
190d6b92ffaSHans Petter Selasky case MLX4_OPCODE_BIND_MW:
191d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_BIND_MW;
192d6b92ffaSHans Petter Selasky break;
193d6b92ffaSHans Petter Selasky default:
194d6b92ffaSHans Petter Selasky /* assume it's a send completion */
195d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_SEND;
196d6b92ffaSHans Petter Selasky break;
197d6b92ffaSHans Petter Selasky }
198d6b92ffaSHans Petter Selasky }
199d6b92ffaSHans Petter Selasky
200d6b92ffaSHans Petter Selasky static inline int mlx4_get_next_cqe(struct mlx4_cq *cq,
201d6b92ffaSHans Petter Selasky struct mlx4_cqe **pcqe)
202d6b92ffaSHans Petter Selasky ALWAYS_INLINE;
mlx4_get_next_cqe(struct mlx4_cq * cq,struct mlx4_cqe ** pcqe)203d6b92ffaSHans Petter Selasky static inline int mlx4_get_next_cqe(struct mlx4_cq *cq,
204d6b92ffaSHans Petter Selasky struct mlx4_cqe **pcqe)
205d6b92ffaSHans Petter Selasky {
206d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe;
207d6b92ffaSHans Petter Selasky
208d6b92ffaSHans Petter Selasky cqe = next_cqe_sw(cq);
209d6b92ffaSHans Petter Selasky if (!cqe)
210d6b92ffaSHans Petter Selasky return CQ_EMPTY;
211d6b92ffaSHans Petter Selasky
212d6b92ffaSHans Petter Selasky if (cq->cqe_size == 64)
213d6b92ffaSHans Petter Selasky ++cqe;
214d6b92ffaSHans Petter Selasky
215d6b92ffaSHans Petter Selasky ++cq->cons_index;
216d6b92ffaSHans Petter Selasky
217d6b92ffaSHans Petter Selasky VALGRIND_MAKE_MEM_DEFINED(cqe, sizeof *cqe);
218d6b92ffaSHans Petter Selasky
219d6b92ffaSHans Petter Selasky /*
220d6b92ffaSHans Petter Selasky * Make sure we read CQ entry contents after we've checked the
221d6b92ffaSHans Petter Selasky * ownership bit.
222d6b92ffaSHans Petter Selasky */
223d6b92ffaSHans Petter Selasky udma_from_device_barrier();
224d6b92ffaSHans Petter Selasky
225d6b92ffaSHans Petter Selasky *pcqe = cqe;
226d6b92ffaSHans Petter Selasky
227d6b92ffaSHans Petter Selasky return CQ_OK;
228d6b92ffaSHans Petter Selasky }
229d6b92ffaSHans Petter Selasky
230d6b92ffaSHans Petter Selasky static inline int mlx4_parse_cqe(struct mlx4_cq *cq,
231d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe,
232d6b92ffaSHans Petter Selasky struct mlx4_qp **cur_qp,
233d6b92ffaSHans Petter Selasky struct ibv_wc *wc, int lazy)
234d6b92ffaSHans Petter Selasky ALWAYS_INLINE;
mlx4_parse_cqe(struct mlx4_cq * cq,struct mlx4_cqe * cqe,struct mlx4_qp ** cur_qp,struct ibv_wc * wc,int lazy)235d6b92ffaSHans Petter Selasky static inline int mlx4_parse_cqe(struct mlx4_cq *cq,
236d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe,
237d6b92ffaSHans Petter Selasky struct mlx4_qp **cur_qp,
238d6b92ffaSHans Petter Selasky struct ibv_wc *wc, int lazy)
239d6b92ffaSHans Petter Selasky {
240d6b92ffaSHans Petter Selasky struct mlx4_wq *wq;
241d6b92ffaSHans Petter Selasky struct mlx4_srq *srq;
242d6b92ffaSHans Petter Selasky uint32_t qpn;
243d6b92ffaSHans Petter Selasky uint32_t g_mlpath_rqpn;
244d6b92ffaSHans Petter Selasky uint64_t *pwr_id;
245d6b92ffaSHans Petter Selasky uint16_t wqe_index;
246d6b92ffaSHans Petter Selasky struct mlx4_err_cqe *ecqe;
247d6b92ffaSHans Petter Selasky struct mlx4_context *mctx;
248d6b92ffaSHans Petter Selasky int is_error;
249d6b92ffaSHans Petter Selasky int is_send;
250d6b92ffaSHans Petter Selasky enum ibv_wc_status *pstatus;
251d6b92ffaSHans Petter Selasky
252d6b92ffaSHans Petter Selasky mctx = to_mctx(cq->ibv_cq.context);
253d6b92ffaSHans Petter Selasky qpn = be32toh(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK;
254d6b92ffaSHans Petter Selasky if (lazy) {
255d6b92ffaSHans Petter Selasky cq->cqe = cqe;
256d6b92ffaSHans Petter Selasky cq->flags &= (~MLX4_CQ_FLAGS_RX_CSUM_VALID);
257d6b92ffaSHans Petter Selasky } else
258d6b92ffaSHans Petter Selasky wc->qp_num = qpn;
259d6b92ffaSHans Petter Selasky
260d6b92ffaSHans Petter Selasky is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;
261d6b92ffaSHans Petter Selasky is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
262d6b92ffaSHans Petter Selasky MLX4_CQE_OPCODE_ERROR;
263d6b92ffaSHans Petter Selasky
264d6b92ffaSHans Petter Selasky if ((qpn & MLX4_XRC_QPN_BIT) && !is_send) {
265d6b92ffaSHans Petter Selasky /*
266d6b92ffaSHans Petter Selasky * We do not have to take the XSRQ table lock here,
267d6b92ffaSHans Petter Selasky * because CQs will be locked while SRQs are removed
268d6b92ffaSHans Petter Selasky * from the table.
269d6b92ffaSHans Petter Selasky */
270d6b92ffaSHans Petter Selasky srq = mlx4_find_xsrq(&mctx->xsrq_table,
271d6b92ffaSHans Petter Selasky be32toh(cqe->g_mlpath_rqpn) & MLX4_CQE_QPN_MASK);
272d6b92ffaSHans Petter Selasky if (!srq)
273d6b92ffaSHans Petter Selasky return CQ_POLL_ERR;
274d6b92ffaSHans Petter Selasky } else {
275d6b92ffaSHans Petter Selasky if (!*cur_qp || (qpn != (*cur_qp)->verbs_qp.qp.qp_num)) {
276d6b92ffaSHans Petter Selasky /*
277d6b92ffaSHans Petter Selasky * We do not have to take the QP table lock here,
278d6b92ffaSHans Petter Selasky * because CQs will be locked while QPs are removed
279d6b92ffaSHans Petter Selasky * from the table.
280d6b92ffaSHans Petter Selasky */
281d6b92ffaSHans Petter Selasky *cur_qp = mlx4_find_qp(mctx, qpn);
282d6b92ffaSHans Petter Selasky if (!*cur_qp)
283d6b92ffaSHans Petter Selasky return CQ_POLL_ERR;
284d6b92ffaSHans Petter Selasky }
285d6b92ffaSHans Petter Selasky srq = ((*cur_qp)->verbs_qp.qp.srq) ? to_msrq((*cur_qp)->verbs_qp.qp.srq) : NULL;
286d6b92ffaSHans Petter Selasky }
287d6b92ffaSHans Petter Selasky
288d6b92ffaSHans Petter Selasky pwr_id = lazy ? &cq->ibv_cq.wr_id : &wc->wr_id;
289d6b92ffaSHans Petter Selasky if (is_send) {
290d6b92ffaSHans Petter Selasky wq = &(*cur_qp)->sq;
291d6b92ffaSHans Petter Selasky wqe_index = be16toh(cqe->wqe_index);
292d6b92ffaSHans Petter Selasky wq->tail += (uint16_t) (wqe_index - (uint16_t) wq->tail);
293d6b92ffaSHans Petter Selasky *pwr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
294d6b92ffaSHans Petter Selasky ++wq->tail;
295d6b92ffaSHans Petter Selasky } else if (srq) {
296d6b92ffaSHans Petter Selasky wqe_index = be16toh(cqe->wqe_index);
297d6b92ffaSHans Petter Selasky *pwr_id = srq->wrid[wqe_index];
298d6b92ffaSHans Petter Selasky mlx4_free_srq_wqe(srq, wqe_index);
299d6b92ffaSHans Petter Selasky } else {
300d6b92ffaSHans Petter Selasky wq = &(*cur_qp)->rq;
301d6b92ffaSHans Petter Selasky *pwr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
302d6b92ffaSHans Petter Selasky ++wq->tail;
303d6b92ffaSHans Petter Selasky }
304d6b92ffaSHans Petter Selasky
305d6b92ffaSHans Petter Selasky pstatus = lazy ? &cq->ibv_cq.status : &wc->status;
306d6b92ffaSHans Petter Selasky if (is_error) {
307d6b92ffaSHans Petter Selasky ecqe = (struct mlx4_err_cqe *)cqe;
308d6b92ffaSHans Petter Selasky *pstatus = mlx4_handle_error_cqe(ecqe);
309d6b92ffaSHans Petter Selasky if (!lazy)
310d6b92ffaSHans Petter Selasky wc->vendor_err = ecqe->vendor_err;
311d6b92ffaSHans Petter Selasky return CQ_OK;
312d6b92ffaSHans Petter Selasky }
313d6b92ffaSHans Petter Selasky
314d6b92ffaSHans Petter Selasky *pstatus = IBV_WC_SUCCESS;
315d6b92ffaSHans Petter Selasky if (lazy) {
316d6b92ffaSHans Petter Selasky if (!is_send)
317d6b92ffaSHans Petter Selasky if ((*cur_qp) && ((*cur_qp)->qp_cap_cache & MLX4_RX_CSUM_VALID))
318d6b92ffaSHans Petter Selasky cq->flags |= MLX4_CQ_FLAGS_RX_CSUM_VALID;
319d6b92ffaSHans Petter Selasky } else if (is_send) {
320d6b92ffaSHans Petter Selasky handle_good_req(wc, cqe);
321d6b92ffaSHans Petter Selasky } else {
322d6b92ffaSHans Petter Selasky wc->byte_len = be32toh(cqe->byte_cnt);
323d6b92ffaSHans Petter Selasky
324d6b92ffaSHans Petter Selasky switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
325d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
326d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_RECV_RDMA_WITH_IMM;
327d6b92ffaSHans Petter Selasky wc->wc_flags = IBV_WC_WITH_IMM;
328d6b92ffaSHans Petter Selasky wc->imm_data = cqe->immed_rss_invalid;
329d6b92ffaSHans Petter Selasky break;
330d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_SEND_INVAL:
331d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_RECV;
332d6b92ffaSHans Petter Selasky wc->wc_flags |= IBV_WC_WITH_INV;
333d6b92ffaSHans Petter Selasky wc->imm_data = be32toh(cqe->immed_rss_invalid);
334d6b92ffaSHans Petter Selasky break;
335d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_SEND:
336d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_RECV;
337d6b92ffaSHans Petter Selasky wc->wc_flags = 0;
338d6b92ffaSHans Petter Selasky break;
339d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_SEND_IMM:
340d6b92ffaSHans Petter Selasky wc->opcode = IBV_WC_RECV;
341d6b92ffaSHans Petter Selasky wc->wc_flags = IBV_WC_WITH_IMM;
342d6b92ffaSHans Petter Selasky wc->imm_data = cqe->immed_rss_invalid;
343d6b92ffaSHans Petter Selasky break;
344d6b92ffaSHans Petter Selasky }
345d6b92ffaSHans Petter Selasky
346d6b92ffaSHans Petter Selasky wc->slid = be16toh(cqe->rlid);
347d6b92ffaSHans Petter Selasky g_mlpath_rqpn = be32toh(cqe->g_mlpath_rqpn);
348d6b92ffaSHans Petter Selasky wc->src_qp = g_mlpath_rqpn & 0xffffff;
349d6b92ffaSHans Petter Selasky wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f;
350d6b92ffaSHans Petter Selasky wc->wc_flags |= g_mlpath_rqpn & 0x80000000 ? IBV_WC_GRH : 0;
351d6b92ffaSHans Petter Selasky wc->pkey_index = be32toh(cqe->immed_rss_invalid) & 0x7f;
352d6b92ffaSHans Petter Selasky /* When working with xrc srqs, don't have qp to check link layer.
353d6b92ffaSHans Petter Selasky * Using IB SL, should consider Roce. (TBD)
354d6b92ffaSHans Petter Selasky */
355d6b92ffaSHans Petter Selasky if ((*cur_qp) && (*cur_qp)->link_layer == IBV_LINK_LAYER_ETHERNET)
356d6b92ffaSHans Petter Selasky wc->sl = be16toh(cqe->sl_vid) >> 13;
357d6b92ffaSHans Petter Selasky else
358d6b92ffaSHans Petter Selasky wc->sl = be16toh(cqe->sl_vid) >> 12;
359d6b92ffaSHans Petter Selasky
360d6b92ffaSHans Petter Selasky if ((*cur_qp) && ((*cur_qp)->qp_cap_cache & MLX4_RX_CSUM_VALID)) {
361d6b92ffaSHans Petter Selasky wc->wc_flags |= ((cqe->status & htobe32(MLX4_CQE_STATUS_IPV4_CSUM_OK)) ==
362d6b92ffaSHans Petter Selasky htobe32(MLX4_CQE_STATUS_IPV4_CSUM_OK)) <<
363d6b92ffaSHans Petter Selasky IBV_WC_IP_CSUM_OK_SHIFT;
364d6b92ffaSHans Petter Selasky }
365d6b92ffaSHans Petter Selasky }
366d6b92ffaSHans Petter Selasky
367d6b92ffaSHans Petter Selasky return CQ_OK;
368d6b92ffaSHans Petter Selasky }
369d6b92ffaSHans Petter Selasky
370d6b92ffaSHans Petter Selasky static inline int mlx4_parse_lazy_cqe(struct mlx4_cq *cq,
371d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe)
372d6b92ffaSHans Petter Selasky ALWAYS_INLINE;
mlx4_parse_lazy_cqe(struct mlx4_cq * cq,struct mlx4_cqe * cqe)373d6b92ffaSHans Petter Selasky static inline int mlx4_parse_lazy_cqe(struct mlx4_cq *cq,
374d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe)
375d6b92ffaSHans Petter Selasky {
376d6b92ffaSHans Petter Selasky return mlx4_parse_cqe(cq, cqe, &cq->cur_qp, NULL, 1);
377d6b92ffaSHans Petter Selasky }
378d6b92ffaSHans Petter Selasky
379d6b92ffaSHans Petter Selasky static inline int mlx4_poll_one(struct mlx4_cq *cq,
380d6b92ffaSHans Petter Selasky struct mlx4_qp **cur_qp,
381d6b92ffaSHans Petter Selasky struct ibv_wc *wc)
382d6b92ffaSHans Petter Selasky ALWAYS_INLINE;
mlx4_poll_one(struct mlx4_cq * cq,struct mlx4_qp ** cur_qp,struct ibv_wc * wc)383d6b92ffaSHans Petter Selasky static inline int mlx4_poll_one(struct mlx4_cq *cq,
384d6b92ffaSHans Petter Selasky struct mlx4_qp **cur_qp,
385d6b92ffaSHans Petter Selasky struct ibv_wc *wc)
386d6b92ffaSHans Petter Selasky {
387d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe;
388d6b92ffaSHans Petter Selasky int err;
389d6b92ffaSHans Petter Selasky
390d6b92ffaSHans Petter Selasky err = mlx4_get_next_cqe(cq, &cqe);
391d6b92ffaSHans Petter Selasky if (err == CQ_EMPTY)
392d6b92ffaSHans Petter Selasky return err;
393d6b92ffaSHans Petter Selasky
394d6b92ffaSHans Petter Selasky return mlx4_parse_cqe(cq, cqe, cur_qp, wc, 0);
395d6b92ffaSHans Petter Selasky }
396d6b92ffaSHans Petter Selasky
mlx4_poll_cq(struct ibv_cq * ibcq,int ne,struct ibv_wc * wc)397d6b92ffaSHans Petter Selasky int mlx4_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc)
398d6b92ffaSHans Petter Selasky {
399d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibcq);
400d6b92ffaSHans Petter Selasky struct mlx4_qp *qp = NULL;
401d6b92ffaSHans Petter Selasky int npolled;
402d6b92ffaSHans Petter Selasky int err = CQ_OK;
403d6b92ffaSHans Petter Selasky
404d6b92ffaSHans Petter Selasky pthread_spin_lock(&cq->lock);
405d6b92ffaSHans Petter Selasky
406d6b92ffaSHans Petter Selasky for (npolled = 0; npolled < ne; ++npolled) {
407d6b92ffaSHans Petter Selasky err = mlx4_poll_one(cq, &qp, wc + npolled);
408d6b92ffaSHans Petter Selasky if (err != CQ_OK)
409d6b92ffaSHans Petter Selasky break;
410d6b92ffaSHans Petter Selasky }
411d6b92ffaSHans Petter Selasky
412d6b92ffaSHans Petter Selasky if (npolled || err == CQ_POLL_ERR)
413d6b92ffaSHans Petter Selasky mlx4_update_cons_index(cq);
414d6b92ffaSHans Petter Selasky
415d6b92ffaSHans Petter Selasky pthread_spin_unlock(&cq->lock);
416d6b92ffaSHans Petter Selasky
417d6b92ffaSHans Petter Selasky return err == CQ_POLL_ERR ? err : npolled;
418d6b92ffaSHans Petter Selasky }
419d6b92ffaSHans Petter Selasky
420d6b92ffaSHans Petter Selasky static inline void _mlx4_end_poll(struct ibv_cq_ex *ibcq, int lock)
421d6b92ffaSHans Petter Selasky ALWAYS_INLINE;
_mlx4_end_poll(struct ibv_cq_ex * ibcq,int lock)422d6b92ffaSHans Petter Selasky static inline void _mlx4_end_poll(struct ibv_cq_ex *ibcq, int lock)
423d6b92ffaSHans Petter Selasky {
424d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
425d6b92ffaSHans Petter Selasky
426d6b92ffaSHans Petter Selasky mlx4_update_cons_index(cq);
427d6b92ffaSHans Petter Selasky
428d6b92ffaSHans Petter Selasky if (lock)
429d6b92ffaSHans Petter Selasky pthread_spin_unlock(&cq->lock);
430d6b92ffaSHans Petter Selasky }
431d6b92ffaSHans Petter Selasky
432d6b92ffaSHans Petter Selasky static inline int _mlx4_start_poll(struct ibv_cq_ex *ibcq,
433d6b92ffaSHans Petter Selasky struct ibv_poll_cq_attr *attr,
434d6b92ffaSHans Petter Selasky int lock)
435d6b92ffaSHans Petter Selasky ALWAYS_INLINE;
_mlx4_start_poll(struct ibv_cq_ex * ibcq,struct ibv_poll_cq_attr * attr,int lock)436d6b92ffaSHans Petter Selasky static inline int _mlx4_start_poll(struct ibv_cq_ex *ibcq,
437d6b92ffaSHans Petter Selasky struct ibv_poll_cq_attr *attr,
438d6b92ffaSHans Petter Selasky int lock)
439d6b92ffaSHans Petter Selasky {
440d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
441d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe;
442d6b92ffaSHans Petter Selasky int err;
443d6b92ffaSHans Petter Selasky
444d6b92ffaSHans Petter Selasky if (unlikely(attr->comp_mask))
445d6b92ffaSHans Petter Selasky return EINVAL;
446d6b92ffaSHans Petter Selasky
447d6b92ffaSHans Petter Selasky if (lock)
448d6b92ffaSHans Petter Selasky pthread_spin_lock(&cq->lock);
449d6b92ffaSHans Petter Selasky
450d6b92ffaSHans Petter Selasky cq->cur_qp = NULL;
451d6b92ffaSHans Petter Selasky
452d6b92ffaSHans Petter Selasky err = mlx4_get_next_cqe(cq, &cqe);
453d6b92ffaSHans Petter Selasky if (err == CQ_EMPTY) {
454d6b92ffaSHans Petter Selasky if (lock)
455d6b92ffaSHans Petter Selasky pthread_spin_unlock(&cq->lock);
456d6b92ffaSHans Petter Selasky return ENOENT;
457d6b92ffaSHans Petter Selasky }
458d6b92ffaSHans Petter Selasky
459d6b92ffaSHans Petter Selasky err = mlx4_parse_lazy_cqe(cq, cqe);
460d6b92ffaSHans Petter Selasky if (lock && err)
461d6b92ffaSHans Petter Selasky pthread_spin_unlock(&cq->lock);
462d6b92ffaSHans Petter Selasky
463d6b92ffaSHans Petter Selasky return err;
464d6b92ffaSHans Petter Selasky }
465d6b92ffaSHans Petter Selasky
mlx4_next_poll(struct ibv_cq_ex * ibcq)466d6b92ffaSHans Petter Selasky static int mlx4_next_poll(struct ibv_cq_ex *ibcq)
467d6b92ffaSHans Petter Selasky {
468d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
469d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe;
470d6b92ffaSHans Petter Selasky int err;
471d6b92ffaSHans Petter Selasky
472d6b92ffaSHans Petter Selasky err = mlx4_get_next_cqe(cq, &cqe);
473d6b92ffaSHans Petter Selasky if (err == CQ_EMPTY)
474d6b92ffaSHans Petter Selasky return ENOENT;
475d6b92ffaSHans Petter Selasky
476d6b92ffaSHans Petter Selasky return mlx4_parse_lazy_cqe(cq, cqe);
477d6b92ffaSHans Petter Selasky }
478d6b92ffaSHans Petter Selasky
mlx4_end_poll(struct ibv_cq_ex * ibcq)479d6b92ffaSHans Petter Selasky static void mlx4_end_poll(struct ibv_cq_ex *ibcq)
480d6b92ffaSHans Petter Selasky {
481d6b92ffaSHans Petter Selasky _mlx4_end_poll(ibcq, 0);
482d6b92ffaSHans Petter Selasky }
483d6b92ffaSHans Petter Selasky
mlx4_end_poll_lock(struct ibv_cq_ex * ibcq)484d6b92ffaSHans Petter Selasky static void mlx4_end_poll_lock(struct ibv_cq_ex *ibcq)
485d6b92ffaSHans Petter Selasky {
486d6b92ffaSHans Petter Selasky _mlx4_end_poll(ibcq, 1);
487d6b92ffaSHans Petter Selasky }
488d6b92ffaSHans Petter Selasky
mlx4_start_poll(struct ibv_cq_ex * ibcq,struct ibv_poll_cq_attr * attr)489d6b92ffaSHans Petter Selasky static int mlx4_start_poll(struct ibv_cq_ex *ibcq,
490d6b92ffaSHans Petter Selasky struct ibv_poll_cq_attr *attr)
491d6b92ffaSHans Petter Selasky {
492d6b92ffaSHans Petter Selasky return _mlx4_start_poll(ibcq, attr, 0);
493d6b92ffaSHans Petter Selasky }
494d6b92ffaSHans Petter Selasky
mlx4_start_poll_lock(struct ibv_cq_ex * ibcq,struct ibv_poll_cq_attr * attr)495d6b92ffaSHans Petter Selasky static int mlx4_start_poll_lock(struct ibv_cq_ex *ibcq,
496d6b92ffaSHans Petter Selasky struct ibv_poll_cq_attr *attr)
497d6b92ffaSHans Petter Selasky {
498d6b92ffaSHans Petter Selasky return _mlx4_start_poll(ibcq, attr, 1);
499d6b92ffaSHans Petter Selasky }
500d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_opcode(struct ibv_cq_ex * ibcq)501d6b92ffaSHans Petter Selasky static enum ibv_wc_opcode mlx4_cq_read_wc_opcode(struct ibv_cq_ex *ibcq)
502d6b92ffaSHans Petter Selasky {
503d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
504d6b92ffaSHans Petter Selasky
505d6b92ffaSHans Petter Selasky if (cq->cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK) {
506d6b92ffaSHans Petter Selasky switch (cq->cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
507d6b92ffaSHans Petter Selasky case MLX4_OPCODE_RDMA_WRITE_IMM:
508d6b92ffaSHans Petter Selasky case MLX4_OPCODE_RDMA_WRITE:
509d6b92ffaSHans Petter Selasky return IBV_WC_RDMA_WRITE;
510d6b92ffaSHans Petter Selasky case MLX4_OPCODE_SEND_INVAL:
511d6b92ffaSHans Petter Selasky case MLX4_OPCODE_SEND_IMM:
512d6b92ffaSHans Petter Selasky case MLX4_OPCODE_SEND:
513d6b92ffaSHans Petter Selasky return IBV_WC_SEND;
514d6b92ffaSHans Petter Selasky case MLX4_OPCODE_RDMA_READ:
515d6b92ffaSHans Petter Selasky return IBV_WC_RDMA_READ;
516d6b92ffaSHans Petter Selasky case MLX4_OPCODE_ATOMIC_CS:
517d6b92ffaSHans Petter Selasky return IBV_WC_COMP_SWAP;
518d6b92ffaSHans Petter Selasky case MLX4_OPCODE_ATOMIC_FA:
519d6b92ffaSHans Petter Selasky return IBV_WC_FETCH_ADD;
520d6b92ffaSHans Petter Selasky case MLX4_OPCODE_LOCAL_INVAL:
521d6b92ffaSHans Petter Selasky return IBV_WC_LOCAL_INV;
522d6b92ffaSHans Petter Selasky case MLX4_OPCODE_BIND_MW:
523d6b92ffaSHans Petter Selasky return IBV_WC_BIND_MW;
524d6b92ffaSHans Petter Selasky }
525d6b92ffaSHans Petter Selasky } else {
526d6b92ffaSHans Petter Selasky switch (cq->cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
527d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
528d6b92ffaSHans Petter Selasky return IBV_WC_RECV_RDMA_WITH_IMM;
529d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_SEND_INVAL:
530d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_SEND_IMM:
531d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_SEND:
532d6b92ffaSHans Petter Selasky return IBV_WC_RECV;
533d6b92ffaSHans Petter Selasky }
534d6b92ffaSHans Petter Selasky }
535d6b92ffaSHans Petter Selasky
536d6b92ffaSHans Petter Selasky return 0;
537d6b92ffaSHans Petter Selasky }
538d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_qp_num(struct ibv_cq_ex * ibcq)539d6b92ffaSHans Petter Selasky static uint32_t mlx4_cq_read_wc_qp_num(struct ibv_cq_ex *ibcq)
540d6b92ffaSHans Petter Selasky {
541d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
542d6b92ffaSHans Petter Selasky
543d6b92ffaSHans Petter Selasky return be32toh(cq->cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK;
544d6b92ffaSHans Petter Selasky }
545d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_flags(struct ibv_cq_ex * ibcq)546d6b92ffaSHans Petter Selasky static int mlx4_cq_read_wc_flags(struct ibv_cq_ex *ibcq)
547d6b92ffaSHans Petter Selasky {
548d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
549d6b92ffaSHans Petter Selasky int is_send = cq->cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;
550d6b92ffaSHans Petter Selasky int wc_flags = 0;
551d6b92ffaSHans Petter Selasky
552d6b92ffaSHans Petter Selasky if (is_send) {
553d6b92ffaSHans Petter Selasky switch (cq->cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
554d6b92ffaSHans Petter Selasky case MLX4_OPCODE_RDMA_WRITE_IMM:
555d6b92ffaSHans Petter Selasky case MLX4_OPCODE_SEND_IMM:
556d6b92ffaSHans Petter Selasky wc_flags |= IBV_WC_WITH_IMM;
557d6b92ffaSHans Petter Selasky break;
558d6b92ffaSHans Petter Selasky }
559d6b92ffaSHans Petter Selasky } else {
560d6b92ffaSHans Petter Selasky if (cq->flags & MLX4_CQ_FLAGS_RX_CSUM_VALID)
561d6b92ffaSHans Petter Selasky wc_flags |= ((cq->cqe->status &
562d6b92ffaSHans Petter Selasky htobe32(MLX4_CQE_STATUS_IPV4_CSUM_OK)) ==
563d6b92ffaSHans Petter Selasky htobe32(MLX4_CQE_STATUS_IPV4_CSUM_OK)) <<
564d6b92ffaSHans Petter Selasky IBV_WC_IP_CSUM_OK_SHIFT;
565d6b92ffaSHans Petter Selasky
566d6b92ffaSHans Petter Selasky switch (cq->cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
567d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
568d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_SEND_IMM:
569d6b92ffaSHans Petter Selasky wc_flags |= IBV_WC_WITH_IMM;
570d6b92ffaSHans Petter Selasky break;
571d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_SEND_INVAL:
572d6b92ffaSHans Petter Selasky wc_flags |= IBV_WC_WITH_INV;
573d6b92ffaSHans Petter Selasky break;
574d6b92ffaSHans Petter Selasky }
575d6b92ffaSHans Petter Selasky wc_flags |= (be32toh(cq->cqe->g_mlpath_rqpn) & 0x80000000) ? IBV_WC_GRH : 0;
576d6b92ffaSHans Petter Selasky }
577d6b92ffaSHans Petter Selasky
578d6b92ffaSHans Petter Selasky return wc_flags;
579d6b92ffaSHans Petter Selasky }
580d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_byte_len(struct ibv_cq_ex * ibcq)581d6b92ffaSHans Petter Selasky static uint32_t mlx4_cq_read_wc_byte_len(struct ibv_cq_ex *ibcq)
582d6b92ffaSHans Petter Selasky {
583d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
584d6b92ffaSHans Petter Selasky
585d6b92ffaSHans Petter Selasky return be32toh(cq->cqe->byte_cnt);
586d6b92ffaSHans Petter Selasky }
587d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_vendor_err(struct ibv_cq_ex * ibcq)588d6b92ffaSHans Petter Selasky static uint32_t mlx4_cq_read_wc_vendor_err(struct ibv_cq_ex *ibcq)
589d6b92ffaSHans Petter Selasky {
590d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
591d6b92ffaSHans Petter Selasky struct mlx4_err_cqe *ecqe = (struct mlx4_err_cqe *)cq->cqe;
592d6b92ffaSHans Petter Selasky
593d6b92ffaSHans Petter Selasky return ecqe->vendor_err;
594d6b92ffaSHans Petter Selasky }
595d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_imm_data(struct ibv_cq_ex * ibcq)596d6b92ffaSHans Petter Selasky static uint32_t mlx4_cq_read_wc_imm_data(struct ibv_cq_ex *ibcq)
597d6b92ffaSHans Petter Selasky {
598d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
599d6b92ffaSHans Petter Selasky
600d6b92ffaSHans Petter Selasky switch (cq->cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
601d6b92ffaSHans Petter Selasky case MLX4_RECV_OPCODE_SEND_INVAL:
602d6b92ffaSHans Petter Selasky return be32toh(cq->cqe->immed_rss_invalid);
603d6b92ffaSHans Petter Selasky default:
604d6b92ffaSHans Petter Selasky return cq->cqe->immed_rss_invalid;
605d6b92ffaSHans Petter Selasky }
606d6b92ffaSHans Petter Selasky }
607d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_slid(struct ibv_cq_ex * ibcq)608d6b92ffaSHans Petter Selasky static uint32_t mlx4_cq_read_wc_slid(struct ibv_cq_ex *ibcq)
609d6b92ffaSHans Petter Selasky {
610d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
611d6b92ffaSHans Petter Selasky
612d6b92ffaSHans Petter Selasky return (uint32_t)be16toh(cq->cqe->rlid);
613d6b92ffaSHans Petter Selasky }
614d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_sl(struct ibv_cq_ex * ibcq)615d6b92ffaSHans Petter Selasky static uint8_t mlx4_cq_read_wc_sl(struct ibv_cq_ex *ibcq)
616d6b92ffaSHans Petter Selasky {
617d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
618d6b92ffaSHans Petter Selasky
619d6b92ffaSHans Petter Selasky if ((cq->cur_qp) && (cq->cur_qp->link_layer == IBV_LINK_LAYER_ETHERNET))
620d6b92ffaSHans Petter Selasky return be16toh(cq->cqe->sl_vid) >> 13;
621d6b92ffaSHans Petter Selasky else
622d6b92ffaSHans Petter Selasky return be16toh(cq->cqe->sl_vid) >> 12;
623d6b92ffaSHans Petter Selasky }
624d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_src_qp(struct ibv_cq_ex * ibcq)625d6b92ffaSHans Petter Selasky static uint32_t mlx4_cq_read_wc_src_qp(struct ibv_cq_ex *ibcq)
626d6b92ffaSHans Petter Selasky {
627d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
628d6b92ffaSHans Petter Selasky
629d6b92ffaSHans Petter Selasky return be32toh(cq->cqe->g_mlpath_rqpn) & 0xffffff;
630d6b92ffaSHans Petter Selasky }
631d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_dlid_path_bits(struct ibv_cq_ex * ibcq)632d6b92ffaSHans Petter Selasky static uint8_t mlx4_cq_read_wc_dlid_path_bits(struct ibv_cq_ex *ibcq)
633d6b92ffaSHans Petter Selasky {
634d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
635d6b92ffaSHans Petter Selasky
636d6b92ffaSHans Petter Selasky return (be32toh(cq->cqe->g_mlpath_rqpn) >> 24) & 0x7f;
637d6b92ffaSHans Petter Selasky }
638d6b92ffaSHans Petter Selasky
mlx4_cq_read_wc_completion_ts(struct ibv_cq_ex * ibcq)639d6b92ffaSHans Petter Selasky static uint64_t mlx4_cq_read_wc_completion_ts(struct ibv_cq_ex *ibcq)
640d6b92ffaSHans Petter Selasky {
641d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq));
642d6b92ffaSHans Petter Selasky
643d6b92ffaSHans Petter Selasky return ((uint64_t)be32toh(cq->cqe->ts_47_16) << 16) |
644d6b92ffaSHans Petter Selasky (cq->cqe->ts_15_8 << 8) |
645d6b92ffaSHans Petter Selasky (cq->cqe->ts_7_0);
646d6b92ffaSHans Petter Selasky }
647d6b92ffaSHans Petter Selasky
mlx4_cq_fill_pfns(struct mlx4_cq * cq,const struct ibv_cq_init_attr_ex * cq_attr)648d6b92ffaSHans Petter Selasky void mlx4_cq_fill_pfns(struct mlx4_cq *cq, const struct ibv_cq_init_attr_ex *cq_attr)
649d6b92ffaSHans Petter Selasky {
650d6b92ffaSHans Petter Selasky
651d6b92ffaSHans Petter Selasky if (cq->flags & MLX4_CQ_FLAGS_SINGLE_THREADED) {
652d6b92ffaSHans Petter Selasky cq->ibv_cq.start_poll = mlx4_start_poll;
653d6b92ffaSHans Petter Selasky cq->ibv_cq.end_poll = mlx4_end_poll;
654d6b92ffaSHans Petter Selasky } else {
655d6b92ffaSHans Petter Selasky cq->ibv_cq.start_poll = mlx4_start_poll_lock;
656d6b92ffaSHans Petter Selasky cq->ibv_cq.end_poll = mlx4_end_poll_lock;
657d6b92ffaSHans Petter Selasky }
658d6b92ffaSHans Petter Selasky cq->ibv_cq.next_poll = mlx4_next_poll;
659d6b92ffaSHans Petter Selasky
660d6b92ffaSHans Petter Selasky cq->ibv_cq.read_opcode = mlx4_cq_read_wc_opcode;
661d6b92ffaSHans Petter Selasky cq->ibv_cq.read_vendor_err = mlx4_cq_read_wc_vendor_err;
662d6b92ffaSHans Petter Selasky cq->ibv_cq.read_wc_flags = mlx4_cq_read_wc_flags;
663d6b92ffaSHans Petter Selasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_BYTE_LEN)
664d6b92ffaSHans Petter Selasky cq->ibv_cq.read_byte_len = mlx4_cq_read_wc_byte_len;
665d6b92ffaSHans Petter Selasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_IMM)
666d6b92ffaSHans Petter Selasky cq->ibv_cq.read_imm_data = mlx4_cq_read_wc_imm_data;
667d6b92ffaSHans Petter Selasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_QP_NUM)
668d6b92ffaSHans Petter Selasky cq->ibv_cq.read_qp_num = mlx4_cq_read_wc_qp_num;
669d6b92ffaSHans Petter Selasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_SRC_QP)
670d6b92ffaSHans Petter Selasky cq->ibv_cq.read_src_qp = mlx4_cq_read_wc_src_qp;
671d6b92ffaSHans Petter Selasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_SLID)
672d6b92ffaSHans Petter Selasky cq->ibv_cq.read_slid = mlx4_cq_read_wc_slid;
673d6b92ffaSHans Petter Selasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_SL)
674d6b92ffaSHans Petter Selasky cq->ibv_cq.read_sl = mlx4_cq_read_wc_sl;
675d6b92ffaSHans Petter Selasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS)
676d6b92ffaSHans Petter Selasky cq->ibv_cq.read_dlid_path_bits = mlx4_cq_read_wc_dlid_path_bits;
677d6b92ffaSHans Petter Selasky if (cq_attr->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP)
678d6b92ffaSHans Petter Selasky cq->ibv_cq.read_completion_ts = mlx4_cq_read_wc_completion_ts;
679d6b92ffaSHans Petter Selasky }
680d6b92ffaSHans Petter Selasky
mlx4_arm_cq(struct ibv_cq * ibvcq,int solicited)681d6b92ffaSHans Petter Selasky int mlx4_arm_cq(struct ibv_cq *ibvcq, int solicited)
682d6b92ffaSHans Petter Selasky {
683d6b92ffaSHans Petter Selasky struct mlx4_cq *cq = to_mcq(ibvcq);
684d6b92ffaSHans Petter Selasky uint32_t doorbell[2];
685d6b92ffaSHans Petter Selasky uint32_t sn;
686d6b92ffaSHans Petter Selasky uint32_t ci;
687d6b92ffaSHans Petter Selasky uint32_t cmd;
688d6b92ffaSHans Petter Selasky
689d6b92ffaSHans Petter Selasky sn = cq->arm_sn & 3;
690d6b92ffaSHans Petter Selasky ci = cq->cons_index & 0xffffff;
691d6b92ffaSHans Petter Selasky cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
692d6b92ffaSHans Petter Selasky
693d6b92ffaSHans Petter Selasky *cq->arm_db = htobe32(sn << 28 | cmd | ci);
694d6b92ffaSHans Petter Selasky
695d6b92ffaSHans Petter Selasky /*
696d6b92ffaSHans Petter Selasky * Make sure that the doorbell record in host memory is
697d6b92ffaSHans Petter Selasky * written before ringing the doorbell via PCI MMIO.
698d6b92ffaSHans Petter Selasky */
699d6b92ffaSHans Petter Selasky udma_to_device_barrier();
700d6b92ffaSHans Petter Selasky
701d6b92ffaSHans Petter Selasky doorbell[0] = htobe32(sn << 28 | cmd | cq->cqn);
702d6b92ffaSHans Petter Selasky doorbell[1] = htobe32(ci);
703d6b92ffaSHans Petter Selasky
704d6b92ffaSHans Petter Selasky mlx4_write64(doorbell, to_mctx(ibvcq->context), MLX4_CQ_DOORBELL);
705d6b92ffaSHans Petter Selasky
706d6b92ffaSHans Petter Selasky return 0;
707d6b92ffaSHans Petter Selasky }
708d6b92ffaSHans Petter Selasky
mlx4_cq_event(struct ibv_cq * cq)709d6b92ffaSHans Petter Selasky void mlx4_cq_event(struct ibv_cq *cq)
710d6b92ffaSHans Petter Selasky {
711d6b92ffaSHans Petter Selasky to_mcq(cq)->arm_sn++;
712d6b92ffaSHans Petter Selasky }
713d6b92ffaSHans Petter Selasky
__mlx4_cq_clean(struct mlx4_cq * cq,uint32_t qpn,struct mlx4_srq * srq)714d6b92ffaSHans Petter Selasky void __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)
715d6b92ffaSHans Petter Selasky {
716d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe, *dest;
717d6b92ffaSHans Petter Selasky uint32_t prod_index;
718d6b92ffaSHans Petter Selasky uint8_t owner_bit;
719d6b92ffaSHans Petter Selasky int nfreed = 0;
720d6b92ffaSHans Petter Selasky int cqe_inc = cq->cqe_size == 64 ? 1 : 0;
721d6b92ffaSHans Petter Selasky
722d6b92ffaSHans Petter Selasky /*
723d6b92ffaSHans Petter Selasky * First we need to find the current producer index, so we
724d6b92ffaSHans Petter Selasky * know where to start cleaning from. It doesn't matter if HW
725d6b92ffaSHans Petter Selasky * adds new entries after this loop -- the QP we're worried
726d6b92ffaSHans Petter Selasky * about is already in RESET, so the new entries won't come
727d6b92ffaSHans Petter Selasky * from our QP and therefore don't need to be checked.
728d6b92ffaSHans Petter Selasky */
729d6b92ffaSHans Petter Selasky for (prod_index = cq->cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
730d6b92ffaSHans Petter Selasky if (prod_index == cq->cons_index + cq->ibv_cq.cqe)
731d6b92ffaSHans Petter Selasky break;
732d6b92ffaSHans Petter Selasky
733d6b92ffaSHans Petter Selasky /*
734d6b92ffaSHans Petter Selasky * Now sweep backwards through the CQ, removing CQ entries
735d6b92ffaSHans Petter Selasky * that match our QP by copying older entries on top of them.
736d6b92ffaSHans Petter Selasky */
737d6b92ffaSHans Petter Selasky while ((int) --prod_index - (int) cq->cons_index >= 0) {
738d6b92ffaSHans Petter Selasky cqe = get_cqe(cq, prod_index & cq->ibv_cq.cqe);
739d6b92ffaSHans Petter Selasky cqe += cqe_inc;
740d6b92ffaSHans Petter Selasky if (srq && srq->ext_srq &&
741d6b92ffaSHans Petter Selasky (be32toh(cqe->g_mlpath_rqpn) & MLX4_CQE_QPN_MASK) == srq->verbs_srq.srq_num &&
742d6b92ffaSHans Petter Selasky !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK)) {
743d6b92ffaSHans Petter Selasky mlx4_free_srq_wqe(srq, be16toh(cqe->wqe_index));
744d6b92ffaSHans Petter Selasky ++nfreed;
745d6b92ffaSHans Petter Selasky } else if ((be32toh(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) == qpn) {
746d6b92ffaSHans Petter Selasky if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
747d6b92ffaSHans Petter Selasky mlx4_free_srq_wqe(srq, be16toh(cqe->wqe_index));
748d6b92ffaSHans Petter Selasky ++nfreed;
749d6b92ffaSHans Petter Selasky } else if (nfreed) {
750d6b92ffaSHans Petter Selasky dest = get_cqe(cq, (prod_index + nfreed) & cq->ibv_cq.cqe);
751d6b92ffaSHans Petter Selasky dest += cqe_inc;
752d6b92ffaSHans Petter Selasky owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
753d6b92ffaSHans Petter Selasky memcpy(dest, cqe, sizeof *cqe);
754d6b92ffaSHans Petter Selasky dest->owner_sr_opcode = owner_bit |
755d6b92ffaSHans Petter Selasky (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
756d6b92ffaSHans Petter Selasky }
757d6b92ffaSHans Petter Selasky }
758d6b92ffaSHans Petter Selasky
759d6b92ffaSHans Petter Selasky if (nfreed) {
760d6b92ffaSHans Petter Selasky cq->cons_index += nfreed;
761d6b92ffaSHans Petter Selasky /*
762d6b92ffaSHans Petter Selasky * Make sure update of buffer contents is done before
763d6b92ffaSHans Petter Selasky * updating consumer index.
764d6b92ffaSHans Petter Selasky */
765d6b92ffaSHans Petter Selasky udma_to_device_barrier();
766d6b92ffaSHans Petter Selasky mlx4_update_cons_index(cq);
767d6b92ffaSHans Petter Selasky }
768d6b92ffaSHans Petter Selasky }
769d6b92ffaSHans Petter Selasky
mlx4_cq_clean(struct mlx4_cq * cq,uint32_t qpn,struct mlx4_srq * srq)770d6b92ffaSHans Petter Selasky void mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq)
771d6b92ffaSHans Petter Selasky {
772d6b92ffaSHans Petter Selasky pthread_spin_lock(&cq->lock);
773d6b92ffaSHans Petter Selasky __mlx4_cq_clean(cq, qpn, srq);
774d6b92ffaSHans Petter Selasky pthread_spin_unlock(&cq->lock);
775d6b92ffaSHans Petter Selasky }
776d6b92ffaSHans Petter Selasky
mlx4_get_outstanding_cqes(struct mlx4_cq * cq)777d6b92ffaSHans Petter Selasky int mlx4_get_outstanding_cqes(struct mlx4_cq *cq)
778d6b92ffaSHans Petter Selasky {
779d6b92ffaSHans Petter Selasky uint32_t i;
780d6b92ffaSHans Petter Selasky
781d6b92ffaSHans Petter Selasky for (i = cq->cons_index; get_sw_cqe(cq, i); ++i)
782d6b92ffaSHans Petter Selasky ;
783d6b92ffaSHans Petter Selasky
784d6b92ffaSHans Petter Selasky return i - cq->cons_index;
785d6b92ffaSHans Petter Selasky }
786d6b92ffaSHans Petter Selasky
mlx4_cq_resize_copy_cqes(struct mlx4_cq * cq,void * buf,int old_cqe)787d6b92ffaSHans Petter Selasky void mlx4_cq_resize_copy_cqes(struct mlx4_cq *cq, void *buf, int old_cqe)
788d6b92ffaSHans Petter Selasky {
789d6b92ffaSHans Petter Selasky struct mlx4_cqe *cqe;
790d6b92ffaSHans Petter Selasky int i;
791d6b92ffaSHans Petter Selasky int cqe_inc = cq->cqe_size == 64 ? 1 : 0;
792d6b92ffaSHans Petter Selasky
793d6b92ffaSHans Petter Selasky i = cq->cons_index;
794d6b92ffaSHans Petter Selasky cqe = get_cqe(cq, (i & old_cqe));
795d6b92ffaSHans Petter Selasky cqe += cqe_inc;
796d6b92ffaSHans Petter Selasky
797d6b92ffaSHans Petter Selasky while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) {
798d6b92ffaSHans Petter Selasky cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) |
799d6b92ffaSHans Petter Selasky (((i + 1) & (cq->ibv_cq.cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0);
800d6b92ffaSHans Petter Selasky memcpy(buf + ((i + 1) & cq->ibv_cq.cqe) * cq->cqe_size,
801d6b92ffaSHans Petter Selasky cqe - cqe_inc, cq->cqe_size);
802d6b92ffaSHans Petter Selasky ++i;
803d6b92ffaSHans Petter Selasky cqe = get_cqe(cq, (i & old_cqe));
804d6b92ffaSHans Petter Selasky cqe += cqe_inc;
805d6b92ffaSHans Petter Selasky }
806d6b92ffaSHans Petter Selasky
807d6b92ffaSHans Petter Selasky ++cq->cons_index;
808d6b92ffaSHans Petter Selasky }
809d6b92ffaSHans Petter Selasky
mlx4_alloc_cq_buf(struct mlx4_device * dev,struct mlx4_buf * buf,int nent,int entry_size)810d6b92ffaSHans Petter Selasky int mlx4_alloc_cq_buf(struct mlx4_device *dev, struct mlx4_buf *buf, int nent,
811d6b92ffaSHans Petter Selasky int entry_size)
812d6b92ffaSHans Petter Selasky {
813d6b92ffaSHans Petter Selasky if (mlx4_alloc_buf(buf, align(nent * entry_size, dev->page_size),
814d6b92ffaSHans Petter Selasky dev->page_size))
815d6b92ffaSHans Petter Selasky return -1;
816d6b92ffaSHans Petter Selasky memset(buf->buf, 0, nent * entry_size);
817d6b92ffaSHans Petter Selasky
818d6b92ffaSHans Petter Selasky return 0;
819d6b92ffaSHans Petter Selasky }
820