197549c34SHans Petter Selasky /*
297549c34SHans Petter Selasky * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
397549c34SHans Petter Selasky * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
497549c34SHans Petter Selasky *
597549c34SHans Petter Selasky * This software is available to you under a choice of one of two
697549c34SHans Petter Selasky * licenses. You may choose to be licensed under the terms of the GNU
797549c34SHans Petter Selasky * General Public License (GPL) Version 2, available from the file
897549c34SHans Petter Selasky * COPYING in the main directory of this source tree, or the
997549c34SHans Petter Selasky * OpenIB.org BSD license below:
1097549c34SHans Petter Selasky *
1197549c34SHans Petter Selasky * Redistribution and use in source and binary forms, with or
1297549c34SHans Petter Selasky * without modification, are permitted provided that the following
1397549c34SHans Petter Selasky * conditions are met:
1497549c34SHans Petter Selasky *
1597549c34SHans Petter Selasky * - Redistributions of source code must retain the above
1697549c34SHans Petter Selasky * copyright notice, this list of conditions and the following
1797549c34SHans Petter Selasky * disclaimer.
1897549c34SHans Petter Selasky *
1997549c34SHans Petter Selasky * - Redistributions in binary form must reproduce the above
2097549c34SHans Petter Selasky * copyright notice, this list of conditions and the following
2197549c34SHans Petter Selasky * disclaimer in the documentation and/or other materials
2297549c34SHans Petter Selasky * provided with the distribution.
2397549c34SHans Petter Selasky *
2497549c34SHans Petter Selasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
2597549c34SHans Petter Selasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
2697549c34SHans Petter Selasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
2797549c34SHans Petter Selasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
2897549c34SHans Petter Selasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
2997549c34SHans Petter Selasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
3097549c34SHans Petter Selasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
3197549c34SHans Petter Selasky * SOFTWARE.
3297549c34SHans Petter Selasky */
3397549c34SHans Petter Selasky
3497549c34SHans Petter Selasky #include <dev/mlx4/cq.h>
3597549c34SHans Petter Selasky #include <dev/mlx4/qp.h>
3697549c34SHans Petter Selasky #include <dev/mlx4/srq.h>
378cc48704SHans Petter Selasky #include <dev/mlx4/driver.h>
3897549c34SHans Petter Selasky #include <linux/slab.h>
3997549c34SHans Petter Selasky
4097549c34SHans Petter Selasky #include "mlx4_ib.h"
418cc48704SHans Petter Selasky #include <rdma/mlx4-abi.h>
42b633e08cSHans Petter Selasky #include <rdma/uverbs_ioctl.h>
4397549c34SHans Petter Selasky
mlx4_ib_cq_comp(struct mlx4_cq * cq)4497549c34SHans Petter Selasky static void mlx4_ib_cq_comp(struct mlx4_cq *cq)
4597549c34SHans Petter Selasky {
4697549c34SHans Petter Selasky struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;
4797549c34SHans Petter Selasky ibcq->comp_handler(ibcq, ibcq->cq_context);
4897549c34SHans Petter Selasky }
4997549c34SHans Petter Selasky
mlx4_ib_cq_event(struct mlx4_cq * cq,enum mlx4_event type)5097549c34SHans Petter Selasky static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type)
5197549c34SHans Petter Selasky {
5297549c34SHans Petter Selasky struct ib_event event;
5397549c34SHans Petter Selasky struct ib_cq *ibcq;
5497549c34SHans Petter Selasky
5597549c34SHans Petter Selasky if (type != MLX4_EVENT_TYPE_CQ_ERROR) {
5697549c34SHans Petter Selasky pr_warn("Unexpected event type %d "
5797549c34SHans Petter Selasky "on CQ %06x\n", type, cq->cqn);
5897549c34SHans Petter Selasky return;
5997549c34SHans Petter Selasky }
6097549c34SHans Petter Selasky
6197549c34SHans Petter Selasky ibcq = &to_mibcq(cq)->ibcq;
6297549c34SHans Petter Selasky if (ibcq->event_handler) {
6397549c34SHans Petter Selasky event.device = ibcq->device;
6497549c34SHans Petter Selasky event.event = IB_EVENT_CQ_ERR;
6597549c34SHans Petter Selasky event.element.cq = ibcq;
6697549c34SHans Petter Selasky ibcq->event_handler(&event, ibcq->cq_context);
6797549c34SHans Petter Selasky }
6897549c34SHans Petter Selasky }
6997549c34SHans Petter Selasky
get_cqe_from_buf(struct mlx4_ib_cq_buf * buf,int n)7097549c34SHans Petter Selasky static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n)
7197549c34SHans Petter Selasky {
7297549c34SHans Petter Selasky return mlx4_buf_offset(&buf->buf, n * buf->entry_size);
7397549c34SHans Petter Selasky }
7497549c34SHans Petter Selasky
get_cqe(struct mlx4_ib_cq * cq,int n)7597549c34SHans Petter Selasky static void *get_cqe(struct mlx4_ib_cq *cq, int n)
7697549c34SHans Petter Selasky {
7797549c34SHans Petter Selasky return get_cqe_from_buf(&cq->buf, n);
7897549c34SHans Petter Selasky }
7997549c34SHans Petter Selasky
get_sw_cqe(struct mlx4_ib_cq * cq,int n)8097549c34SHans Petter Selasky static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n)
8197549c34SHans Petter Selasky {
8297549c34SHans Petter Selasky struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe);
8397549c34SHans Petter Selasky struct mlx4_cqe *tcqe = ((cq->buf.entry_size == 64) ? (cqe + 1) : cqe);
8497549c34SHans Petter Selasky
8597549c34SHans Petter Selasky return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
8697549c34SHans Petter Selasky !!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe;
8797549c34SHans Petter Selasky }
8897549c34SHans Petter Selasky
next_cqe_sw(struct mlx4_ib_cq * cq)8997549c34SHans Petter Selasky static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq)
9097549c34SHans Petter Selasky {
9197549c34SHans Petter Selasky return get_sw_cqe(cq, cq->mcq.cons_index);
9297549c34SHans Petter Selasky }
9397549c34SHans Petter Selasky
mlx4_ib_modify_cq(struct ib_cq * cq,u16 cq_count,u16 cq_period)948cc48704SHans Petter Selasky int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
9597549c34SHans Petter Selasky {
9697549c34SHans Petter Selasky struct mlx4_ib_cq *mcq = to_mcq(cq);
9797549c34SHans Petter Selasky struct mlx4_ib_dev *dev = to_mdev(cq->device);
9897549c34SHans Petter Selasky
998cc48704SHans Petter Selasky return mlx4_cq_modify(dev->dev, &mcq->mcq, cq_count, cq_period);
10097549c34SHans Petter Selasky }
10197549c34SHans Petter Selasky
mlx4_ib_alloc_cq_buf(struct mlx4_ib_dev * dev,struct mlx4_ib_cq_buf * buf,int nent)10297549c34SHans Petter Selasky static int mlx4_ib_alloc_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int nent)
10397549c34SHans Petter Selasky {
10497549c34SHans Petter Selasky int err;
10597549c34SHans Petter Selasky
10697549c34SHans Petter Selasky err = mlx4_buf_alloc(dev->dev, nent * dev->dev->caps.cqe_size,
1078cc48704SHans Petter Selasky PAGE_SIZE * 2, &buf->buf, GFP_KERNEL);
10897549c34SHans Petter Selasky
10997549c34SHans Petter Selasky if (err)
11097549c34SHans Petter Selasky goto out;
11197549c34SHans Petter Selasky
11297549c34SHans Petter Selasky buf->entry_size = dev->dev->caps.cqe_size;
11397549c34SHans Petter Selasky err = mlx4_mtt_init(dev->dev, buf->buf.npages, buf->buf.page_shift,
11497549c34SHans Petter Selasky &buf->mtt);
11597549c34SHans Petter Selasky if (err)
11697549c34SHans Petter Selasky goto err_buf;
11797549c34SHans Petter Selasky
1188cc48704SHans Petter Selasky err = mlx4_buf_write_mtt(dev->dev, &buf->mtt, &buf->buf, GFP_KERNEL);
11997549c34SHans Petter Selasky if (err)
12097549c34SHans Petter Selasky goto err_mtt;
12197549c34SHans Petter Selasky
12297549c34SHans Petter Selasky return 0;
12397549c34SHans Petter Selasky
12497549c34SHans Petter Selasky err_mtt:
12597549c34SHans Petter Selasky mlx4_mtt_cleanup(dev->dev, &buf->mtt);
12697549c34SHans Petter Selasky
12797549c34SHans Petter Selasky err_buf:
12897549c34SHans Petter Selasky mlx4_buf_free(dev->dev, nent * buf->entry_size, &buf->buf);
12997549c34SHans Petter Selasky
13097549c34SHans Petter Selasky out:
13197549c34SHans Petter Selasky return err;
13297549c34SHans Petter Selasky }
13397549c34SHans Petter Selasky
mlx4_ib_free_cq_buf(struct mlx4_ib_dev * dev,struct mlx4_ib_cq_buf * buf,int cqe)13497549c34SHans Petter Selasky static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int cqe)
13597549c34SHans Petter Selasky {
13697549c34SHans Petter Selasky mlx4_buf_free(dev->dev, (cqe + 1) * buf->entry_size, &buf->buf);
13797549c34SHans Petter Selasky }
13897549c34SHans Petter Selasky
mlx4_ib_get_cq_umem(struct mlx4_ib_dev * dev,struct ib_udata * udata,struct mlx4_ib_cq_buf * buf,struct ib_umem ** umem,u64 buf_addr,int cqe)139b633e08cSHans Petter Selasky static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_udata *udata,
140b633e08cSHans Petter Selasky struct mlx4_ib_cq_buf *buf,
141b633e08cSHans Petter Selasky struct ib_umem **umem, u64 buf_addr, int cqe)
14297549c34SHans Petter Selasky {
14397549c34SHans Petter Selasky int err;
14497549c34SHans Petter Selasky int cqe_size = dev->dev->caps.cqe_size;
145b633e08cSHans Petter Selasky struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
146b633e08cSHans Petter Selasky udata, struct mlx4_ib_ucontext, ibucontext);
14797549c34SHans Petter Selasky
148b633e08cSHans Petter Selasky *umem = ib_umem_get(&context->ibucontext, buf_addr, cqe * cqe_size,
14997549c34SHans Petter Selasky IB_ACCESS_LOCAL_WRITE, 1);
15097549c34SHans Petter Selasky if (IS_ERR(*umem))
15197549c34SHans Petter Selasky return PTR_ERR(*umem);
15297549c34SHans Petter Selasky
1538cc48704SHans Petter Selasky err = mlx4_mtt_init(dev->dev, ib_umem_page_count(*umem),
1548cc48704SHans Petter Selasky ilog2((*umem)->page_size), &buf->mtt);
15597549c34SHans Petter Selasky if (err)
15697549c34SHans Petter Selasky goto err_buf;
15797549c34SHans Petter Selasky
15897549c34SHans Petter Selasky err = mlx4_ib_umem_write_mtt(dev, &buf->mtt, *umem);
15997549c34SHans Petter Selasky if (err)
16097549c34SHans Petter Selasky goto err_mtt;
16197549c34SHans Petter Selasky
16297549c34SHans Petter Selasky return 0;
16397549c34SHans Petter Selasky
16497549c34SHans Petter Selasky err_mtt:
16597549c34SHans Petter Selasky mlx4_mtt_cleanup(dev->dev, &buf->mtt);
16697549c34SHans Petter Selasky
16797549c34SHans Petter Selasky err_buf:
16897549c34SHans Petter Selasky ib_umem_release(*umem);
16997549c34SHans Petter Selasky
17097549c34SHans Petter Selasky return err;
17197549c34SHans Petter Selasky }
17297549c34SHans Petter Selasky
1738cc48704SHans Petter Selasky #define CQ_CREATE_FLAGS_SUPPORTED IB_CQ_FLAGS_TIMESTAMP_COMPLETION
mlx4_ib_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct ib_udata * udata)174b633e08cSHans Petter Selasky int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
17597549c34SHans Petter Selasky struct ib_udata *udata)
17697549c34SHans Petter Selasky {
177b633e08cSHans Petter Selasky struct ib_device *ibdev = ibcq->device;
1788cc48704SHans Petter Selasky int entries = attr->cqe;
1798cc48704SHans Petter Selasky int vector = attr->comp_vector;
18097549c34SHans Petter Selasky struct mlx4_ib_dev *dev = to_mdev(ibdev);
181b633e08cSHans Petter Selasky struct mlx4_ib_cq *cq = to_mcq(ibcq);
18297549c34SHans Petter Selasky struct mlx4_uar *uar;
18397549c34SHans Petter Selasky int err;
184b633e08cSHans Petter Selasky struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
185b633e08cSHans Petter Selasky udata, struct mlx4_ib_ucontext, ibucontext);
18697549c34SHans Petter Selasky
18797549c34SHans Petter Selasky if (entries < 1 || entries > dev->dev->caps.max_cqes)
188b633e08cSHans Petter Selasky return -EINVAL;
18997549c34SHans Petter Selasky
19097549c34SHans Petter Selasky if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED)
191b633e08cSHans Petter Selasky return -EINVAL;
19297549c34SHans Petter Selasky
19397549c34SHans Petter Selasky entries = roundup_pow_of_two(entries + 1);
19497549c34SHans Petter Selasky cq->ibcq.cqe = entries - 1;
19597549c34SHans Petter Selasky mutex_init(&cq->resize_mutex);
19697549c34SHans Petter Selasky spin_lock_init(&cq->lock);
19797549c34SHans Petter Selasky cq->resize_buf = NULL;
19897549c34SHans Petter Selasky cq->resize_umem = NULL;
19997549c34SHans Petter Selasky cq->create_flags = attr->flags;
2008cc48704SHans Petter Selasky INIT_LIST_HEAD(&cq->send_qp_list);
2018cc48704SHans Petter Selasky INIT_LIST_HEAD(&cq->recv_qp_list);
20297549c34SHans Petter Selasky
203b633e08cSHans Petter Selasky if (udata) {
20497549c34SHans Petter Selasky struct mlx4_ib_create_cq ucmd;
20597549c34SHans Petter Selasky
20697549c34SHans Petter Selasky if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
20797549c34SHans Petter Selasky err = -EFAULT;
20897549c34SHans Petter Selasky goto err_cq;
20997549c34SHans Petter Selasky }
21097549c34SHans Petter Selasky
211b633e08cSHans Petter Selasky err = mlx4_ib_get_cq_umem(dev, udata, &cq->buf, &cq->umem,
21297549c34SHans Petter Selasky ucmd.buf_addr, entries);
21397549c34SHans Petter Selasky if (err)
21497549c34SHans Petter Selasky goto err_cq;
21597549c34SHans Petter Selasky
216b633e08cSHans Petter Selasky err = mlx4_ib_db_map_user(context, ucmd.db_addr, &cq->db);
21797549c34SHans Petter Selasky if (err)
21897549c34SHans Petter Selasky goto err_mtt;
21997549c34SHans Petter Selasky
220b633e08cSHans Petter Selasky uar = &context->uar;
22197549c34SHans Petter Selasky } else {
2228cc48704SHans Petter Selasky err = mlx4_db_alloc(dev->dev, &cq->db, 1, GFP_KERNEL);
22397549c34SHans Petter Selasky if (err)
22497549c34SHans Petter Selasky goto err_cq;
22597549c34SHans Petter Selasky
22697549c34SHans Petter Selasky cq->mcq.set_ci_db = cq->db.db;
22797549c34SHans Petter Selasky cq->mcq.arm_db = cq->db.db + 1;
22897549c34SHans Petter Selasky *cq->mcq.set_ci_db = 0;
22997549c34SHans Petter Selasky *cq->mcq.arm_db = 0;
23097549c34SHans Petter Selasky
23197549c34SHans Petter Selasky err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries);
23297549c34SHans Petter Selasky if (err)
23397549c34SHans Petter Selasky goto err_db;
23497549c34SHans Petter Selasky
23597549c34SHans Petter Selasky uar = &dev->priv_uar;
23697549c34SHans Petter Selasky }
23797549c34SHans Petter Selasky
23897549c34SHans Petter Selasky if (dev->eq_table)
23997549c34SHans Petter Selasky vector = dev->eq_table[vector % ibdev->num_comp_vectors];
24097549c34SHans Petter Selasky
24197549c34SHans Petter Selasky err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar,
24297549c34SHans Petter Selasky cq->db.dma, &cq->mcq, vector, 0,
2438cc48704SHans Petter Selasky !!(cq->create_flags & IB_CQ_FLAGS_TIMESTAMP_COMPLETION));
24497549c34SHans Petter Selasky if (err)
24597549c34SHans Petter Selasky goto err_dbmap;
24697549c34SHans Petter Selasky
24797549c34SHans Petter Selasky cq->mcq.comp = mlx4_ib_cq_comp;
24897549c34SHans Petter Selasky cq->mcq.event = mlx4_ib_cq_event;
24997549c34SHans Petter Selasky
250b633e08cSHans Petter Selasky if (udata)
25197549c34SHans Petter Selasky if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {
25297549c34SHans Petter Selasky err = -EFAULT;
2538cc48704SHans Petter Selasky goto err_cq_free;
25497549c34SHans Petter Selasky }
25597549c34SHans Petter Selasky
256b633e08cSHans Petter Selasky return 0;
25797549c34SHans Petter Selasky
2588cc48704SHans Petter Selasky err_cq_free:
2598cc48704SHans Petter Selasky mlx4_cq_free(dev->dev, &cq->mcq);
2608cc48704SHans Petter Selasky
26197549c34SHans Petter Selasky err_dbmap:
262b633e08cSHans Petter Selasky if (udata)
263b633e08cSHans Petter Selasky mlx4_ib_db_unmap_user(context, &cq->db);
26497549c34SHans Petter Selasky
26597549c34SHans Petter Selasky err_mtt:
26697549c34SHans Petter Selasky mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);
26797549c34SHans Petter Selasky
26897549c34SHans Petter Selasky ib_umem_release(cq->umem);
269b633e08cSHans Petter Selasky if (!udata)
27097549c34SHans Petter Selasky mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
27197549c34SHans Petter Selasky
27297549c34SHans Petter Selasky err_db:
273b633e08cSHans Petter Selasky if (!udata)
27497549c34SHans Petter Selasky mlx4_db_free(dev->dev, &cq->db);
27597549c34SHans Petter Selasky err_cq:
276b633e08cSHans Petter Selasky return err;
27797549c34SHans Petter Selasky }
27897549c34SHans Petter Selasky
mlx4_alloc_resize_buf(struct mlx4_ib_dev * dev,struct mlx4_ib_cq * cq,int entries)27997549c34SHans Petter Selasky static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,
28097549c34SHans Petter Selasky int entries)
28197549c34SHans Petter Selasky {
28297549c34SHans Petter Selasky int err;
28397549c34SHans Petter Selasky
28497549c34SHans Petter Selasky if (cq->resize_buf)
28597549c34SHans Petter Selasky return -EBUSY;
28697549c34SHans Petter Selasky
2878cc48704SHans Petter Selasky cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL);
28897549c34SHans Petter Selasky if (!cq->resize_buf)
28997549c34SHans Petter Selasky return -ENOMEM;
29097549c34SHans Petter Selasky
29197549c34SHans Petter Selasky err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries);
29297549c34SHans Petter Selasky if (err) {
29397549c34SHans Petter Selasky kfree(cq->resize_buf);
29497549c34SHans Petter Selasky cq->resize_buf = NULL;
29597549c34SHans Petter Selasky return err;
29697549c34SHans Petter Selasky }
29797549c34SHans Petter Selasky
29897549c34SHans Petter Selasky cq->resize_buf->cqe = entries - 1;
29997549c34SHans Petter Selasky
30097549c34SHans Petter Selasky return 0;
30197549c34SHans Petter Selasky }
30297549c34SHans Petter Selasky
mlx4_alloc_resize_umem(struct mlx4_ib_dev * dev,struct mlx4_ib_cq * cq,int entries,struct ib_udata * udata)30397549c34SHans Petter Selasky static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,
30497549c34SHans Petter Selasky int entries, struct ib_udata *udata)
30597549c34SHans Petter Selasky {
30697549c34SHans Petter Selasky struct mlx4_ib_resize_cq ucmd;
30797549c34SHans Petter Selasky int err;
30897549c34SHans Petter Selasky
30997549c34SHans Petter Selasky if (cq->resize_umem)
31097549c34SHans Petter Selasky return -EBUSY;
31197549c34SHans Petter Selasky
31297549c34SHans Petter Selasky if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
31397549c34SHans Petter Selasky return -EFAULT;
31497549c34SHans Petter Selasky
3158cc48704SHans Petter Selasky cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL);
31697549c34SHans Petter Selasky if (!cq->resize_buf)
31797549c34SHans Petter Selasky return -ENOMEM;
31897549c34SHans Petter Selasky
319b633e08cSHans Petter Selasky err = mlx4_ib_get_cq_umem(dev, udata, &cq->resize_buf->buf,
32097549c34SHans Petter Selasky &cq->resize_umem, ucmd.buf_addr, entries);
32197549c34SHans Petter Selasky if (err) {
32297549c34SHans Petter Selasky kfree(cq->resize_buf);
32397549c34SHans Petter Selasky cq->resize_buf = NULL;
32497549c34SHans Petter Selasky return err;
32597549c34SHans Petter Selasky }
32697549c34SHans Petter Selasky
32797549c34SHans Petter Selasky cq->resize_buf->cqe = entries - 1;
32897549c34SHans Petter Selasky
32997549c34SHans Petter Selasky return 0;
33097549c34SHans Petter Selasky }
33197549c34SHans Petter Selasky
mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq * cq)33297549c34SHans Petter Selasky static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq)
33397549c34SHans Petter Selasky {
33497549c34SHans Petter Selasky u32 i;
33597549c34SHans Petter Selasky
33697549c34SHans Petter Selasky i = cq->mcq.cons_index;
33797549c34SHans Petter Selasky while (get_sw_cqe(cq, i))
33897549c34SHans Petter Selasky ++i;
33997549c34SHans Petter Selasky
34097549c34SHans Petter Selasky return i - cq->mcq.cons_index;
34197549c34SHans Petter Selasky }
34297549c34SHans Petter Selasky
mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq * cq)3438cc48704SHans Petter Selasky static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq)
34497549c34SHans Petter Selasky {
34597549c34SHans Petter Selasky struct mlx4_cqe *cqe, *new_cqe;
34697549c34SHans Petter Selasky int i;
34797549c34SHans Petter Selasky int cqe_size = cq->buf.entry_size;
34897549c34SHans Petter Selasky int cqe_inc = cqe_size == 64 ? 1 : 0;
34997549c34SHans Petter Selasky
35097549c34SHans Petter Selasky i = cq->mcq.cons_index;
35197549c34SHans Petter Selasky cqe = get_cqe(cq, i & cq->ibcq.cqe);
35297549c34SHans Petter Selasky cqe += cqe_inc;
35397549c34SHans Petter Selasky
35497549c34SHans Petter Selasky while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) {
35597549c34SHans Petter Selasky new_cqe = get_cqe_from_buf(&cq->resize_buf->buf,
35697549c34SHans Petter Selasky (i + 1) & cq->resize_buf->cqe);
35797549c34SHans Petter Selasky memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size);
35897549c34SHans Petter Selasky new_cqe += cqe_inc;
35997549c34SHans Petter Selasky
36097549c34SHans Petter Selasky new_cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) |
36197549c34SHans Petter Selasky (((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0);
36297549c34SHans Petter Selasky cqe = get_cqe(cq, ++i & cq->ibcq.cqe);
36397549c34SHans Petter Selasky cqe += cqe_inc;
36497549c34SHans Petter Selasky }
36597549c34SHans Petter Selasky ++cq->mcq.cons_index;
36697549c34SHans Petter Selasky }
36797549c34SHans Petter Selasky
mlx4_ib_resize_cq(struct ib_cq * ibcq,int entries,struct ib_udata * udata)36897549c34SHans Petter Selasky int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
36997549c34SHans Petter Selasky {
37097549c34SHans Petter Selasky struct mlx4_ib_dev *dev = to_mdev(ibcq->device);
37197549c34SHans Petter Selasky struct mlx4_ib_cq *cq = to_mcq(ibcq);
37297549c34SHans Petter Selasky struct mlx4_mtt mtt;
37397549c34SHans Petter Selasky int outst_cqe;
37497549c34SHans Petter Selasky int err;
37597549c34SHans Petter Selasky
37697549c34SHans Petter Selasky mutex_lock(&cq->resize_mutex);
37797549c34SHans Petter Selasky if (entries < 1 || entries > dev->dev->caps.max_cqes) {
37897549c34SHans Petter Selasky err = -EINVAL;
37997549c34SHans Petter Selasky goto out;
38097549c34SHans Petter Selasky }
38197549c34SHans Petter Selasky
38297549c34SHans Petter Selasky entries = roundup_pow_of_two(entries + 1);
38397549c34SHans Petter Selasky if (entries == ibcq->cqe + 1) {
38497549c34SHans Petter Selasky err = 0;
38597549c34SHans Petter Selasky goto out;
38697549c34SHans Petter Selasky }
38797549c34SHans Petter Selasky
38897549c34SHans Petter Selasky if (entries > dev->dev->caps.max_cqes + 1) {
38997549c34SHans Petter Selasky err = -EINVAL;
39097549c34SHans Petter Selasky goto out;
39197549c34SHans Petter Selasky }
39297549c34SHans Petter Selasky
39397549c34SHans Petter Selasky if (ibcq->uobject) {
39497549c34SHans Petter Selasky err = mlx4_alloc_resize_umem(dev, cq, entries, udata);
39597549c34SHans Petter Selasky if (err)
39697549c34SHans Petter Selasky goto out;
39797549c34SHans Petter Selasky } else {
39897549c34SHans Petter Selasky /* Can't be smaller than the number of outstanding CQEs */
39997549c34SHans Petter Selasky outst_cqe = mlx4_ib_get_outstanding_cqes(cq);
40097549c34SHans Petter Selasky if (entries < outst_cqe + 1) {
4018cc48704SHans Petter Selasky err = -EINVAL;
40297549c34SHans Petter Selasky goto out;
40397549c34SHans Petter Selasky }
40497549c34SHans Petter Selasky
40597549c34SHans Petter Selasky err = mlx4_alloc_resize_buf(dev, cq, entries);
40697549c34SHans Petter Selasky if (err)
40797549c34SHans Petter Selasky goto out;
40897549c34SHans Petter Selasky }
40997549c34SHans Petter Selasky
41097549c34SHans Petter Selasky mtt = cq->buf.mtt;
41197549c34SHans Petter Selasky
41297549c34SHans Petter Selasky err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt);
41397549c34SHans Petter Selasky if (err)
41497549c34SHans Petter Selasky goto err_buf;
41597549c34SHans Petter Selasky
41697549c34SHans Petter Selasky mlx4_mtt_cleanup(dev->dev, &mtt);
41797549c34SHans Petter Selasky if (ibcq->uobject) {
41897549c34SHans Petter Selasky cq->buf = cq->resize_buf->buf;
41997549c34SHans Petter Selasky cq->ibcq.cqe = cq->resize_buf->cqe;
42097549c34SHans Petter Selasky ib_umem_release(cq->umem);
42197549c34SHans Petter Selasky cq->umem = cq->resize_umem;
42297549c34SHans Petter Selasky
42397549c34SHans Petter Selasky kfree(cq->resize_buf);
42497549c34SHans Petter Selasky cq->resize_buf = NULL;
42597549c34SHans Petter Selasky cq->resize_umem = NULL;
42697549c34SHans Petter Selasky } else {
42797549c34SHans Petter Selasky struct mlx4_ib_cq_buf tmp_buf;
42897549c34SHans Petter Selasky int tmp_cqe = 0;
42997549c34SHans Petter Selasky
43097549c34SHans Petter Selasky spin_lock_irq(&cq->lock);
43197549c34SHans Petter Selasky if (cq->resize_buf) {
4328cc48704SHans Petter Selasky mlx4_ib_cq_resize_copy_cqes(cq);
43397549c34SHans Petter Selasky tmp_buf = cq->buf;
43497549c34SHans Petter Selasky tmp_cqe = cq->ibcq.cqe;
43597549c34SHans Petter Selasky cq->buf = cq->resize_buf->buf;
43697549c34SHans Petter Selasky cq->ibcq.cqe = cq->resize_buf->cqe;
43797549c34SHans Petter Selasky
43897549c34SHans Petter Selasky kfree(cq->resize_buf);
43997549c34SHans Petter Selasky cq->resize_buf = NULL;
44097549c34SHans Petter Selasky }
44197549c34SHans Petter Selasky spin_unlock_irq(&cq->lock);
44297549c34SHans Petter Selasky
44397549c34SHans Petter Selasky if (tmp_cqe)
44497549c34SHans Petter Selasky mlx4_ib_free_cq_buf(dev, &tmp_buf, tmp_cqe);
44597549c34SHans Petter Selasky }
44697549c34SHans Petter Selasky
44797549c34SHans Petter Selasky goto out;
44897549c34SHans Petter Selasky
44997549c34SHans Petter Selasky err_buf:
45097549c34SHans Petter Selasky mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt);
45197549c34SHans Petter Selasky if (!ibcq->uobject)
45297549c34SHans Petter Selasky mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf,
45397549c34SHans Petter Selasky cq->resize_buf->cqe);
45497549c34SHans Petter Selasky
45597549c34SHans Petter Selasky kfree(cq->resize_buf);
45697549c34SHans Petter Selasky cq->resize_buf = NULL;
45797549c34SHans Petter Selasky
45897549c34SHans Petter Selasky ib_umem_release(cq->resize_umem);
45997549c34SHans Petter Selasky cq->resize_umem = NULL;
46097549c34SHans Petter Selasky out:
46197549c34SHans Petter Selasky mutex_unlock(&cq->resize_mutex);
46297549c34SHans Petter Selasky
46397549c34SHans Petter Selasky return err;
46497549c34SHans Petter Selasky }
46597549c34SHans Petter Selasky
mlx4_ib_destroy_cq(struct ib_cq * cq,struct ib_udata * udata)466b633e08cSHans Petter Selasky void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
46797549c34SHans Petter Selasky {
46897549c34SHans Petter Selasky struct mlx4_ib_dev *dev = to_mdev(cq->device);
46997549c34SHans Petter Selasky struct mlx4_ib_cq *mcq = to_mcq(cq);
47097549c34SHans Petter Selasky
47197549c34SHans Petter Selasky mlx4_cq_free(dev->dev, &mcq->mcq);
47297549c34SHans Petter Selasky mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt);
47397549c34SHans Petter Selasky
474b633e08cSHans Petter Selasky if (udata) {
475b633e08cSHans Petter Selasky mlx4_ib_db_unmap_user(
476b633e08cSHans Petter Selasky rdma_udata_to_drv_context(
477b633e08cSHans Petter Selasky udata,
478b633e08cSHans Petter Selasky struct mlx4_ib_ucontext,
479b633e08cSHans Petter Selasky ibucontext),
480b633e08cSHans Petter Selasky &mcq->db);
48197549c34SHans Petter Selasky } else {
48297549c34SHans Petter Selasky mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe);
48397549c34SHans Petter Selasky mlx4_db_free(dev->dev, &mcq->db);
48497549c34SHans Petter Selasky }
485b633e08cSHans Petter Selasky ib_umem_release(mcq->umem);
48697549c34SHans Petter Selasky }
48797549c34SHans Petter Selasky
dump_cqe(void * cqe)48897549c34SHans Petter Selasky static void dump_cqe(void *cqe)
48997549c34SHans Petter Selasky {
49097549c34SHans Petter Selasky __be32 *buf = cqe;
49197549c34SHans Petter Selasky
49297549c34SHans Petter Selasky pr_debug("CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n",
49397549c34SHans Petter Selasky be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]),
49497549c34SHans Petter Selasky be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]),
49597549c34SHans Petter Selasky be32_to_cpu(buf[6]), be32_to_cpu(buf[7]));
49697549c34SHans Petter Selasky }
49797549c34SHans Petter Selasky
mlx4_ib_handle_error_cqe(struct mlx4_err_cqe * cqe,struct ib_wc * wc)49897549c34SHans Petter Selasky static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe,
49997549c34SHans Petter Selasky struct ib_wc *wc)
50097549c34SHans Petter Selasky {
50197549c34SHans Petter Selasky if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) {
50297549c34SHans Petter Selasky pr_debug("local QP operation err "
50397549c34SHans Petter Selasky "(QPN %06x, WQE index %x, vendor syndrome %02x, "
50497549c34SHans Petter Selasky "opcode = %02x)\n",
50597549c34SHans Petter Selasky be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index),
50697549c34SHans Petter Selasky cqe->vendor_err_syndrome,
50797549c34SHans Petter Selasky cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
50897549c34SHans Petter Selasky dump_cqe(cqe);
50997549c34SHans Petter Selasky }
51097549c34SHans Petter Selasky
51197549c34SHans Petter Selasky switch (cqe->syndrome) {
51297549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:
51397549c34SHans Petter Selasky wc->status = IB_WC_LOC_LEN_ERR;
51497549c34SHans Petter Selasky break;
51597549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:
51697549c34SHans Petter Selasky wc->status = IB_WC_LOC_QP_OP_ERR;
51797549c34SHans Petter Selasky break;
51897549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:
51997549c34SHans Petter Selasky wc->status = IB_WC_LOC_PROT_ERR;
52097549c34SHans Petter Selasky break;
52197549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:
52297549c34SHans Petter Selasky wc->status = IB_WC_WR_FLUSH_ERR;
52397549c34SHans Petter Selasky break;
52497549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_MW_BIND_ERR:
52597549c34SHans Petter Selasky wc->status = IB_WC_MW_BIND_ERR;
52697549c34SHans Petter Selasky break;
52797549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_BAD_RESP_ERR:
52897549c34SHans Petter Selasky wc->status = IB_WC_BAD_RESP_ERR;
52997549c34SHans Petter Selasky break;
53097549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:
53197549c34SHans Petter Selasky wc->status = IB_WC_LOC_ACCESS_ERR;
53297549c34SHans Petter Selasky break;
53397549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
53497549c34SHans Petter Selasky wc->status = IB_WC_REM_INV_REQ_ERR;
53597549c34SHans Petter Selasky break;
53697549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:
53797549c34SHans Petter Selasky wc->status = IB_WC_REM_ACCESS_ERR;
53897549c34SHans Petter Selasky break;
53997549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:
54097549c34SHans Petter Selasky wc->status = IB_WC_REM_OP_ERR;
54197549c34SHans Petter Selasky break;
54297549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
54397549c34SHans Petter Selasky wc->status = IB_WC_RETRY_EXC_ERR;
54497549c34SHans Petter Selasky break;
54597549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
54697549c34SHans Petter Selasky wc->status = IB_WC_RNR_RETRY_EXC_ERR;
54797549c34SHans Petter Selasky break;
54897549c34SHans Petter Selasky case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:
54997549c34SHans Petter Selasky wc->status = IB_WC_REM_ABORT_ERR;
55097549c34SHans Petter Selasky break;
55197549c34SHans Petter Selasky default:
55297549c34SHans Petter Selasky wc->status = IB_WC_GENERAL_ERR;
55397549c34SHans Petter Selasky break;
55497549c34SHans Petter Selasky }
55597549c34SHans Petter Selasky
55697549c34SHans Petter Selasky wc->vendor_err = cqe->vendor_err_syndrome;
55797549c34SHans Petter Selasky }
55897549c34SHans Petter Selasky
mlx4_ib_ipoib_csum_ok(__be16 status,__be16 checksum)55997549c34SHans Petter Selasky static int mlx4_ib_ipoib_csum_ok(__be16 status, __be16 checksum)
56097549c34SHans Petter Selasky {
56197549c34SHans Petter Selasky return ((status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 |
56297549c34SHans Petter Selasky MLX4_CQE_STATUS_IPV4F |
56397549c34SHans Petter Selasky MLX4_CQE_STATUS_IPV4OPT |
56497549c34SHans Petter Selasky MLX4_CQE_STATUS_IPV6 |
56597549c34SHans Petter Selasky MLX4_CQE_STATUS_IPOK)) ==
56697549c34SHans Petter Selasky cpu_to_be16(MLX4_CQE_STATUS_IPV4 |
56797549c34SHans Petter Selasky MLX4_CQE_STATUS_IPOK)) &&
56897549c34SHans Petter Selasky (status & cpu_to_be16(MLX4_CQE_STATUS_UDP |
56997549c34SHans Petter Selasky MLX4_CQE_STATUS_TCP)) &&
57097549c34SHans Petter Selasky checksum == cpu_to_be16(0xffff);
57197549c34SHans Petter Selasky }
57297549c34SHans Petter Selasky
use_tunnel_data(struct mlx4_ib_qp * qp,struct mlx4_ib_cq * cq,struct ib_wc * wc,unsigned tail,struct mlx4_cqe * cqe,int is_eth)5738cc48704SHans Petter Selasky static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc,
57497549c34SHans Petter Selasky unsigned tail, struct mlx4_cqe *cqe, int is_eth)
57597549c34SHans Petter Selasky {
57697549c34SHans Petter Selasky struct mlx4_ib_proxy_sqp_hdr *hdr;
57797549c34SHans Petter Selasky
57897549c34SHans Petter Selasky ib_dma_sync_single_for_cpu(qp->ibqp.device,
57997549c34SHans Petter Selasky qp->sqp_proxy_rcv[tail].map,
58097549c34SHans Petter Selasky sizeof (struct mlx4_ib_proxy_sqp_hdr),
58197549c34SHans Petter Selasky DMA_FROM_DEVICE);
58297549c34SHans Petter Selasky hdr = (struct mlx4_ib_proxy_sqp_hdr *) (qp->sqp_proxy_rcv[tail].addr);
58397549c34SHans Petter Selasky wc->pkey_index = be16_to_cpu(hdr->tun.pkey_index);
58497549c34SHans Petter Selasky wc->src_qp = be32_to_cpu(hdr->tun.flags_src_qp) & 0xFFFFFF;
58597549c34SHans Petter Selasky wc->wc_flags |= (hdr->tun.g_ml_path & 0x80) ? (IB_WC_GRH) : 0;
58697549c34SHans Petter Selasky wc->dlid_path_bits = 0;
58797549c34SHans Petter Selasky
58897549c34SHans Petter Selasky if (is_eth) {
58930416d4eSHans Petter Selasky wc->slid = 0;
59097549c34SHans Petter Selasky wc->vlan_id = be16_to_cpu(hdr->tun.sl_vid);
59197549c34SHans Petter Selasky memcpy(&(wc->smac[0]), (char *)&hdr->tun.mac_31_0, 4);
59297549c34SHans Petter Selasky memcpy(&(wc->smac[4]), (char *)&hdr->tun.slid_mac_47_32, 2);
5938cc48704SHans Petter Selasky wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
59497549c34SHans Petter Selasky } else {
59597549c34SHans Petter Selasky wc->slid = be16_to_cpu(hdr->tun.slid_mac_47_32);
59697549c34SHans Petter Selasky wc->sl = (u8) (be16_to_cpu(hdr->tun.sl_vid) >> 12);
59797549c34SHans Petter Selasky }
5988cc48704SHans Petter Selasky }
59997549c34SHans Petter Selasky
mlx4_ib_qp_sw_comp(struct mlx4_ib_qp * qp,int num_entries,struct ib_wc * wc,int * npolled,int is_send)6008cc48704SHans Petter Selasky static void mlx4_ib_qp_sw_comp(struct mlx4_ib_qp *qp, int num_entries,
6018cc48704SHans Petter Selasky struct ib_wc *wc, int *npolled, int is_send)
6028cc48704SHans Petter Selasky {
6038cc48704SHans Petter Selasky struct mlx4_ib_wq *wq;
6048cc48704SHans Petter Selasky unsigned cur;
6058cc48704SHans Petter Selasky int i;
6068cc48704SHans Petter Selasky
6078cc48704SHans Petter Selasky wq = is_send ? &qp->sq : &qp->rq;
6088cc48704SHans Petter Selasky cur = wq->head - wq->tail;
6098cc48704SHans Petter Selasky
6108cc48704SHans Petter Selasky if (cur == 0)
6118cc48704SHans Petter Selasky return;
6128cc48704SHans Petter Selasky
6138cc48704SHans Petter Selasky for (i = 0; i < cur && *npolled < num_entries; i++) {
6148cc48704SHans Petter Selasky wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
6158cc48704SHans Petter Selasky wc->status = IB_WC_WR_FLUSH_ERR;
6168cc48704SHans Petter Selasky wc->vendor_err = MLX4_CQE_SYNDROME_WR_FLUSH_ERR;
6178cc48704SHans Petter Selasky wq->tail++;
6188cc48704SHans Petter Selasky (*npolled)++;
6198cc48704SHans Petter Selasky wc->qp = &qp->ibqp;
6208cc48704SHans Petter Selasky wc++;
6218cc48704SHans Petter Selasky }
6228cc48704SHans Petter Selasky }
6238cc48704SHans Petter Selasky
mlx4_ib_poll_sw_comp(struct mlx4_ib_cq * cq,int num_entries,struct ib_wc * wc,int * npolled)6248cc48704SHans Petter Selasky static void mlx4_ib_poll_sw_comp(struct mlx4_ib_cq *cq, int num_entries,
6258cc48704SHans Petter Selasky struct ib_wc *wc, int *npolled)
6268cc48704SHans Petter Selasky {
6278cc48704SHans Petter Selasky struct mlx4_ib_qp *qp;
6288cc48704SHans Petter Selasky
6298cc48704SHans Petter Selasky *npolled = 0;
6308cc48704SHans Petter Selasky /* Find uncompleted WQEs belonging to that cq and retrun
6318cc48704SHans Petter Selasky * simulated FLUSH_ERR completions
6328cc48704SHans Petter Selasky */
6338cc48704SHans Petter Selasky list_for_each_entry(qp, &cq->send_qp_list, cq_send_list) {
6348cc48704SHans Petter Selasky mlx4_ib_qp_sw_comp(qp, num_entries, wc + *npolled, npolled, 1);
6358cc48704SHans Petter Selasky if (*npolled >= num_entries)
6368cc48704SHans Petter Selasky goto out;
6378cc48704SHans Petter Selasky }
6388cc48704SHans Petter Selasky
6398cc48704SHans Petter Selasky list_for_each_entry(qp, &cq->recv_qp_list, cq_recv_list) {
6408cc48704SHans Petter Selasky mlx4_ib_qp_sw_comp(qp, num_entries, wc + *npolled, npolled, 0);
6418cc48704SHans Petter Selasky if (*npolled >= num_entries)
6428cc48704SHans Petter Selasky goto out;
6438cc48704SHans Petter Selasky }
6448cc48704SHans Petter Selasky
6458cc48704SHans Petter Selasky out:
6468cc48704SHans Petter Selasky return;
64797549c34SHans Petter Selasky }
64897549c34SHans Petter Selasky
mlx4_ib_poll_one(struct mlx4_ib_cq * cq,struct mlx4_ib_qp ** cur_qp,struct ib_wc * wc)64997549c34SHans Petter Selasky static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,
65097549c34SHans Petter Selasky struct mlx4_ib_qp **cur_qp,
65197549c34SHans Petter Selasky struct ib_wc *wc)
65297549c34SHans Petter Selasky {
65397549c34SHans Petter Selasky struct mlx4_cqe *cqe;
65497549c34SHans Petter Selasky struct mlx4_qp *mqp;
65597549c34SHans Petter Selasky struct mlx4_ib_wq *wq;
65697549c34SHans Petter Selasky struct mlx4_ib_srq *srq;
65797549c34SHans Petter Selasky struct mlx4_srq *msrq = NULL;
65897549c34SHans Petter Selasky int is_send;
65997549c34SHans Petter Selasky int is_error;
6608cc48704SHans Petter Selasky int is_eth;
66197549c34SHans Petter Selasky u32 g_mlpath_rqpn;
66297549c34SHans Petter Selasky u16 wqe_ctr;
66397549c34SHans Petter Selasky unsigned tail = 0;
66497549c34SHans Petter Selasky
66597549c34SHans Petter Selasky repoll:
66697549c34SHans Petter Selasky cqe = next_cqe_sw(cq);
66797549c34SHans Petter Selasky if (!cqe)
66897549c34SHans Petter Selasky return -EAGAIN;
66997549c34SHans Petter Selasky
67097549c34SHans Petter Selasky if (cq->buf.entry_size == 64)
67197549c34SHans Petter Selasky cqe++;
67297549c34SHans Petter Selasky
67397549c34SHans Petter Selasky ++cq->mcq.cons_index;
67497549c34SHans Petter Selasky
67597549c34SHans Petter Selasky /*
67697549c34SHans Petter Selasky * Make sure we read CQ entry contents after we've checked the
67797549c34SHans Petter Selasky * ownership bit.
67897549c34SHans Petter Selasky */
67997549c34SHans Petter Selasky rmb();
68097549c34SHans Petter Selasky
68197549c34SHans Petter Selasky is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;
68297549c34SHans Petter Selasky is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
68397549c34SHans Petter Selasky MLX4_CQE_OPCODE_ERROR;
68497549c34SHans Petter Selasky
68597549c34SHans Petter Selasky /* Resize CQ in progress */
68697549c34SHans Petter Selasky if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_RESIZE)) {
68797549c34SHans Petter Selasky if (cq->resize_buf) {
68897549c34SHans Petter Selasky struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device);
68997549c34SHans Petter Selasky
69097549c34SHans Petter Selasky mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
69197549c34SHans Petter Selasky cq->buf = cq->resize_buf->buf;
69297549c34SHans Petter Selasky cq->ibcq.cqe = cq->resize_buf->cqe;
69397549c34SHans Petter Selasky
69497549c34SHans Petter Selasky kfree(cq->resize_buf);
69597549c34SHans Petter Selasky cq->resize_buf = NULL;
69697549c34SHans Petter Selasky }
69797549c34SHans Petter Selasky
69897549c34SHans Petter Selasky goto repoll;
69997549c34SHans Petter Selasky }
70097549c34SHans Petter Selasky
70197549c34SHans Petter Selasky if (!*cur_qp ||
70297549c34SHans Petter Selasky (be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) != (*cur_qp)->mqp.qpn) {
70397549c34SHans Petter Selasky /*
70497549c34SHans Petter Selasky * We do not have to take the QP table lock here,
70597549c34SHans Petter Selasky * because CQs will be locked while QPs are removed
70697549c34SHans Petter Selasky * from the table.
70797549c34SHans Petter Selasky */
70897549c34SHans Petter Selasky mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev,
70997549c34SHans Petter Selasky be32_to_cpu(cqe->vlan_my_qpn));
71097549c34SHans Petter Selasky *cur_qp = to_mibqp(mqp);
71197549c34SHans Petter Selasky }
71297549c34SHans Petter Selasky
71397549c34SHans Petter Selasky wc->qp = &(*cur_qp)->ibqp;
71497549c34SHans Petter Selasky
71597549c34SHans Petter Selasky if (wc->qp->qp_type == IB_QPT_XRC_TGT) {
71697549c34SHans Petter Selasky u32 srq_num;
71797549c34SHans Petter Selasky g_mlpath_rqpn = be32_to_cpu(cqe->g_mlpath_rqpn);
71897549c34SHans Petter Selasky srq_num = g_mlpath_rqpn & 0xffffff;
71997549c34SHans Petter Selasky /* SRQ is also in the radix tree */
72097549c34SHans Petter Selasky msrq = mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev,
72197549c34SHans Petter Selasky srq_num);
72297549c34SHans Petter Selasky }
72397549c34SHans Petter Selasky
72497549c34SHans Petter Selasky if (is_send) {
72597549c34SHans Petter Selasky wq = &(*cur_qp)->sq;
72697549c34SHans Petter Selasky if (!(*cur_qp)->sq_signal_bits) {
72797549c34SHans Petter Selasky wqe_ctr = be16_to_cpu(cqe->wqe_index);
72897549c34SHans Petter Selasky wq->tail += (u16) (wqe_ctr - (u16) wq->tail);
72997549c34SHans Petter Selasky }
73097549c34SHans Petter Selasky wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
73197549c34SHans Petter Selasky ++wq->tail;
73297549c34SHans Petter Selasky } else if ((*cur_qp)->ibqp.srq) {
73397549c34SHans Petter Selasky srq = to_msrq((*cur_qp)->ibqp.srq);
73497549c34SHans Petter Selasky wqe_ctr = be16_to_cpu(cqe->wqe_index);
73597549c34SHans Petter Selasky wc->wr_id = srq->wrid[wqe_ctr];
73697549c34SHans Petter Selasky mlx4_ib_free_srq_wqe(srq, wqe_ctr);
73797549c34SHans Petter Selasky } else if (msrq) {
73897549c34SHans Petter Selasky srq = to_mibsrq(msrq);
73997549c34SHans Petter Selasky wqe_ctr = be16_to_cpu(cqe->wqe_index);
74097549c34SHans Petter Selasky wc->wr_id = srq->wrid[wqe_ctr];
74197549c34SHans Petter Selasky mlx4_ib_free_srq_wqe(srq, wqe_ctr);
74297549c34SHans Petter Selasky } else {
74397549c34SHans Petter Selasky wq = &(*cur_qp)->rq;
74497549c34SHans Petter Selasky tail = wq->tail & (wq->wqe_cnt - 1);
74597549c34SHans Petter Selasky wc->wr_id = wq->wrid[tail];
74697549c34SHans Petter Selasky ++wq->tail;
74797549c34SHans Petter Selasky }
74897549c34SHans Petter Selasky
74997549c34SHans Petter Selasky if (unlikely(is_error)) {
75097549c34SHans Petter Selasky mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);
75197549c34SHans Petter Selasky return 0;
75297549c34SHans Petter Selasky }
75397549c34SHans Petter Selasky
75497549c34SHans Petter Selasky wc->status = IB_WC_SUCCESS;
75597549c34SHans Petter Selasky
75697549c34SHans Petter Selasky if (is_send) {
75797549c34SHans Petter Selasky wc->wc_flags = 0;
75897549c34SHans Petter Selasky switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
75997549c34SHans Petter Selasky case MLX4_OPCODE_RDMA_WRITE_IMM:
76097549c34SHans Petter Selasky wc->wc_flags |= IB_WC_WITH_IMM;
76197549c34SHans Petter Selasky case MLX4_OPCODE_RDMA_WRITE:
76297549c34SHans Petter Selasky wc->opcode = IB_WC_RDMA_WRITE;
76397549c34SHans Petter Selasky break;
76497549c34SHans Petter Selasky case MLX4_OPCODE_SEND_IMM:
76597549c34SHans Petter Selasky wc->wc_flags |= IB_WC_WITH_IMM;
76697549c34SHans Petter Selasky case MLX4_OPCODE_SEND:
76797549c34SHans Petter Selasky case MLX4_OPCODE_SEND_INVAL:
76897549c34SHans Petter Selasky wc->opcode = IB_WC_SEND;
76997549c34SHans Petter Selasky break;
77097549c34SHans Petter Selasky case MLX4_OPCODE_RDMA_READ:
77197549c34SHans Petter Selasky wc->opcode = IB_WC_RDMA_READ;
77297549c34SHans Petter Selasky wc->byte_len = be32_to_cpu(cqe->byte_cnt);
77397549c34SHans Petter Selasky break;
77497549c34SHans Petter Selasky case MLX4_OPCODE_ATOMIC_CS:
77597549c34SHans Petter Selasky wc->opcode = IB_WC_COMP_SWAP;
77697549c34SHans Petter Selasky wc->byte_len = 8;
77797549c34SHans Petter Selasky break;
77897549c34SHans Petter Selasky case MLX4_OPCODE_ATOMIC_FA:
77997549c34SHans Petter Selasky wc->opcode = IB_WC_FETCH_ADD;
78097549c34SHans Petter Selasky wc->byte_len = 8;
78197549c34SHans Petter Selasky break;
78297549c34SHans Petter Selasky case MLX4_OPCODE_MASKED_ATOMIC_CS:
78397549c34SHans Petter Selasky wc->opcode = IB_WC_MASKED_COMP_SWAP;
78497549c34SHans Petter Selasky wc->byte_len = 8;
78597549c34SHans Petter Selasky break;
78697549c34SHans Petter Selasky case MLX4_OPCODE_MASKED_ATOMIC_FA:
78797549c34SHans Petter Selasky wc->opcode = IB_WC_MASKED_FETCH_ADD;
78897549c34SHans Petter Selasky wc->byte_len = 8;
78997549c34SHans Petter Selasky break;
79097549c34SHans Petter Selasky case MLX4_OPCODE_LSO:
79197549c34SHans Petter Selasky wc->opcode = IB_WC_LSO;
79297549c34SHans Petter Selasky break;
79397549c34SHans Petter Selasky case MLX4_OPCODE_FMR:
7948cc48704SHans Petter Selasky wc->opcode = IB_WC_REG_MR;
79597549c34SHans Petter Selasky break;
79697549c34SHans Petter Selasky case MLX4_OPCODE_LOCAL_INVAL:
79797549c34SHans Petter Selasky wc->opcode = IB_WC_LOCAL_INV;
79897549c34SHans Petter Selasky break;
79997549c34SHans Petter Selasky }
80097549c34SHans Petter Selasky } else {
80197549c34SHans Petter Selasky wc->byte_len = be32_to_cpu(cqe->byte_cnt);
80297549c34SHans Petter Selasky
80397549c34SHans Petter Selasky switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
80497549c34SHans Petter Selasky case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
80597549c34SHans Petter Selasky wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
80697549c34SHans Petter Selasky wc->wc_flags = IB_WC_WITH_IMM;
80797549c34SHans Petter Selasky wc->ex.imm_data = cqe->immed_rss_invalid;
80897549c34SHans Petter Selasky break;
80997549c34SHans Petter Selasky case MLX4_RECV_OPCODE_SEND_INVAL:
81097549c34SHans Petter Selasky wc->opcode = IB_WC_RECV;
81197549c34SHans Petter Selasky wc->wc_flags = IB_WC_WITH_INVALIDATE;
81297549c34SHans Petter Selasky wc->ex.invalidate_rkey = be32_to_cpu(cqe->immed_rss_invalid);
81397549c34SHans Petter Selasky break;
81497549c34SHans Petter Selasky case MLX4_RECV_OPCODE_SEND:
81597549c34SHans Petter Selasky wc->opcode = IB_WC_RECV;
81697549c34SHans Petter Selasky wc->wc_flags = 0;
81797549c34SHans Petter Selasky break;
81897549c34SHans Petter Selasky case MLX4_RECV_OPCODE_SEND_IMM:
81997549c34SHans Petter Selasky wc->opcode = IB_WC_RECV;
82097549c34SHans Petter Selasky wc->wc_flags = IB_WC_WITH_IMM;
82197549c34SHans Petter Selasky wc->ex.imm_data = cqe->immed_rss_invalid;
82297549c34SHans Petter Selasky break;
82397549c34SHans Petter Selasky }
82497549c34SHans Petter Selasky
8258cc48704SHans Petter Selasky is_eth = (rdma_port_get_link_layer(wc->qp->device,
8268cc48704SHans Petter Selasky (*cur_qp)->port) ==
8278cc48704SHans Petter Selasky IB_LINK_LAYER_ETHERNET);
82897549c34SHans Petter Selasky if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) {
82997549c34SHans Petter Selasky if ((*cur_qp)->mlx4_ib_qp_type &
83097549c34SHans Petter Selasky (MLX4_IB_QPT_PROXY_SMI_OWNER |
8318cc48704SHans Petter Selasky MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
8328cc48704SHans Petter Selasky use_tunnel_data(*cur_qp, cq, wc, tail, cqe,
8338cc48704SHans Petter Selasky is_eth);
8348cc48704SHans Petter Selasky return 0;
8358cc48704SHans Petter Selasky }
83697549c34SHans Petter Selasky }
83797549c34SHans Petter Selasky
83897549c34SHans Petter Selasky g_mlpath_rqpn = be32_to_cpu(cqe->g_mlpath_rqpn);
83997549c34SHans Petter Selasky wc->src_qp = g_mlpath_rqpn & 0xffffff;
84097549c34SHans Petter Selasky wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f;
84197549c34SHans Petter Selasky wc->wc_flags |= g_mlpath_rqpn & 0x80000000 ? IB_WC_GRH : 0;
84297549c34SHans Petter Selasky wc->pkey_index = be32_to_cpu(cqe->immed_rss_invalid) & 0x7f;
84397549c34SHans Petter Selasky wc->wc_flags |= mlx4_ib_ipoib_csum_ok(cqe->status,
84497549c34SHans Petter Selasky cqe->checksum) ? IB_WC_IP_CSUM_OK : 0;
8458cc48704SHans Petter Selasky if (is_eth) {
84630416d4eSHans Petter Selasky wc->slid = 0;
84797549c34SHans Petter Selasky wc->sl = be16_to_cpu(cqe->sl_vid) >> 13;
8488cc48704SHans Petter Selasky if (be32_to_cpu(cqe->vlan_my_qpn) &
8498cc48704SHans Petter Selasky MLX4_CQE_CVLAN_PRESENT_MASK) {
85097549c34SHans Petter Selasky wc->vlan_id = be16_to_cpu(cqe->sl_vid) &
85197549c34SHans Petter Selasky MLX4_CQE_VID_MASK;
85297549c34SHans Petter Selasky } else {
85397549c34SHans Petter Selasky wc->vlan_id = 0xffff;
85497549c34SHans Petter Selasky }
8558cc48704SHans Petter Selasky memcpy(wc->smac, cqe->smac, ETH_ALEN);
8568cc48704SHans Petter Selasky wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
8578cc48704SHans Petter Selasky } else {
85830416d4eSHans Petter Selasky wc->slid = be16_to_cpu(cqe->rlid);
8598cc48704SHans Petter Selasky wc->sl = be16_to_cpu(cqe->sl_vid) >> 12;
8608cc48704SHans Petter Selasky wc->vlan_id = 0xffff;
86197549c34SHans Petter Selasky }
86297549c34SHans Petter Selasky }
86397549c34SHans Petter Selasky
86497549c34SHans Petter Selasky return 0;
86597549c34SHans Petter Selasky }
86697549c34SHans Petter Selasky
mlx4_ib_poll_cq(struct ib_cq * ibcq,int num_entries,struct ib_wc * wc)86797549c34SHans Petter Selasky int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
86897549c34SHans Petter Selasky {
86997549c34SHans Petter Selasky struct mlx4_ib_cq *cq = to_mcq(ibcq);
87097549c34SHans Petter Selasky struct mlx4_ib_qp *cur_qp = NULL;
87197549c34SHans Petter Selasky unsigned long flags;
87297549c34SHans Petter Selasky int npolled;
8738cc48704SHans Petter Selasky struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device);
87497549c34SHans Petter Selasky
87597549c34SHans Petter Selasky spin_lock_irqsave(&cq->lock, flags);
8766428c27fSHans Petter Selasky if (unlikely(mdev->dev->persist->state &
8776428c27fSHans Petter Selasky MLX4_DEVICE_STATE_INTERNAL_ERROR)) {
8788cc48704SHans Petter Selasky mlx4_ib_poll_sw_comp(cq, num_entries, wc, &npolled);
8798cc48704SHans Petter Selasky goto out;
8808cc48704SHans Petter Selasky }
88197549c34SHans Petter Selasky
88297549c34SHans Petter Selasky for (npolled = 0; npolled < num_entries; ++npolled) {
8838cc48704SHans Petter Selasky if (mlx4_ib_poll_one(cq, &cur_qp, wc + npolled))
88497549c34SHans Petter Selasky break;
88597549c34SHans Petter Selasky }
88697549c34SHans Petter Selasky
88797549c34SHans Petter Selasky mlx4_cq_set_ci(&cq->mcq);
88897549c34SHans Petter Selasky
8898cc48704SHans Petter Selasky out:
89097549c34SHans Petter Selasky spin_unlock_irqrestore(&cq->lock, flags);
89197549c34SHans Petter Selasky
89297549c34SHans Petter Selasky return npolled;
89397549c34SHans Petter Selasky }
89497549c34SHans Petter Selasky
mlx4_ib_arm_cq(struct ib_cq * ibcq,enum ib_cq_notify_flags flags)89597549c34SHans Petter Selasky int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
89697549c34SHans Petter Selasky {
8976428c27fSHans Petter Selasky struct mlx4_ib_cq *cq = to_mcq(ibcq);
8986428c27fSHans Petter Selasky struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device);
8996428c27fSHans Petter Selasky
9006428c27fSHans Petter Selasky if (unlikely(mdev->dev->persist->state &
9016428c27fSHans Petter Selasky MLX4_DEVICE_STATE_INTERNAL_ERROR))
9026428c27fSHans Petter Selasky return -1;
9036428c27fSHans Petter Selasky
9046428c27fSHans Petter Selasky mlx4_cq_arm(&cq->mcq,
90597549c34SHans Petter Selasky (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
90697549c34SHans Petter Selasky MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT,
9076428c27fSHans Petter Selasky mdev->uar_map,
9086428c27fSHans Petter Selasky MLX4_GET_DOORBELL_LOCK(&mdev->uar_lock));
90997549c34SHans Petter Selasky
91097549c34SHans Petter Selasky return 0;
91197549c34SHans Petter Selasky }
91297549c34SHans Petter Selasky
__mlx4_ib_cq_clean(struct mlx4_ib_cq * cq,u32 qpn,struct mlx4_ib_srq * srq)91397549c34SHans Petter Selasky void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
91497549c34SHans Petter Selasky {
91597549c34SHans Petter Selasky u32 prod_index;
91697549c34SHans Petter Selasky int nfreed = 0;
91797549c34SHans Petter Selasky struct mlx4_cqe *cqe, *dest;
91897549c34SHans Petter Selasky u8 owner_bit;
91997549c34SHans Petter Selasky int cqe_inc = cq->buf.entry_size == 64 ? 1 : 0;
92097549c34SHans Petter Selasky
92197549c34SHans Petter Selasky /*
92297549c34SHans Petter Selasky * First we need to find the current producer index, so we
92397549c34SHans Petter Selasky * know where to start cleaning from. It doesn't matter if HW
92497549c34SHans Petter Selasky * adds new entries after this loop -- the QP we're worried
92597549c34SHans Petter Selasky * about is already in RESET, so the new entries won't come
92697549c34SHans Petter Selasky * from our QP and therefore don't need to be checked.
92797549c34SHans Petter Selasky */
92897549c34SHans Petter Selasky for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
92997549c34SHans Petter Selasky if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)
93097549c34SHans Petter Selasky break;
93197549c34SHans Petter Selasky
93297549c34SHans Petter Selasky /*
93397549c34SHans Petter Selasky * Now sweep backwards through the CQ, removing CQ entries
93497549c34SHans Petter Selasky * that match our QP by copying older entries on top of them.
93597549c34SHans Petter Selasky */
93697549c34SHans Petter Selasky while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
93797549c34SHans Petter Selasky cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
93897549c34SHans Petter Selasky cqe += cqe_inc;
93997549c34SHans Petter Selasky
94097549c34SHans Petter Selasky if ((be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) == qpn) {
94197549c34SHans Petter Selasky if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
94297549c34SHans Petter Selasky mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index));
94397549c34SHans Petter Selasky ++nfreed;
94497549c34SHans Petter Selasky } else if (nfreed) {
94597549c34SHans Petter Selasky dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);
94697549c34SHans Petter Selasky dest += cqe_inc;
94797549c34SHans Petter Selasky
94897549c34SHans Petter Selasky owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
94997549c34SHans Petter Selasky memcpy(dest, cqe, sizeof *cqe);
95097549c34SHans Petter Selasky dest->owner_sr_opcode = owner_bit |
95197549c34SHans Petter Selasky (dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
95297549c34SHans Petter Selasky }
95397549c34SHans Petter Selasky }
95497549c34SHans Petter Selasky
95597549c34SHans Petter Selasky if (nfreed) {
95697549c34SHans Petter Selasky cq->mcq.cons_index += nfreed;
95797549c34SHans Petter Selasky /*
95897549c34SHans Petter Selasky * Make sure update of buffer contents is done before
95997549c34SHans Petter Selasky * updating consumer index.
96097549c34SHans Petter Selasky */
96197549c34SHans Petter Selasky wmb();
96297549c34SHans Petter Selasky mlx4_cq_set_ci(&cq->mcq);
96397549c34SHans Petter Selasky }
96497549c34SHans Petter Selasky }
96597549c34SHans Petter Selasky
mlx4_ib_cq_clean(struct mlx4_ib_cq * cq,u32 qpn,struct mlx4_ib_srq * srq)96697549c34SHans Petter Selasky void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
96797549c34SHans Petter Selasky {
96897549c34SHans Petter Selasky spin_lock_irq(&cq->lock);
96997549c34SHans Petter Selasky __mlx4_ib_cq_clean(cq, qpn, srq);
97097549c34SHans Petter Selasky spin_unlock_irq(&cq->lock);
97197549c34SHans Petter Selasky }
972