xref: /freebsd/sys/dev/mlx4/mlx4_ib/mlx4_ib_cq.c (revision 6428c27f)
197549c34SHans Petter Selasky /*
297549c34SHans Petter Selasky  * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
397549c34SHans Petter Selasky  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
497549c34SHans Petter Selasky  *
597549c34SHans Petter Selasky  * This software is available to you under a choice of one of two
697549c34SHans Petter Selasky  * licenses.  You may choose to be licensed under the terms of the GNU
797549c34SHans Petter Selasky  * General Public License (GPL) Version 2, available from the file
897549c34SHans Petter Selasky  * COPYING in the main directory of this source tree, or the
997549c34SHans Petter Selasky  * OpenIB.org BSD license below:
1097549c34SHans Petter Selasky  *
1197549c34SHans Petter Selasky  *     Redistribution and use in source and binary forms, with or
1297549c34SHans Petter Selasky  *     without modification, are permitted provided that the following
1397549c34SHans Petter Selasky  *     conditions are met:
1497549c34SHans Petter Selasky  *
1597549c34SHans Petter Selasky  *      - Redistributions of source code must retain the above
1697549c34SHans Petter Selasky  *        copyright notice, this list of conditions and the following
1797549c34SHans Petter Selasky  *        disclaimer.
1897549c34SHans Petter Selasky  *
1997549c34SHans Petter Selasky  *      - Redistributions in binary form must reproduce the above
2097549c34SHans Petter Selasky  *        copyright notice, this list of conditions and the following
2197549c34SHans Petter Selasky  *        disclaimer in the documentation and/or other materials
2297549c34SHans Petter Selasky  *        provided with the distribution.
2397549c34SHans Petter Selasky  *
2497549c34SHans Petter Selasky  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
2597549c34SHans Petter Selasky  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
2697549c34SHans Petter Selasky  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
2797549c34SHans Petter Selasky  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
2897549c34SHans Petter Selasky  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
2997549c34SHans Petter Selasky  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
3097549c34SHans Petter Selasky  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
3197549c34SHans Petter Selasky  * SOFTWARE.
3297549c34SHans Petter Selasky  */
3397549c34SHans Petter Selasky 
3497549c34SHans Petter Selasky #include <dev/mlx4/cq.h>
3597549c34SHans Petter Selasky #include <dev/mlx4/qp.h>
3697549c34SHans Petter Selasky #include <dev/mlx4/srq.h>
378cc48704SHans Petter Selasky #include <dev/mlx4/driver.h>
3897549c34SHans Petter Selasky #include <linux/slab.h>
3997549c34SHans Petter Selasky 
4097549c34SHans Petter Selasky #include "mlx4_ib.h"
418cc48704SHans Petter Selasky #include <rdma/mlx4-abi.h>
4297549c34SHans Petter Selasky 
4397549c34SHans Petter Selasky static void mlx4_ib_cq_comp(struct mlx4_cq *cq)
4497549c34SHans Petter Selasky {
4597549c34SHans Petter Selasky 	struct ib_cq *ibcq = &to_mibcq(cq)->ibcq;
4697549c34SHans Petter Selasky 	ibcq->comp_handler(ibcq, ibcq->cq_context);
4797549c34SHans Petter Selasky }
4897549c34SHans Petter Selasky 
4997549c34SHans Petter Selasky static void mlx4_ib_cq_event(struct mlx4_cq *cq, enum mlx4_event type)
5097549c34SHans Petter Selasky {
5197549c34SHans Petter Selasky 	struct ib_event event;
5297549c34SHans Petter Selasky 	struct ib_cq *ibcq;
5397549c34SHans Petter Selasky 
5497549c34SHans Petter Selasky 	if (type != MLX4_EVENT_TYPE_CQ_ERROR) {
5597549c34SHans Petter Selasky 		pr_warn("Unexpected event type %d "
5697549c34SHans Petter Selasky 		       "on CQ %06x\n", type, cq->cqn);
5797549c34SHans Petter Selasky 		return;
5897549c34SHans Petter Selasky 	}
5997549c34SHans Petter Selasky 
6097549c34SHans Petter Selasky 	ibcq = &to_mibcq(cq)->ibcq;
6197549c34SHans Petter Selasky 	if (ibcq->event_handler) {
6297549c34SHans Petter Selasky 		event.device     = ibcq->device;
6397549c34SHans Petter Selasky 		event.event      = IB_EVENT_CQ_ERR;
6497549c34SHans Petter Selasky 		event.element.cq = ibcq;
6597549c34SHans Petter Selasky 		ibcq->event_handler(&event, ibcq->cq_context);
6697549c34SHans Petter Selasky 	}
6797549c34SHans Petter Selasky }
6897549c34SHans Petter Selasky 
6997549c34SHans Petter Selasky static void *get_cqe_from_buf(struct mlx4_ib_cq_buf *buf, int n)
7097549c34SHans Petter Selasky {
7197549c34SHans Petter Selasky 	return mlx4_buf_offset(&buf->buf, n * buf->entry_size);
7297549c34SHans Petter Selasky }
7397549c34SHans Petter Selasky 
7497549c34SHans Petter Selasky static void *get_cqe(struct mlx4_ib_cq *cq, int n)
7597549c34SHans Petter Selasky {
7697549c34SHans Petter Selasky 	return get_cqe_from_buf(&cq->buf, n);
7797549c34SHans Petter Selasky }
7897549c34SHans Petter Selasky 
7997549c34SHans Petter Selasky static void *get_sw_cqe(struct mlx4_ib_cq *cq, int n)
8097549c34SHans Petter Selasky {
8197549c34SHans Petter Selasky 	struct mlx4_cqe *cqe = get_cqe(cq, n & cq->ibcq.cqe);
8297549c34SHans Petter Selasky 	struct mlx4_cqe *tcqe = ((cq->buf.entry_size == 64) ? (cqe + 1) : cqe);
8397549c34SHans Petter Selasky 
8497549c34SHans Petter Selasky 	return (!!(tcqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
8597549c34SHans Petter Selasky 		!!(n & (cq->ibcq.cqe + 1))) ? NULL : cqe;
8697549c34SHans Petter Selasky }
8797549c34SHans Petter Selasky 
8897549c34SHans Petter Selasky static struct mlx4_cqe *next_cqe_sw(struct mlx4_ib_cq *cq)
8997549c34SHans Petter Selasky {
9097549c34SHans Petter Selasky 	return get_sw_cqe(cq, cq->mcq.cons_index);
9197549c34SHans Petter Selasky }
9297549c34SHans Petter Selasky 
938cc48704SHans Petter Selasky int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
9497549c34SHans Petter Selasky {
9597549c34SHans Petter Selasky 	struct mlx4_ib_cq *mcq = to_mcq(cq);
9697549c34SHans Petter Selasky 	struct mlx4_ib_dev *dev = to_mdev(cq->device);
9797549c34SHans Petter Selasky 
988cc48704SHans Petter Selasky 	return mlx4_cq_modify(dev->dev, &mcq->mcq, cq_count, cq_period);
9997549c34SHans Petter Selasky }
10097549c34SHans Petter Selasky 
10197549c34SHans Petter Selasky static int mlx4_ib_alloc_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int nent)
10297549c34SHans Petter Selasky {
10397549c34SHans Petter Selasky 	int err;
10497549c34SHans Petter Selasky 
10597549c34SHans Petter Selasky 	err = mlx4_buf_alloc(dev->dev, nent * dev->dev->caps.cqe_size,
1068cc48704SHans Petter Selasky 			     PAGE_SIZE * 2, &buf->buf, GFP_KERNEL);
10797549c34SHans Petter Selasky 
10897549c34SHans Petter Selasky 	if (err)
10997549c34SHans Petter Selasky 		goto out;
11097549c34SHans Petter Selasky 
11197549c34SHans Petter Selasky 	buf->entry_size = dev->dev->caps.cqe_size;
11297549c34SHans Petter Selasky 	err = mlx4_mtt_init(dev->dev, buf->buf.npages, buf->buf.page_shift,
11397549c34SHans Petter Selasky 				    &buf->mtt);
11497549c34SHans Petter Selasky 	if (err)
11597549c34SHans Petter Selasky 		goto err_buf;
11697549c34SHans Petter Selasky 
1178cc48704SHans Petter Selasky 	err = mlx4_buf_write_mtt(dev->dev, &buf->mtt, &buf->buf, GFP_KERNEL);
11897549c34SHans Petter Selasky 	if (err)
11997549c34SHans Petter Selasky 		goto err_mtt;
12097549c34SHans Petter Selasky 
12197549c34SHans Petter Selasky 	return 0;
12297549c34SHans Petter Selasky 
12397549c34SHans Petter Selasky err_mtt:
12497549c34SHans Petter Selasky 	mlx4_mtt_cleanup(dev->dev, &buf->mtt);
12597549c34SHans Petter Selasky 
12697549c34SHans Petter Selasky err_buf:
12797549c34SHans Petter Selasky 	mlx4_buf_free(dev->dev, nent * buf->entry_size, &buf->buf);
12897549c34SHans Petter Selasky 
12997549c34SHans Petter Selasky out:
13097549c34SHans Petter Selasky 	return err;
13197549c34SHans Petter Selasky }
13297549c34SHans Petter Selasky 
13397549c34SHans Petter Selasky static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int cqe)
13497549c34SHans Petter Selasky {
13597549c34SHans Petter Selasky 	mlx4_buf_free(dev->dev, (cqe + 1) * buf->entry_size, &buf->buf);
13697549c34SHans Petter Selasky }
13797549c34SHans Petter Selasky 
13897549c34SHans Petter Selasky static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_ucontext *context,
13997549c34SHans Petter Selasky 			       struct mlx4_ib_cq_buf *buf, struct ib_umem **umem,
14097549c34SHans Petter Selasky 			       u64 buf_addr, int cqe)
14197549c34SHans Petter Selasky {
14297549c34SHans Petter Selasky 	int err;
14397549c34SHans Petter Selasky 	int cqe_size = dev->dev->caps.cqe_size;
14497549c34SHans Petter Selasky 
14597549c34SHans Petter Selasky 	*umem = ib_umem_get(context, buf_addr, cqe * cqe_size,
14697549c34SHans Petter Selasky 			    IB_ACCESS_LOCAL_WRITE, 1);
14797549c34SHans Petter Selasky 	if (IS_ERR(*umem))
14897549c34SHans Petter Selasky 		return PTR_ERR(*umem);
14997549c34SHans Petter Selasky 
1508cc48704SHans Petter Selasky 	err = mlx4_mtt_init(dev->dev, ib_umem_page_count(*umem),
1518cc48704SHans Petter Selasky 			    ilog2((*umem)->page_size), &buf->mtt);
15297549c34SHans Petter Selasky 	if (err)
15397549c34SHans Petter Selasky 		goto err_buf;
15497549c34SHans Petter Selasky 
15597549c34SHans Petter Selasky 	err = mlx4_ib_umem_write_mtt(dev, &buf->mtt, *umem);
15697549c34SHans Petter Selasky 	if (err)
15797549c34SHans Petter Selasky 		goto err_mtt;
15897549c34SHans Petter Selasky 
15997549c34SHans Petter Selasky 	return 0;
16097549c34SHans Petter Selasky 
16197549c34SHans Petter Selasky err_mtt:
16297549c34SHans Petter Selasky 	mlx4_mtt_cleanup(dev->dev, &buf->mtt);
16397549c34SHans Petter Selasky 
16497549c34SHans Petter Selasky err_buf:
16597549c34SHans Petter Selasky 	ib_umem_release(*umem);
16697549c34SHans Petter Selasky 
16797549c34SHans Petter Selasky 	return err;
16897549c34SHans Petter Selasky }
16997549c34SHans Petter Selasky 
1708cc48704SHans Petter Selasky #define CQ_CREATE_FLAGS_SUPPORTED IB_CQ_FLAGS_TIMESTAMP_COMPLETION
17197549c34SHans Petter Selasky struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
1728cc48704SHans Petter Selasky 				const struct ib_cq_init_attr *attr,
17397549c34SHans Petter Selasky 				struct ib_ucontext *context,
17497549c34SHans Petter Selasky 				struct ib_udata *udata)
17597549c34SHans Petter Selasky {
1768cc48704SHans Petter Selasky 	int entries = attr->cqe;
1778cc48704SHans Petter Selasky 	int vector = attr->comp_vector;
17897549c34SHans Petter Selasky 	struct mlx4_ib_dev *dev = to_mdev(ibdev);
17997549c34SHans Petter Selasky 	struct mlx4_ib_cq *cq;
18097549c34SHans Petter Selasky 	struct mlx4_uar *uar;
18197549c34SHans Petter Selasky 	int err;
18297549c34SHans Petter Selasky 
18397549c34SHans Petter Selasky 	if (entries < 1 || entries > dev->dev->caps.max_cqes)
18497549c34SHans Petter Selasky 		return ERR_PTR(-EINVAL);
18597549c34SHans Petter Selasky 
18697549c34SHans Petter Selasky 	if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED)
18797549c34SHans Petter Selasky 		return ERR_PTR(-EINVAL);
18897549c34SHans Petter Selasky 
1898cc48704SHans Petter Selasky 	cq = kmalloc(sizeof *cq, GFP_KERNEL);
19097549c34SHans Petter Selasky 	if (!cq)
19197549c34SHans Petter Selasky 		return ERR_PTR(-ENOMEM);
19297549c34SHans Petter Selasky 
19397549c34SHans Petter Selasky 	entries      = roundup_pow_of_two(entries + 1);
19497549c34SHans Petter Selasky 	cq->ibcq.cqe = entries - 1;
19597549c34SHans Petter Selasky 	mutex_init(&cq->resize_mutex);
19697549c34SHans Petter Selasky 	spin_lock_init(&cq->lock);
19797549c34SHans Petter Selasky 	cq->resize_buf = NULL;
19897549c34SHans Petter Selasky 	cq->resize_umem = NULL;
19997549c34SHans Petter Selasky 	cq->create_flags = attr->flags;
2008cc48704SHans Petter Selasky 	INIT_LIST_HEAD(&cq->send_qp_list);
2018cc48704SHans Petter Selasky 	INIT_LIST_HEAD(&cq->recv_qp_list);
20297549c34SHans Petter Selasky 
20397549c34SHans Petter Selasky 	if (context) {
20497549c34SHans Petter Selasky 		struct mlx4_ib_create_cq ucmd;
20597549c34SHans Petter Selasky 
20697549c34SHans Petter Selasky 		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
20797549c34SHans Petter Selasky 			err = -EFAULT;
20897549c34SHans Petter Selasky 			goto err_cq;
20997549c34SHans Petter Selasky 		}
21097549c34SHans Petter Selasky 
21197549c34SHans Petter Selasky 		err = mlx4_ib_get_cq_umem(dev, context, &cq->buf, &cq->umem,
21297549c34SHans Petter Selasky 					  ucmd.buf_addr, entries);
21397549c34SHans Petter Selasky 		if (err)
21497549c34SHans Petter Selasky 			goto err_cq;
21597549c34SHans Petter Selasky 
21697549c34SHans Petter Selasky 		err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr,
21797549c34SHans Petter Selasky 					  &cq->db);
21897549c34SHans Petter Selasky 		if (err)
21997549c34SHans Petter Selasky 			goto err_mtt;
22097549c34SHans Petter Selasky 
22197549c34SHans Petter Selasky 		uar = &to_mucontext(context)->uar;
22297549c34SHans Petter Selasky 	} else {
2238cc48704SHans Petter Selasky 		err = mlx4_db_alloc(dev->dev, &cq->db, 1, GFP_KERNEL);
22497549c34SHans Petter Selasky 		if (err)
22597549c34SHans Petter Selasky 			goto err_cq;
22697549c34SHans Petter Selasky 
22797549c34SHans Petter Selasky 		cq->mcq.set_ci_db  = cq->db.db;
22897549c34SHans Petter Selasky 		cq->mcq.arm_db     = cq->db.db + 1;
22997549c34SHans Petter Selasky 		*cq->mcq.set_ci_db = 0;
23097549c34SHans Petter Selasky 		*cq->mcq.arm_db    = 0;
23197549c34SHans Petter Selasky 
23297549c34SHans Petter Selasky 		err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries);
23397549c34SHans Petter Selasky 		if (err)
23497549c34SHans Petter Selasky 			goto err_db;
23597549c34SHans Petter Selasky 
23697549c34SHans Petter Selasky 		uar = &dev->priv_uar;
23797549c34SHans Petter Selasky 	}
23897549c34SHans Petter Selasky 
23997549c34SHans Petter Selasky 	if (dev->eq_table)
24097549c34SHans Petter Selasky 		vector = dev->eq_table[vector % ibdev->num_comp_vectors];
24197549c34SHans Petter Selasky 
24297549c34SHans Petter Selasky 	err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar,
24397549c34SHans Petter Selasky 			    cq->db.dma, &cq->mcq, vector, 0,
2448cc48704SHans Petter Selasky 			    !!(cq->create_flags & IB_CQ_FLAGS_TIMESTAMP_COMPLETION));
24597549c34SHans Petter Selasky 	if (err)
24697549c34SHans Petter Selasky 		goto err_dbmap;
24797549c34SHans Petter Selasky 
24897549c34SHans Petter Selasky 	cq->mcq.comp = mlx4_ib_cq_comp;
24997549c34SHans Petter Selasky 	cq->mcq.event = mlx4_ib_cq_event;
25097549c34SHans Petter Selasky 
25197549c34SHans Petter Selasky 	if (context)
25297549c34SHans Petter Selasky 		if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {
25397549c34SHans Petter Selasky 			err = -EFAULT;
2548cc48704SHans Petter Selasky 			goto err_cq_free;
25597549c34SHans Petter Selasky 		}
25697549c34SHans Petter Selasky 
25797549c34SHans Petter Selasky 	return &cq->ibcq;
25897549c34SHans Petter Selasky 
2598cc48704SHans Petter Selasky err_cq_free:
2608cc48704SHans Petter Selasky 	mlx4_cq_free(dev->dev, &cq->mcq);
2618cc48704SHans Petter Selasky 
26297549c34SHans Petter Selasky err_dbmap:
26397549c34SHans Petter Selasky 	if (context)
26497549c34SHans Petter Selasky 		mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db);
26597549c34SHans Petter Selasky 
26697549c34SHans Petter Selasky err_mtt:
26797549c34SHans Petter Selasky 	mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);
26897549c34SHans Petter Selasky 
26997549c34SHans Petter Selasky 	if (context)
27097549c34SHans Petter Selasky 		ib_umem_release(cq->umem);
27197549c34SHans Petter Selasky 	else
27297549c34SHans Petter Selasky 		mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
27397549c34SHans Petter Selasky 
27497549c34SHans Petter Selasky err_db:
27597549c34SHans Petter Selasky 	if (!context)
27697549c34SHans Petter Selasky 		mlx4_db_free(dev->dev, &cq->db);
27797549c34SHans Petter Selasky 
27897549c34SHans Petter Selasky err_cq:
27997549c34SHans Petter Selasky 	kfree(cq);
28097549c34SHans Petter Selasky 
28197549c34SHans Petter Selasky 	return ERR_PTR(err);
28297549c34SHans Petter Selasky }
28397549c34SHans Petter Selasky 
28497549c34SHans Petter Selasky static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,
28597549c34SHans Petter Selasky 				  int entries)
28697549c34SHans Petter Selasky {
28797549c34SHans Petter Selasky 	int err;
28897549c34SHans Petter Selasky 
28997549c34SHans Petter Selasky 	if (cq->resize_buf)
29097549c34SHans Petter Selasky 		return -EBUSY;
29197549c34SHans Petter Selasky 
2928cc48704SHans Petter Selasky 	cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL);
29397549c34SHans Petter Selasky 	if (!cq->resize_buf)
29497549c34SHans Petter Selasky 		return -ENOMEM;
29597549c34SHans Petter Selasky 
29697549c34SHans Petter Selasky 	err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries);
29797549c34SHans Petter Selasky 	if (err) {
29897549c34SHans Petter Selasky 		kfree(cq->resize_buf);
29997549c34SHans Petter Selasky 		cq->resize_buf = NULL;
30097549c34SHans Petter Selasky 		return err;
30197549c34SHans Petter Selasky 	}
30297549c34SHans Petter Selasky 
30397549c34SHans Petter Selasky 	cq->resize_buf->cqe = entries - 1;
30497549c34SHans Petter Selasky 
30597549c34SHans Petter Selasky 	return 0;
30697549c34SHans Petter Selasky }
30797549c34SHans Petter Selasky 
30897549c34SHans Petter Selasky static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,
30997549c34SHans Petter Selasky 				   int entries, struct ib_udata *udata)
31097549c34SHans Petter Selasky {
31197549c34SHans Petter Selasky 	struct mlx4_ib_resize_cq ucmd;
31297549c34SHans Petter Selasky 	int err;
31397549c34SHans Petter Selasky 
31497549c34SHans Petter Selasky 	if (cq->resize_umem)
31597549c34SHans Petter Selasky 		return -EBUSY;
31697549c34SHans Petter Selasky 
31797549c34SHans Petter Selasky 	if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
31897549c34SHans Petter Selasky 		return -EFAULT;
31997549c34SHans Petter Selasky 
3208cc48704SHans Petter Selasky 	cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_KERNEL);
32197549c34SHans Petter Selasky 	if (!cq->resize_buf)
32297549c34SHans Petter Selasky 		return -ENOMEM;
32397549c34SHans Petter Selasky 
32497549c34SHans Petter Selasky 	err = mlx4_ib_get_cq_umem(dev, cq->umem->context, &cq->resize_buf->buf,
32597549c34SHans Petter Selasky 				  &cq->resize_umem, ucmd.buf_addr, entries);
32697549c34SHans Petter Selasky 	if (err) {
32797549c34SHans Petter Selasky 		kfree(cq->resize_buf);
32897549c34SHans Petter Selasky 		cq->resize_buf = NULL;
32997549c34SHans Petter Selasky 		return err;
33097549c34SHans Petter Selasky 	}
33197549c34SHans Petter Selasky 
33297549c34SHans Petter Selasky 	cq->resize_buf->cqe = entries - 1;
33397549c34SHans Petter Selasky 
33497549c34SHans Petter Selasky 	return 0;
33597549c34SHans Petter Selasky }
33697549c34SHans Petter Selasky 
33797549c34SHans Petter Selasky static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq)
33897549c34SHans Petter Selasky {
33997549c34SHans Petter Selasky 	u32 i;
34097549c34SHans Petter Selasky 
34197549c34SHans Petter Selasky 	i = cq->mcq.cons_index;
34297549c34SHans Petter Selasky 	while (get_sw_cqe(cq, i))
34397549c34SHans Petter Selasky 		++i;
34497549c34SHans Petter Selasky 
34597549c34SHans Petter Selasky 	return i - cq->mcq.cons_index;
34697549c34SHans Petter Selasky }
34797549c34SHans Petter Selasky 
3488cc48704SHans Petter Selasky static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq)
34997549c34SHans Petter Selasky {
35097549c34SHans Petter Selasky 	struct mlx4_cqe *cqe, *new_cqe;
35197549c34SHans Petter Selasky 	int i;
35297549c34SHans Petter Selasky 	int cqe_size = cq->buf.entry_size;
35397549c34SHans Petter Selasky 	int cqe_inc = cqe_size == 64 ? 1 : 0;
35497549c34SHans Petter Selasky 
35597549c34SHans Petter Selasky 	i = cq->mcq.cons_index;
35697549c34SHans Petter Selasky 	cqe = get_cqe(cq, i & cq->ibcq.cqe);
35797549c34SHans Petter Selasky 	cqe += cqe_inc;
35897549c34SHans Petter Selasky 
35997549c34SHans Petter Selasky 	while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) {
36097549c34SHans Petter Selasky 		new_cqe = get_cqe_from_buf(&cq->resize_buf->buf,
36197549c34SHans Petter Selasky 					   (i + 1) & cq->resize_buf->cqe);
36297549c34SHans Petter Selasky 		memcpy(new_cqe, get_cqe(cq, i & cq->ibcq.cqe), cqe_size);
36397549c34SHans Petter Selasky 		new_cqe += cqe_inc;
36497549c34SHans Petter Selasky 
36597549c34SHans Petter Selasky 		new_cqe->owner_sr_opcode = (cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK) |
36697549c34SHans Petter Selasky 			(((i + 1) & (cq->resize_buf->cqe + 1)) ? MLX4_CQE_OWNER_MASK : 0);
36797549c34SHans Petter Selasky 		cqe = get_cqe(cq, ++i & cq->ibcq.cqe);
36897549c34SHans Petter Selasky 		cqe += cqe_inc;
36997549c34SHans Petter Selasky 	}
37097549c34SHans Petter Selasky 	++cq->mcq.cons_index;
37197549c34SHans Petter Selasky }
37297549c34SHans Petter Selasky 
37397549c34SHans Petter Selasky int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
37497549c34SHans Petter Selasky {
37597549c34SHans Petter Selasky 	struct mlx4_ib_dev *dev = to_mdev(ibcq->device);
37697549c34SHans Petter Selasky 	struct mlx4_ib_cq *cq = to_mcq(ibcq);
37797549c34SHans Petter Selasky 	struct mlx4_mtt mtt;
37897549c34SHans Petter Selasky 	int outst_cqe;
37997549c34SHans Petter Selasky 	int err;
38097549c34SHans Petter Selasky 
38197549c34SHans Petter Selasky 	mutex_lock(&cq->resize_mutex);
38297549c34SHans Petter Selasky 	if (entries < 1 || entries > dev->dev->caps.max_cqes) {
38397549c34SHans Petter Selasky 		err = -EINVAL;
38497549c34SHans Petter Selasky 		goto out;
38597549c34SHans Petter Selasky 	}
38697549c34SHans Petter Selasky 
38797549c34SHans Petter Selasky 	entries = roundup_pow_of_two(entries + 1);
38897549c34SHans Petter Selasky 	if (entries == ibcq->cqe + 1) {
38997549c34SHans Petter Selasky 		err = 0;
39097549c34SHans Petter Selasky 		goto out;
39197549c34SHans Petter Selasky 	}
39297549c34SHans Petter Selasky 
39397549c34SHans Petter Selasky 	if (entries > dev->dev->caps.max_cqes + 1) {
39497549c34SHans Petter Selasky 		err = -EINVAL;
39597549c34SHans Petter Selasky 		goto out;
39697549c34SHans Petter Selasky 	}
39797549c34SHans Petter Selasky 
39897549c34SHans Petter Selasky 	if (ibcq->uobject) {
39997549c34SHans Petter Selasky 		err = mlx4_alloc_resize_umem(dev, cq, entries, udata);
40097549c34SHans Petter Selasky 		if (err)
40197549c34SHans Petter Selasky 			goto out;
40297549c34SHans Petter Selasky 	} else {
40397549c34SHans Petter Selasky 		/* Can't be smaller than the number of outstanding CQEs */
40497549c34SHans Petter Selasky 		outst_cqe = mlx4_ib_get_outstanding_cqes(cq);
40597549c34SHans Petter Selasky 		if (entries < outst_cqe + 1) {
4068cc48704SHans Petter Selasky 			err = -EINVAL;
40797549c34SHans Petter Selasky 			goto out;
40897549c34SHans Petter Selasky 		}
40997549c34SHans Petter Selasky 
41097549c34SHans Petter Selasky 		err = mlx4_alloc_resize_buf(dev, cq, entries);
41197549c34SHans Petter Selasky 		if (err)
41297549c34SHans Petter Selasky 			goto out;
41397549c34SHans Petter Selasky 	}
41497549c34SHans Petter Selasky 
41597549c34SHans Petter Selasky 	mtt = cq->buf.mtt;
41697549c34SHans Petter Selasky 
41797549c34SHans Petter Selasky 	err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt);
41897549c34SHans Petter Selasky 	if (err)
41997549c34SHans Petter Selasky 		goto err_buf;
42097549c34SHans Petter Selasky 
42197549c34SHans Petter Selasky 	mlx4_mtt_cleanup(dev->dev, &mtt);
42297549c34SHans Petter Selasky 	if (ibcq->uobject) {
42397549c34SHans Petter Selasky 		cq->buf      = cq->resize_buf->buf;
42497549c34SHans Petter Selasky 		cq->ibcq.cqe = cq->resize_buf->cqe;
42597549c34SHans Petter Selasky 		ib_umem_release(cq->umem);
42697549c34SHans Petter Selasky 		cq->umem     = cq->resize_umem;
42797549c34SHans Petter Selasky 
42897549c34SHans Petter Selasky 		kfree(cq->resize_buf);
42997549c34SHans Petter Selasky 		cq->resize_buf = NULL;
43097549c34SHans Petter Selasky 		cq->resize_umem = NULL;
43197549c34SHans Petter Selasky 	} else {
43297549c34SHans Petter Selasky 		struct mlx4_ib_cq_buf tmp_buf;
43397549c34SHans Petter Selasky 		int tmp_cqe = 0;
43497549c34SHans Petter Selasky 
43597549c34SHans Petter Selasky 		spin_lock_irq(&cq->lock);
43697549c34SHans Petter Selasky 		if (cq->resize_buf) {
4378cc48704SHans Petter Selasky 			mlx4_ib_cq_resize_copy_cqes(cq);
43897549c34SHans Petter Selasky 			tmp_buf = cq->buf;
43997549c34SHans Petter Selasky 			tmp_cqe = cq->ibcq.cqe;
44097549c34SHans Petter Selasky 			cq->buf      = cq->resize_buf->buf;
44197549c34SHans Petter Selasky 			cq->ibcq.cqe = cq->resize_buf->cqe;
44297549c34SHans Petter Selasky 
44397549c34SHans Petter Selasky 			kfree(cq->resize_buf);
44497549c34SHans Petter Selasky 			cq->resize_buf = NULL;
44597549c34SHans Petter Selasky 		}
44697549c34SHans Petter Selasky 		spin_unlock_irq(&cq->lock);
44797549c34SHans Petter Selasky 
44897549c34SHans Petter Selasky 		if (tmp_cqe)
44997549c34SHans Petter Selasky 			mlx4_ib_free_cq_buf(dev, &tmp_buf, tmp_cqe);
45097549c34SHans Petter Selasky 	}
45197549c34SHans Petter Selasky 
45297549c34SHans Petter Selasky 	goto out;
45397549c34SHans Petter Selasky 
45497549c34SHans Petter Selasky err_buf:
45597549c34SHans Petter Selasky 	mlx4_mtt_cleanup(dev->dev, &cq->resize_buf->buf.mtt);
45697549c34SHans Petter Selasky 	if (!ibcq->uobject)
45797549c34SHans Petter Selasky 		mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf,
45897549c34SHans Petter Selasky 				    cq->resize_buf->cqe);
45997549c34SHans Petter Selasky 
46097549c34SHans Petter Selasky 	kfree(cq->resize_buf);
46197549c34SHans Petter Selasky 	cq->resize_buf = NULL;
46297549c34SHans Petter Selasky 
46397549c34SHans Petter Selasky 	if (cq->resize_umem) {
46497549c34SHans Petter Selasky 		ib_umem_release(cq->resize_umem);
46597549c34SHans Petter Selasky 		cq->resize_umem = NULL;
46697549c34SHans Petter Selasky 	}
46797549c34SHans Petter Selasky 
46897549c34SHans Petter Selasky out:
46997549c34SHans Petter Selasky 	mutex_unlock(&cq->resize_mutex);
47097549c34SHans Petter Selasky 
47197549c34SHans Petter Selasky 	return err;
47297549c34SHans Petter Selasky }
47397549c34SHans Petter Selasky 
47497549c34SHans Petter Selasky int mlx4_ib_destroy_cq(struct ib_cq *cq)
47597549c34SHans Petter Selasky {
47697549c34SHans Petter Selasky 	struct mlx4_ib_dev *dev = to_mdev(cq->device);
47797549c34SHans Petter Selasky 	struct mlx4_ib_cq *mcq = to_mcq(cq);
47897549c34SHans Petter Selasky 
47997549c34SHans Petter Selasky 	mlx4_cq_free(dev->dev, &mcq->mcq);
48097549c34SHans Petter Selasky 	mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt);
48197549c34SHans Petter Selasky 
48297549c34SHans Petter Selasky 	if (cq->uobject) {
48397549c34SHans Petter Selasky 		mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db);
48497549c34SHans Petter Selasky 		ib_umem_release(mcq->umem);
48597549c34SHans Petter Selasky 	} else {
48697549c34SHans Petter Selasky 		mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe);
48797549c34SHans Petter Selasky 		mlx4_db_free(dev->dev, &mcq->db);
48897549c34SHans Petter Selasky 	}
48997549c34SHans Petter Selasky 
49097549c34SHans Petter Selasky 	kfree(mcq);
49197549c34SHans Petter Selasky 
49297549c34SHans Petter Selasky 	return 0;
49397549c34SHans Petter Selasky }
49497549c34SHans Petter Selasky 
49597549c34SHans Petter Selasky static void dump_cqe(void *cqe)
49697549c34SHans Petter Selasky {
49797549c34SHans Petter Selasky 	__be32 *buf = cqe;
49897549c34SHans Petter Selasky 
49997549c34SHans Petter Selasky 	pr_debug("CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n",
50097549c34SHans Petter Selasky 	       be32_to_cpu(buf[0]), be32_to_cpu(buf[1]), be32_to_cpu(buf[2]),
50197549c34SHans Petter Selasky 	       be32_to_cpu(buf[3]), be32_to_cpu(buf[4]), be32_to_cpu(buf[5]),
50297549c34SHans Petter Selasky 	       be32_to_cpu(buf[6]), be32_to_cpu(buf[7]));
50397549c34SHans Petter Selasky }
50497549c34SHans Petter Selasky 
50597549c34SHans Petter Selasky static void mlx4_ib_handle_error_cqe(struct mlx4_err_cqe *cqe,
50697549c34SHans Petter Selasky 				     struct ib_wc *wc)
50797549c34SHans Petter Selasky {
50897549c34SHans Petter Selasky 	if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR) {
50997549c34SHans Petter Selasky 		pr_debug("local QP operation err "
51097549c34SHans Petter Selasky 		       "(QPN %06x, WQE index %x, vendor syndrome %02x, "
51197549c34SHans Petter Selasky 		       "opcode = %02x)\n",
51297549c34SHans Petter Selasky 		       be32_to_cpu(cqe->my_qpn), be16_to_cpu(cqe->wqe_index),
51397549c34SHans Petter Selasky 		       cqe->vendor_err_syndrome,
51497549c34SHans Petter Selasky 		       cqe->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
51597549c34SHans Petter Selasky 		dump_cqe(cqe);
51697549c34SHans Petter Selasky 	}
51797549c34SHans Petter Selasky 
51897549c34SHans Petter Selasky 	switch (cqe->syndrome) {
51997549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:
52097549c34SHans Petter Selasky 		wc->status = IB_WC_LOC_LEN_ERR;
52197549c34SHans Petter Selasky 		break;
52297549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:
52397549c34SHans Petter Selasky 		wc->status = IB_WC_LOC_QP_OP_ERR;
52497549c34SHans Petter Selasky 		break;
52597549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:
52697549c34SHans Petter Selasky 		wc->status = IB_WC_LOC_PROT_ERR;
52797549c34SHans Petter Selasky 		break;
52897549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:
52997549c34SHans Petter Selasky 		wc->status = IB_WC_WR_FLUSH_ERR;
53097549c34SHans Petter Selasky 		break;
53197549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_MW_BIND_ERR:
53297549c34SHans Petter Selasky 		wc->status = IB_WC_MW_BIND_ERR;
53397549c34SHans Petter Selasky 		break;
53497549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_BAD_RESP_ERR:
53597549c34SHans Petter Selasky 		wc->status = IB_WC_BAD_RESP_ERR;
53697549c34SHans Petter Selasky 		break;
53797549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:
53897549c34SHans Petter Selasky 		wc->status = IB_WC_LOC_ACCESS_ERR;
53997549c34SHans Petter Selasky 		break;
54097549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
54197549c34SHans Petter Selasky 		wc->status = IB_WC_REM_INV_REQ_ERR;
54297549c34SHans Petter Selasky 		break;
54397549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:
54497549c34SHans Petter Selasky 		wc->status = IB_WC_REM_ACCESS_ERR;
54597549c34SHans Petter Selasky 		break;
54697549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:
54797549c34SHans Petter Selasky 		wc->status = IB_WC_REM_OP_ERR;
54897549c34SHans Petter Selasky 		break;
54997549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
55097549c34SHans Petter Selasky 		wc->status = IB_WC_RETRY_EXC_ERR;
55197549c34SHans Petter Selasky 		break;
55297549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
55397549c34SHans Petter Selasky 		wc->status = IB_WC_RNR_RETRY_EXC_ERR;
55497549c34SHans Petter Selasky 		break;
55597549c34SHans Petter Selasky 	case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:
55697549c34SHans Petter Selasky 		wc->status = IB_WC_REM_ABORT_ERR;
55797549c34SHans Petter Selasky 		break;
55897549c34SHans Petter Selasky 	default:
55997549c34SHans Petter Selasky 		wc->status = IB_WC_GENERAL_ERR;
56097549c34SHans Petter Selasky 		break;
56197549c34SHans Petter Selasky 	}
56297549c34SHans Petter Selasky 
56397549c34SHans Petter Selasky 	wc->vendor_err = cqe->vendor_err_syndrome;
56497549c34SHans Petter Selasky }
56597549c34SHans Petter Selasky 
56697549c34SHans Petter Selasky static int mlx4_ib_ipoib_csum_ok(__be16 status, __be16 checksum)
56797549c34SHans Petter Selasky {
56897549c34SHans Petter Selasky 	return ((status & cpu_to_be16(MLX4_CQE_STATUS_IPV4      |
56997549c34SHans Petter Selasky 				      MLX4_CQE_STATUS_IPV4F     |
57097549c34SHans Petter Selasky 				      MLX4_CQE_STATUS_IPV4OPT   |
57197549c34SHans Petter Selasky 				      MLX4_CQE_STATUS_IPV6      |
57297549c34SHans Petter Selasky 				      MLX4_CQE_STATUS_IPOK)) ==
57397549c34SHans Petter Selasky 		cpu_to_be16(MLX4_CQE_STATUS_IPV4        |
57497549c34SHans Petter Selasky 			    MLX4_CQE_STATUS_IPOK))              &&
57597549c34SHans Petter Selasky 		(status & cpu_to_be16(MLX4_CQE_STATUS_UDP       |
57697549c34SHans Petter Selasky 				      MLX4_CQE_STATUS_TCP))     &&
57797549c34SHans Petter Selasky 		checksum == cpu_to_be16(0xffff);
57897549c34SHans Petter Selasky }
57997549c34SHans Petter Selasky 
5808cc48704SHans Petter Selasky static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct ib_wc *wc,
58197549c34SHans Petter Selasky 			    unsigned tail, struct mlx4_cqe *cqe, int is_eth)
58297549c34SHans Petter Selasky {
58397549c34SHans Petter Selasky 	struct mlx4_ib_proxy_sqp_hdr *hdr;
58497549c34SHans Petter Selasky 
58597549c34SHans Petter Selasky 	ib_dma_sync_single_for_cpu(qp->ibqp.device,
58697549c34SHans Petter Selasky 				   qp->sqp_proxy_rcv[tail].map,
58797549c34SHans Petter Selasky 				   sizeof (struct mlx4_ib_proxy_sqp_hdr),
58897549c34SHans Petter Selasky 				   DMA_FROM_DEVICE);
58997549c34SHans Petter Selasky 	hdr = (struct mlx4_ib_proxy_sqp_hdr *) (qp->sqp_proxy_rcv[tail].addr);
59097549c34SHans Petter Selasky 	wc->pkey_index	= be16_to_cpu(hdr->tun.pkey_index);
59197549c34SHans Petter Selasky 	wc->src_qp	= be32_to_cpu(hdr->tun.flags_src_qp) & 0xFFFFFF;
59297549c34SHans Petter Selasky 	wc->wc_flags   |= (hdr->tun.g_ml_path & 0x80) ? (IB_WC_GRH) : 0;
59397549c34SHans Petter Selasky 	wc->dlid_path_bits = 0;
59497549c34SHans Petter Selasky 
59597549c34SHans Petter Selasky 	if (is_eth) {
59697549c34SHans Petter Selasky 		wc->vlan_id = be16_to_cpu(hdr->tun.sl_vid);
59797549c34SHans Petter Selasky 		memcpy(&(wc->smac[0]), (char *)&hdr->tun.mac_31_0, 4);
59897549c34SHans Petter Selasky 		memcpy(&(wc->smac[4]), (char *)&hdr->tun.slid_mac_47_32, 2);
5998cc48704SHans Petter Selasky 		wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
60097549c34SHans Petter Selasky 	} else {
60197549c34SHans Petter Selasky 		wc->slid        = be16_to_cpu(hdr->tun.slid_mac_47_32);
60297549c34SHans Petter Selasky 		wc->sl          = (u8) (be16_to_cpu(hdr->tun.sl_vid) >> 12);
60397549c34SHans Petter Selasky 	}
6048cc48704SHans Petter Selasky }
60597549c34SHans Petter Selasky 
6068cc48704SHans Petter Selasky static void mlx4_ib_qp_sw_comp(struct mlx4_ib_qp *qp, int num_entries,
6078cc48704SHans Petter Selasky 			       struct ib_wc *wc, int *npolled, int is_send)
6088cc48704SHans Petter Selasky {
6098cc48704SHans Petter Selasky 	struct mlx4_ib_wq *wq;
6108cc48704SHans Petter Selasky 	unsigned cur;
6118cc48704SHans Petter Selasky 	int i;
6128cc48704SHans Petter Selasky 
6138cc48704SHans Petter Selasky 	wq = is_send ? &qp->sq : &qp->rq;
6148cc48704SHans Petter Selasky 	cur = wq->head - wq->tail;
6158cc48704SHans Petter Selasky 
6168cc48704SHans Petter Selasky 	if (cur == 0)
6178cc48704SHans Petter Selasky 		return;
6188cc48704SHans Petter Selasky 
6198cc48704SHans Petter Selasky 	for (i = 0;  i < cur && *npolled < num_entries; i++) {
6208cc48704SHans Petter Selasky 		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
6218cc48704SHans Petter Selasky 		wc->status = IB_WC_WR_FLUSH_ERR;
6228cc48704SHans Petter Selasky 		wc->vendor_err = MLX4_CQE_SYNDROME_WR_FLUSH_ERR;
6238cc48704SHans Petter Selasky 		wq->tail++;
6248cc48704SHans Petter Selasky 		(*npolled)++;
6258cc48704SHans Petter Selasky 		wc->qp = &qp->ibqp;
6268cc48704SHans Petter Selasky 		wc++;
6278cc48704SHans Petter Selasky 	}
6288cc48704SHans Petter Selasky }
6298cc48704SHans Petter Selasky 
6308cc48704SHans Petter Selasky static void mlx4_ib_poll_sw_comp(struct mlx4_ib_cq *cq, int num_entries,
6318cc48704SHans Petter Selasky 				 struct ib_wc *wc, int *npolled)
6328cc48704SHans Petter Selasky {
6338cc48704SHans Petter Selasky 	struct mlx4_ib_qp *qp;
6348cc48704SHans Petter Selasky 
6358cc48704SHans Petter Selasky 	*npolled = 0;
6368cc48704SHans Petter Selasky 	/* Find uncompleted WQEs belonging to that cq and retrun
6378cc48704SHans Petter Selasky 	 * simulated FLUSH_ERR completions
6388cc48704SHans Petter Selasky 	 */
6398cc48704SHans Petter Selasky 	list_for_each_entry(qp, &cq->send_qp_list, cq_send_list) {
6408cc48704SHans Petter Selasky 		mlx4_ib_qp_sw_comp(qp, num_entries, wc + *npolled, npolled, 1);
6418cc48704SHans Petter Selasky 		if (*npolled >= num_entries)
6428cc48704SHans Petter Selasky 			goto out;
6438cc48704SHans Petter Selasky 	}
6448cc48704SHans Petter Selasky 
6458cc48704SHans Petter Selasky 	list_for_each_entry(qp, &cq->recv_qp_list, cq_recv_list) {
6468cc48704SHans Petter Selasky 		mlx4_ib_qp_sw_comp(qp, num_entries, wc + *npolled, npolled, 0);
6478cc48704SHans Petter Selasky 		if (*npolled >= num_entries)
6488cc48704SHans Petter Selasky 			goto out;
6498cc48704SHans Petter Selasky 	}
6508cc48704SHans Petter Selasky 
6518cc48704SHans Petter Selasky out:
6528cc48704SHans Petter Selasky 	return;
65397549c34SHans Petter Selasky }
65497549c34SHans Petter Selasky 
65597549c34SHans Petter Selasky static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,
65697549c34SHans Petter Selasky 			    struct mlx4_ib_qp **cur_qp,
65797549c34SHans Petter Selasky 			    struct ib_wc *wc)
65897549c34SHans Petter Selasky {
65997549c34SHans Petter Selasky 	struct mlx4_cqe *cqe;
66097549c34SHans Petter Selasky 	struct mlx4_qp *mqp;
66197549c34SHans Petter Selasky 	struct mlx4_ib_wq *wq;
66297549c34SHans Petter Selasky 	struct mlx4_ib_srq *srq;
66397549c34SHans Petter Selasky 	struct mlx4_srq *msrq = NULL;
66497549c34SHans Petter Selasky 	int is_send;
66597549c34SHans Petter Selasky 	int is_error;
6668cc48704SHans Petter Selasky 	int is_eth;
66797549c34SHans Petter Selasky 	u32 g_mlpath_rqpn;
66897549c34SHans Petter Selasky 	u16 wqe_ctr;
66997549c34SHans Petter Selasky 	unsigned tail = 0;
67097549c34SHans Petter Selasky 
67197549c34SHans Petter Selasky repoll:
67297549c34SHans Petter Selasky 	cqe = next_cqe_sw(cq);
67397549c34SHans Petter Selasky 	if (!cqe)
67497549c34SHans Petter Selasky 		return -EAGAIN;
67597549c34SHans Petter Selasky 
67697549c34SHans Petter Selasky 	if (cq->buf.entry_size == 64)
67797549c34SHans Petter Selasky 		cqe++;
67897549c34SHans Petter Selasky 
67997549c34SHans Petter Selasky 	++cq->mcq.cons_index;
68097549c34SHans Petter Selasky 
68197549c34SHans Petter Selasky 	/*
68297549c34SHans Petter Selasky 	 * Make sure we read CQ entry contents after we've checked the
68397549c34SHans Petter Selasky 	 * ownership bit.
68497549c34SHans Petter Selasky 	 */
68597549c34SHans Petter Selasky 	rmb();
68697549c34SHans Petter Selasky 
68797549c34SHans Petter Selasky 	is_send  = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;
68897549c34SHans Petter Selasky 	is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
68997549c34SHans Petter Selasky 		MLX4_CQE_OPCODE_ERROR;
69097549c34SHans Petter Selasky 
69197549c34SHans Petter Selasky 	/* Resize CQ in progress */
69297549c34SHans Petter Selasky 	if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_RESIZE)) {
69397549c34SHans Petter Selasky 		if (cq->resize_buf) {
69497549c34SHans Petter Selasky 			struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device);
69597549c34SHans Petter Selasky 
69697549c34SHans Petter Selasky 			mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
69797549c34SHans Petter Selasky 			cq->buf      = cq->resize_buf->buf;
69897549c34SHans Petter Selasky 			cq->ibcq.cqe = cq->resize_buf->cqe;
69997549c34SHans Petter Selasky 
70097549c34SHans Petter Selasky 			kfree(cq->resize_buf);
70197549c34SHans Petter Selasky 			cq->resize_buf = NULL;
70297549c34SHans Petter Selasky 		}
70397549c34SHans Petter Selasky 
70497549c34SHans Petter Selasky 		goto repoll;
70597549c34SHans Petter Selasky 	}
70697549c34SHans Petter Selasky 
70797549c34SHans Petter Selasky 	if (!*cur_qp ||
70897549c34SHans Petter Selasky 	    (be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) != (*cur_qp)->mqp.qpn) {
70997549c34SHans Petter Selasky 		/*
71097549c34SHans Petter Selasky 		 * We do not have to take the QP table lock here,
71197549c34SHans Petter Selasky 		 * because CQs will be locked while QPs are removed
71297549c34SHans Petter Selasky 		 * from the table.
71397549c34SHans Petter Selasky 		 */
71497549c34SHans Petter Selasky 		mqp = __mlx4_qp_lookup(to_mdev(cq->ibcq.device)->dev,
71597549c34SHans Petter Selasky 				       be32_to_cpu(cqe->vlan_my_qpn));
71697549c34SHans Petter Selasky 		*cur_qp = to_mibqp(mqp);
71797549c34SHans Petter Selasky 	}
71897549c34SHans Petter Selasky 
71997549c34SHans Petter Selasky 	wc->qp = &(*cur_qp)->ibqp;
72097549c34SHans Petter Selasky 
72197549c34SHans Petter Selasky 	if (wc->qp->qp_type == IB_QPT_XRC_TGT) {
72297549c34SHans Petter Selasky 		u32 srq_num;
72397549c34SHans Petter Selasky 		g_mlpath_rqpn = be32_to_cpu(cqe->g_mlpath_rqpn);
72497549c34SHans Petter Selasky 		srq_num       = g_mlpath_rqpn & 0xffffff;
72597549c34SHans Petter Selasky 		/* SRQ is also in the radix tree */
72697549c34SHans Petter Selasky 		msrq = mlx4_srq_lookup(to_mdev(cq->ibcq.device)->dev,
72797549c34SHans Petter Selasky 				       srq_num);
72897549c34SHans Petter Selasky 	}
72997549c34SHans Petter Selasky 
73097549c34SHans Petter Selasky 	if (is_send) {
73197549c34SHans Petter Selasky 		wq = &(*cur_qp)->sq;
73297549c34SHans Petter Selasky 		if (!(*cur_qp)->sq_signal_bits) {
73397549c34SHans Petter Selasky 			wqe_ctr = be16_to_cpu(cqe->wqe_index);
73497549c34SHans Petter Selasky 			wq->tail += (u16) (wqe_ctr - (u16) wq->tail);
73597549c34SHans Petter Selasky 		}
73697549c34SHans Petter Selasky 		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
73797549c34SHans Petter Selasky 		++wq->tail;
73897549c34SHans Petter Selasky 	} else if ((*cur_qp)->ibqp.srq) {
73997549c34SHans Petter Selasky 		srq = to_msrq((*cur_qp)->ibqp.srq);
74097549c34SHans Petter Selasky 		wqe_ctr = be16_to_cpu(cqe->wqe_index);
74197549c34SHans Petter Selasky 		wc->wr_id = srq->wrid[wqe_ctr];
74297549c34SHans Petter Selasky 		mlx4_ib_free_srq_wqe(srq, wqe_ctr);
74397549c34SHans Petter Selasky 	} else if (msrq) {
74497549c34SHans Petter Selasky 		srq = to_mibsrq(msrq);
74597549c34SHans Petter Selasky 		wqe_ctr = be16_to_cpu(cqe->wqe_index);
74697549c34SHans Petter Selasky 		wc->wr_id = srq->wrid[wqe_ctr];
74797549c34SHans Petter Selasky 		mlx4_ib_free_srq_wqe(srq, wqe_ctr);
74897549c34SHans Petter Selasky 	} else {
74997549c34SHans Petter Selasky 		wq	  = &(*cur_qp)->rq;
75097549c34SHans Petter Selasky 		tail	  = wq->tail & (wq->wqe_cnt - 1);
75197549c34SHans Petter Selasky 		wc->wr_id = wq->wrid[tail];
75297549c34SHans Petter Selasky 		++wq->tail;
75397549c34SHans Petter Selasky 	}
75497549c34SHans Petter Selasky 
75597549c34SHans Petter Selasky 	if (unlikely(is_error)) {
75697549c34SHans Petter Selasky 		mlx4_ib_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);
75797549c34SHans Petter Selasky 		return 0;
75897549c34SHans Petter Selasky 	}
75997549c34SHans Petter Selasky 
76097549c34SHans Petter Selasky 	wc->status = IB_WC_SUCCESS;
76197549c34SHans Petter Selasky 
76297549c34SHans Petter Selasky 	if (is_send) {
76397549c34SHans Petter Selasky 		wc->wc_flags = 0;
76497549c34SHans Petter Selasky 		switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
76597549c34SHans Petter Selasky 		case MLX4_OPCODE_RDMA_WRITE_IMM:
76697549c34SHans Petter Selasky 			wc->wc_flags |= IB_WC_WITH_IMM;
76797549c34SHans Petter Selasky 		case MLX4_OPCODE_RDMA_WRITE:
76897549c34SHans Petter Selasky 			wc->opcode    = IB_WC_RDMA_WRITE;
76997549c34SHans Petter Selasky 			break;
77097549c34SHans Petter Selasky 		case MLX4_OPCODE_SEND_IMM:
77197549c34SHans Petter Selasky 			wc->wc_flags |= IB_WC_WITH_IMM;
77297549c34SHans Petter Selasky 		case MLX4_OPCODE_SEND:
77397549c34SHans Petter Selasky 		case MLX4_OPCODE_SEND_INVAL:
77497549c34SHans Petter Selasky 			wc->opcode    = IB_WC_SEND;
77597549c34SHans Petter Selasky 			break;
77697549c34SHans Petter Selasky 		case MLX4_OPCODE_RDMA_READ:
77797549c34SHans Petter Selasky 			wc->opcode    = IB_WC_RDMA_READ;
77897549c34SHans Petter Selasky 			wc->byte_len  = be32_to_cpu(cqe->byte_cnt);
77997549c34SHans Petter Selasky 			break;
78097549c34SHans Petter Selasky 		case MLX4_OPCODE_ATOMIC_CS:
78197549c34SHans Petter Selasky 			wc->opcode    = IB_WC_COMP_SWAP;
78297549c34SHans Petter Selasky 			wc->byte_len  = 8;
78397549c34SHans Petter Selasky 			break;
78497549c34SHans Petter Selasky 		case MLX4_OPCODE_ATOMIC_FA:
78597549c34SHans Petter Selasky 			wc->opcode    = IB_WC_FETCH_ADD;
78697549c34SHans Petter Selasky 			wc->byte_len  = 8;
78797549c34SHans Petter Selasky 			break;
78897549c34SHans Petter Selasky 		case MLX4_OPCODE_MASKED_ATOMIC_CS:
78997549c34SHans Petter Selasky 			wc->opcode    = IB_WC_MASKED_COMP_SWAP;
79097549c34SHans Petter Selasky 			wc->byte_len  = 8;
79197549c34SHans Petter Selasky 			break;
79297549c34SHans Petter Selasky 		case MLX4_OPCODE_MASKED_ATOMIC_FA:
79397549c34SHans Petter Selasky 			wc->opcode    = IB_WC_MASKED_FETCH_ADD;
79497549c34SHans Petter Selasky 			wc->byte_len  = 8;
79597549c34SHans Petter Selasky 			break;
79697549c34SHans Petter Selasky 		case MLX4_OPCODE_LSO:
79797549c34SHans Petter Selasky 			wc->opcode    = IB_WC_LSO;
79897549c34SHans Petter Selasky 			break;
79997549c34SHans Petter Selasky 		case MLX4_OPCODE_FMR:
8008cc48704SHans Petter Selasky 			wc->opcode    = IB_WC_REG_MR;
80197549c34SHans Petter Selasky 			break;
80297549c34SHans Petter Selasky 		case MLX4_OPCODE_LOCAL_INVAL:
80397549c34SHans Petter Selasky 			wc->opcode    = IB_WC_LOCAL_INV;
80497549c34SHans Petter Selasky 			break;
80597549c34SHans Petter Selasky 		}
80697549c34SHans Petter Selasky 	} else {
80797549c34SHans Petter Selasky 		wc->byte_len = be32_to_cpu(cqe->byte_cnt);
80897549c34SHans Petter Selasky 
80997549c34SHans Petter Selasky 		switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
81097549c34SHans Petter Selasky 		case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
81197549c34SHans Petter Selasky 			wc->opcode	= IB_WC_RECV_RDMA_WITH_IMM;
81297549c34SHans Petter Selasky 			wc->wc_flags	= IB_WC_WITH_IMM;
81397549c34SHans Petter Selasky 			wc->ex.imm_data = cqe->immed_rss_invalid;
81497549c34SHans Petter Selasky 			break;
81597549c34SHans Petter Selasky 		case MLX4_RECV_OPCODE_SEND_INVAL:
81697549c34SHans Petter Selasky 			wc->opcode	= IB_WC_RECV;
81797549c34SHans Petter Selasky 			wc->wc_flags	= IB_WC_WITH_INVALIDATE;
81897549c34SHans Petter Selasky 			wc->ex.invalidate_rkey = be32_to_cpu(cqe->immed_rss_invalid);
81997549c34SHans Petter Selasky 			break;
82097549c34SHans Petter Selasky 		case MLX4_RECV_OPCODE_SEND:
82197549c34SHans Petter Selasky 			wc->opcode   = IB_WC_RECV;
82297549c34SHans Petter Selasky 			wc->wc_flags = 0;
82397549c34SHans Petter Selasky 			break;
82497549c34SHans Petter Selasky 		case MLX4_RECV_OPCODE_SEND_IMM:
82597549c34SHans Petter Selasky 			wc->opcode	= IB_WC_RECV;
82697549c34SHans Petter Selasky 			wc->wc_flags	= IB_WC_WITH_IMM;
82797549c34SHans Petter Selasky 			wc->ex.imm_data = cqe->immed_rss_invalid;
82897549c34SHans Petter Selasky 			break;
82997549c34SHans Petter Selasky 		}
83097549c34SHans Petter Selasky 
8318cc48704SHans Petter Selasky 		is_eth = (rdma_port_get_link_layer(wc->qp->device,
8328cc48704SHans Petter Selasky 						  (*cur_qp)->port) ==
8338cc48704SHans Petter Selasky 			  IB_LINK_LAYER_ETHERNET);
83497549c34SHans Petter Selasky 		if (mlx4_is_mfunc(to_mdev(cq->ibcq.device)->dev)) {
83597549c34SHans Petter Selasky 			if ((*cur_qp)->mlx4_ib_qp_type &
83697549c34SHans Petter Selasky 			    (MLX4_IB_QPT_PROXY_SMI_OWNER |
8378cc48704SHans Petter Selasky 			     MLX4_IB_QPT_PROXY_SMI | MLX4_IB_QPT_PROXY_GSI)) {
8388cc48704SHans Petter Selasky 				use_tunnel_data(*cur_qp, cq, wc, tail, cqe,
8398cc48704SHans Petter Selasky 						is_eth);
8408cc48704SHans Petter Selasky 				return 0;
8418cc48704SHans Petter Selasky 			}
84297549c34SHans Petter Selasky 		}
84397549c34SHans Petter Selasky 
84497549c34SHans Petter Selasky 		wc->slid	   = be16_to_cpu(cqe->rlid);
84597549c34SHans Petter Selasky 		g_mlpath_rqpn	   = be32_to_cpu(cqe->g_mlpath_rqpn);
84697549c34SHans Petter Selasky 		wc->src_qp	   = g_mlpath_rqpn & 0xffffff;
84797549c34SHans Petter Selasky 		wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f;
84897549c34SHans Petter Selasky 		wc->wc_flags	  |= g_mlpath_rqpn & 0x80000000 ? IB_WC_GRH : 0;
84997549c34SHans Petter Selasky 		wc->pkey_index     = be32_to_cpu(cqe->immed_rss_invalid) & 0x7f;
85097549c34SHans Petter Selasky 		wc->wc_flags	  |= mlx4_ib_ipoib_csum_ok(cqe->status,
85197549c34SHans Petter Selasky 					cqe->checksum) ? IB_WC_IP_CSUM_OK : 0;
8528cc48704SHans Petter Selasky 		if (is_eth) {
85397549c34SHans Petter Selasky 			wc->sl  = be16_to_cpu(cqe->sl_vid) >> 13;
8548cc48704SHans Petter Selasky 			if (be32_to_cpu(cqe->vlan_my_qpn) &
8558cc48704SHans Petter Selasky 					MLX4_CQE_CVLAN_PRESENT_MASK) {
85697549c34SHans Petter Selasky 				wc->vlan_id = be16_to_cpu(cqe->sl_vid) &
85797549c34SHans Petter Selasky 					MLX4_CQE_VID_MASK;
85897549c34SHans Petter Selasky 			} else {
85997549c34SHans Petter Selasky 				wc->vlan_id = 0xffff;
86097549c34SHans Petter Selasky 			}
8618cc48704SHans Petter Selasky 			memcpy(wc->smac, cqe->smac, ETH_ALEN);
8628cc48704SHans Petter Selasky 			wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
8638cc48704SHans Petter Selasky 		} else {
8648cc48704SHans Petter Selasky 			wc->sl  = be16_to_cpu(cqe->sl_vid) >> 12;
8658cc48704SHans Petter Selasky 			wc->vlan_id = 0xffff;
86697549c34SHans Petter Selasky 		}
86797549c34SHans Petter Selasky 	}
86897549c34SHans Petter Selasky 
86997549c34SHans Petter Selasky 	return 0;
87097549c34SHans Petter Selasky }
87197549c34SHans Petter Selasky 
87297549c34SHans Petter Selasky int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
87397549c34SHans Petter Selasky {
87497549c34SHans Petter Selasky 	struct mlx4_ib_cq *cq = to_mcq(ibcq);
87597549c34SHans Petter Selasky 	struct mlx4_ib_qp *cur_qp = NULL;
87697549c34SHans Petter Selasky 	unsigned long flags;
87797549c34SHans Petter Selasky 	int npolled;
8788cc48704SHans Petter Selasky 	struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device);
87997549c34SHans Petter Selasky 
88097549c34SHans Petter Selasky 	spin_lock_irqsave(&cq->lock, flags);
8816428c27fSHans Petter Selasky 	if (unlikely(mdev->dev->persist->state &
8826428c27fSHans Petter Selasky 		     MLX4_DEVICE_STATE_INTERNAL_ERROR)) {
8838cc48704SHans Petter Selasky 		mlx4_ib_poll_sw_comp(cq, num_entries, wc, &npolled);
8848cc48704SHans Petter Selasky 		goto out;
8858cc48704SHans Petter Selasky 	}
88697549c34SHans Petter Selasky 
88797549c34SHans Petter Selasky 	for (npolled = 0; npolled < num_entries; ++npolled) {
8888cc48704SHans Petter Selasky 		if (mlx4_ib_poll_one(cq, &cur_qp, wc + npolled))
88997549c34SHans Petter Selasky 			break;
89097549c34SHans Petter Selasky 	}
89197549c34SHans Petter Selasky 
89297549c34SHans Petter Selasky 	mlx4_cq_set_ci(&cq->mcq);
89397549c34SHans Petter Selasky 
8948cc48704SHans Petter Selasky out:
89597549c34SHans Petter Selasky 	spin_unlock_irqrestore(&cq->lock, flags);
89697549c34SHans Petter Selasky 
89797549c34SHans Petter Selasky 	return npolled;
89897549c34SHans Petter Selasky }
89997549c34SHans Petter Selasky 
90097549c34SHans Petter Selasky int mlx4_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
90197549c34SHans Petter Selasky {
9026428c27fSHans Petter Selasky 	struct mlx4_ib_cq *cq = to_mcq(ibcq);
9036428c27fSHans Petter Selasky 	struct mlx4_ib_dev *mdev = to_mdev(cq->ibcq.device);
9046428c27fSHans Petter Selasky 
9056428c27fSHans Petter Selasky 	if (unlikely(mdev->dev->persist->state &
9066428c27fSHans Petter Selasky 		     MLX4_DEVICE_STATE_INTERNAL_ERROR))
9076428c27fSHans Petter Selasky 		return -1;
9086428c27fSHans Petter Selasky 
9096428c27fSHans Petter Selasky 	mlx4_cq_arm(&cq->mcq,
91097549c34SHans Petter Selasky 		    (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
91197549c34SHans Petter Selasky 		    MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT,
9126428c27fSHans Petter Selasky 		    mdev->uar_map,
9136428c27fSHans Petter Selasky 		    MLX4_GET_DOORBELL_LOCK(&mdev->uar_lock));
91497549c34SHans Petter Selasky 
91597549c34SHans Petter Selasky 	return 0;
91697549c34SHans Petter Selasky }
91797549c34SHans Petter Selasky 
91897549c34SHans Petter Selasky void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
91997549c34SHans Petter Selasky {
92097549c34SHans Petter Selasky 	u32 prod_index;
92197549c34SHans Petter Selasky 	int nfreed = 0;
92297549c34SHans Petter Selasky 	struct mlx4_cqe *cqe, *dest;
92397549c34SHans Petter Selasky 	u8 owner_bit;
92497549c34SHans Petter Selasky 	int cqe_inc = cq->buf.entry_size == 64 ? 1 : 0;
92597549c34SHans Petter Selasky 
92697549c34SHans Petter Selasky 	/*
92797549c34SHans Petter Selasky 	 * First we need to find the current producer index, so we
92897549c34SHans Petter Selasky 	 * know where to start cleaning from.  It doesn't matter if HW
92997549c34SHans Petter Selasky 	 * adds new entries after this loop -- the QP we're worried
93097549c34SHans Petter Selasky 	 * about is already in RESET, so the new entries won't come
93197549c34SHans Petter Selasky 	 * from our QP and therefore don't need to be checked.
93297549c34SHans Petter Selasky 	 */
93397549c34SHans Petter Selasky 	for (prod_index = cq->mcq.cons_index; get_sw_cqe(cq, prod_index); ++prod_index)
93497549c34SHans Petter Selasky 		if (prod_index == cq->mcq.cons_index + cq->ibcq.cqe)
93597549c34SHans Petter Selasky 			break;
93697549c34SHans Petter Selasky 
93797549c34SHans Petter Selasky 	/*
93897549c34SHans Petter Selasky 	 * Now sweep backwards through the CQ, removing CQ entries
93997549c34SHans Petter Selasky 	 * that match our QP by copying older entries on top of them.
94097549c34SHans Petter Selasky 	 */
94197549c34SHans Petter Selasky 	while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
94297549c34SHans Petter Selasky 		cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
94397549c34SHans Petter Selasky 		cqe += cqe_inc;
94497549c34SHans Petter Selasky 
94597549c34SHans Petter Selasky 		if ((be32_to_cpu(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK) == qpn) {
94697549c34SHans Petter Selasky 			if (srq && !(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK))
94797549c34SHans Petter Selasky 				mlx4_ib_free_srq_wqe(srq, be16_to_cpu(cqe->wqe_index));
94897549c34SHans Petter Selasky 			++nfreed;
94997549c34SHans Petter Selasky 		} else if (nfreed) {
95097549c34SHans Petter Selasky 			dest = get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe);
95197549c34SHans Petter Selasky 			dest += cqe_inc;
95297549c34SHans Petter Selasky 
95397549c34SHans Petter Selasky 			owner_bit = dest->owner_sr_opcode & MLX4_CQE_OWNER_MASK;
95497549c34SHans Petter Selasky 			memcpy(dest, cqe, sizeof *cqe);
95597549c34SHans Petter Selasky 			dest->owner_sr_opcode = owner_bit |
95697549c34SHans Petter Selasky 				(dest->owner_sr_opcode & ~MLX4_CQE_OWNER_MASK);
95797549c34SHans Petter Selasky 		}
95897549c34SHans Petter Selasky 	}
95997549c34SHans Petter Selasky 
96097549c34SHans Petter Selasky 	if (nfreed) {
96197549c34SHans Petter Selasky 		cq->mcq.cons_index += nfreed;
96297549c34SHans Petter Selasky 		/*
96397549c34SHans Petter Selasky 		 * Make sure update of buffer contents is done before
96497549c34SHans Petter Selasky 		 * updating consumer index.
96597549c34SHans Petter Selasky 		 */
96697549c34SHans Petter Selasky 		wmb();
96797549c34SHans Petter Selasky 		mlx4_cq_set_ci(&cq->mcq);
96897549c34SHans Petter Selasky 	}
96997549c34SHans Petter Selasky }
97097549c34SHans Petter Selasky 
97197549c34SHans Petter Selasky void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
97297549c34SHans Petter Selasky {
97397549c34SHans Petter Selasky 	spin_lock_irq(&cq->lock);
97497549c34SHans Petter Selasky 	__mlx4_ib_cq_clean(cq, qpn, srq);
97597549c34SHans Petter Selasky 	spin_unlock_irq(&cq->lock);
97697549c34SHans Petter Selasky }
977