1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0 3 * 4 * Copyright (c) 2017 Mellanox Technologies Ltd. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include <linux/module.h> 39 #include <linux/err.h> 40 #include <linux/slab.h> 41 42 #include <rdma/ib_verbs.h> 43 44 #define IB_CQ_POLL_MAX 16 45 /* maximum number of completions per poll loop */ 46 #define IB_CQ_POLL_BUDGET 65536 47 #define IB_CQ_POLL_FLAGS (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS) 48 49 static void 50 ib_cq_poll_work(struct work_struct *work) 51 { 52 struct ib_wc ib_wc[IB_CQ_POLL_MAX]; 53 struct ib_cq *cq = container_of(work, struct ib_cq, work); 54 int total = 0; 55 int i; 56 int n; 57 58 while (1) { 59 n = ib_poll_cq(cq, IB_CQ_POLL_MAX, ib_wc); 60 for (i = 0; i < n; i++) { 61 struct ib_wc *wc = ib_wc + i; 62 63 if (wc->wr_cqe != NULL) 64 wc->wr_cqe->done(cq, wc); 65 } 66 67 if (n != IB_CQ_POLL_MAX) { 68 if (ib_req_notify_cq(cq, IB_CQ_POLL_FLAGS) > 0) 69 break; 70 else 71 return; 72 } 73 total += n; 74 if (total >= IB_CQ_POLL_BUDGET) 75 break; 76 } 77 78 /* give other work structs a chance */ 79 queue_work(ib_comp_wq, &cq->work); 80 } 81 82 static void 83 ib_cq_completion_workqueue(struct ib_cq *cq, void *private) 84 { 85 queue_work(ib_comp_wq, &cq->work); 86 } 87 88 struct ib_cq * 89 __ib_alloc_cq_user(struct ib_device *dev, void *private, 90 int nr_cqe, int comp_vector, 91 enum ib_poll_context poll_ctx, 92 const char *caller, struct ib_udata *udata) 93 { 94 struct ib_cq_init_attr cq_attr = { 95 .cqe = nr_cqe, 96 .comp_vector = comp_vector, 97 }; 98 struct ib_cq *cq; 99 int ret; 100 101 /* 102 * Check for invalid parameters early on to avoid 103 * extra error handling code: 104 */ 105 switch (poll_ctx) { 106 case IB_POLL_DIRECT: 107 case IB_POLL_SOFTIRQ: 108 case IB_POLL_WORKQUEUE: 109 break; 110 default: 111 return (ERR_PTR(-EINVAL)); 112 } 113 114 cq = rdma_zalloc_drv_obj(dev, ib_cq); 115 if (!cq) 116 return ERR_PTR(-ENOMEM); 117 118 cq->device = dev; 119 cq->cq_context = private; 120 cq->poll_ctx = poll_ctx; 121 atomic_set(&cq->usecnt, 0); 122 123 ret = dev->create_cq(cq, &cq_attr, NULL); 124 if (ret) 125 goto out_free_cq; 126 127 switch (poll_ctx) { 128 case IB_POLL_DIRECT: 129 cq->comp_handler = NULL; /* no hardware completions */ 130 break; 131 case IB_POLL_SOFTIRQ: 132 case IB_POLL_WORKQUEUE: 133 cq->comp_handler = ib_cq_completion_workqueue; 134 INIT_WORK(&cq->work, ib_cq_poll_work); 135 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 136 break; 137 default: 138 break; 139 } 140 return (cq); 141 142 out_free_cq: 143 kfree(cq); 144 return (ERR_PTR(ret)); 145 } 146 EXPORT_SYMBOL(__ib_alloc_cq_user); 147 148 void 149 ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) 150 { 151 152 if (WARN_ON_ONCE(atomic_read(&cq->usecnt) != 0)) 153 return; 154 155 switch (cq->poll_ctx) { 156 case IB_POLL_DIRECT: 157 break; 158 case IB_POLL_SOFTIRQ: 159 case IB_POLL_WORKQUEUE: 160 flush_work(&cq->work); 161 break; 162 default: 163 break; 164 } 165 166 cq->device->destroy_cq(cq, udata); 167 kfree(cq); 168 } 169 EXPORT_SYMBOL(ib_free_cq_user); 170