1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
3 *
4 * Copyright (c) 2017 Mellanox Technologies Ltd. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include <sys/cdefs.h>
36 #include <linux/module.h>
37 #include <linux/err.h>
38 #include <linux/slab.h>
39
40 #include <rdma/ib_verbs.h>
41
42 #define IB_CQ_POLL_MAX 16
43 /* maximum number of completions per poll loop */
44 #define IB_CQ_POLL_BUDGET 65536
45 #define IB_CQ_POLL_FLAGS (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)
46
47 static void
ib_cq_poll_work(struct work_struct * work)48 ib_cq_poll_work(struct work_struct *work)
49 {
50 struct ib_wc ib_wc[IB_CQ_POLL_MAX];
51 struct ib_cq *cq = container_of(work, struct ib_cq, work);
52 int total = 0;
53 int i;
54 int n;
55
56 while (1) {
57 n = ib_poll_cq(cq, IB_CQ_POLL_MAX, ib_wc);
58 for (i = 0; i < n; i++) {
59 struct ib_wc *wc = ib_wc + i;
60
61 if (wc->wr_cqe != NULL)
62 wc->wr_cqe->done(cq, wc);
63 }
64
65 if (n != IB_CQ_POLL_MAX) {
66 if (ib_req_notify_cq(cq, IB_CQ_POLL_FLAGS) > 0)
67 break;
68 else
69 return;
70 }
71 total += n;
72 if (total >= IB_CQ_POLL_BUDGET)
73 break;
74 }
75
76 /* give other work structs a chance */
77 queue_work(ib_comp_wq, &cq->work);
78 }
79
80 static void
ib_cq_completion_workqueue(struct ib_cq * cq,void * private)81 ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
82 {
83 queue_work(ib_comp_wq, &cq->work);
84 }
85
86 struct ib_cq *
__ib_alloc_cq_user(struct ib_device * dev,void * private,int nr_cqe,int comp_vector,enum ib_poll_context poll_ctx,const char * caller,struct ib_udata * udata)87 __ib_alloc_cq_user(struct ib_device *dev, void *private,
88 int nr_cqe, int comp_vector,
89 enum ib_poll_context poll_ctx,
90 const char *caller, struct ib_udata *udata)
91 {
92 struct ib_cq_init_attr cq_attr = {
93 .cqe = nr_cqe,
94 .comp_vector = comp_vector,
95 };
96 struct ib_cq *cq;
97 int ret;
98
99 /*
100 * Check for invalid parameters early on to avoid
101 * extra error handling code:
102 */
103 switch (poll_ctx) {
104 case IB_POLL_DIRECT:
105 case IB_POLL_SOFTIRQ:
106 case IB_POLL_WORKQUEUE:
107 break;
108 default:
109 return (ERR_PTR(-EINVAL));
110 }
111
112 cq = rdma_zalloc_drv_obj(dev, ib_cq);
113 if (!cq)
114 return ERR_PTR(-ENOMEM);
115
116 cq->device = dev;
117 cq->cq_context = private;
118 cq->poll_ctx = poll_ctx;
119 atomic_set(&cq->usecnt, 0);
120
121 ret = dev->create_cq(cq, &cq_attr, NULL);
122 if (ret)
123 goto out_free_cq;
124
125 switch (poll_ctx) {
126 case IB_POLL_DIRECT:
127 cq->comp_handler = NULL; /* no hardware completions */
128 break;
129 case IB_POLL_SOFTIRQ:
130 case IB_POLL_WORKQUEUE:
131 cq->comp_handler = ib_cq_completion_workqueue;
132 INIT_WORK(&cq->work, ib_cq_poll_work);
133 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
134 break;
135 default:
136 break;
137 }
138 return (cq);
139
140 out_free_cq:
141 kfree(cq);
142 return (ERR_PTR(ret));
143 }
144 EXPORT_SYMBOL(__ib_alloc_cq_user);
145
146 void
ib_free_cq_user(struct ib_cq * cq,struct ib_udata * udata)147 ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
148 {
149
150 if (WARN_ON_ONCE(atomic_read(&cq->usecnt) != 0))
151 return;
152
153 switch (cq->poll_ctx) {
154 case IB_POLL_DIRECT:
155 break;
156 case IB_POLL_SOFTIRQ:
157 case IB_POLL_WORKQUEUE:
158 flush_work(&cq->work);
159 break;
160 default:
161 break;
162 }
163
164 cq->device->destroy_cq(cq, udata);
165 kfree(cq);
166 }
167 EXPORT_SYMBOL(ib_free_cq_user);
168