1 /*
2 * Copyright (c) 2010 Lawrence Livermore National Laboratory
3 * Copyright (c) 2011 Mellanox Technologies LTD. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 *
33 */
34
35 #if HAVE_CONFIG_H
36 # include <config.h>
37 #endif /* HAVE_CONFIG_H */
38
39 #include <errno.h>
40 #include <infiniband/ibnetdisc.h>
41 #include <infiniband/umad.h>
42 #include "internal.h"
43
44 extern int mlnx_ext_port_info_err(smp_engine_t * engine, ibnd_smp_t * smp,
45 uint8_t * mad, void *cb_data);
46
queue_smp(smp_engine_t * engine,ibnd_smp_t * smp)47 static void queue_smp(smp_engine_t * engine, ibnd_smp_t * smp)
48 {
49 smp->qnext = NULL;
50 if (!engine->smp_queue_head) {
51 engine->smp_queue_head = smp;
52 engine->smp_queue_tail = smp;
53 } else {
54 engine->smp_queue_tail->qnext = smp;
55 engine->smp_queue_tail = smp;
56 }
57 }
58
get_smp(smp_engine_t * engine)59 static ibnd_smp_t *get_smp(smp_engine_t * engine)
60 {
61 ibnd_smp_t *head = engine->smp_queue_head;
62 ibnd_smp_t *tail = engine->smp_queue_tail;
63 ibnd_smp_t *rc = head;
64 if (head) {
65 if (tail == head)
66 engine->smp_queue_tail = NULL;
67 engine->smp_queue_head = head->qnext;
68 }
69 return rc;
70 }
71
send_smp(ibnd_smp_t * smp,smp_engine_t * engine)72 static int send_smp(ibnd_smp_t * smp, smp_engine_t * engine)
73 {
74 int rc = 0;
75 uint8_t umad[1024];
76 ib_rpc_t *rpc = &smp->rpc;
77 int agent = 0;
78
79 memset(umad, 0, umad_size() + IB_MAD_SIZE);
80
81 if (rpc->mgtclass == IB_SMI_CLASS) {
82 agent = engine->smi_agent;
83 } else if (rpc->mgtclass == IB_SMI_DIRECT_CLASS) {
84 agent = engine->smi_dir_agent;
85 } else {
86 IBND_ERROR("Invalid class for RPC\n");
87 return (-EIO);
88 }
89
90 if ((rc = mad_build_pkt(umad, &smp->rpc, &smp->path, NULL, NULL))
91 < 0) {
92 IBND_ERROR("mad_build_pkt failed; %d\n", rc);
93 return rc;
94 }
95
96 if ((rc = umad_send(engine->umad_fd, agent, umad, IB_MAD_SIZE,
97 engine->cfg->timeout_ms, engine->cfg->retries)) < 0) {
98 IBND_ERROR("send failed; %d\n", rc);
99 return rc;
100 }
101
102 return 0;
103 }
104
process_smp_queue(smp_engine_t * engine)105 static int process_smp_queue(smp_engine_t * engine)
106 {
107 int rc = 0;
108 ibnd_smp_t *smp;
109 while (cl_qmap_count(&engine->smps_on_wire)
110 < engine->cfg->max_smps) {
111 smp = get_smp(engine);
112 if (!smp)
113 return 0;
114
115 if ((rc = send_smp(smp, engine)) != 0) {
116 free(smp);
117 return rc;
118 }
119 cl_qmap_insert(&engine->smps_on_wire, (uint32_t) smp->rpc.trid,
120 (cl_map_item_t *) smp);
121 engine->total_smps++;
122 }
123 return 0;
124 }
125
issue_smp(smp_engine_t * engine,ib_portid_t * portid,unsigned attrid,unsigned mod,smp_comp_cb_t cb,void * cb_data)126 int issue_smp(smp_engine_t * engine, ib_portid_t * portid,
127 unsigned attrid, unsigned mod, smp_comp_cb_t cb, void *cb_data)
128 {
129 ibnd_smp_t *smp = calloc(1, sizeof *smp);
130 if (!smp) {
131 IBND_ERROR("OOM\n");
132 return -ENOMEM;
133 }
134
135 smp->cb = cb;
136 smp->cb_data = cb_data;
137 smp->path = *portid;
138 smp->rpc.method = IB_MAD_METHOD_GET;
139 smp->rpc.attr.id = attrid;
140 smp->rpc.attr.mod = mod;
141 smp->rpc.timeout = engine->cfg->timeout_ms;
142 smp->rpc.datasz = IB_SMP_DATA_SIZE;
143 smp->rpc.dataoffs = IB_SMP_DATA_OFFS;
144 smp->rpc.trid = mad_trid();
145 smp->rpc.mkey = engine->cfg->mkey;
146
147 if (portid->lid <= 0 || portid->drpath.drslid == 0xffff ||
148 portid->drpath.drdlid == 0xffff)
149 smp->rpc.mgtclass = IB_SMI_DIRECT_CLASS; /* direct SMI */
150 else
151 smp->rpc.mgtclass = IB_SMI_CLASS; /* Lid routed SMI */
152
153 portid->sl = 0;
154 portid->qp = 0;
155
156 queue_smp(engine, smp);
157 return process_smp_queue(engine);
158 }
159
process_one_recv(smp_engine_t * engine)160 static int process_one_recv(smp_engine_t * engine)
161 {
162 int rc = 0;
163 int status = 0;
164 ibnd_smp_t *smp;
165 uint8_t *mad;
166 uint32_t trid;
167 uint8_t umad[sizeof(struct ib_user_mad) + IB_MAD_SIZE];
168 int length = umad_size() + IB_MAD_SIZE;
169
170 memset(umad, 0, sizeof(umad));
171
172 /* wait for the next message */
173 if ((rc = umad_recv(engine->umad_fd, umad, &length,
174 -1)) < 0) {
175 IBND_ERROR("umad_recv failed: %d\n", rc);
176 return -1;
177 }
178
179 mad = umad_get_mad(umad);
180 trid = (uint32_t) mad_get_field64(mad, 0, IB_MAD_TRID_F);
181
182 smp = (ibnd_smp_t *) cl_qmap_remove(&engine->smps_on_wire, trid);
183 if ((cl_map_item_t *) smp == cl_qmap_end(&engine->smps_on_wire)) {
184 IBND_ERROR("Failed to find matching smp for trid (%x)\n", trid);
185 return -1;
186 }
187
188 rc = process_smp_queue(engine);
189 if (rc)
190 goto error;
191
192 if ((status = umad_status(umad))) {
193 IBND_ERROR("umad (%s Attr 0x%x:%u) bad status %d; %s\n",
194 portid2str(&smp->path), smp->rpc.attr.id,
195 smp->rpc.attr.mod, status, strerror(status));
196 if (smp->rpc.attr.id == IB_ATTR_MLNX_EXT_PORT_INFO)
197 rc = mlnx_ext_port_info_err(engine, smp, mad,
198 smp->cb_data);
199 } else if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F))) {
200 IBND_ERROR("mad (%s Attr 0x%x:%u) bad status 0x%x\n",
201 portid2str(&smp->path), smp->rpc.attr.id,
202 smp->rpc.attr.mod, status);
203 if (smp->rpc.attr.id == IB_ATTR_MLNX_EXT_PORT_INFO)
204 rc = mlnx_ext_port_info_err(engine, smp, mad,
205 smp->cb_data);
206 } else
207 rc = smp->cb(engine, smp, mad, smp->cb_data);
208
209 error:
210 free(smp);
211 return rc;
212 }
213
smp_engine_init(smp_engine_t * engine,char * ca_name,int ca_port,void * user_data,ibnd_config_t * cfg)214 int smp_engine_init(smp_engine_t * engine, char * ca_name, int ca_port,
215 void *user_data, ibnd_config_t *cfg)
216 {
217 memset(engine, 0, sizeof(*engine));
218
219 if (umad_init() < 0) {
220 IBND_ERROR("umad_init failed\n");
221 return -EIO;
222 }
223
224 engine->umad_fd = umad_open_port(ca_name, ca_port);
225 if (engine->umad_fd < 0) {
226 IBND_ERROR("can't open UMAD port (%s:%d)\n", ca_name, ca_port);
227 return -EIO;
228 }
229
230 if ((engine->smi_agent = umad_register(engine->umad_fd,
231 IB_SMI_CLASS, 1, 0, 0)) < 0) {
232 IBND_ERROR("Failed to register SMI agent on (%s:%d)\n",
233 ca_name, ca_port);
234 goto eio_close;
235 }
236
237 if ((engine->smi_dir_agent = umad_register(engine->umad_fd,
238 IB_SMI_DIRECT_CLASS, 1, 0, 0)) < 0) {
239 IBND_ERROR("Failed to register SMI_DIRECT agent on (%s:%d)\n",
240 ca_name, ca_port);
241 goto eio_close;
242 }
243
244 engine->user_data = user_data;
245 cl_qmap_init(&engine->smps_on_wire);
246 engine->cfg = cfg;
247 return (0);
248
249 eio_close:
250 umad_close_port(engine->umad_fd);
251 return (-EIO);
252 }
253
smp_engine_destroy(smp_engine_t * engine)254 void smp_engine_destroy(smp_engine_t * engine)
255 {
256 cl_map_item_t *item;
257 ibnd_smp_t *smp;
258
259 /* remove queued smps */
260 smp = get_smp(engine);
261 if (smp)
262 IBND_ERROR("outstanding SMP's\n");
263 for ( /* */ ; smp; smp = get_smp(engine))
264 free(smp);
265
266 /* remove smps from the wire queue */
267 item = cl_qmap_head(&engine->smps_on_wire);
268 if (item != cl_qmap_end(&engine->smps_on_wire))
269 IBND_ERROR("outstanding SMP's on wire\n");
270 for ( /* */ ; item != cl_qmap_end(&engine->smps_on_wire);
271 item = cl_qmap_head(&engine->smps_on_wire)) {
272 cl_qmap_remove_item(&engine->smps_on_wire, item);
273 free(item);
274 }
275
276 umad_close_port(engine->umad_fd);
277 }
278
process_mads(smp_engine_t * engine)279 int process_mads(smp_engine_t * engine)
280 {
281 int rc;
282 while (!cl_is_qmap_empty(&engine->smps_on_wire))
283 if ((rc = process_one_recv(engine)) != 0)
284 return rc;
285 return 0;
286 }
287