1d6b92ffaSHans Petter Selasky /*
2d6b92ffaSHans Petter Selasky * Copyright (c) 2010 Lawrence Livermore National Laboratory
3d6b92ffaSHans Petter Selasky * Copyright (c) 2011 Mellanox Technologies LTD. All rights reserved.
4d6b92ffaSHans Petter Selasky *
5d6b92ffaSHans Petter Selasky * This software is available to you under a choice of one of two
6d6b92ffaSHans Petter Selasky * licenses. You may choose to be licensed under the terms of the GNU
7d6b92ffaSHans Petter Selasky * General Public License (GPL) Version 2, available from the file
8d6b92ffaSHans Petter Selasky * COPYING in the main directory of this source tree, or the
9d6b92ffaSHans Petter Selasky * OpenIB.org BSD license below:
10d6b92ffaSHans Petter Selasky *
11d6b92ffaSHans Petter Selasky * Redistribution and use in source and binary forms, with or
12d6b92ffaSHans Petter Selasky * without modification, are permitted provided that the following
13d6b92ffaSHans Petter Selasky * conditions are met:
14d6b92ffaSHans Petter Selasky *
15d6b92ffaSHans Petter Selasky * - Redistributions of source code must retain the above
16d6b92ffaSHans Petter Selasky * copyright notice, this list of conditions and the following
17d6b92ffaSHans Petter Selasky * disclaimer.
18d6b92ffaSHans Petter Selasky *
19d6b92ffaSHans Petter Selasky * - Redistributions in binary form must reproduce the above
20d6b92ffaSHans Petter Selasky * copyright notice, this list of conditions and the following
21d6b92ffaSHans Petter Selasky * disclaimer in the documentation and/or other materials
22d6b92ffaSHans Petter Selasky * provided with the distribution.
23d6b92ffaSHans Petter Selasky *
24d6b92ffaSHans Petter Selasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25d6b92ffaSHans Petter Selasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26d6b92ffaSHans Petter Selasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27d6b92ffaSHans Petter Selasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28d6b92ffaSHans Petter Selasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29d6b92ffaSHans Petter Selasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30d6b92ffaSHans Petter Selasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31d6b92ffaSHans Petter Selasky * SOFTWARE.
32d6b92ffaSHans Petter Selasky *
33d6b92ffaSHans Petter Selasky */
34d6b92ffaSHans Petter Selasky
35d6b92ffaSHans Petter Selasky #if HAVE_CONFIG_H
36d6b92ffaSHans Petter Selasky # include <config.h>
37d6b92ffaSHans Petter Selasky #endif /* HAVE_CONFIG_H */
38d6b92ffaSHans Petter Selasky
39d6b92ffaSHans Petter Selasky #include <errno.h>
40d6b92ffaSHans Petter Selasky #include <infiniband/ibnetdisc.h>
41d6b92ffaSHans Petter Selasky #include <infiniband/umad.h>
42d6b92ffaSHans Petter Selasky #include "internal.h"
43d6b92ffaSHans Petter Selasky
44d6b92ffaSHans Petter Selasky extern int mlnx_ext_port_info_err(smp_engine_t * engine, ibnd_smp_t * smp,
45d6b92ffaSHans Petter Selasky uint8_t * mad, void *cb_data);
46d6b92ffaSHans Petter Selasky
queue_smp(smp_engine_t * engine,ibnd_smp_t * smp)47d6b92ffaSHans Petter Selasky static void queue_smp(smp_engine_t * engine, ibnd_smp_t * smp)
48d6b92ffaSHans Petter Selasky {
49d6b92ffaSHans Petter Selasky smp->qnext = NULL;
50d6b92ffaSHans Petter Selasky if (!engine->smp_queue_head) {
51d6b92ffaSHans Petter Selasky engine->smp_queue_head = smp;
52d6b92ffaSHans Petter Selasky engine->smp_queue_tail = smp;
53d6b92ffaSHans Petter Selasky } else {
54d6b92ffaSHans Petter Selasky engine->smp_queue_tail->qnext = smp;
55d6b92ffaSHans Petter Selasky engine->smp_queue_tail = smp;
56d6b92ffaSHans Petter Selasky }
57d6b92ffaSHans Petter Selasky }
58d6b92ffaSHans Petter Selasky
get_smp(smp_engine_t * engine)59d6b92ffaSHans Petter Selasky static ibnd_smp_t *get_smp(smp_engine_t * engine)
60d6b92ffaSHans Petter Selasky {
61d6b92ffaSHans Petter Selasky ibnd_smp_t *head = engine->smp_queue_head;
62d6b92ffaSHans Petter Selasky ibnd_smp_t *tail = engine->smp_queue_tail;
63d6b92ffaSHans Petter Selasky ibnd_smp_t *rc = head;
64d6b92ffaSHans Petter Selasky if (head) {
65d6b92ffaSHans Petter Selasky if (tail == head)
66d6b92ffaSHans Petter Selasky engine->smp_queue_tail = NULL;
67d6b92ffaSHans Petter Selasky engine->smp_queue_head = head->qnext;
68d6b92ffaSHans Petter Selasky }
69d6b92ffaSHans Petter Selasky return rc;
70d6b92ffaSHans Petter Selasky }
71d6b92ffaSHans Petter Selasky
send_smp(ibnd_smp_t * smp,smp_engine_t * engine)72d6b92ffaSHans Petter Selasky static int send_smp(ibnd_smp_t * smp, smp_engine_t * engine)
73d6b92ffaSHans Petter Selasky {
74d6b92ffaSHans Petter Selasky int rc = 0;
75d6b92ffaSHans Petter Selasky uint8_t umad[1024];
76d6b92ffaSHans Petter Selasky ib_rpc_t *rpc = &smp->rpc;
77d6b92ffaSHans Petter Selasky int agent = 0;
78d6b92ffaSHans Petter Selasky
79d6b92ffaSHans Petter Selasky memset(umad, 0, umad_size() + IB_MAD_SIZE);
80d6b92ffaSHans Petter Selasky
81d6b92ffaSHans Petter Selasky if (rpc->mgtclass == IB_SMI_CLASS) {
82d6b92ffaSHans Petter Selasky agent = engine->smi_agent;
83d6b92ffaSHans Petter Selasky } else if (rpc->mgtclass == IB_SMI_DIRECT_CLASS) {
84d6b92ffaSHans Petter Selasky agent = engine->smi_dir_agent;
85d6b92ffaSHans Petter Selasky } else {
86d6b92ffaSHans Petter Selasky IBND_ERROR("Invalid class for RPC\n");
87d6b92ffaSHans Petter Selasky return (-EIO);
88d6b92ffaSHans Petter Selasky }
89d6b92ffaSHans Petter Selasky
90d6b92ffaSHans Petter Selasky if ((rc = mad_build_pkt(umad, &smp->rpc, &smp->path, NULL, NULL))
91d6b92ffaSHans Petter Selasky < 0) {
92d6b92ffaSHans Petter Selasky IBND_ERROR("mad_build_pkt failed; %d\n", rc);
93d6b92ffaSHans Petter Selasky return rc;
94d6b92ffaSHans Petter Selasky }
95d6b92ffaSHans Petter Selasky
96d6b92ffaSHans Petter Selasky if ((rc = umad_send(engine->umad_fd, agent, umad, IB_MAD_SIZE,
97d6b92ffaSHans Petter Selasky engine->cfg->timeout_ms, engine->cfg->retries)) < 0) {
98d6b92ffaSHans Petter Selasky IBND_ERROR("send failed; %d\n", rc);
99d6b92ffaSHans Petter Selasky return rc;
100d6b92ffaSHans Petter Selasky }
101d6b92ffaSHans Petter Selasky
102d6b92ffaSHans Petter Selasky return 0;
103d6b92ffaSHans Petter Selasky }
104d6b92ffaSHans Petter Selasky
process_smp_queue(smp_engine_t * engine)105d6b92ffaSHans Petter Selasky static int process_smp_queue(smp_engine_t * engine)
106d6b92ffaSHans Petter Selasky {
107d6b92ffaSHans Petter Selasky int rc = 0;
108d6b92ffaSHans Petter Selasky ibnd_smp_t *smp;
109d6b92ffaSHans Petter Selasky while (cl_qmap_count(&engine->smps_on_wire)
110d6b92ffaSHans Petter Selasky < engine->cfg->max_smps) {
111d6b92ffaSHans Petter Selasky smp = get_smp(engine);
112d6b92ffaSHans Petter Selasky if (!smp)
113d6b92ffaSHans Petter Selasky return 0;
114d6b92ffaSHans Petter Selasky
115d6b92ffaSHans Petter Selasky if ((rc = send_smp(smp, engine)) != 0) {
116d6b92ffaSHans Petter Selasky free(smp);
117d6b92ffaSHans Petter Selasky return rc;
118d6b92ffaSHans Petter Selasky }
119d6b92ffaSHans Petter Selasky cl_qmap_insert(&engine->smps_on_wire, (uint32_t) smp->rpc.trid,
120d6b92ffaSHans Petter Selasky (cl_map_item_t *) smp);
121d6b92ffaSHans Petter Selasky engine->total_smps++;
122d6b92ffaSHans Petter Selasky }
123d6b92ffaSHans Petter Selasky return 0;
124d6b92ffaSHans Petter Selasky }
125d6b92ffaSHans Petter Selasky
issue_smp(smp_engine_t * engine,ib_portid_t * portid,unsigned attrid,unsigned mod,smp_comp_cb_t cb,void * cb_data)126d6b92ffaSHans Petter Selasky int issue_smp(smp_engine_t * engine, ib_portid_t * portid,
127d6b92ffaSHans Petter Selasky unsigned attrid, unsigned mod, smp_comp_cb_t cb, void *cb_data)
128d6b92ffaSHans Petter Selasky {
129d6b92ffaSHans Petter Selasky ibnd_smp_t *smp = calloc(1, sizeof *smp);
130d6b92ffaSHans Petter Selasky if (!smp) {
131d6b92ffaSHans Petter Selasky IBND_ERROR("OOM\n");
132d6b92ffaSHans Petter Selasky return -ENOMEM;
133d6b92ffaSHans Petter Selasky }
134d6b92ffaSHans Petter Selasky
135d6b92ffaSHans Petter Selasky smp->cb = cb;
136d6b92ffaSHans Petter Selasky smp->cb_data = cb_data;
137d6b92ffaSHans Petter Selasky smp->path = *portid;
138d6b92ffaSHans Petter Selasky smp->rpc.method = IB_MAD_METHOD_GET;
139d6b92ffaSHans Petter Selasky smp->rpc.attr.id = attrid;
140d6b92ffaSHans Petter Selasky smp->rpc.attr.mod = mod;
141d6b92ffaSHans Petter Selasky smp->rpc.timeout = engine->cfg->timeout_ms;
142d6b92ffaSHans Petter Selasky smp->rpc.datasz = IB_SMP_DATA_SIZE;
143d6b92ffaSHans Petter Selasky smp->rpc.dataoffs = IB_SMP_DATA_OFFS;
144d6b92ffaSHans Petter Selasky smp->rpc.trid = mad_trid();
145d6b92ffaSHans Petter Selasky smp->rpc.mkey = engine->cfg->mkey;
146d6b92ffaSHans Petter Selasky
147d6b92ffaSHans Petter Selasky if (portid->lid <= 0 || portid->drpath.drslid == 0xffff ||
148d6b92ffaSHans Petter Selasky portid->drpath.drdlid == 0xffff)
149d6b92ffaSHans Petter Selasky smp->rpc.mgtclass = IB_SMI_DIRECT_CLASS; /* direct SMI */
150d6b92ffaSHans Petter Selasky else
151d6b92ffaSHans Petter Selasky smp->rpc.mgtclass = IB_SMI_CLASS; /* Lid routed SMI */
152d6b92ffaSHans Petter Selasky
153d6b92ffaSHans Petter Selasky portid->sl = 0;
154d6b92ffaSHans Petter Selasky portid->qp = 0;
155d6b92ffaSHans Petter Selasky
156d6b92ffaSHans Petter Selasky queue_smp(engine, smp);
157d6b92ffaSHans Petter Selasky return process_smp_queue(engine);
158d6b92ffaSHans Petter Selasky }
159d6b92ffaSHans Petter Selasky
process_one_recv(smp_engine_t * engine)160d6b92ffaSHans Petter Selasky static int process_one_recv(smp_engine_t * engine)
161d6b92ffaSHans Petter Selasky {
162d6b92ffaSHans Petter Selasky int rc = 0;
163d6b92ffaSHans Petter Selasky int status = 0;
164d6b92ffaSHans Petter Selasky ibnd_smp_t *smp;
165d6b92ffaSHans Petter Selasky uint8_t *mad;
166d6b92ffaSHans Petter Selasky uint32_t trid;
167d6b92ffaSHans Petter Selasky uint8_t umad[sizeof(struct ib_user_mad) + IB_MAD_SIZE];
168d6b92ffaSHans Petter Selasky int length = umad_size() + IB_MAD_SIZE;
169d6b92ffaSHans Petter Selasky
170d6b92ffaSHans Petter Selasky memset(umad, 0, sizeof(umad));
171d6b92ffaSHans Petter Selasky
172d6b92ffaSHans Petter Selasky /* wait for the next message */
173d6b92ffaSHans Petter Selasky if ((rc = umad_recv(engine->umad_fd, umad, &length,
174d6b92ffaSHans Petter Selasky -1)) < 0) {
175d6b92ffaSHans Petter Selasky IBND_ERROR("umad_recv failed: %d\n", rc);
176d6b92ffaSHans Petter Selasky return -1;
177d6b92ffaSHans Petter Selasky }
178d6b92ffaSHans Petter Selasky
179d6b92ffaSHans Petter Selasky mad = umad_get_mad(umad);
180d6b92ffaSHans Petter Selasky trid = (uint32_t) mad_get_field64(mad, 0, IB_MAD_TRID_F);
181d6b92ffaSHans Petter Selasky
182d6b92ffaSHans Petter Selasky smp = (ibnd_smp_t *) cl_qmap_remove(&engine->smps_on_wire, trid);
183d6b92ffaSHans Petter Selasky if ((cl_map_item_t *) smp == cl_qmap_end(&engine->smps_on_wire)) {
184d6b92ffaSHans Petter Selasky IBND_ERROR("Failed to find matching smp for trid (%x)\n", trid);
185d6b92ffaSHans Petter Selasky return -1;
186d6b92ffaSHans Petter Selasky }
187d6b92ffaSHans Petter Selasky
188d6b92ffaSHans Petter Selasky rc = process_smp_queue(engine);
189d6b92ffaSHans Petter Selasky if (rc)
190d6b92ffaSHans Petter Selasky goto error;
191d6b92ffaSHans Petter Selasky
192d6b92ffaSHans Petter Selasky if ((status = umad_status(umad))) {
193d6b92ffaSHans Petter Selasky IBND_ERROR("umad (%s Attr 0x%x:%u) bad status %d; %s\n",
194d6b92ffaSHans Petter Selasky portid2str(&smp->path), smp->rpc.attr.id,
195d6b92ffaSHans Petter Selasky smp->rpc.attr.mod, status, strerror(status));
196d6b92ffaSHans Petter Selasky if (smp->rpc.attr.id == IB_ATTR_MLNX_EXT_PORT_INFO)
197d6b92ffaSHans Petter Selasky rc = mlnx_ext_port_info_err(engine, smp, mad,
198d6b92ffaSHans Petter Selasky smp->cb_data);
199d6b92ffaSHans Petter Selasky } else if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F))) {
200d6b92ffaSHans Petter Selasky IBND_ERROR("mad (%s Attr 0x%x:%u) bad status 0x%x\n",
201d6b92ffaSHans Petter Selasky portid2str(&smp->path), smp->rpc.attr.id,
202d6b92ffaSHans Petter Selasky smp->rpc.attr.mod, status);
203d6b92ffaSHans Petter Selasky if (smp->rpc.attr.id == IB_ATTR_MLNX_EXT_PORT_INFO)
204d6b92ffaSHans Petter Selasky rc = mlnx_ext_port_info_err(engine, smp, mad,
205d6b92ffaSHans Petter Selasky smp->cb_data);
206d6b92ffaSHans Petter Selasky } else
207d6b92ffaSHans Petter Selasky rc = smp->cb(engine, smp, mad, smp->cb_data);
208d6b92ffaSHans Petter Selasky
209d6b92ffaSHans Petter Selasky error:
210d6b92ffaSHans Petter Selasky free(smp);
211d6b92ffaSHans Petter Selasky return rc;
212d6b92ffaSHans Petter Selasky }
213d6b92ffaSHans Petter Selasky
smp_engine_init(smp_engine_t * engine,char * ca_name,int ca_port,void * user_data,ibnd_config_t * cfg)214d6b92ffaSHans Petter Selasky int smp_engine_init(smp_engine_t * engine, char * ca_name, int ca_port,
215d6b92ffaSHans Petter Selasky void *user_data, ibnd_config_t *cfg)
216d6b92ffaSHans Petter Selasky {
217d6b92ffaSHans Petter Selasky memset(engine, 0, sizeof(*engine));
218d6b92ffaSHans Petter Selasky
219d6b92ffaSHans Petter Selasky if (umad_init() < 0) {
220d6b92ffaSHans Petter Selasky IBND_ERROR("umad_init failed\n");
221d6b92ffaSHans Petter Selasky return -EIO;
222d6b92ffaSHans Petter Selasky }
223d6b92ffaSHans Petter Selasky
224d6b92ffaSHans Petter Selasky engine->umad_fd = umad_open_port(ca_name, ca_port);
225d6b92ffaSHans Petter Selasky if (engine->umad_fd < 0) {
226d6b92ffaSHans Petter Selasky IBND_ERROR("can't open UMAD port (%s:%d)\n", ca_name, ca_port);
227d6b92ffaSHans Petter Selasky return -EIO;
228d6b92ffaSHans Petter Selasky }
229d6b92ffaSHans Petter Selasky
230d6b92ffaSHans Petter Selasky if ((engine->smi_agent = umad_register(engine->umad_fd,
231d6b92ffaSHans Petter Selasky IB_SMI_CLASS, 1, 0, 0)) < 0) {
232d6b92ffaSHans Petter Selasky IBND_ERROR("Failed to register SMI agent on (%s:%d)\n",
233d6b92ffaSHans Petter Selasky ca_name, ca_port);
234d6b92ffaSHans Petter Selasky goto eio_close;
235d6b92ffaSHans Petter Selasky }
236d6b92ffaSHans Petter Selasky
237d6b92ffaSHans Petter Selasky if ((engine->smi_dir_agent = umad_register(engine->umad_fd,
238d6b92ffaSHans Petter Selasky IB_SMI_DIRECT_CLASS, 1, 0, 0)) < 0) {
239d6b92ffaSHans Petter Selasky IBND_ERROR("Failed to register SMI_DIRECT agent on (%s:%d)\n",
240d6b92ffaSHans Petter Selasky ca_name, ca_port);
241d6b92ffaSHans Petter Selasky goto eio_close;
242d6b92ffaSHans Petter Selasky }
243d6b92ffaSHans Petter Selasky
244d6b92ffaSHans Petter Selasky engine->user_data = user_data;
245d6b92ffaSHans Petter Selasky cl_qmap_init(&engine->smps_on_wire);
246d6b92ffaSHans Petter Selasky engine->cfg = cfg;
247d6b92ffaSHans Petter Selasky return (0);
248d6b92ffaSHans Petter Selasky
249d6b92ffaSHans Petter Selasky eio_close:
250d6b92ffaSHans Petter Selasky umad_close_port(engine->umad_fd);
251d6b92ffaSHans Petter Selasky return (-EIO);
252d6b92ffaSHans Petter Selasky }
253d6b92ffaSHans Petter Selasky
smp_engine_destroy(smp_engine_t * engine)254d6b92ffaSHans Petter Selasky void smp_engine_destroy(smp_engine_t * engine)
255d6b92ffaSHans Petter Selasky {
256d6b92ffaSHans Petter Selasky cl_map_item_t *item;
257d6b92ffaSHans Petter Selasky ibnd_smp_t *smp;
258d6b92ffaSHans Petter Selasky
259d6b92ffaSHans Petter Selasky /* remove queued smps */
260d6b92ffaSHans Petter Selasky smp = get_smp(engine);
261d6b92ffaSHans Petter Selasky if (smp)
262d6b92ffaSHans Petter Selasky IBND_ERROR("outstanding SMP's\n");
263d6b92ffaSHans Petter Selasky for ( /* */ ; smp; smp = get_smp(engine))
264d6b92ffaSHans Petter Selasky free(smp);
265d6b92ffaSHans Petter Selasky
266d6b92ffaSHans Petter Selasky /* remove smps from the wire queue */
267d6b92ffaSHans Petter Selasky item = cl_qmap_head(&engine->smps_on_wire);
268d6b92ffaSHans Petter Selasky if (item != cl_qmap_end(&engine->smps_on_wire))
269d6b92ffaSHans Petter Selasky IBND_ERROR("outstanding SMP's on wire\n");
270d6b92ffaSHans Petter Selasky for ( /* */ ; item != cl_qmap_end(&engine->smps_on_wire);
271d6b92ffaSHans Petter Selasky item = cl_qmap_head(&engine->smps_on_wire)) {
272d6b92ffaSHans Petter Selasky cl_qmap_remove_item(&engine->smps_on_wire, item);
273d6b92ffaSHans Petter Selasky free(item);
274d6b92ffaSHans Petter Selasky }
275d6b92ffaSHans Petter Selasky
276d6b92ffaSHans Petter Selasky umad_close_port(engine->umad_fd);
277d6b92ffaSHans Petter Selasky }
278d6b92ffaSHans Petter Selasky
process_mads(smp_engine_t * engine)279d6b92ffaSHans Petter Selasky int process_mads(smp_engine_t * engine)
280d6b92ffaSHans Petter Selasky {
281d6b92ffaSHans Petter Selasky int rc;
282d6b92ffaSHans Petter Selasky while (!cl_is_qmap_empty(&engine->smps_on_wire))
283d6b92ffaSHans Petter Selasky if ((rc = process_one_recv(engine)) != 0)
284d6b92ffaSHans Petter Selasky return rc;
285d6b92ffaSHans Petter Selasky return 0;
286d6b92ffaSHans Petter Selasky }
287