1 /*
2 * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
3 *
4 * LICENSE_BEGIN
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
28 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
29 * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
30 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
31 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
32 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
33 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
35 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 * POSSIBILITY OF SUCH DAMAGE.
37 *
38 * LICENSE_END
39 *
40 *
41 */
42
43 #include <stdio.h>
44 #include <stdlib.h>
45 #include <unistd.h>
46 #include <string.h>
47 #include <fcntl.h>
48 #include <dirent.h>
49 #include <pthread.h>
50 #include <errno.h>
51 #include <sys/stat.h>
52
53 #include "usd.h"
54 #include "usd_ib_sysfs.h"
55 #include "usd_util.h"
56
57 /*
58 * usnic_direct routines that depend on Infiniband /sysfs directory structure
59 */
60
61 /*
62 * Perform one-time initialization
63 */
64 int
usd_ib_get_devlist(struct usd_ib_dev ** dev_list)65 usd_ib_get_devlist(
66 struct usd_ib_dev **dev_list)
67 {
68 char *class_path = "/sys/class/infiniband_verbs";
69 DIR *class_dir;
70 struct dirent *dent;
71 struct stat sbuf;
72 char *dev_path = NULL;
73 char *ibdev_path = NULL;
74 char ibdev_buf[32];
75 struct usd_ib_dev *idp;
76 struct usd_ib_dev *last_idp;
77 int fd;
78 int rc;
79 int n;
80
81 /*
82 * For now, we are glomming onto Infiniband driver for setup
83 */
84 class_dir = opendir(class_path);
85 if (class_dir == NULL) {
86 return -ENODEV;
87 }
88
89 /* Check dir entries for USNIC devices */
90 last_idp = NULL;
91 fd = -1;
92 while ((dent = readdir(class_dir)) != NULL) {
93 /* skip "." and ".." */
94 if (dent->d_name[0] == '.')
95 continue;
96
97 /* build path to entry */
98 if (asprintf(&dev_path, "%s/%s", class_path,
99 dent->d_name) <= 0) {
100 rc = -errno;
101 usd_perror("failed to asprintf");
102 goto out;
103 }
104
105 /* see if it's a dir */
106 rc = stat(dev_path, &sbuf);
107 if (rc != 0) {
108 usd_perror(dev_path);
109 rc = -errno;
110 goto out;
111 }
112
113 /* Must be a directory */
114 if (!S_ISDIR(sbuf.st_mode))
115 continue;
116
117 /* read the ibdev */
118 if (asprintf(&ibdev_path, "%s/ibdev", dev_path) <= 0) {
119 rc = -errno;
120 usd_perror(ibdev_path);
121 goto out;
122 }
123 fd = open(ibdev_path, O_RDONLY);
124 if (fd == -1) {
125 usd_perror(ibdev_path);
126 rc = -errno;
127 goto out;
128 }
129 memset(ibdev_buf, 0, sizeof(ibdev_buf));
130 n = read(fd, ibdev_buf, sizeof(ibdev_buf) - 1);
131 if (n == -1) {
132 usd_perror("reading ibdev");
133 rc = -errno;
134 goto out;
135 }
136 close(fd);
137 fd = -1;
138 if (n > 0 && ibdev_buf[n - 1] == '\n') {
139 ibdev_buf[n - 1] = '\0'; /* newline -> EOF */
140 }
141
142 /* If USNIC device, remember this one */
143 if (strncmp(ibdev_buf, "usnic", 5) == 0) {
144 idp = calloc(sizeof(*idp), 1);
145 if (idp == NULL) {
146 usd_perror("calloc IB device");
147 rc = -errno;
148 goto out;
149 }
150 strncpy(idp->id_name, dent->d_name, sizeof(idp->id_name) - 1);
151 strncpy(idp->id_usnic_name, ibdev_buf,
152 sizeof(idp->id_usnic_name) - 1);
153 snprintf(idp->id_dev_path, sizeof(idp->id_dev_path) - 1,
154 "/dev/infiniband/%s", idp->id_name);
155 snprintf(idp->id_class_path, sizeof(idp->id_class_path) - 1,
156 "%s/device/infiniband/%s", dev_path, ibdev_buf);
157
158 if (last_idp == NULL) {
159 *dev_list = idp;
160 } else {
161 last_idp->id_next = idp;
162 }
163 idp->id_next = NULL;
164 last_idp = idp;
165 }
166 free(dev_path);
167 dev_path = NULL;
168 free(ibdev_path);
169 ibdev_path = NULL;
170 }
171 rc = 0;
172
173 out:
174 /* clean up */
175 free(dev_path);
176 free(ibdev_path);
177 if (class_dir != NULL) {
178 closedir(class_dir);
179 }
180 if (fd != -1) {
181 close(fd);
182 }
183
184 return rc;
185 }
186
187 /*
188 * Find MAC for a device
189 * (we assume port 0)
190 */
191 int
usd_get_mac(struct usd_device * dev,uint8_t * mac)192 usd_get_mac(
193 struct usd_device *dev,
194 uint8_t * mac)
195 {
196 char name[PATH_MAX + 128];
197 char gid[80];
198 char *p;
199 uint16_t v;
200 struct usd_ib_dev *idp;
201 int fd;
202 int n;
203
204 idp = dev->ud_ctx->ucx_ib_dev;
205 snprintf(name, sizeof(name), "%s/ports/1/gids/0", idp->id_class_path);
206
207 fd = open(name, O_RDONLY);
208 if (fd == -1) {
209 usd_perror(name);
210 return -errno;
211 }
212
213 n = read(fd, gid, sizeof(gid) - 1);
214 close(fd);
215 if (n < 0) {
216 usd_perror("reading GID");
217 return -errno;
218 }
219 gid[n] = '\0';
220
221 p = gid + 20;
222 sscanf(p, "%hx", &v);
223 *mac++ = (v >> 8) ^ 2;
224 *mac++ = v & 0xFF;
225 p += 5;
226 sscanf(p, "%hx", &v);
227 *mac++ = v >> 8;
228 p += 5;
229 sscanf(p, "%hx", &v);
230 *mac++ = v & 0xFF;
231 p += 5;
232 sscanf(p, "%hx", &v);
233 *mac++ = v >> 8;
234 *mac++ = v & 0xFF;
235
236 return 0;
237 }
238
239 /*
240 * Find interface for a device
241 */
242 int
usd_get_iface(struct usd_device * dev)243 usd_get_iface(
244 struct usd_device *dev)
245 {
246 char name[PATH_MAX + 128];
247 struct usd_ib_dev *idp;
248 int fd;
249 int n;
250
251 idp = dev->ud_ctx->ucx_ib_dev;
252 snprintf(name, sizeof(name), "%s/iface", idp->id_class_path);
253
254 fd = open(name, O_RDONLY);
255 if (fd == -1) {
256 usd_perror(name);
257 dev->ud_attrs.uda_ifname[0] = '\0';
258 return -errno;
259 }
260
261 n = read(fd, dev->ud_attrs.uda_ifname,
262 sizeof(dev->ud_attrs.uda_ifname));
263 close(fd);
264 if (n < 0) {
265 usd_perror("reading iface");
266 return -errno;
267 }
268
269 dev->ud_attrs.uda_ifname[n - 1] = '\0';
270
271 return 0;
272 }
273
274 /*
275 * Read an integer from a sysfs entry
276 */
277 static int
usd_ib_sysfs_get_int(struct usd_device * dev,char * entry,int * result)278 usd_ib_sysfs_get_int(
279 struct usd_device *dev,
280 char *entry,
281 int *result)
282 {
283 char name[PATH_MAX + 128];
284 char buf[32];
285 struct usd_ib_dev *idp;
286 int fd;
287 int n;
288
289 idp = dev->ud_ctx->ucx_ib_dev;
290 snprintf(name, sizeof(name), "%s/%s", idp->id_class_path, entry);
291
292 fd = open(name, O_RDONLY);
293 if (fd == -1) {
294 usd_perror(name);
295 return -errno;
296 }
297
298 n = read(fd, buf, sizeof(buf));
299 close(fd);
300 if (n < 0) {
301 fprintf(stderr, "Error %d reading %s\n", errno, entry);
302 return -errno;
303 }
304
305 *result = atoi(buf);
306 return 0;
307 }
308
309 /*
310 * Get usNIC configuration
311 */
312 int
usd_get_usnic_config(struct usd_device * dev)313 usd_get_usnic_config(
314 struct usd_device *dev)
315 {
316 int v;
317 int ret;
318
319 ret = usd_ib_sysfs_get_int(dev, "max_vf", &v);
320 if (ret != 0)
321 return ret;
322 dev->ud_attrs.uda_num_vf = v;
323
324 ret = usd_ib_sysfs_get_int(dev, "qp_per_vf", &v);
325 if (ret != 0)
326 return ret;
327 dev->ud_attrs.uda_qp_per_vf = v;
328
329 ret = usd_ib_sysfs_get_int(dev, "cq_per_vf", &v);
330 if (ret != 0)
331 return ret;
332 dev->ud_attrs.uda_cq_per_vf = v;
333
334 ret = usd_ib_sysfs_get_int(dev, "intr_per_vf", &v);
335 if (ret != 0) {
336 /* older kernels did not export this sysfs node */
337 if (ret == -ENOENT) {
338 dev->ud_attrs.uda_intr_per_vf = 0;
339 ret = 0;
340 }
341 else {
342 return ret;
343 }
344 } else {
345 dev->ud_attrs.uda_intr_per_vf = v;
346 }
347
348 return ret;
349 }
350
351 /*
352 * Find firmware version
353 */
354 int
usd_get_firmware(struct usd_device * dev)355 usd_get_firmware(
356 struct usd_device *dev)
357 {
358 char name[PATH_MAX + 128];
359 struct usd_ib_dev *idp;
360 char *fw;
361 int fd;
362 int n;
363
364 idp = dev->ud_ctx->ucx_ib_dev;
365 snprintf(name, sizeof(name), "%s/fw_ver", idp->id_class_path);
366
367 fd = open(name, O_RDONLY);
368 if (fd == -1) {
369 usd_perror(name);
370 return -errno;
371 }
372
373 fw = &dev->ud_attrs.uda_firmware[0];
374 n = read(fd, fw, sizeof(dev->ud_attrs.uda_firmware));
375 close(fd);
376 if (n < 0) {
377 usd_perror("reading fw_ver");
378 return -errno;
379 }
380 fw[n - 1] = '\0';
381
382 return 0;
383 }
384