1 /*
2  * Copyright (c) 2014, Cisco Systems, Inc. All rights reserved.
3  *
4  * LICENSE_BEGIN
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
28  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
29  * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
30  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
31  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
32  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
33  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
35  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36  * POSSIBILITY OF SUCH DAMAGE.
37  *
38  * LICENSE_END
39  *
40  *
41  */
42 
43 #include <stdio.h>
44 #include <stdlib.h>
45 #include <unistd.h>
46 #include <string.h>
47 #include <fcntl.h>
48 #include <dirent.h>
49 #include <pthread.h>
50 #include <errno.h>
51 #include <sys/stat.h>
52 
53 #include "usd.h"
54 #include "usd_ib_sysfs.h"
55 #include "usd_util.h"
56 
57 /*
58  * usnic_direct routines that depend on Infiniband /sysfs directory structure
59  */
60 
61 /*
62  * Perform one-time initialization
63  */
64 int
usd_ib_get_devlist(struct usd_ib_dev ** dev_list)65 usd_ib_get_devlist(
66     struct usd_ib_dev **dev_list)
67 {
68     char *class_path = "/sys/class/infiniband_verbs";
69     DIR *class_dir;
70     struct dirent *dent;
71     struct stat sbuf;
72     char *dev_path = NULL;
73     char *ibdev_path = NULL;
74     char ibdev_buf[32];
75     struct usd_ib_dev *idp;
76     struct usd_ib_dev *last_idp;
77     int fd;
78     int rc;
79     int n;
80 
81     /*
82      * For now, we are glomming onto Infiniband driver for setup
83      */
84     class_dir = opendir(class_path);
85     if (class_dir == NULL) {
86         return -ENODEV;
87     }
88 
89     /* Check dir entries for USNIC devices */
90     last_idp = NULL;
91     fd = -1;
92     while ((dent = readdir(class_dir)) != NULL) {
93         /* skip "." and ".." */
94         if (dent->d_name[0] == '.')
95             continue;
96 
97         /* build path to entry */
98         if (asprintf(&dev_path, "%s/%s", class_path,
99                      dent->d_name) <= 0) {
100             rc = -errno;
101             usd_perror("failed to asprintf");
102             goto out;
103         }
104 
105         /* see if it's a dir */
106         rc = stat(dev_path, &sbuf);
107         if (rc != 0) {
108             usd_perror(dev_path);
109             rc = -errno;
110             goto out;
111         }
112 
113         /* Must be a directory */
114         if (!S_ISDIR(sbuf.st_mode))
115             continue;
116 
117         /* read the ibdev */
118         if (asprintf(&ibdev_path, "%s/ibdev", dev_path) <= 0) {
119             rc = -errno;
120             usd_perror(ibdev_path);
121             goto out;
122         }
123         fd = open(ibdev_path, O_RDONLY);
124         if (fd == -1) {
125             usd_perror(ibdev_path);
126             rc = -errno;
127             goto out;
128         }
129         memset(ibdev_buf, 0, sizeof(ibdev_buf));
130         n = read(fd, ibdev_buf, sizeof(ibdev_buf) - 1);
131         if (n == -1) {
132             usd_perror("reading ibdev");
133             rc = -errno;
134             goto out;
135         }
136         close(fd);
137         fd = -1;
138         if (n > 0 && ibdev_buf[n - 1] == '\n') {
139             ibdev_buf[n - 1] = '\0';       /* newline -> EOF */
140         }
141 
142         /* If USNIC device, remember this one */
143         if (strncmp(ibdev_buf, "usnic", 5) == 0) {
144             idp = calloc(sizeof(*idp), 1);
145             if (idp == NULL) {
146                 usd_perror("calloc IB device");
147                 rc = -errno;
148                 goto out;
149             }
150             strncpy(idp->id_name, dent->d_name, sizeof(idp->id_name) - 1);
151             strncpy(idp->id_usnic_name, ibdev_buf,
152                     sizeof(idp->id_usnic_name) - 1);
153             snprintf(idp->id_dev_path, sizeof(idp->id_dev_path) - 1,
154                      "/dev/infiniband/%s", idp->id_name);
155             snprintf(idp->id_class_path, sizeof(idp->id_class_path) - 1,
156                      "%s/device/infiniband/%s", dev_path, ibdev_buf);
157 
158             if (last_idp == NULL) {
159                 *dev_list = idp;
160             } else {
161                 last_idp->id_next = idp;
162             }
163             idp->id_next = NULL;
164             last_idp = idp;
165         }
166         free(dev_path);
167         dev_path = NULL;
168         free(ibdev_path);
169         ibdev_path = NULL;
170     }
171     rc = 0;
172 
173 out:
174     /* clean up */
175     free(dev_path);
176     free(ibdev_path);
177     if (class_dir != NULL) {
178         closedir(class_dir);
179     }
180     if (fd != -1) {
181         close(fd);
182     }
183 
184     return rc;
185 }
186 
187 /*
188  * Find MAC for a device
189  * (we assume port 0)
190  */
191 int
usd_get_mac(struct usd_device * dev,uint8_t * mac)192 usd_get_mac(
193     struct usd_device *dev,
194     uint8_t * mac)
195 {
196     char name[PATH_MAX + 128];
197     char gid[80];
198     char *p;
199     uint16_t v;
200     struct usd_ib_dev *idp;
201     int fd;
202     int n;
203 
204     idp = dev->ud_ctx->ucx_ib_dev;
205     snprintf(name, sizeof(name), "%s/ports/1/gids/0", idp->id_class_path);
206 
207     fd = open(name, O_RDONLY);
208     if (fd == -1) {
209         usd_perror(name);
210         return -errno;
211     }
212 
213     n = read(fd, gid, sizeof(gid) - 1);
214     close(fd);
215     if (n < 0) {
216         usd_perror("reading GID");
217         return -errno;
218     }
219     gid[n] = '\0';
220 
221     p = gid + 20;
222     sscanf(p, "%hx", &v);
223     *mac++ = (v >> 8) ^ 2;
224     *mac++ = v & 0xFF;
225     p += 5;
226     sscanf(p, "%hx", &v);
227     *mac++ = v >> 8;
228     p += 5;
229     sscanf(p, "%hx", &v);
230     *mac++ = v & 0xFF;
231     p += 5;
232     sscanf(p, "%hx", &v);
233     *mac++ = v >> 8;
234     *mac++ = v & 0xFF;
235 
236     return 0;
237 }
238 
239 /*
240  * Find interface for a device
241  */
242 int
usd_get_iface(struct usd_device * dev)243 usd_get_iface(
244     struct usd_device *dev)
245 {
246     char name[PATH_MAX + 128];
247     struct usd_ib_dev *idp;
248     int fd;
249     int n;
250 
251     idp = dev->ud_ctx->ucx_ib_dev;
252     snprintf(name, sizeof(name), "%s/iface", idp->id_class_path);
253 
254     fd = open(name, O_RDONLY);
255     if (fd == -1) {
256         usd_perror(name);
257         dev->ud_attrs.uda_ifname[0] = '\0';
258         return -errno;
259     }
260 
261     n = read(fd, dev->ud_attrs.uda_ifname,
262              sizeof(dev->ud_attrs.uda_ifname));
263     close(fd);
264     if (n < 0) {
265         usd_perror("reading iface");
266         return -errno;
267     }
268 
269     dev->ud_attrs.uda_ifname[n - 1] = '\0';
270 
271     return 0;
272 }
273 
274 /*
275  * Read an integer from a sysfs entry
276  */
277 static int
usd_ib_sysfs_get_int(struct usd_device * dev,char * entry,int * result)278 usd_ib_sysfs_get_int(
279     struct usd_device *dev,
280     char *entry,
281     int *result)
282 {
283     char name[PATH_MAX + 128];
284     char buf[32];
285     struct usd_ib_dev *idp;
286     int fd;
287     int n;
288 
289     idp = dev->ud_ctx->ucx_ib_dev;
290     snprintf(name, sizeof(name), "%s/%s", idp->id_class_path, entry);
291 
292     fd = open(name, O_RDONLY);
293     if (fd == -1) {
294         usd_perror(name);
295         return -errno;
296     }
297 
298     n = read(fd, buf, sizeof(buf));
299     close(fd);
300     if (n < 0) {
301         fprintf(stderr, "Error %d reading %s\n", errno, entry);
302         return -errno;
303     }
304 
305     *result = atoi(buf);
306     return 0;
307 }
308 
309 /*
310  * Get usNIC configuration
311  */
312 int
usd_get_usnic_config(struct usd_device * dev)313 usd_get_usnic_config(
314     struct usd_device *dev)
315 {
316     int v;
317     int ret;
318 
319     ret = usd_ib_sysfs_get_int(dev, "max_vf", &v);
320     if (ret != 0)
321         return ret;
322     dev->ud_attrs.uda_num_vf = v;
323 
324     ret = usd_ib_sysfs_get_int(dev, "qp_per_vf", &v);
325     if (ret != 0)
326         return ret;
327     dev->ud_attrs.uda_qp_per_vf = v;
328 
329     ret = usd_ib_sysfs_get_int(dev, "cq_per_vf", &v);
330     if (ret != 0)
331         return ret;
332     dev->ud_attrs.uda_cq_per_vf = v;
333 
334     ret = usd_ib_sysfs_get_int(dev, "intr_per_vf", &v);
335     if (ret != 0) {
336         /* older kernels did not export this sysfs node */
337         if (ret == -ENOENT) {
338             dev->ud_attrs.uda_intr_per_vf = 0;
339             ret = 0;
340         }
341         else {
342             return ret;
343         }
344     } else {
345         dev->ud_attrs.uda_intr_per_vf = v;
346     }
347 
348     return ret;
349 }
350 
351 /*
352  * Find firmware version
353  */
354 int
usd_get_firmware(struct usd_device * dev)355 usd_get_firmware(
356     struct usd_device *dev)
357 {
358     char name[PATH_MAX + 128];
359     struct usd_ib_dev *idp;
360     char *fw;
361     int fd;
362     int n;
363 
364     idp = dev->ud_ctx->ucx_ib_dev;
365     snprintf(name, sizeof(name), "%s/fw_ver", idp->id_class_path);
366 
367     fd = open(name, O_RDONLY);
368     if (fd == -1) {
369         usd_perror(name);
370         return -errno;
371     }
372 
373     fw = &dev->ud_attrs.uda_firmware[0];
374     n = read(fd, fw, sizeof(dev->ud_attrs.uda_firmware));
375     close(fd);
376     if (n < 0) {
377         usd_perror("reading fw_ver");
378         return -errno;
379     }
380     fw[n - 1] = '\0';
381 
382     return 0;
383 }
384