1 /*
2  * Copyright (c) 2004-2007 Voltaire Inc.  All rights reserved.
3  * Copyright (c) 2007 Xsigo Systems Inc.  All rights reserved.
4  * Copyright (c) 2008 Lawrence Livermore National Laboratory
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35 
36 #if HAVE_CONFIG_H
37 #include <config.h>
38 #endif				/* HAVE_CONFIG_H */
39 
40 #define _GNU_SOURCE
41 #include <stdio.h>
42 #include <stdlib.h>
43 #include <sys/types.h>
44 #include <sys/stat.h>
45 #include <unistd.h>
46 #include <fcntl.h>
47 #include <string.h>
48 #include <errno.h>
49 #include <inttypes.h>
50 
51 #include <infiniband/ibnetdisc.h>
52 
53 #include "internal.h"
54 #include "chassis.h"
55 
56 /* For this caching lib, we always cache little endian */
57 
58 /* Cache format
59  *
60  * Bytes 1-4 - magic number
61  * Bytes 5-8 - version number
62  * Bytes 9-12 - node count
63  * Bytes 13-16 - port count
64  * Bytes 17-24 - "from node" guid
65  * Bytes 25-28 - maxhops discovered
66  * Bytes X-Y - nodes (variable length)
67  * Bytes X-Y - ports (variable length)
68  *
69  * Nodes are cached as
70  *
71  * 2 bytes - smalid
72  * 1 byte - smalmc
73  * 1 byte - smaenhsp0 flag
74  * IB_SMP_DATA_SIZE bytes - switchinfo
75  * 8 bytes - guid
76  * 1 byte - type
77  * 1 byte - numports
78  * IB_SMP_DATA_SIZE bytes - info
79  * IB_SMP_DATA_SIZE bytes - nodedesc
80  * 1 byte - number of ports stored
81  * 8 bytes - portguid A
82  * 1 byte - port num A
83  * 8 bytes - portguid B
84  * 1 byte - port num B
85  * ... etc., depending on number of ports stored
86  *
87  * Ports are cached as
88  *
89  * 8 bytes - guid
90  * 1 byte - portnum
91  * 1 byte - external portnum
92  * 2 bytes - base lid
93  * 1 byte - lmc
94  * IB_SMP_DATA_SIZE bytes - info
95  * 8 bytes - node guid port "owned" by
96  * 1 byte - flag indicating if remote port exists
97  * 8 bytes - port guid remotely connected to
98  * 1 byte - port num remotely connected to
99  */
100 
101 /* Structs that hold cache info temporarily before
102  * the real structs can be reconstructed.
103  */
104 
105 typedef struct ibnd_port_cache_key {
106 	uint64_t guid;
107 	uint8_t portnum;
108 } ibnd_port_cache_key_t;
109 
110 typedef struct ibnd_node_cache {
111 	ibnd_node_t *node;
112 	uint8_t ports_stored_count;
113 	ibnd_port_cache_key_t *port_cache_keys;
114 	struct ibnd_node_cache *next;
115 	struct ibnd_node_cache *htnext;
116 	int node_stored_to_fabric;
117 } ibnd_node_cache_t;
118 
119 typedef struct ibnd_port_cache {
120 	ibnd_port_t *port;
121 	uint64_t node_guid;
122 	uint8_t remoteport_flag;
123 	ibnd_port_cache_key_t remoteport_cache_key;
124 	struct ibnd_port_cache *next;
125 	struct ibnd_port_cache *htnext;
126 	int port_stored_to_fabric;
127 } ibnd_port_cache_t;
128 
129 typedef struct ibnd_fabric_cache {
130 	f_internal_t *f_int;
131 	uint64_t from_node_guid;
132 	ibnd_node_cache_t *nodes_cache;
133 	ibnd_port_cache_t *ports_cache;
134 	ibnd_node_cache_t *nodescachetbl[HTSZ];
135 	ibnd_port_cache_t *portscachetbl[HTSZ];
136 } ibnd_fabric_cache_t;
137 
138 #define IBND_FABRIC_CACHE_BUFLEN  4096
139 #define IBND_FABRIC_CACHE_MAGIC   0x8FE7832B
140 #define IBND_FABRIC_CACHE_VERSION 0x00000001
141 
142 #define IBND_FABRIC_CACHE_COUNT_OFFSET 8
143 
144 #define IBND_FABRIC_CACHE_HEADER_LEN   (28)
145 #define IBND_NODE_CACHE_HEADER_LEN     (15 + IB_SMP_DATA_SIZE*3)
146 #define IBND_PORT_CACHE_KEY_LEN        (8 + 1)
147 #define IBND_PORT_CACHE_LEN            (31 + IB_SMP_DATA_SIZE)
148 
149 static ssize_t ibnd_read(int fd, void *buf, size_t count)
150 {
151 	size_t count_done = 0;
152 	ssize_t ret;
153 
154 	while ((count - count_done) > 0) {
155 		ret = read(fd, ((char *) buf) + count_done, count - count_done);
156 		if (ret < 0) {
157 			if (errno == EINTR)
158 				continue;
159 			else {
160 				IBND_DEBUG("read: %s\n", strerror(errno));
161 				return -1;
162 			}
163 		}
164 		if (!ret)
165 			break;
166 		count_done += ret;
167 	}
168 
169 	if (count_done != count) {
170 		IBND_DEBUG("read: read short\n");
171 		return -1;
172 	}
173 
174 	return count_done;
175 }
176 
177 static size_t _unmarshall8(uint8_t * inbuf, uint8_t * num)
178 {
179 	(*num) = inbuf[0];
180 
181 	return (sizeof(*num));
182 }
183 
184 static size_t _unmarshall16(uint8_t * inbuf, uint16_t * num)
185 {
186 	(*num) = ((uint16_t) inbuf[1] << 8) | inbuf[0];
187 
188 	return (sizeof(*num));
189 }
190 
191 static size_t _unmarshall32(uint8_t * inbuf, uint32_t * num)
192 {
193 	(*num) = (uint32_t) inbuf[0];
194 	(*num) |= ((uint32_t) inbuf[1] << 8);
195 	(*num) |= ((uint32_t) inbuf[2] << 16);
196 	(*num) |= ((uint32_t) inbuf[3] << 24);
197 
198 	return (sizeof(*num));
199 }
200 
201 static size_t _unmarshall64(uint8_t * inbuf, uint64_t * num)
202 {
203 	(*num) = (uint64_t) inbuf[0];
204 	(*num) |= ((uint64_t) inbuf[1] << 8);
205 	(*num) |= ((uint64_t) inbuf[2] << 16);
206 	(*num) |= ((uint64_t) inbuf[3] << 24);
207 	(*num) |= ((uint64_t) inbuf[4] << 32);
208 	(*num) |= ((uint64_t) inbuf[5] << 40);
209 	(*num) |= ((uint64_t) inbuf[6] << 48);
210 	(*num) |= ((uint64_t) inbuf[7] << 56);
211 
212 	return (sizeof(*num));
213 }
214 
215 static size_t _unmarshall_buf(const void *inbuf, void *outbuf, unsigned int len)
216 {
217 	memcpy(outbuf, inbuf, len);
218 
219 	return len;
220 }
221 
222 static int _load_header_info(int fd, ibnd_fabric_cache_t * fabric_cache,
223 			     unsigned int *node_count, unsigned int *port_count)
224 {
225 	uint8_t buf[IBND_FABRIC_CACHE_BUFLEN];
226 	uint32_t magic = 0;
227 	uint32_t version = 0;
228 	size_t offset = 0;
229 	uint32_t tmp32;
230 
231 	if (ibnd_read(fd, buf, IBND_FABRIC_CACHE_HEADER_LEN) < 0)
232 		return -1;
233 
234 	offset += _unmarshall32(buf + offset, &magic);
235 
236 	if (magic != IBND_FABRIC_CACHE_MAGIC) {
237 		IBND_DEBUG("invalid fabric cache file\n");
238 		return -1;
239 	}
240 
241 	offset += _unmarshall32(buf + offset, &version);
242 
243 	if (version != IBND_FABRIC_CACHE_VERSION) {
244 		IBND_DEBUG("invalid fabric cache version\n");
245 		return -1;
246 	}
247 
248 	offset += _unmarshall32(buf + offset, node_count);
249 	offset += _unmarshall32(buf + offset, port_count);
250 
251 	offset += _unmarshall64(buf + offset, &fabric_cache->from_node_guid);
252 	offset += _unmarshall32(buf + offset, &tmp32);
253 	fabric_cache->f_int->fabric.maxhops_discovered = tmp32;
254 
255 	return 0;
256 }
257 
258 static void _destroy_ibnd_node_cache(ibnd_node_cache_t * node_cache)
259 {
260 	free(node_cache->port_cache_keys);
261 	if (!node_cache->node_stored_to_fabric && node_cache->node)
262 		destroy_node(node_cache->node);
263 	free(node_cache);
264 }
265 
266 static void _destroy_ibnd_fabric_cache(ibnd_fabric_cache_t * fabric_cache)
267 {
268 	ibnd_node_cache_t *node_cache;
269 	ibnd_node_cache_t *node_cache_next;
270 	ibnd_port_cache_t *port_cache;
271 	ibnd_port_cache_t *port_cache_next;
272 
273 	if (!fabric_cache)
274 		return;
275 
276 	node_cache = fabric_cache->nodes_cache;
277 	while (node_cache) {
278 		node_cache_next = node_cache->next;
279 
280 		_destroy_ibnd_node_cache(node_cache);
281 
282 		node_cache = node_cache_next;
283 	}
284 
285 	port_cache = fabric_cache->ports_cache;
286 	while (port_cache) {
287 		port_cache_next = port_cache->next;
288 
289 		if (!port_cache->port_stored_to_fabric && port_cache->port)
290 			free(port_cache->port);
291 		free(port_cache);
292 
293 		port_cache = port_cache_next;
294 	}
295 
296 	free(fabric_cache);
297 }
298 
299 static void store_node_cache(ibnd_node_cache_t * node_cache,
300 			     ibnd_fabric_cache_t * fabric_cache)
301 {
302 	int hash_indx = HASHGUID(node_cache->node->guid) % HTSZ;
303 
304 	node_cache->next = fabric_cache->nodes_cache;
305 	fabric_cache->nodes_cache = node_cache;
306 
307 	node_cache->htnext = fabric_cache->nodescachetbl[hash_indx];
308 	fabric_cache->nodescachetbl[hash_indx] = node_cache;
309 }
310 
311 static int _load_node(int fd, ibnd_fabric_cache_t * fabric_cache)
312 {
313 	uint8_t buf[IBND_FABRIC_CACHE_BUFLEN];
314 	ibnd_node_cache_t *node_cache = NULL;
315 	ibnd_node_t *node = NULL;
316 	size_t offset = 0;
317 	uint8_t tmp8;
318 
319 	node_cache = (ibnd_node_cache_t *) malloc(sizeof(ibnd_node_cache_t));
320 	if (!node_cache) {
321 		IBND_DEBUG("OOM: node_cache\n");
322 		return -1;
323 	}
324 	memset(node_cache, '\0', sizeof(ibnd_node_cache_t));
325 
326 	node = (ibnd_node_t *) malloc(sizeof(ibnd_node_t));
327 	if (!node) {
328 		IBND_DEBUG("OOM: node\n");
329 		free(node_cache);
330 		return -1;
331 	}
332 	memset(node, '\0', sizeof(ibnd_node_t));
333 
334 	node_cache->node = node;
335 
336 	if (ibnd_read(fd, buf, IBND_NODE_CACHE_HEADER_LEN) < 0)
337 		goto cleanup;
338 
339 	offset += _unmarshall16(buf + offset, &node->smalid);
340 	offset += _unmarshall8(buf + offset, &node->smalmc);
341 	offset += _unmarshall8(buf + offset, &tmp8);
342 	node->smaenhsp0 = tmp8;
343 	offset += _unmarshall_buf(buf + offset, node->switchinfo,
344 				  IB_SMP_DATA_SIZE);
345 	offset += _unmarshall64(buf + offset, &node->guid);
346 	offset += _unmarshall8(buf + offset, &tmp8);
347 	node->type = tmp8;
348 	offset += _unmarshall8(buf + offset, &tmp8);
349 	node->numports = tmp8;
350 	offset += _unmarshall_buf(buf + offset, node->info, IB_SMP_DATA_SIZE);
351 	offset += _unmarshall_buf(buf + offset, node->nodedesc,
352 				  IB_SMP_DATA_SIZE);
353 
354 	offset += _unmarshall8(buf + offset, &node_cache->ports_stored_count);
355 
356 	if (node_cache->ports_stored_count) {
357 		unsigned int tomalloc = 0;
358 		unsigned int toread = 0;
359 		unsigned int i;
360 
361 		tomalloc =
362 		    sizeof(ibnd_port_cache_key_t) *
363 		    node_cache->ports_stored_count;
364 
365 		toread =
366 		    IBND_PORT_CACHE_KEY_LEN * node_cache->ports_stored_count;
367 
368 		node_cache->port_cache_keys =
369 		    (ibnd_port_cache_key_t *) malloc(tomalloc);
370 		if (!node_cache->port_cache_keys) {
371 			IBND_DEBUG("OOM: node_cache port_cache_keys\n");
372 			goto cleanup;
373 		}
374 
375 		if (ibnd_read(fd, buf, toread) < 0)
376 			goto cleanup;
377 
378 		offset = 0;
379 
380 		for (i = 0; i < node_cache->ports_stored_count; i++) {
381 			offset +=
382 			    _unmarshall64(buf + offset,
383 					  &node_cache->port_cache_keys[i].guid);
384 			offset +=
385 			    _unmarshall8(buf + offset,
386 					 &node_cache->
387 					 port_cache_keys[i].portnum);
388 		}
389 	}
390 
391 	store_node_cache(node_cache, fabric_cache);
392 
393 	return 0;
394 
395 cleanup:
396 	_destroy_ibnd_node_cache(node_cache);
397 	return -1;
398 }
399 
400 static void store_port_cache(ibnd_port_cache_t * port_cache,
401 			     ibnd_fabric_cache_t * fabric_cache)
402 {
403 	int hash_indx = HASHGUID(port_cache->port->guid) % HTSZ;
404 
405 	port_cache->next = fabric_cache->ports_cache;
406 	fabric_cache->ports_cache = port_cache;
407 
408 	port_cache->htnext = fabric_cache->portscachetbl[hash_indx];
409 	fabric_cache->portscachetbl[hash_indx] = port_cache;
410 }
411 
412 static int _load_port(int fd, ibnd_fabric_cache_t * fabric_cache)
413 {
414 	uint8_t buf[IBND_FABRIC_CACHE_BUFLEN];
415 	ibnd_port_cache_t *port_cache = NULL;
416 	ibnd_port_t *port = NULL;
417 	size_t offset = 0;
418 	uint8_t tmp8;
419 
420 	port_cache = (ibnd_port_cache_t *) malloc(sizeof(ibnd_port_cache_t));
421 	if (!port_cache) {
422 		IBND_DEBUG("OOM: port_cache\n");
423 		return -1;
424 	}
425 	memset(port_cache, '\0', sizeof(ibnd_port_cache_t));
426 
427 	port = (ibnd_port_t *) malloc(sizeof(ibnd_port_t));
428 	if (!port) {
429 		IBND_DEBUG("OOM: port\n");
430 		free(port_cache);
431 		return -1;
432 	}
433 	memset(port, '\0', sizeof(ibnd_port_t));
434 
435 	port_cache->port = port;
436 
437 	if (ibnd_read(fd, buf, IBND_PORT_CACHE_LEN) < 0)
438 		goto cleanup;
439 
440 	offset += _unmarshall64(buf + offset, &port->guid);
441 	offset += _unmarshall8(buf + offset, &tmp8);
442 	port->portnum = tmp8;
443 	offset += _unmarshall8(buf + offset, &tmp8);
444 	port->ext_portnum = tmp8;
445 	offset += _unmarshall16(buf + offset, &port->base_lid);
446 	offset += _unmarshall8(buf + offset, &port->lmc);
447 	offset += _unmarshall_buf(buf + offset, port->info, IB_SMP_DATA_SIZE);
448 	offset += _unmarshall64(buf + offset, &port_cache->node_guid);
449 	offset += _unmarshall8(buf + offset, &port_cache->remoteport_flag);
450 	offset +=
451 	    _unmarshall64(buf + offset, &port_cache->remoteport_cache_key.guid);
452 	offset +=
453 	    _unmarshall8(buf + offset,
454 			 &port_cache->remoteport_cache_key.portnum);
455 
456 	store_port_cache(port_cache, fabric_cache);
457 
458 	return 0;
459 
460 cleanup:
461 	free(port);
462 	free(port_cache);
463 	return -1;
464 }
465 
466 static ibnd_port_cache_t *_find_port(ibnd_fabric_cache_t * fabric_cache,
467 				     ibnd_port_cache_key_t * port_cache_key)
468 {
469 	int hash_indx = HASHGUID(port_cache_key->guid) % HTSZ;
470 	ibnd_port_cache_t *port_cache;
471 
472 	for (port_cache = fabric_cache->portscachetbl[hash_indx];
473 	     port_cache; port_cache = port_cache->htnext) {
474 		if (port_cache->port->guid == port_cache_key->guid
475 		    && port_cache->port->portnum == port_cache_key->portnum)
476 			return port_cache;
477 	}
478 
479 	return NULL;
480 }
481 
482 static ibnd_node_cache_t *_find_node(ibnd_fabric_cache_t * fabric_cache,
483 				     uint64_t guid)
484 {
485 	int hash_indx = HASHGUID(guid) % HTSZ;
486 	ibnd_node_cache_t *node_cache;
487 
488 	for (node_cache = fabric_cache->nodescachetbl[hash_indx];
489 	     node_cache; node_cache = node_cache->htnext) {
490 		if (node_cache->node->guid == guid)
491 			return node_cache;
492 	}
493 
494 	return NULL;
495 }
496 
497 static int _fill_port(ibnd_fabric_cache_t * fabric_cache, ibnd_node_t * node,
498 		      ibnd_port_cache_key_t * port_cache_key)
499 {
500 	ibnd_port_cache_t *port_cache;
501 
502 	if (!(port_cache = _find_port(fabric_cache, port_cache_key))) {
503 		IBND_DEBUG("Cache invalid: cannot find port\n");
504 		return -1;
505 	}
506 
507 	if (port_cache->port_stored_to_fabric) {
508 		IBND_DEBUG("Cache invalid: duplicate port discovered\n");
509 		return -1;
510 	}
511 
512 	node->ports[port_cache->port->portnum] = port_cache->port;
513 	port_cache->port_stored_to_fabric++;
514 
515 	/* achu: needed if user wishes to re-cache a loaded fabric.
516 	 * Otherwise, mostly unnecessary to do this.
517 	 */
518 	int rc = add_to_portguid_hash(port_cache->port,
519 				      fabric_cache->f_int->fabric.portstbl);
520 	if (rc) {
521 		IBND_DEBUG("Error Occurred when trying"
522 			   " to insert new port guid 0x%016" PRIx64 " to DB\n",
523 			   port_cache->port->guid);
524 	}
525 	return 0;
526 }
527 
528 static int _rebuild_nodes(ibnd_fabric_cache_t * fabric_cache)
529 {
530 	ibnd_node_cache_t *node_cache;
531 	ibnd_node_cache_t *node_cache_next;
532 
533 	node_cache = fabric_cache->nodes_cache;
534 	while (node_cache) {
535 		ibnd_node_t *node;
536 		int i;
537 
538 		node_cache_next = node_cache->next;
539 
540 		node = node_cache->node;
541 
542 		/* Insert node into appropriate data structures */
543 
544 		node->next = fabric_cache->f_int->fabric.nodes;
545 		fabric_cache->f_int->fabric.nodes = node;
546 
547 		int rc = add_to_nodeguid_hash(node_cache->node,
548 					      fabric_cache->
549 					      f_int->
550 					      fabric.nodestbl);
551 		if (rc) {
552 			IBND_DEBUG("Error Occurred when trying"
553 				   " to insert new node guid 0x%016" PRIx64 " to DB\n",
554 				   node_cache->node->guid);
555 		}
556 
557 		add_to_type_list(node_cache->node, fabric_cache->f_int);
558 
559 		node_cache->node_stored_to_fabric++;
560 
561 		/* Rebuild node ports array */
562 
563 		if (!(node->ports =
564 		      calloc(sizeof(*node->ports), node->numports + 1))) {
565 			IBND_DEBUG("OOM: node->ports\n");
566 			return -1;
567 		}
568 
569 		for (i = 0; i < node_cache->ports_stored_count; i++) {
570 			if (_fill_port(fabric_cache, node,
571 				       &node_cache->port_cache_keys[i]) < 0)
572 				return -1;
573 		}
574 
575 		node_cache = node_cache_next;
576 	}
577 
578 	return 0;
579 }
580 
581 static int _rebuild_ports(ibnd_fabric_cache_t * fabric_cache)
582 {
583 	ibnd_port_cache_t *port_cache;
584 	ibnd_port_cache_t *port_cache_next;
585 
586 	port_cache = fabric_cache->ports_cache;
587 	while (port_cache) {
588 		ibnd_node_cache_t *node_cache;
589 		ibnd_port_cache_t *remoteport_cache;
590 		ibnd_port_t *port;
591 
592 		port_cache_next = port_cache->next;
593 
594 		port = port_cache->port;
595 
596 		if (!(node_cache =
597 		      _find_node(fabric_cache, port_cache->node_guid))) {
598 			IBND_DEBUG("Cache invalid: cannot find node\n");
599 			return -1;
600 		}
601 
602 		port->node = node_cache->node;
603 
604 		if (port_cache->remoteport_flag) {
605 			if (!(remoteport_cache = _find_port(fabric_cache,
606 							    &port_cache->remoteport_cache_key)))
607 			{
608 				IBND_DEBUG
609 				    ("Cache invalid: cannot find remote port\n");
610 				return -1;
611 			}
612 
613 			port->remoteport = remoteport_cache->port;
614 		} else
615 			port->remoteport = NULL;
616 
617 		add_to_portlid_hash(port, fabric_cache->f_int->lid2guid);
618 		port_cache = port_cache_next;
619 	}
620 
621 	return 0;
622 }
623 
624 ibnd_fabric_t *ibnd_load_fabric(const char *file, unsigned int flags)
625 {
626 	unsigned int node_count = 0;
627 	unsigned int port_count = 0;
628 	ibnd_fabric_cache_t *fabric_cache = NULL;
629 	f_internal_t *f_int = NULL;
630 	ibnd_node_cache_t *node_cache = NULL;
631 	int fd = -1;
632 	unsigned int i;
633 
634 	if (!file) {
635 		IBND_DEBUG("file parameter NULL\n");
636 		return NULL;
637 	}
638 
639 	if ((fd = open(file, O_RDONLY)) < 0) {
640 		IBND_DEBUG("open: %s\n", strerror(errno));
641 		return NULL;
642 	}
643 
644 	fabric_cache =
645 	    (ibnd_fabric_cache_t *) malloc(sizeof(ibnd_fabric_cache_t));
646 	if (!fabric_cache) {
647 		IBND_DEBUG("OOM: fabric_cache\n");
648 		goto cleanup;
649 	}
650 	memset(fabric_cache, '\0', sizeof(ibnd_fabric_cache_t));
651 
652 	f_int = allocate_fabric_internal();
653 	if (!f_int) {
654 		IBND_DEBUG("OOM: fabric\n");
655 		goto cleanup;
656 	}
657 
658 	fabric_cache->f_int = f_int;
659 
660 	if (_load_header_info(fd, fabric_cache, &node_count, &port_count) < 0)
661 		goto cleanup;
662 
663 	for (i = 0; i < node_count; i++) {
664 		if (_load_node(fd, fabric_cache) < 0)
665 			goto cleanup;
666 	}
667 
668 	for (i = 0; i < port_count; i++) {
669 		if (_load_port(fd, fabric_cache) < 0)
670 			goto cleanup;
671 	}
672 
673 	/* Special case - find from node */
674 	if (!(node_cache =
675 	      _find_node(fabric_cache, fabric_cache->from_node_guid))) {
676 		IBND_DEBUG("Cache invalid: cannot find from node\n");
677 		goto cleanup;
678 	}
679 	f_int->fabric.from_node = node_cache->node;
680 
681 	if (_rebuild_nodes(fabric_cache) < 0)
682 		goto cleanup;
683 
684 	if (_rebuild_ports(fabric_cache) < 0)
685 		goto cleanup;
686 
687 	if (group_nodes(&f_int->fabric))
688 		goto cleanup;
689 
690 	_destroy_ibnd_fabric_cache(fabric_cache);
691 	close(fd);
692 	return (ibnd_fabric_t *)&f_int->fabric;
693 
694 cleanup:
695 	ibnd_destroy_fabric((ibnd_fabric_t *)f_int);
696 	_destroy_ibnd_fabric_cache(fabric_cache);
697 	close(fd);
698 	return NULL;
699 }
700 
701 static ssize_t ibnd_write(int fd, const void *buf, size_t count)
702 {
703 	size_t count_done = 0;
704 	ssize_t ret;
705 
706 	while ((count - count_done) > 0) {
707 		ret = write(fd, ((char *) buf) + count_done, count - count_done);
708 		if (ret < 0) {
709 			if (errno == EINTR)
710 				continue;
711 			else {
712 				IBND_DEBUG("write: %s\n", strerror(errno));
713 				return -1;
714 			}
715 		}
716 		count_done += ret;
717 	}
718 	return count_done;
719 }
720 
721 static size_t _marshall8(uint8_t * outbuf, uint8_t num)
722 {
723 	outbuf[0] = num;
724 
725 	return (sizeof(num));
726 }
727 
728 static size_t _marshall16(uint8_t * outbuf, uint16_t num)
729 {
730 	outbuf[0] = num & 0x00FF;
731 	outbuf[1] = (num & 0xFF00) >> 8;
732 
733 	return (sizeof(num));
734 }
735 
736 static size_t _marshall32(uint8_t * outbuf, uint32_t num)
737 {
738 	outbuf[0] = num & 0x000000FF;
739 	outbuf[1] = (num & 0x0000FF00) >> 8;
740 	outbuf[2] = (num & 0x00FF0000) >> 16;
741 	outbuf[3] = (num & 0xFF000000) >> 24;
742 
743 	return (sizeof(num));
744 }
745 
746 static size_t _marshall64(uint8_t * outbuf, uint64_t num)
747 {
748 	outbuf[0] = (uint8_t) num;
749 	outbuf[1] = (uint8_t) (num >> 8);
750 	outbuf[2] = (uint8_t) (num >> 16);
751 	outbuf[3] = (uint8_t) (num >> 24);
752 	outbuf[4] = (uint8_t) (num >> 32);
753 	outbuf[5] = (uint8_t) (num >> 40);
754 	outbuf[6] = (uint8_t) (num >> 48);
755 	outbuf[7] = (uint8_t) (num >> 56);
756 
757 	return (sizeof(num));
758 }
759 
760 static size_t _marshall_buf(void *outbuf, const void *inbuf, unsigned int len)
761 {
762 	memcpy(outbuf, inbuf, len);
763 
764 	return len;
765 }
766 
767 static int _cache_header_info(int fd, ibnd_fabric_t * fabric)
768 {
769 	uint8_t buf[IBND_FABRIC_CACHE_BUFLEN];
770 	size_t offset = 0;
771 
772 	/* Store magic number, version, and other important info */
773 	/* For this caching lib, we always assume cached as little endian */
774 
775 	offset += _marshall32(buf + offset, IBND_FABRIC_CACHE_MAGIC);
776 	offset += _marshall32(buf + offset, IBND_FABRIC_CACHE_VERSION);
777 	/* save space for node count */
778 	offset += _marshall32(buf + offset, 0);
779 	/* save space for port count */
780 	offset += _marshall32(buf + offset, 0);
781 	offset += _marshall64(buf + offset, fabric->from_node->guid);
782 	offset += _marshall32(buf + offset, fabric->maxhops_discovered);
783 
784 	if (ibnd_write(fd, buf, offset) < 0)
785 		return -1;
786 
787 	return 0;
788 }
789 
790 static int _cache_header_counts(int fd, unsigned int node_count,
791 				unsigned int port_count)
792 {
793 	uint8_t buf[IBND_FABRIC_CACHE_BUFLEN];
794 	size_t offset = 0;
795 
796 	offset += _marshall32(buf + offset, node_count);
797 	offset += _marshall32(buf + offset, port_count);
798 
799 	if (lseek(fd, IBND_FABRIC_CACHE_COUNT_OFFSET, SEEK_SET) < 0) {
800 		IBND_DEBUG("lseek: %s\n", strerror(errno));
801 		return -1;
802 	}
803 
804 	if (ibnd_write(fd, buf, offset) < 0)
805 		return -1;
806 
807 	return 0;
808 }
809 
810 static int _cache_node(int fd, ibnd_node_t * node)
811 {
812 	uint8_t buf[IBND_FABRIC_CACHE_BUFLEN];
813 	size_t offset = 0;
814 	size_t ports_stored_offset = 0;
815 	uint8_t ports_stored_count = 0;
816 	int i;
817 
818 	offset += _marshall16(buf + offset, node->smalid);
819 	offset += _marshall8(buf + offset, node->smalmc);
820 	offset += _marshall8(buf + offset, (uint8_t) node->smaenhsp0);
821 	offset += _marshall_buf(buf + offset, node->switchinfo,
822 				IB_SMP_DATA_SIZE);
823 	offset += _marshall64(buf + offset, node->guid);
824 	offset += _marshall8(buf + offset, (uint8_t) node->type);
825 	offset += _marshall8(buf + offset, (uint8_t) node->numports);
826 	offset += _marshall_buf(buf + offset, node->info, IB_SMP_DATA_SIZE);
827 	offset += _marshall_buf(buf + offset, node->nodedesc, IB_SMP_DATA_SIZE);
828 	/* need to come back later and store number of stored ports
829 	 * because port entries can be NULL or (in the case of switches)
830 	 * there is an additional port 0 not accounted for in numports.
831 	 */
832 	ports_stored_offset = offset;
833 	offset += sizeof(uint8_t);
834 
835 	for (i = 0; i <= node->numports; i++) {
836 		if (node->ports[i]) {
837 			offset += _marshall64(buf + offset,
838 					      node->ports[i]->guid);
839 			offset += _marshall8(buf + offset,
840 					     (uint8_t) node->ports[i]->portnum);
841 			ports_stored_count++;
842 		}
843 	}
844 
845 	/* go back and store number of port keys stored */
846 	_marshall8(buf + ports_stored_offset, ports_stored_count);
847 
848 	if (ibnd_write(fd, buf, offset) < 0)
849 		return -1;
850 
851 	return 0;
852 }
853 
854 static int _cache_port(int fd, ibnd_port_t * port)
855 {
856 	uint8_t buf[IBND_FABRIC_CACHE_BUFLEN];
857 	size_t offset = 0;
858 
859 	offset += _marshall64(buf + offset, port->guid);
860 	offset += _marshall8(buf + offset, (uint8_t) port->portnum);
861 	offset += _marshall8(buf + offset, (uint8_t) port->ext_portnum);
862 	offset += _marshall16(buf + offset, port->base_lid);
863 	offset += _marshall8(buf + offset, port->lmc);
864 	offset += _marshall_buf(buf + offset, port->info, IB_SMP_DATA_SIZE);
865 	offset += _marshall64(buf + offset, port->node->guid);
866 	if (port->remoteport) {
867 		offset += _marshall8(buf + offset, 1);
868 		offset += _marshall64(buf + offset, port->remoteport->guid);
869 		offset += _marshall8(buf + offset, (uint8_t) port->remoteport->portnum);
870 	} else {
871 		offset += _marshall8(buf + offset, 0);
872 		offset += _marshall64(buf + offset, 0);
873 		offset += _marshall8(buf + offset, 0);
874 	}
875 
876 	if (ibnd_write(fd, buf, offset) < 0)
877 		return -1;
878 
879 	return 0;
880 }
881 
882 int ibnd_cache_fabric(ibnd_fabric_t * fabric, const char *file,
883 		      unsigned int flags)
884 {
885 	struct stat statbuf;
886 	ibnd_node_t *node = NULL;
887 	ibnd_node_t *node_next = NULL;
888 	unsigned int node_count = 0;
889 	ibnd_port_t *port = NULL;
890 	ibnd_port_t *port_next = NULL;
891 	unsigned int port_count = 0;
892 	int fd;
893 	int i;
894 
895 	if (!fabric) {
896 		IBND_DEBUG("fabric parameter NULL\n");
897 		return -1;
898 	}
899 
900 	if (!file) {
901 		IBND_DEBUG("file parameter NULL\n");
902 		return -1;
903 	}
904 
905 	if (!(flags & IBND_CACHE_FABRIC_FLAG_NO_OVERWRITE)) {
906 		if (!stat(file, &statbuf)) {
907 			if (unlink(file) < 0) {
908 				IBND_DEBUG("error removing '%s': %s\n",
909 					   file, strerror(errno));
910 				return -1;
911 			}
912 		}
913 	}
914 	else {
915 		if (!stat(file, &statbuf)) {
916 			IBND_DEBUG("file '%s' already exists\n", file);
917 			return -1;
918 		}
919 	}
920 
921 	if ((fd = open(file, O_CREAT | O_EXCL | O_WRONLY, 0644)) < 0) {
922 		IBND_DEBUG("open: %s\n", strerror(errno));
923 		return -1;
924 	}
925 
926 	if (_cache_header_info(fd, fabric) < 0)
927 		goto cleanup;
928 
929 	node = fabric->nodes;
930 	while (node) {
931 		node_next = node->next;
932 
933 		if (_cache_node(fd, node) < 0)
934 			goto cleanup;
935 
936 		node_count++;
937 		node = node_next;
938 	}
939 
940 	for (i = 0; i < HTSZ; i++) {
941 		port = fabric->portstbl[i];
942 		while (port) {
943 			port_next = port->htnext;
944 
945 			if (_cache_port(fd, port) < 0)
946 				goto cleanup;
947 
948 			port_count++;
949 			port = port_next;
950 		}
951 	}
952 
953 	if (_cache_header_counts(fd, node_count, port_count) < 0)
954 		goto cleanup;
955 
956 	if (close(fd) < 0) {
957 		IBND_DEBUG("close: %s\n", strerror(errno));
958 		goto cleanup;
959 	}
960 
961 	return 0;
962 
963 cleanup:
964 	unlink(file);
965 	close(fd);
966 	return -1;
967 }
968