1 /*
2  * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  * 02110-1301, USA.
18  *
19  * You can also choose to distribute this program under the terms of
20  * the Unmodified Binary Distribution Licence (as given in the file
21  * COPYING.UBDL), provided that you have satisfied its requirements.
22  */
23 
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25 
26 #include <string.h>
27 #include <stdio.h>
28 #include <errno.h>
29 #include <byteswap.h>
30 #include <ipxe/version.h>
31 #include <ipxe/timer.h>
32 #include <ipxe/malloc.h>
33 #include <ipxe/iobuf.h>
34 #include <ipxe/retry.h>
35 #include <ipxe/process.h>
36 #include <ipxe/settings.h>
37 #include <ipxe/infiniband.h>
38 #include <ipxe/ib_service.h>
39 #include <ipxe/ib_cmrc.h>
40 #include <ipxe/if_ether.h>
41 #include <ipxe/ethernet.h>
42 #include <ipxe/eoib.h>
43 #include <ipxe/xsigo.h>
44 
45 /** @file
46  *
47  * Xsigo virtual Ethernet devices
48  *
49  */
50 
51 /** A Xsigo device */
52 struct xsigo_device {
53 	/** Reference count */
54 	struct refcnt refcnt;
55 	/** Underlying Infiniband device */
56 	struct ib_device *ibdev;
57 	/** List of Xsigo devices */
58 	struct list_head list;
59 	/** Device name */
60 	const char *name;
61 
62 	/** Link opener timer */
63 	struct retry_timer opener;
64 
65 	/** Discovery timer */
66 	struct retry_timer discovery;
67 	/** Discovery management transaction (if any) */
68 	struct ib_mad_transaction *madx;
69 
70 	/** List of configuration managers */
71 	struct list_head managers;
72 };
73 
74 /** A Xsigo configuration manager */
75 struct xsigo_manager {
76 	/** Reference count */
77 	struct refcnt refcnt;
78 	/** Xsigo device */
79 	struct xsigo_device *xdev;
80 	/** List of managers */
81 	struct list_head list;
82 	/** Device name */
83 	char name[16];
84 	/** Manager ID */
85 	struct xsigo_manager_id id;
86 
87 	/** Data transfer interface */
88 	struct interface xfer;
89 	/** Connection timer */
90 	struct retry_timer reopen;
91 	/** Keepalive timer */
92 	struct retry_timer keepalive;
93 	/** Transmission process */
94 	struct process process;
95 	/** Pending transmissions */
96 	unsigned int pending;
97 	/** Transmit sequence number */
98 	uint32_t seq;
99 
100 	/** List of virtual Ethernet devices */
101 	struct list_head nics;
102 };
103 
104 /** Configuration manager pending transmissions */
105 enum xsigo_manager_pending {
106 	/** Send connection request */
107 	XCM_TX_CONNECT = 0x0001,
108 	/** Send registration message */
109 	XCM_TX_REGISTER = 0x0002,
110 };
111 
112 /** A Xsigo virtual Ethernet device */
113 struct xsigo_nic {
114 	/** Configuration manager */
115 	struct xsigo_manager *xcm;
116 	/** List of virtual Ethernet devices */
117 	struct list_head list;
118 	/** Device name */
119 	char name[16];
120 
121 	/** Resource identifier */
122 	union ib_guid resource;
123 	/** MAC address */
124 	uint8_t mac[ETH_ALEN];
125 	/** Network ID */
126 	unsigned long network;
127 };
128 
129 /** Configuration manager service ID */
130 static union ib_guid xcm_service_id = {
131 	.bytes = XCM_SERVICE_ID,
132 };
133 
134 /** List of all Xsigo devices */
135 static LIST_HEAD ( xsigo_devices );
136 
137 /**
138  * Free Xsigo device
139  *
140  * @v refcnt		Reference count
141  */
xsigo_free(struct refcnt * refcnt)142 static void xsigo_free ( struct refcnt *refcnt ) {
143 	struct xsigo_device *xdev =
144 		container_of ( refcnt, struct xsigo_device, refcnt );
145 
146 	/* Sanity checks */
147 	assert ( ! timer_running ( &xdev->opener ) );
148 	assert ( ! timer_running ( &xdev->discovery ) );
149 	assert ( xdev->madx == NULL );
150 	assert ( list_empty ( &xdev->managers ) );
151 
152 	/* Drop reference to Infiniband device */
153 	ibdev_put ( xdev->ibdev );
154 
155 	/* Free device */
156 	free ( xdev );
157 }
158 
159 /**
160  * Free configuration manager
161  *
162  * @v refcnt		Reference count
163  */
xcm_free(struct refcnt * refcnt)164 static void xcm_free ( struct refcnt *refcnt ) {
165 	struct xsigo_manager *xcm =
166 		container_of ( refcnt, struct xsigo_manager, refcnt );
167 
168 	/* Sanity checks */
169 	assert ( ! timer_running ( &xcm->reopen ) );
170 	assert ( ! timer_running ( &xcm->keepalive ) );
171 	assert ( ! process_running ( &xcm->process ) );
172 	assert ( list_empty ( &xcm->nics ) );
173 
174 	/* Drop reference to Xsigo device */
175 	ref_put ( &xcm->xdev->refcnt );
176 
177 	/* Free manager */
178 	free ( xcm );
179 }
180 
181 /****************************************************************************
182  *
183  * Virtual Ethernet (XVE) devices
184  *
185  ****************************************************************************
186  */
187 
188 /**
189  * Create virtual Ethernet device
190  *
191  * @v xcm		Configuration manager
192  * @v resource		Resource identifier
193  * @v mac		Ethernet MAC
194  * @v network		Network identifier
195  * @v name		Device name
196  * @ret rc		Return status code
197  */
xve_create(struct xsigo_manager * xcm,union ib_guid * resource,const uint8_t * mac,unsigned long network,unsigned long qkey,const char * name)198 static int xve_create ( struct xsigo_manager *xcm, union ib_guid *resource,
199 			const uint8_t *mac, unsigned long network,
200 			unsigned long qkey, const char *name ) {
201 	struct xsigo_device *xdev = xcm->xdev;
202 	struct ib_device *ibdev = xdev->ibdev;
203 	struct xsigo_nic *xve;
204 	struct ib_address_vector broadcast;
205 	int rc;
206 
207 	/* Allocate and initialise structure */
208 	xve = zalloc ( sizeof ( *xve ) );
209 	if ( ! xve ) {
210 		rc = -ENOMEM;
211 		goto err_alloc;
212 	}
213 	xve->xcm = xcm;
214 	snprintf ( xve->name, sizeof ( xve->name ), "%s", name );
215 	memcpy ( &xve->resource, resource, sizeof ( xve->resource ) );
216 	memcpy ( xve->mac, mac, ETH_ALEN );
217 	xve->network = network;
218 	DBGC ( xve, "XVE %s created for %s " IB_GUID_FMT "\n",
219 	       xve->name, xcm->name, IB_GUID_ARGS ( resource ) );
220 	DBGC ( xve, "XVE %s is MAC %s on network %ld\n",
221 	       xve->name, eth_ntoa ( mac ), network );
222 
223 	/* Construct broadcast address vector */
224 	memset ( &broadcast, 0, sizeof ( broadcast ) );
225 	broadcast.qpn = IB_QPN_BROADCAST;
226 	broadcast.qkey = qkey;
227 	broadcast.gid_present = 1;
228 	broadcast.gid.dwords[0] = htonl ( XVE_PREFIX );
229 	broadcast.gid.words[2] = htons ( ibdev->pkey );
230 	broadcast.gid.dwords[3] = htonl ( network );
231 
232 	/* Create EoIB device */
233 	if ( ( rc = eoib_create ( ibdev, xve->mac, &broadcast,
234 				  xve->name ) ) != 0 ) {
235 		DBGC ( xve, "XVE %s could not create EoIB device: %s\n",
236 		       xve->name, strerror ( rc ) );
237 		goto err_create;
238 	}
239 
240 	/* Add to list of virtual Ethernet devices.  Do this only
241 	 * after creating the EoIB device, so that our net device
242 	 * notifier won't attempt to send an operational state update
243 	 * before we have acknowledged the installation.
244 	 */
245 	list_add ( &xve->list, &xcm->nics );
246 
247 	return 0;
248 
249 	list_del ( &xve->list );
250  err_create:
251 	free ( xve );
252  err_alloc:
253 	return rc;
254 }
255 
256 /**
257  * Find virtual Ethernet device
258  *
259  * @v xcm		Configuration manager
260  * @v resource		Resource identifier
261  * @ret xve		Virtual Ethernet device, or NULL
262  */
xve_find(struct xsigo_manager * xcm,union ib_guid * resource)263 static struct xsigo_nic * xve_find ( struct xsigo_manager *xcm,
264 				     union ib_guid *resource ) {
265 	struct xsigo_nic *xve;
266 
267 	list_for_each_entry ( xve, &xcm->nics, list ) {
268 		if ( memcmp ( resource, &xve->resource,
269 			      sizeof ( *resource ) ) == 0 )
270 			return xve;
271 	}
272 	return NULL;
273 }
274 
275 /**
276  * Destroy virtual Ethernet device
277  *
278  * @v xve		Virtual Ethernet device
279  */
xve_destroy(struct xsigo_nic * xve)280 static void xve_destroy ( struct xsigo_nic *xve ) {
281 	struct xsigo_manager *xcm = xve->xcm;
282 	struct xsigo_device *xdev = xcm->xdev;
283 	struct ib_device *ibdev = xdev->ibdev;
284 	struct eoib_device *eoib;
285 
286 	/* Destroy corresponding EoIB device, if any */
287 	if ( ( eoib = eoib_find ( ibdev, xve->mac ) ) )
288 		eoib_destroy ( eoib );
289 
290 	/* Remove from list of virtual Ethernet devices */
291 	list_del ( &xve->list );
292 
293 	/* Free virtual Ethernet device */
294 	DBGC ( xve, "XVE %s destroyed\n", xve->name );
295 	free ( xve );
296 }
297 
298 /**
299  * Update virtual Ethernet device MTU
300  *
301  * @v xve		Virtual Ethernet device
302  * @v eoib		EoIB device
303  * @v mtu		New MTU (excluding Ethernet and EoIB headers)
304  * @ret rc		Return status code
305  */
xve_update_mtu(struct xsigo_nic * xve,struct eoib_device * eoib,size_t mtu)306 static int xve_update_mtu ( struct xsigo_nic *xve, struct eoib_device *eoib,
307 			    size_t mtu ) {
308 	struct net_device *netdev = eoib->netdev;
309 	size_t max;
310 
311 	/* Check that we can support this MTU */
312 	max = ( IB_MAX_PAYLOAD_SIZE - ( sizeof ( struct ethhdr ) +
313 					sizeof ( struct eoib_header ) ) );
314 	if ( mtu > max ) {
315 		DBGC ( xve, "XVE %s cannot support MTU %zd (max %zd)\n",
316 		       xve->name, mtu, max );
317 		return -ERANGE;
318 	}
319 
320 	/* Update MTU.  No need to close/reopen the network device,
321 	 * since our Infiniband stack uses a fixed MTU anyway.  Note
322 	 * that the network device sees the Ethernet frame header but
323 	 * not the EoIB header.
324 	 */
325 	netdev->max_pkt_len = ( mtu + sizeof ( struct ethhdr ) );
326 	netdev->mtu = mtu;
327 	DBGC ( xve, "XVE %s has MTU %zd\n", xve->name, mtu );
328 
329 	return 0;
330 }
331 
332 /**
333  * Open virtual Ethernet device
334  *
335  * @v xve		Virtual Ethernet device
336  * @v eoib		EoIB device
337  * @v open		New administrative state
338  * @ret rc		Return status code
339  */
xve_open(struct xsigo_nic * xve,struct eoib_device * eoib)340 static int xve_open ( struct xsigo_nic *xve, struct eoib_device *eoib ) {
341 	struct net_device *netdev = eoib->netdev;
342 	int rc;
343 
344 	/* Do nothing if network device is already open */
345 	if ( netdev_is_open ( netdev ) )
346 		return 0;
347 	DBGC ( xve, "XVE %s opening network device\n", xve->name );
348 
349 	/* Open network device */
350 	if ( ( rc = netdev_open ( netdev ) ) != 0 ) {
351 		DBGC ( xve, "XVE %s could not open: %s\n",
352 		       xve->name, strerror ( rc ) );
353 		return rc;
354 	}
355 
356 	return 0;
357 }
358 
359 /**
360  * Close virtual Ethernet device
361  *
362  * @v xve		Virtual Ethernet device
363  * @v eoib		EoIB device
364  */
xve_close(struct xsigo_nic * xve,struct eoib_device * eoib)365 static void xve_close ( struct xsigo_nic *xve, struct eoib_device *eoib ) {
366 	struct net_device *netdev = eoib->netdev;
367 
368 	/* Do nothing if network device is already closed */
369 	if ( ! netdev_is_open ( netdev ) )
370 		return;
371 
372 	/* Close network device */
373 	netdev_close ( netdev );
374 	DBGC ( xve, "XVE %s closed network device\n", xve->name );
375 }
376 
377 /**
378  * Update virtual Ethernet device administrative state
379  *
380  * @v xve		Virtual Ethernet device
381  * @v eoib		EoIB device
382  * @v open		New administrative state
383  * @ret rc		Return status code
384  */
xve_update_state(struct xsigo_nic * xve,struct eoib_device * eoib,int open)385 static int xve_update_state ( struct xsigo_nic *xve, struct eoib_device *eoib,
386 			      int open ) {
387 
388 	/* Open or close device, as applicable */
389 	if ( open ) {
390 		return xve_open ( xve, eoib );
391 	} else {
392 		xve_close ( xve, eoib );
393 		return 0;
394 	}
395 }
396 
397 /**
398  * Update gateway (TCA)
399  *
400  * @v xve		Virtual Ethernet device
401  * @v eoib		EoIB device
402  * @v av		Address vector, or NULL if no gateway
403  * @ret rc		Return status code
404  */
xve_update_tca(struct xsigo_nic * xve,struct eoib_device * eoib,struct ib_address_vector * av)405 static int xve_update_tca ( struct xsigo_nic *xve, struct eoib_device *eoib,
406 			    struct ib_address_vector *av ) {
407 
408 	/* Update gateway address */
409 	eoib_set_gateway ( eoib, av );
410 	if ( av ) {
411 		DBGC ( xve, "XVE %s has TCA " IB_GID_FMT " data %#lx qkey "
412 		       "%#lx\n", xve->name, IB_GID_ARGS ( &av->gid ), av->qpn,
413 		       av->qkey );
414 	} else {
415 		DBGC ( xve, "XVE %s has no TCA\n", xve->name );
416 	}
417 
418 	/* The Linux driver will modify the local device's link state
419 	 * to reflect the EoIB-to-Ethernet gateway's link state, but
420 	 * this seems philosophically incorrect since communication
421 	 * within the EoIB broadcast domain still works regardless of
422 	 * the state of the gateway.
423 	 */
424 
425 	return 0;
426 }
427 
428 /****************************************************************************
429  *
430  * Server management protocol (XSMP) session messages
431  *
432  ****************************************************************************
433  */
434 
435 /**
436  * Get session message name (for debugging)
437  *
438  * @v type		Message type
439  * @ret name		Message name
440  */
xsmp_session_type(unsigned int type)441 static const char * xsmp_session_type ( unsigned int type ) {
442 	static char buf[16];
443 
444 	switch ( type ) {
445 	case XSMP_SESSION_TYPE_HELLO:		return "HELLO";
446 	case XSMP_SESSION_TYPE_REGISTER:	return "REGISTER";
447 	case XSMP_SESSION_TYPE_CONFIRM:		return "CONFIRM";
448 	case XSMP_SESSION_TYPE_REJECT:		return "REJECT";
449 	case XSMP_SESSION_TYPE_SHUTDOWN:	return "SHUTDOWN";
450 	default:
451 		snprintf ( buf, sizeof ( buf ), "UNKNOWN<%d>", type );
452 		return buf;
453 	}
454 }
455 
456 /**
457  * Extract chassis name (for debugging)
458  *
459  * @v msg		Session message
460  * @ret chassis		Chassis name
461  */
xsmp_chassis_name(struct xsmp_session_message * msg)462 static const char * xsmp_chassis_name ( struct xsmp_session_message *msg ) {
463 	static char chassis[ sizeof ( msg->chassis ) + 1 /* NUL */ ];
464 
465 	memcpy ( chassis, msg->chassis, sizeof ( msg->chassis ) );
466 	return chassis;
467 }
468 
469 /**
470  * Extract session name (for debugging)
471  *
472  * @v msg		Session message
473  * @ret session		Session name
474  */
xsmp_session_name(struct xsmp_session_message * msg)475 static const char * xsmp_session_name ( struct xsmp_session_message *msg ) {
476 	static char session[ sizeof ( msg->session ) + 1 /* NUL */ ];
477 
478 	memcpy ( session, msg->session, sizeof ( msg->session ) );
479 	return session;
480 }
481 
482 /**
483  * Send session message
484  *
485  * @v xcm		Configuration manager
486  * @v type		Message type
487  * @ret rc		Return status code
488  */
xsmp_tx_session(struct xsigo_manager * xcm,unsigned int type)489 static int xsmp_tx_session ( struct xsigo_manager *xcm, unsigned int type ) {
490 	struct xsigo_device *xdev = xcm->xdev;
491 	struct ib_device *ibdev = xdev->ibdev;
492 	struct xsmp_session_message msg;
493 	int rc;
494 
495 	/* Construct session message */
496 	memset ( &msg, 0, sizeof ( msg ) );
497 	msg.hdr.type = XSMP_TYPE_SESSION;
498 	msg.hdr.len = htons ( sizeof ( msg ) );
499 	msg.hdr.seq = htonl ( ++xcm->seq );
500 	memcpy ( &msg.hdr.src.guid, &ibdev->gid.s.guid,
501 		 sizeof ( msg.hdr.src.guid ) );
502 	memcpy ( &msg.hdr.dst.guid, &xcm->id.guid,
503 		 sizeof ( msg.hdr.dst.guid ) );
504 	msg.type = type;
505 	msg.len = htons ( sizeof ( msg ) - sizeof ( msg.hdr ) );
506 	msg.os_type = XSIGO_OS_TYPE_GENERIC;
507 	msg.resources = htons ( XSIGO_RESOURCE_XVE |
508 				XSIGO_RESOURCE_NO_HA );
509 	msg.boot = htonl ( XSMP_BOOT_PXE );
510 	DBGCP ( xcm, "XCM %s TX[%d] session %s\n", xcm->name,
511 		ntohl ( msg.hdr.seq ), xsmp_session_type ( msg.type ) );
512 	DBGCP_HDA ( xcm, 0, &msg, sizeof ( msg ) );
513 
514 	/* Send session message */
515 	if ( ( rc = xfer_deliver_raw ( &xcm->xfer, &msg,
516 				       sizeof ( msg ) ) ) != 0 ) {
517 		DBGC ( xcm, "XCM %s TX session %s failed: %s\n", xcm->name,
518 		       xsmp_session_type ( msg.type ), strerror ( rc ) );
519 		return rc;
520 	}
521 
522 	return 0;
523 }
524 
525 /**
526  * Send registration message
527  *
528  * @v xcm		Configuration manager
529  * @ret rc		Return status code
530  */
xsmp_tx_session_register(struct xsigo_manager * xcm)531 static inline int xsmp_tx_session_register ( struct xsigo_manager *xcm ) {
532 
533 	DBGC ( xcm, "XCM %s registering with " IB_GUID_FMT "\n",
534 	       xcm->name, IB_GUID_ARGS ( &xcm->id.guid ) );
535 
536 	/* Send registration message */
537 	return xsmp_tx_session ( xcm, XSMP_SESSION_TYPE_REGISTER );
538 }
539 
540 /**
541  * Send keepalive message
542  *
543  * @v xcm		Configuration manager
544  * @ret rc		Return status code
545  */
xsmp_tx_session_hello(struct xsigo_manager * xcm)546 static int xsmp_tx_session_hello ( struct xsigo_manager *xcm ) {
547 
548 	/* Send keepalive message */
549 	return xsmp_tx_session ( xcm, XSMP_SESSION_TYPE_HELLO );
550 }
551 
552 /**
553  * Handle received keepalive message
554  *
555  * @v xcm		Configuration manager
556  * @v msg		Keepalive message
557  * @ret rc		Return status code
558  */
xsmp_rx_session_hello(struct xsigo_manager * xcm,struct xsmp_session_message * msg __unused)559 static int xsmp_rx_session_hello ( struct xsigo_manager *xcm,
560 				   struct xsmp_session_message *msg __unused ) {
561 
562 	/* Respond to keepalive message.  Note that the XCM doesn't
563 	 * seem to actually ever send these.
564 	 */
565 	return xsmp_tx_session_hello ( xcm );
566 }
567 
568 /**
569  * Handle received registration confirmation message
570  *
571  * @v xcm		Configuration manager
572  * @v msg		Registration confirmation message
573  * @ret rc		Return status code
574  */
xsmp_rx_session_confirm(struct xsigo_manager * xcm,struct xsmp_session_message * msg)575 static int xsmp_rx_session_confirm ( struct xsigo_manager *xcm,
576 				     struct xsmp_session_message *msg ) {
577 
578 	DBGC ( xcm, "XCM %s registered with \"%s\" as \"%s\"\n", xcm->name,
579 	       xsmp_chassis_name ( msg ), xsmp_session_name ( msg ) );
580 
581 	return 0;
582 }
583 
584 /**
585  * Handle received registration rejection message
586  *
587  * @v xcm		Configuration manager
588  * @v msg		Registration confirmation message
589  * @ret rc		Return status code
590  */
xsmp_rx_session_reject(struct xsigo_manager * xcm,struct xsmp_session_message * msg)591 static int xsmp_rx_session_reject ( struct xsigo_manager *xcm,
592 				    struct xsmp_session_message *msg ) {
593 
594 	DBGC ( xcm, "XCM %s rejected by \"%s\":\n",
595 	       xcm->name, xsmp_chassis_name ( msg ) );
596 	DBGC_HDA ( xcm, 0, msg, sizeof ( *msg ) );
597 
598 	return -EPERM;
599 }
600 
601 /**
602  * Handle received shutdown message
603  *
604  * @v xcm		Configuration manager
605  * @v msg		Registration confirmation message
606  * @ret rc		Return status code
607  */
xsmp_rx_session_shutdown(struct xsigo_manager * xcm,struct xsmp_session_message * msg)608 static int xsmp_rx_session_shutdown ( struct xsigo_manager *xcm,
609 				      struct xsmp_session_message *msg ) {
610 
611 	DBGC ( xcm, "XCM %s shut down by \"%s\":\n",
612 	       xcm->name, xsmp_chassis_name ( msg ) );
613 	DBGC_HDA ( xcm, 0, msg, sizeof ( *msg ) );
614 
615 	return -ENOTCONN;
616 }
617 
618 /**
619  * Handle received session message
620  *
621  * @v xcm		Configuration manager
622  * @v msg		Session message
623  * @ret rc		Return status code
624  */
xsmp_rx_session(struct xsigo_manager * xcm,struct xsmp_session_message * msg)625 static int xsmp_rx_session ( struct xsigo_manager *xcm,
626 			     struct xsmp_session_message *msg ) {
627 
628 	DBGCP ( xcm, "XCM %s RX[%d] session %s\n", xcm->name,
629 		ntohl ( msg->hdr.seq ), xsmp_session_type ( msg->type ) );
630 	DBGCP_HDA ( xcm, 0, msg, sizeof ( *msg ) );
631 
632 	/* Handle message according to type */
633 	switch ( msg->type ) {
634 	case XSMP_SESSION_TYPE_HELLO:
635 		return xsmp_rx_session_hello ( xcm, msg );
636 	case XSMP_SESSION_TYPE_CONFIRM:
637 		return xsmp_rx_session_confirm ( xcm, msg );
638 	case XSMP_SESSION_TYPE_REJECT:
639 		return xsmp_rx_session_reject ( xcm, msg );
640 	case XSMP_SESSION_TYPE_SHUTDOWN:
641 		return xsmp_rx_session_shutdown ( xcm, msg );
642 	default:
643 		DBGC ( xcm, "XCM %s RX[%d] session unexpected %s:\n", xcm->name,
644 		       ntohl ( msg->hdr.seq ), xsmp_session_type ( msg->type ));
645 		DBGC_HDA ( xcm, 0, msg, sizeof ( *msg ) );
646 		return -EPROTO;
647 	}
648 }
649 
650 /****************************************************************************
651  *
652  * Server management protocol (XSMP) virtual Ethernet (XVE) messages
653  *
654  ****************************************************************************
655  */
656 
657 /**
658  * Get virtual Ethernet message name (for debugging)
659  *
660  * @v type		Message type
661  * @ret name		Message name
662  */
xsmp_xve_type(unsigned int type)663 static const char * xsmp_xve_type ( unsigned int type ) {
664 	static char buf[16];
665 
666 	switch ( type ) {
667 	case XSMP_XVE_TYPE_INSTALL:		return "INSTALL";
668 	case XSMP_XVE_TYPE_DELETE:		return "DELETE";
669 	case XSMP_XVE_TYPE_UPDATE:		return "UPDATE";
670 	case XSMP_XVE_TYPE_OPER_UP:		return "OPER_UP";
671 	case XSMP_XVE_TYPE_OPER_DOWN:		return "OPER_DOWN";
672 	case XSMP_XVE_TYPE_OPER_REQ:		return "OPER_REQ";
673 	case XSMP_XVE_TYPE_READY:		return "READY";
674 	default:
675 		snprintf ( buf, sizeof ( buf ), "UNKNOWN<%d>", type );
676 		return buf;
677 	}
678 }
679 
680 /**
681  * Send virtual Ethernet message
682  *
683  * @v xcm		Configuration manager
684  * @v msg		Partial message
685  * @ret rc		Return status code
686  */
xsmp_tx_xve(struct xsigo_manager * xcm,struct xsmp_xve_message * msg)687 static int xsmp_tx_xve ( struct xsigo_manager *xcm,
688 			 struct xsmp_xve_message *msg ) {
689 	struct xsigo_device *xdev = xcm->xdev;
690 	struct ib_device *ibdev = xdev->ibdev;
691 	int rc;
692 
693 	/* Fill in common header fields */
694 	msg->hdr.type = XSMP_TYPE_XVE;
695 	msg->hdr.len = htons ( sizeof ( *msg ) );
696 	msg->hdr.seq = htonl ( ++xcm->seq );
697 	memcpy ( &msg->hdr.src.guid, &ibdev->gid.s.guid,
698 		 sizeof ( msg->hdr.src.guid ) );
699 	memcpy ( &msg->hdr.dst.guid, &xcm->id.guid,
700 		 sizeof ( msg->hdr.dst.guid ) );
701 	msg->len = htons ( sizeof ( *msg ) - sizeof ( msg->hdr ) );
702 	DBGCP ( xcm, "XCM %s TX[%d] xve %s code %#02x\n", xcm->name,
703 		ntohl ( msg->hdr.seq ), xsmp_xve_type ( msg->type ),
704 		msg->code );
705 	DBGCP_HDA ( xcm, 0, msg, sizeof ( *msg ) );
706 
707 	/* Send virtual Ethernet message */
708 	if ( ( rc = xfer_deliver_raw ( &xcm->xfer, msg,
709 				       sizeof ( *msg ) ) ) != 0 ) {
710 		DBGC ( xcm, "XCM %s TX xve %s failed: %s\n", xcm->name,
711 		       xsmp_xve_type ( msg->type ), strerror ( rc ) );
712 		return rc;
713 	}
714 
715 	return 0;
716 }
717 
718 /**
719  * Send virtual Ethernet message including current device parameters
720  *
721  * @v xcm		Configuration manager
722  * @v msg		Partial virtual Ethernet message
723  * @v xve		Virtual Ethernet device
724  * @v eoib		EoIB device
725  * @ret rc		Return status code
726  */
xsmp_tx_xve_params(struct xsigo_manager * xcm,struct xsmp_xve_message * msg,struct xsigo_nic * xve,struct eoib_device * eoib)727 static int xsmp_tx_xve_params ( struct xsigo_manager *xcm,
728 				struct xsmp_xve_message *msg,
729 				struct xsigo_nic *xve,
730 				struct eoib_device *eoib ) {
731 	struct xsigo_device *xdev = xcm->xdev;
732 	struct ib_device *ibdev = xdev->ibdev;
733 	struct net_device *netdev = eoib->netdev;
734 
735 	/* Set successful response code */
736 	msg->code = 0;
737 
738 	/* Include network identifier, MTU, and current HCA parameters */
739 	msg->network = htonl ( xve->network );
740 	msg->mtu = htons ( netdev->max_pkt_len - sizeof ( struct ethhdr ) );
741 	msg->hca.prefix_le.qword = bswap_64 ( ibdev->gid.s.prefix.qword );
742 	msg->hca.pkey = htons ( ibdev->pkey );
743 	msg->hca.qkey = msg->tca.qkey;
744 	if ( eoib->qp ) {
745 		msg->hca.data = htonl ( eoib->qp->ext_qpn );
746 		msg->hca.qkey = htons ( eoib->qp->qkey );
747 	}
748 
749 	/* The message type field is (ab)used to return the current
750 	 * operational status.
751 	 */
752 	if ( msg->type == XSMP_XVE_TYPE_OPER_REQ ) {
753 		msg->type = ( netdev_is_open ( netdev ) ?
754 			      XSMP_XVE_TYPE_OPER_UP : XSMP_XVE_TYPE_OPER_DOWN );
755 	}
756 
757 	/* Send message */
758 	DBGC ( xve, "XVE %s network %d MTU %d ctrl %#x data %#x qkey %#04x "
759 	       "%s\n", xve->name, ntohl ( msg->network ), ntohs ( msg->mtu ),
760 	       ntohl ( msg->hca.ctrl ), ntohl ( msg->hca.data ),
761 	       ntohs ( msg->hca.qkey ), xsmp_xve_type ( msg->type ) );
762 
763 	return xsmp_tx_xve ( xcm, msg );
764 }
765 
766 /**
767  * Send virtual Ethernet error response
768  *
769  * @v xcm		Configuration manager
770  * @v msg		Partial virtual Ethernet message
771  * @ret rc		Return status code
772  */
xsmp_tx_xve_nack(struct xsigo_manager * xcm,struct xsmp_xve_message * msg)773 static inline int xsmp_tx_xve_nack ( struct xsigo_manager *xcm,
774 				     struct xsmp_xve_message *msg ) {
775 
776 	/* Set error response code.  (There aren't any meaningful
777 	 * detailed response codes defined by the wire protocol.)
778 	 */
779 	msg->code = XSMP_XVE_CODE_ERROR;
780 
781 	/* Send message */
782 	return xsmp_tx_xve ( xcm, msg );
783 }
784 
785 /**
786  * Send virtual Ethernet notification
787  *
788  * @v xcm		Configuration manager
789  * @v type		Message type
790  * @v xve		Virtual Ethernet device
791  * @v eoib		EoIB device
792  * @ret rc		Return status code
793  */
xsmp_tx_xve_notify(struct xsigo_manager * xcm,unsigned int type,struct xsigo_nic * xve,struct eoib_device * eoib)794 static int xsmp_tx_xve_notify ( struct xsigo_manager *xcm,
795 				unsigned int type,
796 				struct xsigo_nic *xve,
797 				struct eoib_device *eoib ) {
798 	struct xsmp_xve_message msg;
799 
800 	/* Construct message */
801 	memset ( &msg, 0, sizeof ( msg ) );
802 	msg.type = type;
803 	memcpy ( &msg.resource, &xve->resource, sizeof ( msg.resource ) );
804 
805 	/* Send message */
806 	return xsmp_tx_xve_params ( xcm, &msg, xve, eoib );
807 }
808 
809 /**
810  * Send virtual Ethernet current operational state
811  *
812  * @v xcm		Configuration manager
813  * @v xve		Virtual Ethernet device
814  * @v eoib		EoIB device
815  * @ret rc		Return status code
816  */
xsmp_tx_xve_oper(struct xsigo_manager * xcm,struct xsigo_nic * xve,struct eoib_device * eoib)817 static inline int xsmp_tx_xve_oper ( struct xsigo_manager *xcm,
818 				     struct xsigo_nic *xve,
819 				     struct eoib_device *eoib ) {
820 
821 	/* Send notification */
822 	return xsmp_tx_xve_notify ( xcm, XSMP_XVE_TYPE_OPER_REQ, xve, eoib );
823 }
824 
825 /**
826  * Handle received virtual Ethernet modification message
827  *
828  * @v xcm		Configuration manager
829  * @v msg		Virtual Ethernet message
830  * @v update		Update bitmask
831  * @ret rc		Return status code
832  */
xsmp_rx_xve_modify(struct xsigo_manager * xcm,struct xsmp_xve_message * msg,unsigned int update)833 static int xsmp_rx_xve_modify ( struct xsigo_manager *xcm,
834 				struct xsmp_xve_message *msg,
835 				unsigned int update ) {
836 	struct xsigo_device *xdev = xcm->xdev;
837 	struct ib_device *ibdev = xdev->ibdev;
838 	struct xsigo_nic *xve;
839 	struct eoib_device *eoib;
840 	struct ib_address_vector tca;
841 	size_t mtu;
842 	int rc;
843 
844 	/* Avoid returning uninitialised HCA parameters in response */
845 	memset ( &msg->hca, 0, sizeof ( msg->hca ) );
846 
847 	/* Find virtual Ethernet device */
848 	xve = xve_find ( xcm, &msg->resource );
849 	if ( ! xve ) {
850 		DBGC ( xcm, "XCM %s unrecognised resource " IB_GUID_FMT "\n",
851 		       xcm->name, IB_GUID_ARGS ( &msg->resource ) );
852 		rc = -ENOENT;
853 		goto err_no_xve;
854 	}
855 
856 	/* Find corresponding EoIB device */
857 	eoib = eoib_find ( ibdev, xve->mac );
858 	if ( ! eoib ) {
859 		DBGC ( xve, "XVE %s has no EoIB device\n", xve->name );
860 		rc = -EPIPE;
861 		goto err_no_eoib;
862 	}
863 
864 	/* The Xsigo management software fails to create the EoIB
865 	 * multicast group.  This is a fundamental design flaw.
866 	 */
867 	eoib_force_group_creation ( eoib );
868 
869 	/* Extract modifiable parameters.  Note that the TCA GID is
870 	 * erroneously transmitted as little-endian.
871 	 */
872 	mtu = ntohs ( msg->mtu );
873 	tca.qpn = ntohl ( msg->tca.data );
874 	tca.qkey = ntohs ( msg->tca.qkey );
875 	tca.gid_present = 1;
876 	tca.gid.s.prefix.qword = bswap_64 ( msg->tca.prefix_le.qword );
877 	tca.gid.s.guid.qword = bswap_64 ( msg->guid_le.qword );
878 
879 	/* Update MTU, if applicable */
880 	if ( ( update & XSMP_XVE_UPDATE_MTU ) &&
881 	     ( ( rc = xve_update_mtu ( xve, eoib, mtu ) ) != 0 ) )
882 		goto err_mtu;
883 	update &= ~XSMP_XVE_UPDATE_MTU;
884 
885 	/* Update admin state, if applicable */
886 	if ( ( update & XSMP_XVE_UPDATE_STATE ) &&
887 	     ( ( rc = xve_update_state ( xve, eoib, msg->state ) ) != 0 ) )
888 		goto err_state;
889 	update &= ~XSMP_XVE_UPDATE_STATE;
890 
891 	/* Remove gateway, if applicable */
892 	if ( ( update & XSMP_XVE_UPDATE_GW_DOWN ) &&
893 	     ( ( rc = xve_update_tca ( xve, eoib, NULL ) ) != 0 ) )
894 		goto err_gw_down;
895 	update &= ~XSMP_XVE_UPDATE_GW_DOWN;
896 
897 	/* Update gateway, if applicable */
898 	if ( ( update & XSMP_XVE_UPDATE_GW_CHANGE ) &&
899 	     ( ( rc = xve_update_tca ( xve, eoib, &tca ) ) != 0 ) )
900 		goto err_gw_change;
901 	update &= ~XSMP_XVE_UPDATE_GW_CHANGE;
902 
903 	/* Warn about unexpected updates */
904 	if ( update ) {
905 		DBGC ( xve, "XVE %s unrecognised update(s) %#08x\n",
906 		       xve->name, update );
907 	}
908 
909 	xsmp_tx_xve_params ( xcm, msg, xve, eoib );
910 	return 0;
911 
912  err_gw_change:
913  err_gw_down:
914  err_state:
915  err_mtu:
916  err_no_eoib:
917  err_no_xve:
918 	/* Send NACK */
919 	xsmp_tx_xve_nack ( xcm, msg );
920 	return rc;
921 }
922 
923 /**
924  * Handle received virtual Ethernet installation message
925  *
926  * @v xcm		Configuration manager
927  * @v msg		Virtual Ethernet message
928  * @ret rc		Return status code
929  */
xsmp_rx_xve_install(struct xsigo_manager * xcm,struct xsmp_xve_message * msg)930 static int xsmp_rx_xve_install ( struct xsigo_manager *xcm,
931 				 struct xsmp_xve_message *msg ) {
932 	union {
933 		struct xsmp_xve_mac msg;
934 		uint8_t raw[ETH_ALEN];
935 	} mac;
936 	char name[ sizeof ( msg->name ) + 1 /* NUL */ ];
937 	unsigned long network;
938 	unsigned long qkey;
939 	unsigned int update;
940 	int rc;
941 
942 	/* Demangle MAC address (which is erroneously transmitted as
943 	 * little-endian).
944 	 */
945 	mac.msg.high = bswap_16 ( msg->mac_le.high );
946 	mac.msg.low = bswap_32 ( msg->mac_le.low );
947 
948 	/* Extract interface name (which may not be NUL-terminated) */
949 	memcpy ( name, msg->name, ( sizeof ( name ) - 1 /* NUL */ ) );
950 	name[ sizeof ( name ) - 1 /* NUL */ ] = '\0';
951 
952 	/* Extract remaining message parameters */
953 	network = ntohl ( msg->network );
954 	qkey = ntohs ( msg->tca.qkey );
955 	DBGC2 ( xcm, "XCM %s " IB_GUID_FMT " install \"%s\" %s net %ld qkey "
956 		"%#lx\n", xcm->name, IB_GUID_ARGS ( &msg->resource ), name,
957 		eth_ntoa ( mac.raw ), network, qkey );
958 
959 	/* Create virtual Ethernet device, if applicable */
960 	if ( ( xve_find ( xcm, &msg->resource ) == NULL ) &&
961 	     ( ( rc = xve_create ( xcm, &msg->resource, mac.raw, network,
962 				   qkey, name ) ) != 0 ) )
963 		goto err_create;
964 
965 	/* Handle remaining parameters as for a modification message */
966 	update = XSMP_XVE_UPDATE_MTU;
967 	if ( msg->uplink == XSMP_XVE_UPLINK )
968 		update |= XSMP_XVE_UPDATE_GW_CHANGE;
969 	return xsmp_rx_xve_modify ( xcm, msg, update );
970 
971  err_create:
972 	/* Send NACK */
973 	xsmp_tx_xve_nack ( xcm, msg );
974 	return rc;
975 }
976 
977 /**
978  * Handle received virtual Ethernet deletion message
979  *
980  * @v xcm		Configuration manager
981  * @v msg		Virtual Ethernet message
982  * @ret rc		Return status code
983  */
xsmp_rx_xve_delete(struct xsigo_manager * xcm,struct xsmp_xve_message * msg)984 static int xsmp_rx_xve_delete ( struct xsigo_manager *xcm,
985 				struct xsmp_xve_message *msg ) {
986 	struct xsigo_nic *xve;
987 
988 	DBGC2 ( xcm, "XCM %s " IB_GUID_FMT " delete\n",
989 		xcm->name, IB_GUID_ARGS ( &msg->resource ) );
990 
991 	/* Destroy virtual Ethernet device (if any) */
992 	if ( ( xve = xve_find ( xcm, &msg->resource ) ) )
993 		xve_destroy ( xve );
994 
995 	/* Send ACK */
996 	msg->code = 0;
997 	xsmp_tx_xve ( xcm, msg );
998 
999 	return 0;
1000 }
1001 
1002 /**
1003  * Handle received virtual Ethernet update message
1004  *
1005  * @v xcm		Configuration manager
1006  * @v msg		Virtual Ethernet message
1007  * @ret rc		Return status code
1008  */
xsmp_rx_xve_update(struct xsigo_manager * xcm,struct xsmp_xve_message * msg)1009 static int xsmp_rx_xve_update ( struct xsigo_manager *xcm,
1010 				struct xsmp_xve_message *msg ) {
1011 	unsigned int update = ntohl ( msg->update );
1012 
1013 	DBGC2 ( xcm, "XCM %s " IB_GUID_FMT " update (%08x)\n",
1014 		xcm->name, IB_GUID_ARGS ( &msg->resource ), update );
1015 
1016 	/* Handle as a modification message */
1017 	return xsmp_rx_xve_modify ( xcm, msg, update );
1018 }
1019 
1020 /**
1021  * Handle received virtual Ethernet operational request message
1022  *
1023  * @v xcm		Configuration manager
1024  * @v msg		Virtual Ethernet message
1025  * @ret rc		Return status code
1026  */
xsmp_rx_xve_oper_req(struct xsigo_manager * xcm,struct xsmp_xve_message * msg)1027 static int xsmp_rx_xve_oper_req ( struct xsigo_manager *xcm,
1028 				  struct xsmp_xve_message *msg ) {
1029 
1030 	DBGC2 ( xcm, "XCM %s " IB_GUID_FMT " operational request\n",
1031 		xcm->name, IB_GUID_ARGS ( &msg->resource ) );
1032 
1033 	/* Handle as a nullipotent modification message */
1034 	return xsmp_rx_xve_modify ( xcm, msg, 0 );
1035 }
1036 
1037 /**
1038  * Handle received virtual Ethernet readiness message
1039  *
1040  * @v xcm		Configuration manager
1041  * @v msg		Virtual Ethernet message
1042  * @ret rc		Return status code
1043  */
xsmp_rx_xve_ready(struct xsigo_manager * xcm,struct xsmp_xve_message * msg)1044 static int xsmp_rx_xve_ready ( struct xsigo_manager *xcm,
1045 			       struct xsmp_xve_message *msg ) {
1046 	int rc;
1047 
1048 	DBGC2 ( xcm, "XCM %s " IB_GUID_FMT " ready\n",
1049 		xcm->name, IB_GUID_ARGS ( &msg->resource ) );
1050 
1051 	/* Handle as a nullipotent modification message */
1052 	if ( ( rc = xsmp_rx_xve_modify ( xcm, msg, 0 ) ) != 0 )
1053 		return rc;
1054 
1055 	/* Send an unsolicited operational state update, since there
1056 	 * is no other way to convey the current operational state.
1057 	 */
1058 	msg->type = XSMP_XVE_TYPE_OPER_REQ;
1059 	if ( ( rc = xsmp_rx_xve_modify ( xcm, msg, 0 ) ) != 0 )
1060 		return rc;
1061 
1062 	return 0;
1063 }
1064 
1065 /**
1066  * Handle received virtual Ethernet message
1067  *
1068  * @v xcm		Configuration manager
1069  * @v msg		Virtual Ethernet message
1070  * @ret rc		Return status code
1071  */
xsmp_rx_xve(struct xsigo_manager * xcm,struct xsmp_xve_message * msg)1072 static int xsmp_rx_xve ( struct xsigo_manager *xcm,
1073 			 struct xsmp_xve_message *msg ) {
1074 
1075 	DBGCP ( xcm, "XCM %s RX[%d] xve %s\n", xcm->name,
1076 		ntohl ( msg->hdr.seq ), xsmp_xve_type ( msg->type ) );
1077 	DBGCP_HDA ( xcm, 0, msg, sizeof ( *msg ) );
1078 
1079 	/* Handle message according to type */
1080 	switch ( msg->type ) {
1081 	case XSMP_XVE_TYPE_INSTALL:
1082 		return xsmp_rx_xve_install ( xcm, msg );
1083 	case XSMP_XVE_TYPE_DELETE:
1084 		return xsmp_rx_xve_delete ( xcm, msg );
1085 	case XSMP_XVE_TYPE_UPDATE:
1086 		return xsmp_rx_xve_update ( xcm, msg );
1087 	case XSMP_XVE_TYPE_OPER_REQ:
1088 		return xsmp_rx_xve_oper_req ( xcm, msg );
1089 	case XSMP_XVE_TYPE_READY:
1090 		return xsmp_rx_xve_ready ( xcm, msg );
1091 	default:
1092 		DBGC ( xcm, "XCM %s RX[%d] xve unexpected %s:\n", xcm->name,
1093 		       ntohl ( msg->hdr.seq ), xsmp_xve_type ( msg->type ) );
1094 		DBGC_HDA ( xcm, 0, msg, sizeof ( *msg ) );
1095 		return -EPROTO;
1096 	}
1097 }
1098 
1099 /****************************************************************************
1100  *
1101  * Configuration managers (XCM)
1102  *
1103  ****************************************************************************
1104  */
1105 
1106 /**
1107  * Close configuration manager connection
1108  *
1109  * @v xcm		Configuration manager
1110  * @v rc		Reason for close
1111  */
xcm_close(struct xsigo_manager * xcm,int rc)1112 static void xcm_close ( struct xsigo_manager *xcm, int rc ) {
1113 
1114 	DBGC ( xcm, "XCM %s closed: %s\n", xcm->name, strerror ( rc ) );
1115 
1116 	/* Stop transmission process */
1117 	process_del ( &xcm->process );
1118 
1119 	/* Stop keepalive timer */
1120 	stop_timer ( &xcm->keepalive );
1121 
1122 	/* Restart data transfer interface */
1123 	intf_restart ( &xcm->xfer, rc );
1124 
1125 	/* Schedule reconnection attempt */
1126 	start_timer ( &xcm->reopen );
1127 }
1128 
1129 /**
1130  * Send data to configuration manager
1131  *
1132  * @v xcm		Configuration manager
1133  */
xcm_step(struct xsigo_manager * xcm)1134 static void xcm_step ( struct xsigo_manager *xcm ) {
1135 	int rc;
1136 
1137 	/* Do nothing unless we have something to send */
1138 	if ( ! xcm->pending )
1139 		return;
1140 
1141 	/* Send (empty) connection request, if applicable */
1142 	if ( xcm->pending & XCM_TX_CONNECT ) {
1143 		if ( ( rc = xfer_deliver_raw ( &xcm->xfer, NULL, 0 ) ) != 0 ) {
1144 			DBGC ( xcm, "XCM %s could not send connection request: "
1145 			       "%s\n", xcm->name, strerror ( rc ) );
1146 			goto err;
1147 		}
1148 		xcm->pending &= ~XCM_TX_CONNECT;
1149 		return;
1150 	}
1151 
1152 	/* Wait until data transfer interface is connected */
1153 	if ( ! xfer_window ( &xcm->xfer ) )
1154 		return;
1155 
1156 	/* Send registration message, if applicable */
1157 	if ( xcm->pending & XCM_TX_REGISTER ) {
1158 		if ( ( rc = xsmp_tx_session_register ( xcm ) ) != 0 )
1159 			goto err;
1160 		xcm->pending &= ~XCM_TX_REGISTER;
1161 		return;
1162 	}
1163 
1164 	return;
1165 
1166  err:
1167 	xcm_close ( xcm, rc );
1168 }
1169 
1170 /**
1171  * Receive data from configuration manager
1172  *
1173  * @v xcm		Configuration manager
1174  * @v iobuf		I/O buffer
1175  * @v meta		Data transfer metadata
1176  * @ret rc		Return status code
1177  */
xcm_deliver(struct xsigo_manager * xcm,struct io_buffer * iobuf,struct xfer_metadata * meta __unused)1178 static int xcm_deliver ( struct xsigo_manager *xcm, struct io_buffer *iobuf,
1179 			 struct xfer_metadata *meta __unused ) {
1180 	union xsmp_message *msg;
1181 	size_t len = iob_len ( iobuf );
1182 	int rc;
1183 
1184 	/* Sanity check */
1185 	if ( len < sizeof ( msg->hdr ) ) {
1186 		DBGC ( xcm, "XCM %s underlength message:\n", xcm->name );
1187 		DBGC_HDA ( xcm, 0, iobuf->data, iob_len ( iobuf ) );
1188 		rc = -EPROTO;
1189 		goto out;
1190 	}
1191 	msg = iobuf->data;
1192 
1193 	/* Handle message according to type */
1194 	if ( ! msg->hdr.type ) {
1195 
1196 		/* Ignore unused communication manager private data blocks */
1197 		rc = 0;
1198 
1199 	} else if ( ( msg->hdr.type == XSMP_TYPE_SESSION ) &&
1200 		    ( len >= sizeof ( msg->sess ) ) ) {
1201 
1202 		/* Session message */
1203 		rc = xsmp_rx_session ( xcm, &msg->sess );
1204 
1205 	} else if ( ( msg->hdr.type == XSMP_TYPE_XVE ) &&
1206 		    ( len >= sizeof ( msg->xve ) ) ) {
1207 
1208 		/* Virtual Ethernet message */
1209 		xsmp_rx_xve ( xcm, &msg->xve );
1210 
1211 		/* Virtual Ethernet message errors are non-fatal */
1212 		rc = 0;
1213 
1214 	} else {
1215 
1216 		/* Unknown message */
1217 		DBGC ( xcm, "XCM %s unexpected message type %d:\n",
1218 		       xcm->name, msg->hdr.type );
1219 		DBGC_HDA ( xcm, 0, iobuf->data, iob_len ( iobuf ) );
1220 		rc = -EPROTO;
1221 	}
1222 
1223  out:
1224 	free_iob ( iobuf );
1225 	if ( rc != 0 )
1226 		xcm_close ( xcm, rc );
1227 	return rc;
1228 }
1229 
1230 /** Configuration manager data transfer interface operations */
1231 static struct interface_operation xcm_xfer_op[] = {
1232 	INTF_OP ( xfer_deliver, struct xsigo_manager *, xcm_deliver ),
1233 	INTF_OP ( xfer_window_changed, struct xsigo_manager *, xcm_step ),
1234 	INTF_OP ( intf_close, struct xsigo_manager *, xcm_close ),
1235 };
1236 
1237 /** Configuration manager data transfer interface descriptor */
1238 static struct interface_descriptor xcm_xfer_desc =
1239 	INTF_DESC ( struct xsigo_manager, xfer, xcm_xfer_op );
1240 
1241 /** Configuration manager process descriptor */
1242 static struct process_descriptor xcm_process_desc =
1243 	PROC_DESC_ONCE ( struct xsigo_manager, process, xcm_step );
1244 
1245 /**
1246  * Handle configuration manager connection timer expiry
1247  *
1248  * @v timer		Connection timer
1249  * @v fail		Failure indicator
1250  */
xcm_reopen(struct retry_timer * timer,int fail __unused)1251 static void xcm_reopen ( struct retry_timer *timer, int fail __unused ) {
1252 	struct xsigo_manager *xcm =
1253 		container_of ( timer, struct xsigo_manager, reopen );
1254 	struct xsigo_device *xdev = xcm->xdev;
1255 	struct ib_device *ibdev = xdev->ibdev;
1256 	union ib_gid gid;
1257 	int rc;
1258 
1259 	/* Stop transmission process */
1260 	process_del ( &xcm->process );
1261 
1262 	/* Stop keepalive timer */
1263 	stop_timer ( &xcm->keepalive );
1264 
1265 	/* Restart data transfer interface */
1266 	intf_restart ( &xcm->xfer, -ECANCELED );
1267 
1268 	/* Reset sequence number */
1269 	xcm->seq = 0;
1270 
1271 	/* Construct GID */
1272 	memcpy ( &gid.s.prefix, &ibdev->gid.s.prefix, sizeof ( gid.s.prefix ) );
1273 	memcpy ( &gid.s.guid, &xcm->id.guid, sizeof ( gid.s.guid ) );
1274 	DBGC ( xcm, "XCM %s connecting to " IB_GID_FMT "\n",
1275 	       xcm->name, IB_GID_ARGS ( &gid ) );
1276 
1277 	/* Open CMRC connection */
1278 	if ( ( rc = ib_cmrc_open ( &xcm->xfer, ibdev, &gid,
1279 				   &xcm_service_id, xcm->name ) ) != 0 ) {
1280 		DBGC ( xcm, "XCM %s could not open CMRC connection: %s\n",
1281 		       xcm->name, strerror ( rc ) );
1282 		start_timer ( &xcm->reopen );
1283 		return;
1284 	}
1285 
1286 	/* Schedule transmissions */
1287 	xcm->pending |= ( XCM_TX_CONNECT | XCM_TX_REGISTER );
1288 	process_add ( &xcm->process );
1289 
1290 	/* Start keepalive timer */
1291 	start_timer_fixed ( &xcm->keepalive, XSIGO_KEEPALIVE_INTERVAL );
1292 
1293 	return;
1294 }
1295 
1296 /**
1297  * Handle configuration manager keepalive timer expiry
1298  *
1299  * @v timer		Connection timer
1300  * @v fail		Failure indicator
1301  */
xcm_keepalive(struct retry_timer * timer,int fail __unused)1302 static void xcm_keepalive ( struct retry_timer *timer, int fail __unused ) {
1303 	struct xsigo_manager *xcm =
1304 		container_of ( timer, struct xsigo_manager, keepalive );
1305 	int rc;
1306 
1307 	/* Send keepalive message.  The server won't actually respond
1308 	 * to these, but it gives the RC queue pair a chance to
1309 	 * complain if it doesn't ever at least get an ACK.
1310 	 */
1311 	if ( ( rc = xsmp_tx_session_hello ( xcm ) ) != 0 ) {
1312 		xcm_close ( xcm, rc );
1313 		return;
1314 	}
1315 
1316 	/* Restart keepalive timer */
1317 	start_timer_fixed ( &xcm->keepalive, XSIGO_KEEPALIVE_INTERVAL );
1318 }
1319 
1320 /**
1321  * Create configuration manager
1322  *
1323  * @v xsigo		Xsigo device
1324  * @v id		Configuration manager ID
1325  * @ret rc		Return status code
1326  */
xcm_create(struct xsigo_device * xdev,struct xsigo_manager_id * id)1327 static int xcm_create ( struct xsigo_device *xdev,
1328 			struct xsigo_manager_id *id ) {
1329 	struct xsigo_manager *xcm;
1330 
1331 	/* Allocate and initialise structure */
1332 	xcm = zalloc ( sizeof ( *xcm ) );
1333 	if ( ! xcm )
1334 		return -ENOMEM;
1335 	ref_init ( &xcm->refcnt, xcm_free );
1336 	xcm->xdev = xdev;
1337 	ref_get ( &xcm->xdev->refcnt );
1338 	snprintf ( xcm->name, sizeof ( xcm->name ), "%s:xcm-%d",
1339 		   xdev->name, ntohs ( id->lid ) );
1340 	memcpy ( &xcm->id, id, sizeof ( xcm->id ) );
1341 	intf_init ( &xcm->xfer, &xcm_xfer_desc, &xcm->refcnt );
1342 	timer_init ( &xcm->keepalive, xcm_keepalive, &xcm->refcnt );
1343 	timer_init ( &xcm->reopen, xcm_reopen, &xcm->refcnt );
1344 	process_init_stopped ( &xcm->process, &xcm_process_desc, &xcm->refcnt );
1345 	INIT_LIST_HEAD ( &xcm->nics );
1346 
1347 	/* Start timer to open connection */
1348 	start_timer_nodelay ( &xcm->reopen );
1349 
1350 	/* Add to list of managers and transfer reference to list */
1351 	list_add ( &xcm->list, &xdev->managers );
1352 	DBGC ( xcm, "XCM %s created for " IB_GUID_FMT " (LID %d)\n", xcm->name,
1353 	       IB_GUID_ARGS ( &xcm->id.guid ), ntohs ( id->lid ) );
1354 	return 0;
1355 }
1356 
1357 /**
1358  * Find configuration manager
1359  *
1360  * @v xsigo		Xsigo device
1361  * @v id		Configuration manager ID
1362  * @ret xcm		Configuration manager, or NULL
1363  */
xcm_find(struct xsigo_device * xdev,struct xsigo_manager_id * id)1364 static struct xsigo_manager * xcm_find ( struct xsigo_device *xdev,
1365 					 struct xsigo_manager_id *id ) {
1366 	struct xsigo_manager *xcm;
1367 	union ib_guid *guid = &id->guid;
1368 
1369 	/* Find configuration manager */
1370 	list_for_each_entry ( xcm, &xdev->managers, list ) {
1371 		if ( memcmp ( guid, &xcm->id.guid, sizeof ( *guid ) ) == 0 )
1372 			return xcm;
1373 	}
1374 	return NULL;
1375 }
1376 
1377 /**
1378  * Destroy configuration manager
1379  *
1380  * @v xcm		Configuration manager
1381  */
xcm_destroy(struct xsigo_manager * xcm)1382 static void xcm_destroy ( struct xsigo_manager *xcm ) {
1383 	struct xsigo_nic *xve;
1384 
1385 	/* Remove all EoIB NICs */
1386 	while ( ( xve = list_first_entry ( &xcm->nics, struct xsigo_nic,
1387 					   list ) ) ) {
1388 		xve_destroy ( xve );
1389 	}
1390 
1391 	/* Stop transmission process */
1392 	process_del ( &xcm->process );
1393 
1394 	/* Stop timers */
1395 	stop_timer ( &xcm->keepalive );
1396 	stop_timer ( &xcm->reopen );
1397 
1398 	/* Shut down data transfer interface */
1399 	intf_shutdown ( &xcm->xfer, 0 );
1400 
1401 	/* Remove from list of managers and drop list's reference */
1402 	DBGC ( xcm, "XCM %s destroyed\n", xcm->name );
1403 	list_del ( &xcm->list );
1404 	ref_put ( &xcm->refcnt );
1405 }
1406 
1407 /**
1408  * Synchronise list of configuration managers
1409  *
1410  * @v xdev		Xsigo device
1411  * @v ids		List of manager IDs
1412  * @v count		Number of manager IDs
1413  * @ret rc		Return status code
1414  */
xcm_list(struct xsigo_device * xdev,struct xsigo_manager_id * ids,unsigned int count)1415 static int xcm_list ( struct xsigo_device *xdev, struct xsigo_manager_id *ids,
1416 		      unsigned int count ) {
1417 	struct xsigo_manager_id *id;
1418 	struct xsigo_manager *xcm;
1419 	struct xsigo_manager *tmp;
1420 	struct list_head list;
1421 	unsigned int i;
1422 	int rc;
1423 
1424 	/* Create list of managers to be retained */
1425 	INIT_LIST_HEAD ( &list );
1426 	for ( i = 0, id = ids ; i < count ; i++, id++ ) {
1427 		if ( ( xcm = xcm_find ( xdev, id ) ) ) {
1428 			list_del ( &xcm->list );
1429 			list_add_tail ( &xcm->list, &list );
1430 		}
1431 	}
1432 
1433 	/* Destroy any managers not in the list */
1434 	list_for_each_entry_safe ( xcm, tmp, &xdev->managers, list )
1435 		xcm_destroy ( xcm );
1436 	list_splice ( &list, &xdev->managers );
1437 
1438 	/* Create any new managers in the list, and force reconnection
1439 	 * for any changed LIDs.
1440 	 */
1441 	for ( i = 0, id = ids ; i < count ; i++, id++ ) {
1442 		if ( ( xcm = xcm_find ( xdev, id ) ) ) {
1443 			if ( xcm->id.lid != id->lid )
1444 				start_timer_nodelay ( &xcm->reopen );
1445 			continue;
1446 		}
1447 		if ( ( rc = xcm_create ( xdev, id ) ) != 0 ) {
1448 			DBGC ( xdev, "XDEV %s could not create manager: %s\n",
1449 			       xdev->name, strerror ( rc ) );
1450 			return rc;
1451 		}
1452 	}
1453 
1454 	return 0;
1455 }
1456 
1457 /****************************************************************************
1458  *
1459  * Configuration manager discovery
1460  *
1461  ****************************************************************************
1462  */
1463 
1464 /** A stage of discovery */
1465 struct xsigo_discovery {
1466 	/** Name */
1467 	const char *name;
1468 	/** Management transaction operations */
1469 	struct ib_mad_transaction_operations op;
1470 };
1471 
1472 /**
1473  * Handle configuration manager lookup completion
1474  *
1475  * @v ibdev		Infiniband device
1476  * @v mi		Management interface
1477  * @v madx		Management transaction
1478  * @v rc		Status code
1479  * @v mad		Received MAD (or NULL on error)
1480  * @v av		Source address vector (or NULL on error)
1481  */
xsigo_xcm_complete(struct ib_device * ibdev,struct ib_mad_interface * mi __unused,struct ib_mad_transaction * madx,int rc,union ib_mad * mad,struct ib_address_vector * av __unused)1482 static void xsigo_xcm_complete ( struct ib_device *ibdev,
1483 				 struct ib_mad_interface *mi __unused,
1484 				 struct ib_mad_transaction *madx,
1485 				 int rc, union ib_mad *mad,
1486 				 struct ib_address_vector *av __unused ) {
1487 	struct xsigo_device *xdev = ib_madx_get_ownerdata ( madx );
1488 	union xsigo_mad *xsmad = container_of ( mad, union xsigo_mad, mad );
1489 	struct xsigo_managers_reply *reply = &xsmad->reply;
1490 
1491 	/* Check for failures */
1492 	if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) )
1493 		rc = -ENODEV;
1494 	if ( rc != 0 ) {
1495 		DBGC ( xdev, "XDEV %s manager lookup failed: %s\n",
1496 		       xdev->name, strerror ( rc ) );
1497 		goto out;
1498 	}
1499 
1500 	/* Sanity checks */
1501 	if ( reply->count > ( sizeof ( reply->manager ) /
1502 			      sizeof ( reply->manager[0] ) ) ) {
1503 		DBGC ( xdev, "XDEV %s has too many managers (%d)\n",
1504 		       xdev->name, reply->count );
1505 		goto out;
1506 	}
1507 
1508 	/* Synchronise list of managers */
1509 	if ( ( rc = xcm_list ( xdev, reply->manager, reply->count ) ) != 0 )
1510 		goto out;
1511 
1512 	/* Report an empty list of managers */
1513 	if ( reply->count == 0 )
1514 		DBGC ( xdev, "XDEV %s has no managers\n", xdev->name );
1515 
1516 	/* Delay next discovery attempt */
1517 	start_timer_fixed ( &xdev->discovery, XSIGO_DISCOVERY_SUCCESS_DELAY );
1518 
1519 out:
1520 	/* Destroy the completed transaction */
1521 	ib_destroy_madx ( ibdev, ibdev->gsi, madx );
1522 	xdev->madx = NULL;
1523 }
1524 
1525 /** Configuration manager lookup discovery stage */
1526 static struct xsigo_discovery xsigo_xcm_discovery = {
1527 	.name = "manager",
1528 	.op = {
1529 		.complete = xsigo_xcm_complete,
1530 	},
1531 };
1532 
1533 /**
1534  * Handle directory service lookup completion
1535  *
1536  * @v ibdev		Infiniband device
1537  * @v mi		Management interface
1538  * @v madx		Management transaction
1539  * @v rc		Status code
1540  * @v mad		Received MAD (or NULL on error)
1541  * @v av		Source address vector (or NULL on error)
1542  */
xsigo_xds_complete(struct ib_device * ibdev,struct ib_mad_interface * mi __unused,struct ib_mad_transaction * madx,int rc,union ib_mad * mad,struct ib_address_vector * av __unused)1543 static void xsigo_xds_complete ( struct ib_device *ibdev,
1544 				 struct ib_mad_interface *mi __unused,
1545 				 struct ib_mad_transaction *madx,
1546 				 int rc, union ib_mad *mad,
1547 				 struct ib_address_vector *av __unused ) {
1548 	struct xsigo_device *xdev = ib_madx_get_ownerdata ( madx );
1549 	union xsigo_mad *xsmad = container_of ( mad, union xsigo_mad, mad );
1550 	struct xsigo_managers_request *request = &xsmad->request;
1551 	struct ib_service_record *svc;
1552 	struct ib_address_vector dest;
1553 	union ib_guid *guid;
1554 
1555 	/* Allow for reuse of transaction pointer */
1556 	xdev->madx = NULL;
1557 
1558 	/* Check for failures */
1559 	if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ) )
1560 		rc = -ENODEV;
1561 	if ( rc != 0 ) {
1562 		DBGC ( xdev, "XDEV %s directory lookup failed: %s\n",
1563 		       xdev->name, strerror ( rc ) );
1564 		goto out;
1565 	}
1566 
1567 	/* Construct address vector */
1568 	memset ( &dest, 0, sizeof ( dest ) );
1569 	svc = &mad->sa.sa_data.service_record;
1570 	dest.lid = ntohs ( svc->data16[0] );
1571 	dest.sl = ibdev->sm_sl;
1572 	dest.qpn = IB_QPN_GSI;
1573 	dest.qkey = IB_QKEY_GSI;
1574 	guid = ( ( union ib_guid * ) &svc->data64[0] );
1575 	DBGC2 ( xdev, "XDEV %s found directory at LID %d GUID " IB_GUID_FMT
1576 		"\n", xdev->name, dest.lid, IB_GUID_ARGS ( guid ) );
1577 
1578 	/* Construct request (reusing MAD buffer) */
1579 	memset ( request, 0, sizeof ( *request ) );
1580 	request->mad_hdr.mgmt_class = XSIGO_MGMT_CLASS;
1581 	request->mad_hdr.class_version = XSIGO_MGMT_CLASS_VERSION;
1582 	request->mad_hdr.method = IB_MGMT_METHOD_GET;
1583 	request->mad_hdr.attr_id = htons ( XSIGO_ATTR_XCM_REQUEST );
1584 	memcpy ( &request->server.guid, &ibdev->gid.s.guid,
1585 		 sizeof ( request->server.guid ) );
1586 	snprintf ( request->os_version, sizeof ( request->os_version ),
1587 		   "%s %s", product_short_name, product_version );
1588 	snprintf ( request->arch, sizeof ( request->arch ), _S2 ( ARCH ) );
1589 	request->os_type = XSIGO_OS_TYPE_GENERIC;
1590 	request->resources = htons ( XSIGO_RESOURCES_PRESENT |
1591 				     XSIGO_RESOURCE_XVE |
1592 				     XSIGO_RESOURCE_NO_HA );
1593 
1594 	/* The handling of this request on the server side is a
1595 	 * textbook example of how not to design a wire protocol.  The
1596 	 * server uses the _driver_ version number to determine which
1597 	 * fields are present.
1598 	 */
1599 	request->driver_version = htonl ( 0x2a2a2a );
1600 
1601 	/* The build version field is ignored unless it happens to
1602 	 * contain the substring "xg-".
1603 	 */
1604 	snprintf ( request->build, sizeof ( request->build ),
1605 		   "not-xg-%08lx", build_id );
1606 
1607 	/* The server side user interface occasionally has no way to
1608 	 * refer to an entry with an empty hostname.
1609 	 */
1610 	fetch_string_setting ( NULL, &hostname_setting, request->hostname,
1611 			       sizeof ( request->hostname ) );
1612 	if ( ! request->hostname[0] ) {
1613 		snprintf ( request->hostname, sizeof ( request->hostname ),
1614 			   "%s-" IB_GUID_FMT, product_short_name,
1615 			   IB_GUID_ARGS ( &ibdev->gid.s.guid ) );
1616 	}
1617 
1618 	/* Start configuration manager lookup */
1619 	xdev->madx = ib_create_madx ( ibdev, ibdev->gsi, mad, &dest,
1620 				      &xsigo_xcm_discovery.op );
1621 	if ( ! xdev->madx ) {
1622 		DBGC ( xdev, "XDEV %s could not start manager lookup\n",
1623 		       xdev->name );
1624 		goto out;
1625 	}
1626 	ib_madx_set_ownerdata ( xdev->madx, xdev );
1627 
1628 out:
1629 	/* Destroy the completed transaction */
1630 	ib_destroy_madx ( ibdev, ibdev->gsi, madx );
1631 }
1632 
1633 /** Directory service lookup discovery stage */
1634 static struct xsigo_discovery xsigo_xds_discovery = {
1635 	.name = "directory",
1636 	.op = {
1637 		.complete = xsigo_xds_complete,
1638 	},
1639 };
1640 
1641 /**
1642  * Discover configuration managers
1643  *
1644  * @v timer		Retry timer
1645  * @v over		Failure indicator
1646  */
xsigo_discover(struct retry_timer * timer,int over __unused)1647 static void xsigo_discover ( struct retry_timer *timer, int over __unused ) {
1648 	struct xsigo_device *xdev =
1649 		container_of ( timer, struct xsigo_device, discovery );
1650 	struct ib_device *ibdev = xdev->ibdev;
1651 	struct xsigo_discovery *discovery;
1652 
1653 	/* Restart timer */
1654 	start_timer_fixed ( &xdev->discovery, XSIGO_DISCOVERY_FAILURE_DELAY );
1655 
1656 	/* Cancel any pending discovery transaction */
1657 	if ( xdev->madx ) {
1658 		discovery = container_of ( xdev->madx->op,
1659 					   struct xsigo_discovery, op );
1660 		DBGC ( xdev, "XDEV %s timed out waiting for %s lookup\n",
1661 		       xdev->name, discovery->name );
1662 		ib_destroy_madx ( ibdev, ibdev->gsi, xdev->madx );
1663 		xdev->madx = NULL;
1664 	}
1665 
1666 	/* Start directory service lookup */
1667 	xdev->madx = ib_create_service_madx ( ibdev, ibdev->gsi,
1668 					      XDS_SERVICE_NAME,
1669 					      &xsigo_xds_discovery.op );
1670 	if ( ! xdev->madx ) {
1671 		DBGC ( xdev, "XDEV %s could not start directory lookup\n",
1672 		       xdev->name );
1673 		return;
1674 	}
1675 	ib_madx_set_ownerdata ( xdev->madx, xdev );
1676 }
1677 
1678 /****************************************************************************
1679  *
1680  * Infiniband device driver
1681  *
1682  ****************************************************************************
1683  */
1684 
1685 /**
1686  * Open link and start discovery
1687  *
1688  * @v opener		Link opener
1689  * @v over		Failure indicator
1690  */
xsigo_ib_open(struct retry_timer * opener,int over __unused)1691 static void xsigo_ib_open ( struct retry_timer *opener, int over __unused ) {
1692 	struct xsigo_device *xdev =
1693 		container_of ( opener, struct xsigo_device, opener );
1694 	struct ib_device *ibdev = xdev->ibdev;
1695 	int rc;
1696 
1697 	/* Open Infiniband device */
1698 	if ( ( rc = ib_open ( ibdev ) ) != 0 ) {
1699 		DBGC ( xdev, "XDEV %s could not open: %s\n",
1700 		       xdev->name, strerror ( rc ) );
1701 		/* Delay and try again */
1702 		start_timer_fixed ( &xdev->opener, XSIGO_OPEN_RETRY_DELAY );
1703 		return;
1704 	}
1705 
1706 	/* If link is already up, then start discovery */
1707 	if ( ib_link_ok ( ibdev ) )
1708 		start_timer_nodelay ( &xdev->discovery );
1709 }
1710 
1711 /**
1712  * Probe Xsigo device
1713  *
1714  * @v ibdev		Infiniband device
1715  * @ret rc		Return status code
1716  */
xsigo_ib_probe(struct ib_device * ibdev)1717 static int xsigo_ib_probe ( struct ib_device *ibdev ) {
1718 	struct xsigo_device *xdev;
1719 
1720 	/* Allocate and initialise structure */
1721 	xdev = zalloc ( sizeof ( *xdev ) );
1722 	if ( ! xdev )
1723 		return -ENOMEM;
1724 	ref_init ( &xdev->refcnt, xsigo_free );
1725 	xdev->ibdev = ibdev_get ( ibdev );
1726 	xdev->name = ibdev->name;
1727 	timer_init ( &xdev->opener, xsigo_ib_open, &xdev->refcnt );
1728 	timer_init ( &xdev->discovery, xsigo_discover, &xdev->refcnt );
1729 	INIT_LIST_HEAD ( &xdev->managers );
1730 
1731 	/* Start timer to open Infiniband device.  (We are currently
1732 	 * within the Infiniband device probe callback list; opening
1733 	 * the device here would have interesting side-effects.)
1734 	 */
1735 	start_timer_nodelay ( &xdev->opener );
1736 
1737 	/* Add to list of devices and transfer reference to list */
1738 	list_add_tail ( &xdev->list, &xsigo_devices );
1739 	DBGC ( xdev, "XDEV %s created for " IB_GUID_FMT "\n",
1740 	       xdev->name, IB_GUID_ARGS ( &ibdev->gid.s.guid ) );
1741 	return 0;
1742 }
1743 
1744 /**
1745  * Handle device or link status change
1746  *
1747  * @v ibdev		Infiniband device
1748  */
xsigo_ib_notify(struct ib_device * ibdev)1749 static void xsigo_ib_notify ( struct ib_device *ibdev ) {
1750 	struct xsigo_device *xdev;
1751 
1752 	/* Stop/restart discovery on any attached devices */
1753 	list_for_each_entry ( xdev, &xsigo_devices, list ) {
1754 
1755 		/* Skip non-attached devices */
1756 		if ( xdev->ibdev != ibdev )
1757 			continue;
1758 
1759 		/* Stop any ongoing discovery */
1760 		if ( xdev->madx ) {
1761 			ib_destroy_madx ( ibdev, ibdev->gsi, xdev->madx );
1762 			xdev->madx = NULL;
1763 		}
1764 		stop_timer ( &xdev->discovery );
1765 
1766 		/* If link is up, then start discovery */
1767 		if ( ib_link_ok ( ibdev ) )
1768 			start_timer_nodelay ( &xdev->discovery );
1769 	}
1770 }
1771 
1772 /**
1773  * Remove Xsigo device
1774  *
1775  * @v ibdev		Infiniband device
1776  */
xsigo_ib_remove(struct ib_device * ibdev)1777 static void xsigo_ib_remove ( struct ib_device *ibdev ) {
1778 	struct xsigo_device *xdev;
1779 	struct xsigo_device *tmp;
1780 
1781 	/* Remove any attached Xsigo devices */
1782 	list_for_each_entry_safe ( xdev, tmp, &xsigo_devices, list ) {
1783 
1784 		/* Skip non-attached devices */
1785 		if ( xdev->ibdev != ibdev )
1786 			continue;
1787 
1788 		/* Stop any ongoing discovery */
1789 		if ( xdev->madx ) {
1790 			ib_destroy_madx ( ibdev, ibdev->gsi, xdev->madx );
1791 			xdev->madx = NULL;
1792 		}
1793 		stop_timer ( &xdev->discovery );
1794 
1795 		/* Destroy all configuration managers */
1796 		xcm_list ( xdev, NULL, 0 );
1797 
1798 		/* Close Infiniband device, if applicable */
1799 		if ( ! timer_running ( &xdev->opener ) )
1800 			ib_close ( xdev->ibdev );
1801 
1802 		/* Stop link opener */
1803 		stop_timer ( &xdev->opener );
1804 
1805 		/* Remove from list of devices and drop list's reference */
1806 		DBGC ( xdev, "XDEV %s destroyed\n", xdev->name );
1807 		list_del ( &xdev->list );
1808 		ref_put ( &xdev->refcnt );
1809 	}
1810 }
1811 
1812 /** Xsigo Infiniband driver */
1813 struct ib_driver xsigo_ib_driver __ib_driver = {
1814 	.name = "Xsigo",
1815 	.probe = xsigo_ib_probe,
1816 	.notify = xsigo_ib_notify,
1817 	.remove = xsigo_ib_remove,
1818 };
1819 
1820 /****************************************************************************
1821  *
1822  * Network device driver
1823  *
1824  ****************************************************************************
1825  */
1826 
1827 /**
1828  * Handle device or link status change
1829  *
1830  * @v netdev		Network device
1831  */
xsigo_net_notify(struct net_device * netdev)1832 static void xsigo_net_notify ( struct net_device *netdev ) {
1833 	struct xsigo_device *xdev;
1834 	struct ib_device *ibdev;
1835 	struct xsigo_manager *xcm;
1836 	struct xsigo_nic *xve;
1837 	struct eoib_device *eoib;
1838 
1839 	/* Send current operational state to XCM, if applicable */
1840 	list_for_each_entry ( xdev, &xsigo_devices, list ) {
1841 		ibdev = xdev->ibdev;
1842 		list_for_each_entry ( xcm, &xdev->managers, list ) {
1843 			list_for_each_entry ( xve, &xcm->nics, list ) {
1844 				eoib = eoib_find ( ibdev, xve->mac );
1845 				if ( ! eoib )
1846 					continue;
1847 				if ( eoib->netdev != netdev )
1848 					continue;
1849 				xsmp_tx_xve_oper ( xcm, xve, eoib );
1850 			}
1851 		}
1852 	}
1853 }
1854 
1855 /** Xsigo network driver */
1856 struct net_driver xsigo_net_driver __net_driver = {
1857 	.name = "Xsigo",
1858 	.notify = xsigo_net_notify,
1859 };
1860