1 /*
2  * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation; either version 2 of the
7  * License, or any later version.
8  *
9  * This program is distributed in the hope that it will be useful, but
10  * WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17  * 02110-1301, USA.
18  *
19  * You can also choose to distribute this program under the terms of
20  * the Unmodified Binary Distribution Licence (as given in the file
21  * COPYING.UBDL), provided that you have satisfied its requirements.
22  */
23 
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25 
26 #include <stdint.h>
27 #include <string.h>
28 #include <byteswap.h>
29 #include <errno.h>
30 #include <ipxe/list.h>
31 #include <ipxe/infiniband.h>
32 #include <ipxe/ib_mi.h>
33 #include <ipxe/ib_mcast.h>
34 
35 /** @file
36  *
37  * Infiniband multicast groups
38  *
39  */
40 
41 /**
42  * Generate multicast membership MAD
43  *
44  * @v ibdev		Infiniband device
45  * @v av		Address vector
46  * @v method		Method (IB_MGMT_METHOD_SET or IB_MGMT_METHOD_DELETE)
47  * @v mask		Additional component mask
48  * @v mad		MAD to fill in
49  */
ib_mcast_mad(struct ib_device * ibdev,struct ib_address_vector * av,unsigned int method,unsigned int mask,union ib_mad * mad)50 static void ib_mcast_mad ( struct ib_device *ibdev,
51 			   struct ib_address_vector *av,
52 			   unsigned int method, unsigned int mask,
53 			   union ib_mad *mad ) {
54 	struct ib_mad_sa *sa = &mad->sa;
55 
56 	/* Construct multicast membership record request */
57 	memset ( sa, 0, sizeof ( *sa ) );
58 	sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
59 	sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
60 	sa->mad_hdr.method = method;
61 	sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC );
62 	sa->sa_hdr.comp_mask[1] =
63 		htonl ( IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
64 			IB_SA_MCMEMBER_REC_JOIN_STATE | mask );
65 	sa->sa_data.mc_member_record.qkey = htonl ( av->qkey );
66 	sa->sa_data.mc_member_record.pkey =
67 		htons ( ibdev->pkey | IB_PKEY_FULL );
68 	sa->sa_data.mc_member_record.rate_selector__rate = av->rate;
69 	sa->sa_data.mc_member_record.sl__flow_label__hop_limit =
70 		htonl ( av->sl << 28 );
71 	sa->sa_data.mc_member_record.scope__join_state = 0x01;
72 	memcpy ( &sa->sa_data.mc_member_record.mgid, &av->gid,
73 		 sizeof ( sa->sa_data.mc_member_record.mgid ) );
74 	memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid,
75 		 sizeof ( sa->sa_data.mc_member_record.port_gid ) );
76 }
77 
78 /**
79  * Handle multicast membership record join response
80  *
81  * @v ibdev		Infiniband device
82  * @v mi		Management interface
83  * @v madx		Management transaction
84  * @v rc		Status code
85  * @v mad		Received MAD (or NULL on error)
86  * @v src		Source address vector (or NULL on error)
87  */
ib_mcast_complete(struct ib_device * ibdev,struct ib_mad_interface * mi __unused,struct ib_mad_transaction * madx,int rc,union ib_mad * mad,struct ib_address_vector * src __unused)88 static void ib_mcast_complete ( struct ib_device *ibdev,
89 				struct ib_mad_interface *mi __unused,
90 				struct ib_mad_transaction *madx,
91 				int rc, union ib_mad *mad,
92 				struct ib_address_vector *src __unused ) {
93 	struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx );
94 	struct ib_queue_pair *qp = membership->qp;
95 	struct ib_address_vector *av = membership->av;
96 	struct ib_mc_member_record *mc_member_record =
97 		&mad->sa.sa_data.mc_member_record;
98 	int joined;
99 
100 	/* Report failures */
101 	if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ))
102 		rc = -ENOTCONN;
103 	if ( rc != 0 ) {
104 		DBGC ( ibdev, "IBDEV %s QPN %#lx join failed: %s\n",
105 		       ibdev->name, qp->qpn, strerror ( rc ) );
106 		goto out;
107 	}
108 
109 	/* Extract values from MAD */
110 	joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP );
111 	av->qkey = ntohl ( mc_member_record->qkey );
112 	av->lid = ntohs ( mc_member_record->mlid );
113 	av->rate = ( mc_member_record->rate_selector__rate & 0x3f );
114 	av->sl = ( ( ntohl ( mc_member_record->sl__flow_label__hop_limit )
115 		     >> 28 ) & 0x0f );
116 	DBGC ( ibdev, "IBDEV %s QPN %#lx %s " IB_GID_FMT " qkey %#lx\n",
117 	       ibdev->name, qp->qpn, ( joined ? "joined" : "left" ),
118 	       IB_GID_ARGS ( &av->gid ), av->qkey );
119 
120 	/* Set queue key */
121 	qp->qkey = av->qkey;
122 	if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
123 		DBGC ( ibdev, "IBDEV %s QPN %#lx could not modify qkey: %s\n",
124 		       ibdev->name, qp->qpn, strerror ( rc ) );
125 		goto out;
126 	}
127 
128  out:
129 	/* Destroy the completed transaction */
130 	ib_destroy_madx ( ibdev, mi, madx );
131 	membership->madx = NULL;
132 
133 	/* Hand off to upper completion handler */
134 	membership->complete ( membership, rc );
135 }
136 
137 /** Multicast membership management transaction completion operations */
138 static struct ib_mad_transaction_operations ib_mcast_op = {
139 	.complete = ib_mcast_complete,
140 };
141 
142 /**
143  * Join multicast group
144  *
145  * @v ibdev		Infiniband device
146  * @v qp		Queue pair
147  * @v membership	Multicast group membership
148  * @v av		Address vector to fill in
149  * @v joined		Join completion handler
150  * @ret rc		Return status code
151  */
ib_mcast_join(struct ib_device * ibdev,struct ib_queue_pair * qp,struct ib_mc_membership * membership,struct ib_address_vector * av,unsigned int mask,void (* complete)(struct ib_mc_membership * membership,int rc))152 int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
153 		    struct ib_mc_membership *membership,
154 		    struct ib_address_vector *av, unsigned int mask,
155 		    void ( * complete ) ( struct ib_mc_membership *membership,
156 					  int rc ) ) {
157 	union ib_mad mad;
158 	int rc;
159 
160 	DBGC ( ibdev, "IBDEV %s QPN %#lx joining " IB_GID_FMT "\n",
161 	       ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
162 
163 	/* Sanity checks */
164 	assert ( qp != NULL );
165 	assert ( ! membership->attached );
166 
167 	/* Initialise structure */
168 	membership->qp = qp;
169 	membership->av = av;
170 	membership->complete = complete;
171 
172 	/* Attach queue pair to multicast GID */
173 	if ( ( rc = ib_mcast_attach ( ibdev, qp, &av->gid ) ) != 0 ) {
174 		DBGC ( ibdev, "IBDEV %s QPN %#lx could not attach: %s\n",
175 		       ibdev->name, qp->qpn, strerror ( rc ) );
176 		goto err_mcast_attach;
177 	}
178 	membership->attached = 1;
179 
180 	/* Initiate multicast membership join */
181 	ib_mcast_mad ( ibdev, av, IB_MGMT_METHOD_SET, mask, &mad );
182 	membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
183 					    &ib_mcast_op );
184 	if ( ! membership->madx ) {
185 		DBGC ( ibdev, "IBDEV %s QPN %#lx could not create join "
186 		       "transaction\n", ibdev->name, qp->qpn );
187 		rc = -ENOMEM;
188 		goto err_create_madx;
189 	}
190 	ib_madx_set_ownerdata ( membership->madx, membership );
191 
192 	return 0;
193 
194 	ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
195  err_create_madx:
196 	ib_mcast_detach ( ibdev, qp, &av->gid );
197 	membership->attached = 0;
198  err_mcast_attach:
199 	return rc;
200 }
201 
202 /**
203  * Leave multicast group
204  *
205  * @v ibdev		Infiniband device
206  * @v qp		Queue pair
207  * @v membership	Multicast group membership
208  */
ib_mcast_leave(struct ib_device * ibdev,struct ib_queue_pair * qp,struct ib_mc_membership * membership)209 void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
210 		      struct ib_mc_membership *membership ) {
211 	struct ib_address_vector *av = membership->av;
212 	union ib_mad mad;
213 	int rc;
214 
215 	/* Do nothing if we are already detached from the multicast GID */
216 	if ( ! membership->attached )
217 		return;
218 
219 	DBGC ( ibdev, "IBDEV %s QPN %#lx leaving " IB_GID_FMT "\n",
220 	       ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
221 
222 	/* Sanity check */
223 	assert ( qp != NULL );
224 
225 	/* Detach from multicast GID */
226 	ib_mcast_detach ( ibdev, qp, &av->gid );
227 	membership->attached = 0;
228 
229 	/* Cancel multicast membership join, if applicable */
230 	if ( membership->madx ) {
231 		ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
232 		membership->madx = NULL;
233 	}
234 
235 	/* Send a single group leave MAD */
236 	ib_mcast_mad ( ibdev, av, IB_MGMT_METHOD_DELETE, 0, &mad );
237 	if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) {
238 		DBGC ( ibdev, "IBDEV %s QPN %#lx could not send leave request: "
239 		       "%s\n", ibdev->name, qp->qpn, strerror ( rc ) );
240 	}
241 }
242