1 /*
2 * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License as
6 * published by the Free Software Foundation; either version 2 of the
7 * License, or any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17 * 02110-1301, USA.
18 *
19 * You can also choose to distribute this program under the terms of
20 * the Unmodified Binary Distribution Licence (as given in the file
21 * COPYING.UBDL), provided that you have satisfied its requirements.
22 */
23
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25
26 #include <stdint.h>
27 #include <string.h>
28 #include <byteswap.h>
29 #include <errno.h>
30 #include <ipxe/list.h>
31 #include <ipxe/infiniband.h>
32 #include <ipxe/ib_mi.h>
33 #include <ipxe/ib_mcast.h>
34
35 /** @file
36 *
37 * Infiniband multicast groups
38 *
39 */
40
41 /**
42 * Generate multicast membership MAD
43 *
44 * @v ibdev Infiniband device
45 * @v av Address vector
46 * @v method Method (IB_MGMT_METHOD_SET or IB_MGMT_METHOD_DELETE)
47 * @v mask Additional component mask
48 * @v mad MAD to fill in
49 */
ib_mcast_mad(struct ib_device * ibdev,struct ib_address_vector * av,unsigned int method,unsigned int mask,union ib_mad * mad)50 static void ib_mcast_mad ( struct ib_device *ibdev,
51 struct ib_address_vector *av,
52 unsigned int method, unsigned int mask,
53 union ib_mad *mad ) {
54 struct ib_mad_sa *sa = &mad->sa;
55
56 /* Construct multicast membership record request */
57 memset ( sa, 0, sizeof ( *sa ) );
58 sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
59 sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
60 sa->mad_hdr.method = method;
61 sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC );
62 sa->sa_hdr.comp_mask[1] =
63 htonl ( IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
64 IB_SA_MCMEMBER_REC_JOIN_STATE | mask );
65 sa->sa_data.mc_member_record.qkey = htonl ( av->qkey );
66 sa->sa_data.mc_member_record.pkey =
67 htons ( ibdev->pkey | IB_PKEY_FULL );
68 sa->sa_data.mc_member_record.rate_selector__rate = av->rate;
69 sa->sa_data.mc_member_record.sl__flow_label__hop_limit =
70 htonl ( av->sl << 28 );
71 sa->sa_data.mc_member_record.scope__join_state = 0x01;
72 memcpy ( &sa->sa_data.mc_member_record.mgid, &av->gid,
73 sizeof ( sa->sa_data.mc_member_record.mgid ) );
74 memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid,
75 sizeof ( sa->sa_data.mc_member_record.port_gid ) );
76 }
77
78 /**
79 * Handle multicast membership record join response
80 *
81 * @v ibdev Infiniband device
82 * @v mi Management interface
83 * @v madx Management transaction
84 * @v rc Status code
85 * @v mad Received MAD (or NULL on error)
86 * @v src Source address vector (or NULL on error)
87 */
ib_mcast_complete(struct ib_device * ibdev,struct ib_mad_interface * mi __unused,struct ib_mad_transaction * madx,int rc,union ib_mad * mad,struct ib_address_vector * src __unused)88 static void ib_mcast_complete ( struct ib_device *ibdev,
89 struct ib_mad_interface *mi __unused,
90 struct ib_mad_transaction *madx,
91 int rc, union ib_mad *mad,
92 struct ib_address_vector *src __unused ) {
93 struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx );
94 struct ib_queue_pair *qp = membership->qp;
95 struct ib_address_vector *av = membership->av;
96 struct ib_mc_member_record *mc_member_record =
97 &mad->sa.sa_data.mc_member_record;
98 int joined;
99
100 /* Report failures */
101 if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ))
102 rc = -ENOTCONN;
103 if ( rc != 0 ) {
104 DBGC ( ibdev, "IBDEV %s QPN %#lx join failed: %s\n",
105 ibdev->name, qp->qpn, strerror ( rc ) );
106 goto out;
107 }
108
109 /* Extract values from MAD */
110 joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP );
111 av->qkey = ntohl ( mc_member_record->qkey );
112 av->lid = ntohs ( mc_member_record->mlid );
113 av->rate = ( mc_member_record->rate_selector__rate & 0x3f );
114 av->sl = ( ( ntohl ( mc_member_record->sl__flow_label__hop_limit )
115 >> 28 ) & 0x0f );
116 DBGC ( ibdev, "IBDEV %s QPN %#lx %s " IB_GID_FMT " qkey %#lx\n",
117 ibdev->name, qp->qpn, ( joined ? "joined" : "left" ),
118 IB_GID_ARGS ( &av->gid ), av->qkey );
119
120 /* Set queue key */
121 qp->qkey = av->qkey;
122 if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
123 DBGC ( ibdev, "IBDEV %s QPN %#lx could not modify qkey: %s\n",
124 ibdev->name, qp->qpn, strerror ( rc ) );
125 goto out;
126 }
127
128 out:
129 /* Destroy the completed transaction */
130 ib_destroy_madx ( ibdev, mi, madx );
131 membership->madx = NULL;
132
133 /* Hand off to upper completion handler */
134 membership->complete ( membership, rc );
135 }
136
137 /** Multicast membership management transaction completion operations */
138 static struct ib_mad_transaction_operations ib_mcast_op = {
139 .complete = ib_mcast_complete,
140 };
141
142 /**
143 * Join multicast group
144 *
145 * @v ibdev Infiniband device
146 * @v qp Queue pair
147 * @v membership Multicast group membership
148 * @v av Address vector to fill in
149 * @v joined Join completion handler
150 * @ret rc Return status code
151 */
ib_mcast_join(struct ib_device * ibdev,struct ib_queue_pair * qp,struct ib_mc_membership * membership,struct ib_address_vector * av,unsigned int mask,void (* complete)(struct ib_mc_membership * membership,int rc))152 int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
153 struct ib_mc_membership *membership,
154 struct ib_address_vector *av, unsigned int mask,
155 void ( * complete ) ( struct ib_mc_membership *membership,
156 int rc ) ) {
157 union ib_mad mad;
158 int rc;
159
160 DBGC ( ibdev, "IBDEV %s QPN %#lx joining " IB_GID_FMT "\n",
161 ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
162
163 /* Sanity checks */
164 assert ( qp != NULL );
165 assert ( ! membership->attached );
166
167 /* Initialise structure */
168 membership->qp = qp;
169 membership->av = av;
170 membership->complete = complete;
171
172 /* Attach queue pair to multicast GID */
173 if ( ( rc = ib_mcast_attach ( ibdev, qp, &av->gid ) ) != 0 ) {
174 DBGC ( ibdev, "IBDEV %s QPN %#lx could not attach: %s\n",
175 ibdev->name, qp->qpn, strerror ( rc ) );
176 goto err_mcast_attach;
177 }
178 membership->attached = 1;
179
180 /* Initiate multicast membership join */
181 ib_mcast_mad ( ibdev, av, IB_MGMT_METHOD_SET, mask, &mad );
182 membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
183 &ib_mcast_op );
184 if ( ! membership->madx ) {
185 DBGC ( ibdev, "IBDEV %s QPN %#lx could not create join "
186 "transaction\n", ibdev->name, qp->qpn );
187 rc = -ENOMEM;
188 goto err_create_madx;
189 }
190 ib_madx_set_ownerdata ( membership->madx, membership );
191
192 return 0;
193
194 ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
195 err_create_madx:
196 ib_mcast_detach ( ibdev, qp, &av->gid );
197 membership->attached = 0;
198 err_mcast_attach:
199 return rc;
200 }
201
202 /**
203 * Leave multicast group
204 *
205 * @v ibdev Infiniband device
206 * @v qp Queue pair
207 * @v membership Multicast group membership
208 */
ib_mcast_leave(struct ib_device * ibdev,struct ib_queue_pair * qp,struct ib_mc_membership * membership)209 void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
210 struct ib_mc_membership *membership ) {
211 struct ib_address_vector *av = membership->av;
212 union ib_mad mad;
213 int rc;
214
215 /* Do nothing if we are already detached from the multicast GID */
216 if ( ! membership->attached )
217 return;
218
219 DBGC ( ibdev, "IBDEV %s QPN %#lx leaving " IB_GID_FMT "\n",
220 ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
221
222 /* Sanity check */
223 assert ( qp != NULL );
224
225 /* Detach from multicast GID */
226 ib_mcast_detach ( ibdev, qp, &av->gid );
227 membership->attached = 0;
228
229 /* Cancel multicast membership join, if applicable */
230 if ( membership->madx ) {
231 ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
232 membership->madx = NULL;
233 }
234
235 /* Send a single group leave MAD */
236 ib_mcast_mad ( ibdev, av, IB_MGMT_METHOD_DELETE, 0, &mad );
237 if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) {
238 DBGC ( ibdev, "IBDEV %s QPN %#lx could not send leave request: "
239 "%s\n", ibdev->name, qp->qpn, strerror ( rc ) );
240 }
241 }
242