1 /*
2 * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License as
6 * published by the Free Software Foundation; either version 2 of the
7 * License, or any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
17 * 02110-1301, USA.
18 *
19 * You can also choose to distribute this program under the terms of
20 * the Unmodified Binary Distribution Licence (as given in the file
21 * COPYING.UBDL), provided that you have satisfied its requirements.
22 */
23
24 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
25
26 #include <stdint.h>
27 #include <stdlib.h>
28 #include <string.h>
29 #include <errno.h>
30 #include <byteswap.h>
31 #include <ipxe/iobuf.h>
32 #include <ipxe/infiniband.h>
33 #include <ipxe/ib_packet.h>
34
35 /**
36 * @file
37 *
38 * Infiniband Packet Formats
39 *
40 */
41
42 /**
43 * Add IB headers
44 *
45 * @v ibdev Infiniband device
46 * @v iobuf I/O buffer to contain headers
47 * @v qp Queue pair
48 * @v payload_len Payload length
49 * @v dest Destination address vector
50 * @ret rc Return status code
51 */
ib_push(struct ib_device * ibdev,struct io_buffer * iobuf,struct ib_queue_pair * qp,size_t payload_len,const struct ib_address_vector * dest)52 int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
53 struct ib_queue_pair *qp, size_t payload_len,
54 const struct ib_address_vector *dest ) {
55 struct ib_local_route_header *lrh;
56 struct ib_global_route_header *grh;
57 struct ib_base_transport_header *bth;
58 struct ib_datagram_extended_transport_header *deth;
59 size_t orig_iob_len = iob_len ( iobuf );
60 size_t pad_len;
61 size_t lrh_len;
62 size_t grh_len;
63 unsigned int vl;
64 unsigned int lnh;
65
66 DBGC2 ( ibdev, "IBDEV %s TX %04x:%08lx => %04x:%08lx (key %08lx)\n",
67 ibdev->name, ibdev->lid, qp->ext_qpn, dest->lid, dest->qpn,
68 dest->qkey );
69
70 /* Calculate packet length */
71 pad_len = ( (-payload_len) & 0x3 );
72 payload_len += pad_len;
73 payload_len += 4; /* ICRC */
74
75 /* Reserve space for headers */
76 orig_iob_len = iob_len ( iobuf );
77 deth = iob_push ( iobuf, sizeof ( *deth ) );
78 bth = iob_push ( iobuf, sizeof ( *bth ) );
79 grh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
80 grh = ( dest->gid_present ?
81 iob_push ( iobuf, sizeof ( *grh ) ) : NULL );
82 lrh = iob_push ( iobuf, sizeof ( *lrh ) );
83 lrh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
84
85 /* Construct LRH */
86 vl = ( ( qp->ext_qpn == IB_QPN_SMI ) ? IB_VL_SMP : IB_VL_DEFAULT );
87 lrh->vl__lver = ( vl << 4 );
88 lnh = ( grh ? IB_LNH_GRH : IB_LNH_BTH );
89 lrh->sl__lnh = ( ( dest->sl << 4 ) | lnh );
90 lrh->dlid = htons ( dest->lid );
91 lrh->length = htons ( lrh_len >> 2 );
92 lrh->slid = htons ( ibdev->lid );
93
94 /* Construct GRH, if required */
95 if ( grh ) {
96 grh->ipver__tclass__flowlabel =
97 htonl ( IB_GRH_IPVER_IPv6 << 28 );
98 grh->paylen = htons ( grh_len );
99 grh->nxthdr = IB_GRH_NXTHDR_IBA;
100 grh->hoplmt = 0;
101 memcpy ( &grh->sgid, &ibdev->gid, sizeof ( grh->sgid ) );
102 memcpy ( &grh->dgid, &dest->gid, sizeof ( grh->dgid ) );
103 }
104
105 /* Construct BTH */
106 bth->opcode = BTH_OPCODE_UD_SEND;
107 bth->se__m__padcnt__tver = ( pad_len << 4 );
108 bth->pkey = htons ( ibdev->pkey );
109 bth->dest_qp = htonl ( dest->qpn );
110 bth->ack__psn = htonl ( ( qp->send.psn++ ) & 0xffffffUL );
111
112 /* Construct DETH */
113 deth->qkey = htonl ( dest->qkey );
114 deth->src_qp = htonl ( qp->ext_qpn );
115
116 DBGCP_HDA ( ibdev, 0, iobuf->data,
117 ( iob_len ( iobuf ) - orig_iob_len ) );
118
119 return 0;
120 }
121
122 /**
123 * Remove IB headers
124 *
125 * @v ibdev Infiniband device
126 * @v iobuf I/O buffer containing headers
127 * @v qp Queue pair to fill in, or NULL
128 * @v payload_len Payload length to fill in, or NULL
129 * @v dest Destination address vector to fill in
130 * @v source Source address vector to fill in
131 * @ret rc Return status code
132 */
ib_pull(struct ib_device * ibdev,struct io_buffer * iobuf,struct ib_queue_pair ** qp,size_t * payload_len,struct ib_address_vector * dest,struct ib_address_vector * source)133 int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf,
134 struct ib_queue_pair **qp, size_t *payload_len,
135 struct ib_address_vector *dest,
136 struct ib_address_vector *source ) {
137 struct ib_local_route_header *lrh;
138 struct ib_global_route_header *grh;
139 struct ib_base_transport_header *bth;
140 struct ib_datagram_extended_transport_header *deth;
141 size_t orig_iob_len = iob_len ( iobuf );
142 unsigned int lnh;
143 size_t pad_len;
144
145 /* Clear return values */
146 if ( qp )
147 *qp = NULL;
148 if ( payload_len )
149 *payload_len = 0;
150 memset ( dest, 0, sizeof ( *dest ) );
151 memset ( source, 0, sizeof ( *source ) );
152
153 /* Extract LRH */
154 if ( iob_len ( iobuf ) < sizeof ( *lrh ) ) {
155 DBGC ( ibdev, "IBDEV %s RX too short (%zd bytes) for LRH\n",
156 ibdev->name, iob_len ( iobuf ) );
157 return -EINVAL;
158 }
159 lrh = iobuf->data;
160 iob_pull ( iobuf, sizeof ( *lrh ) );
161 dest->lid = ntohs ( lrh->dlid );
162 dest->sl = ( lrh->sl__lnh >> 4 );
163 source->lid = ntohs ( lrh->slid );
164 source->sl = ( lrh->sl__lnh >> 4 );
165 lnh = ( lrh->sl__lnh & 0x3 );
166
167 /* Reject unsupported packets */
168 if ( ! ( ( lnh == IB_LNH_BTH ) || ( lnh == IB_LNH_GRH ) ) ) {
169 DBGC ( ibdev, "IBDEV %s RX unsupported LNH %x\n",
170 ibdev->name, lnh );
171 return -ENOTSUP;
172 }
173
174 /* Extract GRH, if present */
175 if ( lnh == IB_LNH_GRH ) {
176 if ( iob_len ( iobuf ) < sizeof ( *grh ) ) {
177 DBGC ( ibdev, "IBDEV %s RX too short (%zd bytes) "
178 "for GRH\n", ibdev->name, iob_len ( iobuf ) );
179 return -EINVAL;
180 }
181 grh = iobuf->data;
182 iob_pull ( iobuf, sizeof ( *grh ) );
183 dest->gid_present = 1;
184 memcpy ( &dest->gid, &grh->dgid, sizeof ( dest->gid ) );
185 source->gid_present = 1;
186 memcpy ( &source->gid, &grh->sgid, sizeof ( source->gid ) );
187 } else {
188 grh = NULL;
189 }
190
191 /* Extract BTH */
192 if ( iob_len ( iobuf ) < sizeof ( *bth ) ) {
193 DBGC ( ibdev, "IBDEV %s RX too short (%zd bytes) for BTH\n",
194 ibdev->name, iob_len ( iobuf ) );
195 return -EINVAL;
196 }
197 bth = iobuf->data;
198 iob_pull ( iobuf, sizeof ( *bth ) );
199 if ( bth->opcode != BTH_OPCODE_UD_SEND ) {
200 DBGC ( ibdev, "IBDEV %s unsupported BTH opcode %x\n",
201 ibdev->name, bth->opcode );
202 return -ENOTSUP;
203 }
204 dest->qpn = ntohl ( bth->dest_qp );
205
206 /* Extract DETH */
207 if ( iob_len ( iobuf ) < sizeof ( *deth ) ) {
208 DBGC ( ibdev, "IBDEV %s RX too short (%zd bytes) for DETH\n",
209 ibdev->name, iob_len ( iobuf ) );
210 return -EINVAL;
211 }
212 deth = iobuf->data;
213 iob_pull ( iobuf, sizeof ( *deth ) );
214 source->qpn = ntohl ( deth->src_qp );
215 source->qkey = ntohl ( deth->qkey );
216
217 /* Calculate payload length, if applicable */
218 if ( payload_len ) {
219 pad_len = ( ( bth->se__m__padcnt__tver >> 4 ) & 0x3 );
220 *payload_len = ( ( ntohs ( lrh->length ) << 2 )
221 - ( orig_iob_len - iob_len ( iobuf ) )
222 - pad_len - 4 /* ICRC */ );
223 }
224
225 /* Determine destination QP, if applicable */
226 if ( qp ) {
227 if ( IB_LID_MULTICAST ( dest->lid ) && grh ) {
228 if ( ! ( *qp = ib_find_qp_mgid ( ibdev, &grh->dgid ))){
229 DBGC ( ibdev, "IBDEV %s RX for unknown MGID "
230 IB_GID_FMT "\n", ibdev->name,
231 IB_GID_ARGS ( &grh->dgid ) );
232 return -ENODEV;
233 }
234 } else {
235 if ( ! ( *qp = ib_find_qp_qpn ( ibdev, dest->qpn ) ) ) {
236 DBGC ( ibdev, "IBDEV %s RX for nonexistent "
237 "QPN %#lx\n", ibdev->name, dest->qpn );
238 return -ENODEV;
239 }
240 }
241 assert ( *qp );
242 }
243
244 DBGC2 ( ibdev, "IBDEV %s RX %04x:%08lx <= %04x:%08lx (key %08x)\n",
245 ibdev->name, dest->lid,
246 ( IB_LID_MULTICAST ( dest->lid ) ?
247 ( qp ? (*qp)->ext_qpn : -1UL ) : dest->qpn ),
248 source->lid, source->qpn, ntohl ( deth->qkey ) );
249 DBGCP_HDA ( ibdev, 0,
250 ( iobuf->data - ( orig_iob_len - iob_len ( iobuf ) ) ),
251 ( orig_iob_len - iob_len ( iobuf ) ) );
252
253 return 0;
254 }
255