1 /* packet-nvme-rdma.c
2 * Routines for NVM Express over Fabrics(RDMA) dissection
3 * Copyright 2016
4 * Code by Parav Pandit
5 *
6 * Wireshark - Network traffic analyzer
7 * By Gerald Combs <gerald@wireshark.org>
8 * Copyright 1998 Gerald Combs
9 *
10 * SPDX-License-Identifier: GPL-2.0-or-later
11 */
12
13 /*
14 NVM Express is high speed interface for accessing solid state drives.
15 NVM Express specifications are maintained by NVM Express industry
16 association at http://www.nvmexpress.org.
17
18 This file adds support to dissect NVM Express over fabrics packets
19 for RDMA. This adds very basic support for dissecting commands
20 completions.
21
22 Current dissection supports dissection of
23 (a) NVMe cmd and cqe
24 (b) NVMe Fabric command and cqe
25 As part of it, it also calculates cmd completion latencies.
26
27 This protocol is similar to iSCSI and SCSI dissection where iSCSI is
28 transport protocol for carying SCSI commands and responses. Similarly
29 NVMe Fabrics - RDMA transport protocol carries NVMe commands.
30
31 +----------+
32 | NVMe |
33 +------+---+
34 |
35 +-----------+---------+
36 | NVMe Fabrics |
37 +----+-----------+----+
38 | |
39 +----+---+ +---+----+
40 | RDMA | | FC |
41 +--------+ +--------+
42
43 References:
44 NVMe Express fabrics specification is located at
45 http://www.nvmexpress.org/wp-content/uploads/NVMe_over_Fabrics_1_0_Gold_20160605.pdf
46
47 NVMe Express specification is located at
48 http://www.nvmexpress.org/wp-content/uploads/NVM-Express-1_2a.pdf
49
50 NVM Express RDMA TCP port assigned by IANA that maps to RDMA IP service
51 TCP port can be found at
52 http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=NVM+Express
53
54 */
55 #include "config.h"
56
57 #include <stdlib.h>
58 #include <errno.h>
59
60 #include <epan/packet.h>
61 #include <epan/prefs.h>
62 #include <epan/conversation.h>
63 #include <epan/addr_resolv.h>
64
65 #include "packet-infiniband.h"
66 #include "packet-nvme.h"
67
68 #define SID_ULP_MASK 0x00000000FF000000
69 #define SID_PROTO_MASK 0x0000000000FF0000
70 #define SID_PORT_MASK 0x000000000000FFFF
71
72 #define SID_ULP 0x01
73 #define SID_PROTO_TCP 0x06
74 #define NVME_RDMA_TCP_PORT_RANGE "4420" /* IANA registered */
75
76 #define SID_MASK (SID_ULP_MASK | SID_PROTO_MASK)
77 #define SID_ULP_TCP ((SID_ULP << 3 * 8) | (SID_PROTO_TCP << 2 * 8))
78
79 #define NVME_FABRICS_RDMA "NVMe Fabrics RDMA"
80
81 #define NVME_FABRIC_CMD_SIZE NVME_CMD_SIZE
82 #define NVME_FABRIC_CQE_SIZE NVME_CQE_SIZE
83
84 struct nvme_rdma_cmd_ctx;
85
86 /* The idea of RDMA context matching is as follows:
87 * addresses, sizes, and keys are registred with nvme_add_data_request()
88 * at RDMA request, the packet is matched to queue (this is already done)
89 * at RDMA request, we see address, size, key, and find command with nvme_lookup_data_request()
90 * we store comand context and packet sequence in the queue
dissect_tpcp(tvbuff_t * tvb,packet_info * pinfo,proto_tree * tree,void * data _U_)91 * the next RDMA transfer with the same sequence number will find a macth from queue to the command
92 * knowing command context, we can decode the buffer
93 * We expect all RDMA transfers to be done in order, so storing in queue context is OK
94 */
95 struct nvme_rdma_q_ctx {
96 struct nvme_q_ctx n_q_ctx;
97 struct {
98 struct nvme_rdma_cmd_ctx *cmd_ctx;
99 guint32 pkt_seq;
100 } rdma_ctx;
101 };
102
103 struct nvme_rdma_cmd_ctx {
104 struct nvme_cmd_ctx n_cmd_ctx;
105 };
106
107 void proto_reg_handoff_nvme_rdma(void);
108 void proto_register_nvme_rdma(void);
109
110 static int proto_nvme_rdma = -1;
111 static dissector_handle_t ib_handler;
112 static int proto_ib = -1;
113
114 /* NVMe Fabrics RDMA CM Private data */
115 static int hf_nvmeof_rdma_cm_req_recfmt = -1;
116 static int hf_nvmeof_rdma_cm_req_qid = -1;
117 static int hf_nvmeof_rdma_cm_req_hrqsize = -1;
118 static int hf_nvmeof_rdma_cm_req_hsqsize = -1;
119 static int hf_nvmeof_rdma_cm_req_cntlid = -1;
120 static int hf_nvmeof_rdma_cm_req_reserved = -1;
121
122 static int hf_nvmeof_rdma_cm_rsp_recfmt = -1;
123 static int hf_nvmeof_rdma_cm_rsp_crqsize = -1;
124 static int hf_nvmeof_rdma_cm_rsp_reserved = -1;
125
126 static int hf_nvmeof_rdma_cm_rej_recfmt = -1;
127 static int hf_nvmeof_rdma_cm_rej_status = -1;
128
129 /* Data Transfers */
130 static int hf_nvmeof_from_host_unknown_data = -1;
131 static int hf_nvmeof_read_to_host_req = -1;
132 static int hf_nvmeof_read_to_host_unmatched = -1;
133 static int hf_nvmeof_read_from_host_resp = -1;
134 static int hf_nvmeof_read_from_host_unmatched = -1;
135 static int hf_nvmeof_write_to_host_req = -1;
136 static int hf_nvmeof_write_to_host_unmatched = -1;
137 static int hf_nvmeof_to_host_unknown_data = -1;
138
139 /* Tracking commands, transfers and CQEs */
140 static int hf_nvmeof_data_resp = -1;
141 static int hf_nvmeof_cmd_qid = -1;
142
143
144 /* Initialize the subtree pointers */
145 static gint ett_cm = -1;
146 static gint ett_data = -1;
147
148 static range_t *gPORT_RANGE;
149
150 static struct nvme_rdma_cmd_ctx* nvme_cmd_to_nvme_rdma_cmd(struct nvme_cmd_ctx *nvme_cmd)
151 {
152 return (struct nvme_rdma_cmd_ctx*)(((char *)nvme_cmd) - offsetof(struct nvme_rdma_cmd_ctx, n_cmd_ctx));
153 }
154
155 static conversation_infiniband_data *get_conversion_data(conversation_t *conv)
156 {
proto_register_tpcp(void)157 conversation_infiniband_data *conv_data;
158
159 conv_data = (conversation_infiniband_data *)conversation_get_proto_data(conv, proto_ib);
160 if (!conv_data)
161 return NULL;
162
163 if ((conv_data->service_id & SID_MASK) != SID_ULP_TCP)
164 return NULL; /* the service id doesn't match that of TCP ULP - nothing for us to do here */
165
166 if (!(value_is_in_range(gPORT_RANGE, (guint32)(conv_data->service_id & SID_PORT_MASK))))
167 return NULL; /* the port doesn't match that of NVM Express Fabrics - nothing for us to do here */
168 return conv_data;
169 }
170
171 static conversation_t*
172 find_ib_conversation(packet_info *pinfo, conversation_infiniband_data **uni_conv_data)
173 {
174 conversation_t *conv;
175 conversation_infiniband_data *conv_data;
176
177 conv = find_conversation(pinfo->num, &pinfo->dst, &pinfo->dst,
178 ENDPOINT_IBQP, pinfo->destport, pinfo->destport,
179 NO_ADDR_B|NO_PORT_B);
180 if (!conv)
181 return NULL; /* nothing to do with no conversation context */
182
183 conv_data = get_conversion_data(conv);
184 *uni_conv_data = conv_data;
185 if (!conv_data)
186 return NULL;
187
188 /* now that we found unidirectional conversation, find bidirectional
189 * conversation, so that we can relate to nvme q.
190 */
191 return find_conversation(pinfo->num, &pinfo->src, &pinfo->dst,
192 ENDPOINT_IBQP, pinfo->srcport, pinfo->destport, 0);
193 }
194
195 static guint16 find_nvme_qid(packet_info *pinfo)
196 {
197 conversation_t *conv;
198 conversation_infiniband_data *conv_data;
199 guint16 qid;
200
201 conv = find_conversation(pinfo->num, &pinfo->dst, &pinfo->dst,
202 ENDPOINT_IBQP, pinfo->destport, pinfo->destport,
203 NO_ADDR_B|NO_PORT_B);
204 if (!conv)
205 return 0; /* nothing to do with no conversation context */
206
207 conv_data = get_conversion_data(conv);
208 if (!conv_data)
209 return 0;
210
211 if (conv_data->client_to_server == FALSE) {
212 memcpy(&qid, &conv_data->mad_private_data[178], 2);
213 return qid;
214 }
215 conv = find_conversation(pinfo->num, &pinfo->src, &pinfo->src,
216 ENDPOINT_IBQP, conv_data->src_qp, conv_data->src_qp,
217 NO_ADDR_B|NO_PORT_B);
218 if (!conv)
219 return 0;
220 conv_data = get_conversion_data(conv);
221 if (!conv_data)
222 return 0;
223 memcpy(&qid, &conv_data->mad_private_data[178], 2);
224 return qid;
225 }
226
227 static struct nvme_rdma_q_ctx*
228 find_add_q_ctx(packet_info *pinfo, conversation_t *conv)
proto_reg_handoff_tpcp(void)229 {
230 struct nvme_rdma_q_ctx *q_ctx;
231 guint16 qid;
232
233 q_ctx = (struct nvme_rdma_q_ctx*)conversation_get_proto_data(conv, proto_nvme_rdma);
234 if (!q_ctx) {
235 qid = find_nvme_qid(pinfo);
236 q_ctx = wmem_new0(wmem_file_scope(), struct nvme_rdma_q_ctx);
237 q_ctx->n_q_ctx.pending_cmds = wmem_tree_new(wmem_file_scope());
238 q_ctx->n_q_ctx.done_cmds = wmem_tree_new(wmem_file_scope());
239 q_ctx->n_q_ctx.data_requests = wmem_tree_new(wmem_file_scope());
240 q_ctx->n_q_ctx.data_responses = wmem_tree_new(wmem_file_scope());
241 q_ctx->n_q_ctx.qid = qid;
242 conversation_add_proto_data(conv, proto_nvme_rdma, q_ctx);
243 }
244 return q_ctx;
245 }
246
247 static conversation_infiniband_data*
248 find_ib_cm_conversation(packet_info *pinfo)
249 {
250 conversation_t *conv;
251
252 conv = find_conversation(pinfo->num, &pinfo->src, &pinfo->dst,
253 ENDPOINT_IBQP, pinfo->srcport, pinfo->destport, 0);
254 if (!conv)
255 return NULL;
256
257 return get_conversion_data(conv);
258 }
259
260 static void add_rdma_cm_qid(gchar *result, guint32 val)
261 {
262 g_snprintf(result, ITEM_LABEL_LENGTH, "%x (%s)", val, val ? "IOQ" : "AQ");
263 }
264
265 static void add_zero_base(gchar *result, guint32 val)
266 {
267 g_snprintf(result, ITEM_LABEL_LENGTH, "%u", val+1);
268 }
269
270 static void dissect_rdma_cm_req_packet(tvbuff_t *tvb, proto_tree *tree)
271 {
272 proto_tree *cm_tree;
273 proto_item *ti;
274 /* NVME-RDMA connect private data starts at offset 0 of RDMA-CM
275 * private data
276 */
277
278 /* create display subtree for private data */
279 ti = proto_tree_add_item(tree, proto_nvme_rdma, tvb, 0, 32, ENC_NA);
280 cm_tree = proto_item_add_subtree(ti, ett_cm);
281
282 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_req_recfmt, tvb,
283 0, 2, ENC_LITTLE_ENDIAN);
284
285 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_req_qid, tvb,
286 2, 2, ENC_LITTLE_ENDIAN);
287 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_req_hrqsize, tvb,
288 4, 2, ENC_LITTLE_ENDIAN);
289 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_req_hsqsize, tvb,
290 6, 2, ENC_LITTLE_ENDIAN);
291 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_req_cntlid, tvb,
292 8, 2, ENC_LITTLE_ENDIAN);
293 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_req_reserved, tvb,
294 10, 22, ENC_NA);
295 }
296
297 static void dissect_rdma_cm_rsp_packet(tvbuff_t *tvb, proto_tree *tree)
298 {
299 proto_tree *cm_tree;
300 proto_item *ti;
301
302 /* create display subtree for the private datat that start at offset 0 */
303 ti = proto_tree_add_item(tree, proto_nvme_rdma, tvb, 0, 32, ENC_NA);
304 cm_tree = proto_item_add_subtree(ti, ett_cm);
305
306 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_rsp_recfmt, tvb,
307 0, 2, ENC_LITTLE_ENDIAN);
308 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_rsp_crqsize, tvb,
309 2, 2, ENC_LITTLE_ENDIAN);
310 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_rsp_reserved, tvb,
311 4, 28, ENC_NA);
312 }
313
314 static void dissect_rdma_cm_rej_packet(tvbuff_t *tvb, proto_tree *tree)
315 {
316 proto_tree *cm_tree;
317 proto_item *ti;
318
319 /* create display subtree for the private datat that start at offset 0 */
320 ti = proto_tree_add_item(tree, proto_nvme_rdma, tvb, 0, 4, ENC_NA);
321 cm_tree = proto_item_add_subtree(ti, ett_cm);
322
323 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_rej_recfmt, tvb,
324 0, 2, ENC_LITTLE_ENDIAN);
325 proto_tree_add_item(cm_tree, hf_nvmeof_rdma_cm_rej_status, tvb,
326 2, 2, ENC_LITTLE_ENDIAN);
327 }
328
329 static int dissect_rdma_cm_packet(tvbuff_t *tvb, proto_tree *tree,
330 guint16 cm_attribute_id)
331 {
332 switch (cm_attribute_id) {
333 case ATTR_CM_REQ:
334 dissect_rdma_cm_req_packet(tvb, tree);
335 break;
336 case ATTR_CM_REP:
337 dissect_rdma_cm_rsp_packet(tvb, tree);
338 break;
339 case ATTR_CM_REJ:
340 dissect_rdma_cm_rej_packet(tvb, tree);
341 break;
342 default:
343 break;
344 }
345 return TRUE;
346 }
347
348 static int
349 dissect_nvme_ib_cm(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree,
350 void *data)
351 {
352 /* infiniband dissector dissects RDMA-CM header and passes RDMA-CM
353 * private data for further decoding, so we start at RDMA-CM
354 * private data here
355 */
356 conversation_infiniband_data *conv_data = NULL;
357 struct infinibandinfo *info = (struct infinibandinfo *)data;
358
359 conv_data = find_ib_cm_conversation(pinfo);
360 if (!conv_data)
361 return FALSE;
362
363 col_set_str(pinfo->cinfo, COL_PROTOCOL, NVME_FABRICS_RDMA);
364 return dissect_rdma_cm_packet(tvb, tree, info->cm_attribute_id);
365 }
366
367
368 static struct nvme_rdma_cmd_ctx*
369 bind_cmd_to_qctx(packet_info *pinfo, struct nvme_q_ctx *q_ctx,
370 guint16 cmd_id)
371 {
372 struct nvme_rdma_cmd_ctx *ctx;
373
374 if (!PINFO_FD_VISITED(pinfo)) {
375 ctx = wmem_new0(wmem_file_scope(), struct nvme_rdma_cmd_ctx);
376
377 nvme_add_cmd_to_pending_list(pinfo, q_ctx,
378 &ctx->n_cmd_ctx, (void*)ctx, cmd_id);
379 } else {
380 /* Already visited this frame */
381 ctx = (struct nvme_rdma_cmd_ctx*)
382 nvme_lookup_cmd_in_done_list(pinfo, q_ctx, cmd_id);
383 /* if we have already visited frame but haven't found completion yet,
384 * we won't find cmd in done q, so allocate a dummy ctx for doing
385 * rest of the processing.
386 */
387 if (!ctx)
388 ctx = wmem_new0(wmem_file_scope(), struct nvme_rdma_cmd_ctx);
389 }
390 return ctx;
391 }
392
393 static void
394 dissect_nvme_rdma_cmd(tvbuff_t *nvme_tvb, packet_info *pinfo, proto_tree *root_tree,
395 proto_tree *nvme_tree, struct nvme_rdma_q_ctx *q_ctx)
396 {
397 struct nvme_rdma_cmd_ctx *cmd_ctx;
398 guint16 cmd_id;
399 guint8 opcode;
400
401 opcode = tvb_get_guint8(nvme_tvb, 0);
402 cmd_id = tvb_get_guint16(nvme_tvb, 2, ENC_LITTLE_ENDIAN);
403 cmd_ctx = bind_cmd_to_qctx(pinfo, &q_ctx->n_q_ctx, cmd_id);
404 if (opcode == NVME_FABRIC_OPC) {
405 cmd_ctx->n_cmd_ctx.fabric = TRUE;
406 dissect_nvmeof_fabric_cmd(nvme_tvb, pinfo, nvme_tree, &q_ctx->n_q_ctx, &cmd_ctx->n_cmd_ctx, 0, TRUE);
407 } else {
408 cmd_ctx->n_cmd_ctx.fabric = FALSE;
409 dissect_nvme_cmd(nvme_tvb, pinfo, root_tree, &q_ctx->n_q_ctx, &cmd_ctx->n_cmd_ctx);
410 }
411 }
412
413 static void dissect_rdma_read_transfer(tvbuff_t *data_tvb, packet_info *pinfo, proto_tree *data_tree,
414 struct nvme_rdma_q_ctx *q_ctx, struct nvme_rdma_cmd_ctx *rdma_cmd, guint len)
415 {
416 if (rdma_cmd->n_cmd_ctx.fabric == TRUE)
417 dissect_nvmeof_cmd_data(data_tvb, pinfo, data_tree, 0, &rdma_cmd->n_cmd_ctx, len);
418 else
419 dissect_nvme_data_response(data_tvb, pinfo, data_tree, &q_ctx->n_q_ctx, &rdma_cmd->n_cmd_ctx, len, FALSE);
420 }
421
422 static void
423 dissect_nvme_from_host(tvbuff_t *nvme_tvb, packet_info *pinfo,
424 proto_tree *root_tree, proto_tree *nvme_tree,
425 struct infinibandinfo *info,
426 struct nvme_rdma_q_ctx *q_ctx,
427 guint len)
428
429 {
430 switch (info->opCode) {
431 case RC_RDMA_READ_RESPONSE_FIRST:
432 case RC_RDMA_READ_RESPONSE_ONLY:
433 {
434 struct nvme_cmd_ctx *cmd = NULL;
435 /* try fast path - is this transaction cached? */
436 if (q_ctx->rdma_ctx.pkt_seq == info->packet_seq_num) {
437 cmd = &q_ctx->rdma_ctx.cmd_ctx->n_cmd_ctx;
438 if (!PINFO_FD_VISITED(pinfo))
439 nvme_add_data_response(&q_ctx->n_q_ctx, cmd, info->packet_seq_num, 0);
440 } else {
441 cmd = nvme_lookup_data_response(&q_ctx->n_q_ctx, info->packet_seq_num, 0);
442 }
443 if (cmd) {
444 struct nvme_rdma_cmd_ctx *rdma_cmd = nvme_cmd_to_nvme_rdma_cmd(cmd);
445 proto_item *ti = proto_tree_add_item(nvme_tree,
446 hf_nvmeof_read_from_host_resp, nvme_tvb, 0, len, ENC_NA);
447 proto_tree *rdma_tree = proto_item_add_subtree(ti, ett_data);
448 cmd->data_resp_pkt_num = pinfo->num;
449 nvme_publish_to_data_req_link(rdma_tree, nvme_tvb,
450 hf_nvmeof_data_req, cmd);
451 nvme_publish_to_cmd_link(rdma_tree, nvme_tvb,
452 hf_nvmeof_cmd_pkt, cmd);
453 q_ctx->rdma_ctx.cmd_ctx = nvme_cmd_to_nvme_rdma_cmd(cmd);
454 q_ctx->rdma_ctx.pkt_seq = info->packet_seq_num;
455 dissect_rdma_read_transfer(nvme_tvb, pinfo, rdma_tree, q_ctx, rdma_cmd, len);
456 } else {
457 proto_tree_add_item(nvme_tree, hf_nvmeof_read_from_host_unmatched,
458 nvme_tvb, 0, len, ENC_NA);
459 }
460 break;
461 }
462 case RC_SEND_ONLY:
463 if (len >= NVME_FABRIC_CMD_SIZE)
464 dissect_nvme_rdma_cmd(nvme_tvb, pinfo, root_tree, nvme_tree, q_ctx);
465 else
466 proto_tree_add_item(nvme_tree, hf_nvmeof_from_host_unknown_data,
467 nvme_tvb, 0, len, ENC_NA);
468 break;
469 default:
470 proto_tree_add_item(nvme_tree, hf_nvmeof_from_host_unknown_data, nvme_tvb,
471 0, len, ENC_NA);
472 break;
473 }
474 }
475
476 static void
477 dissect_nvme_rdma_cqe(tvbuff_t *nvme_tvb, packet_info *pinfo,
478 proto_tree *root_tree, proto_tree *nvme_tree,
479 struct nvme_rdma_q_ctx *q_ctx)
480 {
481 struct nvme_rdma_cmd_ctx *cmd_ctx;
482 guint16 cmd_id;
483
484 cmd_id = tvb_get_guint16(nvme_tvb, 12, ENC_LITTLE_ENDIAN);
485
486 if (!PINFO_FD_VISITED(pinfo)) {
487
488 cmd_ctx = (struct nvme_rdma_cmd_ctx*)
489 nvme_lookup_cmd_in_pending_list(&q_ctx->n_q_ctx, cmd_id);
490 if (!cmd_ctx)
491 goto not_found;
492
493 /* we have already seen this cqe, or an identical one */
494 if (cmd_ctx->n_cmd_ctx.cqe_pkt_num)
495 goto not_found;
496
497 cmd_ctx->n_cmd_ctx.cqe_pkt_num = pinfo->num;
498 nvme_add_cmd_cqe_to_done_list(&q_ctx->n_q_ctx, &cmd_ctx->n_cmd_ctx, cmd_id);
499 } else {
500 /* Already visited this frame */
501 cmd_ctx = (struct nvme_rdma_cmd_ctx*)
502 nvme_lookup_cmd_in_done_list(pinfo, &q_ctx->n_q_ctx, cmd_id);
503 if (!cmd_ctx)
504 goto not_found;
505 }
506
507 nvme_update_cmd_end_info(pinfo, &cmd_ctx->n_cmd_ctx);
508
509 if (cmd_ctx->n_cmd_ctx.fabric)
510 dissect_nvmeof_fabric_cqe(nvme_tvb, pinfo, nvme_tree, &cmd_ctx->n_cmd_ctx, 0);
511 else
512 dissect_nvme_cqe(nvme_tvb, pinfo, root_tree, &q_ctx->n_q_ctx, &cmd_ctx->n_cmd_ctx);
513 return;
514
515 not_found:
516 proto_tree_add_item(nvme_tree, hf_nvmeof_to_host_unknown_data, nvme_tvb,
517 0, NVME_FABRIC_CQE_SIZE, ENC_NA);
518 }
519
520 static void
521 dissect_nvme_to_host(tvbuff_t *nvme_tvb, packet_info *pinfo,
522 proto_tree *root_tree, proto_tree *nvme_tree,
523 struct infinibandinfo *info,
524 struct nvme_rdma_q_ctx *q_ctx, guint len)
525 {
526 struct nvme_rdma_cmd_ctx *cmd_ctx = NULL;
527
528 switch (info->opCode) {
529 case RC_RDMA_READ_REQUEST:
530 {
531 struct keyed_data_req req = {
532 .addr = info->reth_remote_address,
533 .key = info->reth_remote_key,
534 .size = info->reth_dma_length
535 };
536 struct nvme_cmd_ctx *cmd = nvme_lookup_data_request(&q_ctx->n_q_ctx, &req);
537 if (cmd) {
538 proto_item *ti = proto_tree_add_item(nvme_tree,
539 hf_nvmeof_read_to_host_req, nvme_tvb, 0, 0, ENC_NA);
540 proto_tree *rdma_tree = proto_item_add_subtree(ti, ett_data);
541 cmd->data_req_pkt_num = pinfo->num;
542 nvme_publish_to_data_resp_link(rdma_tree, nvme_tvb,
543 hf_nvmeof_data_resp, cmd);
544 nvme_publish_to_cmd_link(rdma_tree, nvme_tvb,
545 hf_nvmeof_cmd_pkt, cmd);
546 q_ctx->rdma_ctx.cmd_ctx = nvme_cmd_to_nvme_rdma_cmd(cmd);
547 q_ctx->rdma_ctx.pkt_seq = info->packet_seq_num;
548 nvme_update_transfer_request(pinfo, cmd, &q_ctx->n_q_ctx);
549 } else {
550 proto_tree_add_item(nvme_tree, hf_nvmeof_read_to_host_unmatched,
551 nvme_tvb, 0, len, ENC_NA);
552 }
553 break;
554 }
555 case RC_SEND_ONLY:
556 case RC_SEND_ONLY_INVAL:
557 if (len == NVME_FABRIC_CQE_SIZE)
558 dissect_nvme_rdma_cqe(nvme_tvb, pinfo, root_tree, nvme_tree, q_ctx);
559 else
560 proto_tree_add_item(nvme_tree, hf_nvmeof_to_host_unknown_data, nvme_tvb,
561 0, len, ENC_NA);
562 break;
563 case RC_RDMA_WRITE_ONLY:
564 case RC_RDMA_WRITE_FIRST:
565 {
566 struct nvme_cmd_ctx *cmd;
567 struct keyed_data_req req = {
568 .addr = info->reth_remote_address,
569 .key = info->reth_remote_key,
570 .size = info->reth_dma_length
571 };
572 cmd = nvme_lookup_data_request(&q_ctx->n_q_ctx, &req);
573 if (cmd) {
574 proto_item *ti = proto_tree_add_item(nvme_tree,
575 hf_nvmeof_write_to_host_req, nvme_tvb, 0, 0, ENC_NA);
576 proto_tree *rdma_tree = proto_item_add_subtree(ti, ett_data);
577 cmd->data_req_pkt_num = pinfo->num;
578 nvme_publish_to_data_resp_link(rdma_tree, nvme_tvb,
579 hf_nvmeof_data_resp, cmd);
580 nvme_publish_to_cmd_link(rdma_tree, nvme_tvb, hf_nvmeof_cmd_pkt, cmd);
581 q_ctx->rdma_ctx.cmd_ctx = nvme_cmd_to_nvme_rdma_cmd(cmd);
582 q_ctx->rdma_ctx.pkt_seq = info->packet_seq_num;
583 cmd_ctx = nvme_cmd_to_nvme_rdma_cmd(cmd);
584 } else {
585 proto_tree_add_item(nvme_tree, hf_nvmeof_write_to_host_unmatched,
586 nvme_tvb, 0, len, ENC_NA);
587 }
588
589 if (cmd_ctx)
590 dissect_nvme_data_response(nvme_tvb, pinfo, root_tree, &q_ctx->n_q_ctx,
591 &cmd_ctx->n_cmd_ctx, len, FALSE);
592 break;
593 }
594 default:
595 proto_tree_add_item(nvme_tree, hf_nvmeof_to_host_unknown_data, nvme_tvb,
596 0, len, ENC_NA);
597 break;
598 }
599 }
600
601 static int
602 dissect_nvme_ib(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree, void *data)
603 {
604 struct infinibandinfo *info = (struct infinibandinfo *)data;
605 conversation_infiniband_data *conv_data = NULL;
606 conversation_t *conv;
607 proto_tree *nvme_tree;
608 proto_item *ti;
609 struct nvme_rdma_q_ctx *q_ctx;
610 guint len = tvb_reported_length(tvb);
611
612 conv = find_ib_conversation(pinfo, &conv_data);
613 if (!conv)
614 return FALSE;
615
616 q_ctx = find_add_q_ctx(pinfo, conv);
617 if (!q_ctx)
618 return FALSE;
619
620 col_set_str(pinfo->cinfo, COL_PROTOCOL, NVME_FABRICS_RDMA);
621
622 ti = proto_tree_add_item(tree, proto_nvme_rdma, tvb, 0, len, ENC_NA);
623 nvme_tree = proto_item_add_subtree(ti, ett_data);
624
625 nvme_publish_qid(nvme_tree, hf_nvmeof_cmd_qid, q_ctx->n_q_ctx.qid);
626
627 if (conv_data->client_to_server)
628 dissect_nvme_from_host(tvb, pinfo, tree, nvme_tree, info, q_ctx, len);
629 else
630 dissect_nvme_to_host(tvb, pinfo, tree, nvme_tree, info, q_ctx, len);
631
632 return TRUE;
633 }
634
635 void
636 proto_register_nvme_rdma(void)
637 {
638 module_t *nvme_rdma_module;
639 static hf_register_info hf[] = {
640 /* IB RDMA CM fields */
641 { &hf_nvmeof_rdma_cm_req_recfmt,
642 { "Record Format", "nvme-rdma.cm.req.recfmt",
643 FT_UINT16, BASE_DEC, NULL, 0x0, NULL, HFILL}
644 },
645 { &hf_nvmeof_rdma_cm_req_qid,
646 { "Queue Id", "nvme-rdma.cm.req.qid",
647 FT_UINT16, BASE_CUSTOM, CF_FUNC(add_rdma_cm_qid), 0x0, NULL, HFILL}
648 },
649 { &hf_nvmeof_rdma_cm_req_hrqsize,
650 { "RDMA QP Host Receive Queue Size", "nvme-rdma.cm.req.hrqsize",
651 FT_UINT16, BASE_DEC, NULL, 0x0, NULL, HFILL}
652 },
653 { &hf_nvmeof_rdma_cm_req_hsqsize,
654 { "RDMA QP Host Send Queue Size", "nvme-rdma.cm.req.hsqsize",
655 FT_UINT16, BASE_CUSTOM, CF_FUNC(add_zero_base), 0x0, NULL, HFILL}
656 },
657 { &hf_nvmeof_rdma_cm_req_cntlid,
658 { "Controller ID", "nvme-rdma.cm.req.cntlid",
659 FT_UINT16, BASE_HEX, NULL, 0x0, NULL, HFILL}
660 },
661 { &hf_nvmeof_rdma_cm_req_reserved,
662 { "Reserved", "nvme-rdma.cm.req.reserved",
663 FT_BYTES, BASE_NONE, NULL, 0x0, NULL, HFILL}
664 },
665 { &hf_nvmeof_rdma_cm_rsp_recfmt,
666 { "Record Format", "nvme-rdma.cm.rsp.recfmt",
667 FT_UINT16, BASE_DEC, NULL, 0x0, NULL, HFILL}
668 },
669 { &hf_nvmeof_rdma_cm_rsp_crqsize,
670 { "RDMA QP Controller Receive Queue Size", "nvme-rdma.cm.rsp.crqsize",
671 FT_UINT16, BASE_DEC, NULL, 0x0, NULL, HFILL}
672 },
673 { &hf_nvmeof_rdma_cm_rsp_reserved,
674 { "Reserved", "nvme-rdma.cm.rsp.reserved",
675 FT_BYTES, BASE_NONE, NULL, 0x0, NULL, HFILL}
676 },
677 { &hf_nvmeof_rdma_cm_rej_recfmt,
678 { "Record Format", "nvme-rdma.cm.rej.recfmt",
679 FT_UINT16, BASE_DEC, NULL, 0x0, NULL, HFILL}
680 },
681 { &hf_nvmeof_rdma_cm_rej_status,
682 { "Status", "nvme-rdma.cm.rej.status",
683 FT_UINT16, BASE_HEX, NULL, 0x0, NULL, HFILL}
684 },
685 { &hf_nvmeof_from_host_unknown_data,
686 { "Dissection unsupported", "nvme-rdma.unknown_data",
687 FT_BYTES, BASE_NONE, NULL, 0x0, NULL, HFILL}
688 },
689 { &hf_nvmeof_read_to_host_req,
690 { "RDMA Read Request Sent to Host", "nvme-rdma.read_to_host_req",
691 FT_NONE, BASE_NONE, NULL, 0x0, NULL, HFILL}
692 },
693 { &hf_nvmeof_read_to_host_unmatched,
694 { "RDMA Read Request Sent to Host (no Command Match)", "nvme-rdma.read_to_host_req",
695 FT_NONE, BASE_NONE, NULL, 0x0, NULL, HFILL}
696 },
697 { &hf_nvmeof_read_from_host_resp,
698 { "RDMA Read Transfer Sent from Host", "nvme-rdma.read_from_host_resp",
699 FT_BYTES, BASE_NONE, NULL, 0x0, NULL, HFILL}
700 },
701 { &hf_nvmeof_read_from_host_unmatched,
702 { "RDMA Read Transfer Sent from Host (no Command Match)", "nvme-rdma.read_from_host_resp",
703 FT_BYTES, BASE_NONE, NULL, 0x0, NULL, HFILL}
704 },
705 { &hf_nvmeof_write_to_host_req,
706 { "RDMA Write Request Sent to Host", "nvme-rdma.write_to_host_req",
707 FT_NONE, BASE_NONE, NULL, 0x0, NULL, HFILL}
708 },
709 { &hf_nvmeof_write_to_host_unmatched,
710 { "RDMA Write Request Sent to Host (no Command Match)", "nvme-rdma.write_to_host_req",
711 FT_NONE, BASE_NONE, NULL, 0x0, NULL, HFILL}
712 },
713 { &hf_nvmeof_to_host_unknown_data,
714 { "Dissection unsupported", "nvme-rdma.unknown_data",
715 FT_BYTES, BASE_NONE, NULL, 0x0, NULL, HFILL}
716 },
717 { &hf_nvmeof_data_resp,
718 { "DATA Transfer Response", "nvme-rdma.data_resp",
719 FT_FRAMENUM, BASE_NONE, NULL, 0,
720 "DATA transfer response for this transaction is in this frame", HFILL }
721 },
722 { &hf_nvmeof_cmd_qid,
723 { "Cmd Qid", "nvme-rdma.cmd.qid",
724 FT_UINT16, BASE_HEX, NULL, 0x0,
725 "Qid on which command is issued", HFILL }
726 },
727 };
728 static gint *ett[] = {
729 &ett_cm,
730 &ett_data,
731 };
732
733 proto_nvme_rdma = proto_register_protocol("NVM Express Fabrics RDMA",
734 NVME_FABRICS_RDMA, "nvme-rdma");
735
736 proto_register_field_array(proto_nvme_rdma, hf, array_length(hf));
737 proto_register_subtree_array(ett, array_length(ett));
738
739 /* Register preferences */
740 //nvme_rdma_module = prefs_register_protocol(proto_nvme_rdma, proto_reg_handoff_nvme_rdma);
741 nvme_rdma_module = prefs_register_protocol(proto_nvme_rdma, NULL);
742
743 range_convert_str(wmem_epan_scope(), &gPORT_RANGE, NVME_RDMA_TCP_PORT_RANGE, MAX_TCP_PORT);
744 prefs_register_range_preference(nvme_rdma_module,
745 "subsystem_ports",
746 "Subsystem Ports Range",
747 "Range of NVMe Subsystem ports"
748 "(default " NVME_RDMA_TCP_PORT_RANGE ")",
749 &gPORT_RANGE, MAX_TCP_PORT);
750 }
751
752 void
753 proto_reg_handoff_nvme_rdma(void)
754 {
755 heur_dissector_add("infiniband.mad.cm.private", dissect_nvme_ib_cm,
756 "NVMe Fabrics RDMA CM packets",
757 "nvme_rdma_cm_private", proto_nvme_rdma, HEURISTIC_ENABLE);
758 heur_dissector_add("infiniband.payload", dissect_nvme_ib,
759 "NVMe Fabrics RDMA packets",
760 "nvme_rdma", proto_nvme_rdma, HEURISTIC_ENABLE);
761 ib_handler = find_dissector_add_dependency("infiniband", proto_nvme_rdma);
762 proto_ib = dissector_handle_get_protocol_index(ib_handler);
763 }
764
765 /*
766 * Editor modelines - https://www.wireshark.org/tools/modelines.html
767 *
768 * Local variables:
769 * c-basic-offset: 4
770 * tab-width: 8
771 * indent-tabs-mode: nil
772 * End:
773 *
774 * vi: set shiftwidth=4 tabstop=8 expandtab:
775 * :indentSize=4:tabSize=8:noTabs=true:
776 */
777