1 /* 2 * Copyright (c) 2016-2017 Intel Corporation. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #ifndef _OFI_PROTO_H_ 34 #define _OFI_PROTO_H_ 35 36 #include "config.h" 37 38 #include <stdint.h> 39 #include <stddef.h> 40 41 #include <rdma/fi_rma.h> 42 43 44 #ifdef __cplusplus 45 extern "C" { 46 #endif 47 48 49 #define OFI_CTRL_VERSION 2 50 51 /* ofi_ctrl_hdr::type */ 52 enum { 53 ofi_ctrl_connreq, 54 ofi_ctrl_connresp, 55 ofi_ctrl_start_data, 56 ofi_ctrl_data, 57 ofi_ctrl_large_data, 58 ofi_ctrl_ack, 59 ofi_ctrl_nack, 60 ofi_ctrl_discard, 61 ofi_ctrl_seg_data, 62 ofi_ctrl_atomic, 63 ofi_ctrl_atomic_resp, 64 }; 65 66 /* 67 * Control message header. For segmentation and reassembly, reliability, 68 * rendezvous protocol, acks, and communication setup. 69 * 70 * version: OFI_CTRL_VERSION 71 * type 72 * conn_id: Communication identifier. Conn_id values are exchanged between 73 * peer endpoints as part of communication setup. This field is valid 74 * as part of the first message in any data transfer. 75 * msg_id: This is the sender's identifier for a message. 76 * Unique number identifying all segments of a message 77 * Message id can be formed using an equation similar to: 78 * (seq_no++ << tx size) | tx_key 79 * seg_size: 80 * Data packets - size of current message, in bytes. 81 * Large data packets - size of current message, 2 ^ seg_size, in bytes 82 * Ctrl packets - number of segments in window allowed past seg_no. 83 * seg_no: 84 * Data packets - position 0..(n-1) of segment in current message. 85 * Ctrl packets - last segment ack'ed. 86 * conn_data: Connection specific data. This may be set to the index 87 * of the transmit endpoint's address in its local AV, which may 88 * be used as a hint at the Rx side to locate the Tx EP address in 89 * its AV. The assumption is that all addresses were inserted into 90 * all AVs across the fabric using a copied array. (This is an 91 * optimization hint only; the peer validates the actual entry.) 92 * rx_key: This is the receiver's identifier for a message (receive side 93 * equivalent of msg_id). Key returned by the Rx side, that the 94 * Tx side includes in subsequent packets. This field is used for 95 * rendezvous protocol. 96 * The rx_key may be formed similar to message_id. 97 * ctrl_data: This is provider specific data for remote side 98 */ 99 struct ofi_ctrl_hdr { 100 uint8_t version; 101 uint8_t type; 102 uint16_t seg_size; 103 uint32_t seg_no; 104 uint64_t conn_id; 105 uint64_t msg_id; 106 union { 107 uint64_t conn_data; 108 uint64_t rx_key; 109 uint64_t ctrl_data; 110 }; 111 }; 112 113 114 #define OFI_OP_VERSION 2 115 116 /* 117 * Basic command opcode. ofi_op_hdr::op 118 * Intent is that RX can use opcode + control as indices into a function 119 * pointer array for message processing (after validating values). 120 */ 121 enum { 122 ofi_op_msg, 123 ofi_op_tagged, 124 ofi_op_read_req, 125 ofi_op_read_rsp, 126 ofi_op_write, 127 ofi_op_write_async, 128 ofi_op_atomic, 129 ofi_op_atomic_fetch, 130 ofi_op_atomic_compare, 131 ofi_op_read_async, 132 ofi_op_max, 133 }; 134 135 #define OFI_REMOTE_CQ_DATA (1 << 0) 136 #define OFI_TRANSMIT_COMPLETE (1 << 1) 137 #define OFI_DELIVERY_COMPLETE (1 << 2) 138 #define OFI_COMMIT_COMPLETE (1 << 3) 139 140 /* 141 * Common command header 142 * 143 * version: OFI_OP_VERSION 144 * rxid: RX index for scalable endpoints 145 * op: 146 * op_data: implementation specific 147 * tx_key: Tx request identifier for command 148 * flags: Command flags 149 * size: Size of data transfer 150 * data: Remote CQ data, if available 151 * tag: Message tag, used for tagged operations only 152 * iov_count: Count of destination iov, used for RMA operations 153 * atomic: Control fields for atomic operations 154 * remote_idx: Tx request identifier of remote side 155 * resv: Reserved, used for msg operations 156 */ 157 struct ofi_op_hdr { 158 uint8_t version; 159 uint8_t rx_index; 160 uint8_t op; 161 uint8_t op_data; 162 uint32_t flags; 163 164 uint64_t size; 165 uint64_t data; 166 union { 167 uint64_t tag; 168 uint8_t iov_count; 169 struct { 170 uint8_t datatype; 171 uint8_t op; 172 uint8_t ioc_count; 173 } atomic; 174 uint64_t remote_idx; 175 uint64_t resv; 176 }; 177 }; 178 179 struct ofi_iov { 180 uint64_t addr; 181 uint64_t len; 182 }; 183 184 struct ofi_rma_iov { 185 uint64_t addr; 186 uint64_t len; 187 uint64_t key; 188 }; 189 190 struct ofi_rma_ioc { 191 uint64_t addr; 192 uint64_t count; 193 uint64_t key; 194 }; 195 196 #define OFI_CMD_SIZE 64 /* to align with 64-byte cache line */ 197 #define OFI_CMD_DATA_LEN (OFI_CMD_SIZE - sizeof(struct ofi_ctrl_hdr)) 198 199 200 #ifdef __cplusplus 201 } 202 #endif 203 204 #endif /* _OFI_PROTO_H_ */ 205