1d6b92ffaSHans Petter Selasky /*
2d6b92ffaSHans Petter Selasky * Copyright (c) 2017 Mellanox Technologies, Inc. All rights reserved.
3d6b92ffaSHans Petter Selasky *
4d6b92ffaSHans Petter Selasky * This software is available to you under a choice of one of two
5d6b92ffaSHans Petter Selasky * licenses. You may choose to be licensed under the terms of the GNU
6d6b92ffaSHans Petter Selasky * General Public License (GPL) Version 2, available from the file
7d6b92ffaSHans Petter Selasky * COPYING in the main directory of this source tree, or the
8d6b92ffaSHans Petter Selasky * OpenIB.org BSD license below:
9d6b92ffaSHans Petter Selasky *
10d6b92ffaSHans Petter Selasky * Redistribution and use in source and binary forms, with or
11d6b92ffaSHans Petter Selasky * without modification, are permitted provided that the following
12d6b92ffaSHans Petter Selasky * conditions are met:
13d6b92ffaSHans Petter Selasky *
14d6b92ffaSHans Petter Selasky * - Redistributions of source code must retain the above
15d6b92ffaSHans Petter Selasky * copyright notice, this list of conditions and the following
16d6b92ffaSHans Petter Selasky * disclaimer.
17d6b92ffaSHans Petter Selasky *
18d6b92ffaSHans Petter Selasky * - Redistributions in binary form must reproduce the above
19d6b92ffaSHans Petter Selasky * copyright notice, this list of conditions and the following
20d6b92ffaSHans Petter Selasky * disclaimer in the documentation and/or other materials
21d6b92ffaSHans Petter Selasky * provided with the distribution.
22d6b92ffaSHans Petter Selasky *
23d6b92ffaSHans Petter Selasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24d6b92ffaSHans Petter Selasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25d6b92ffaSHans Petter Selasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26d6b92ffaSHans Petter Selasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27d6b92ffaSHans Petter Selasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28d6b92ffaSHans Petter Selasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29d6b92ffaSHans Petter Selasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30d6b92ffaSHans Petter Selasky * SOFTWARE.
31d6b92ffaSHans Petter Selasky */
32d6b92ffaSHans Petter Selasky
33d6b92ffaSHans Petter Selasky #ifndef _MLX5DV_H_
34d6b92ffaSHans Petter Selasky #define _MLX5DV_H_
35d6b92ffaSHans Petter Selasky
36d6b92ffaSHans Petter Selasky #include <infiniband/types.h> /* For the __be64 type */
37d6b92ffaSHans Petter Selasky #include <infiniband/endian.h>
38d6b92ffaSHans Petter Selasky
39d6b92ffaSHans Petter Selasky #if defined(__SSE3__)
40d6b92ffaSHans Petter Selasky #include <emmintrin.h>
41d6b92ffaSHans Petter Selasky #include <tmmintrin.h>
42d6b92ffaSHans Petter Selasky #endif /* defined(__SSE3__) */
43d6b92ffaSHans Petter Selasky
44d6b92ffaSHans Petter Selasky #include <infiniband/verbs.h>
45d6b92ffaSHans Petter Selasky
46d6b92ffaSHans Petter Selasky /* Always inline the functions */
47d6b92ffaSHans Petter Selasky #ifdef __GNUC__
48d6b92ffaSHans Petter Selasky #define MLX5DV_ALWAYS_INLINE inline __attribute__((always_inline))
49d6b92ffaSHans Petter Selasky #else
50d6b92ffaSHans Petter Selasky #define MLX5DV_ALWAYS_INLINE inline
51d6b92ffaSHans Petter Selasky #endif
52d6b92ffaSHans Petter Selasky
53d6b92ffaSHans Petter Selasky enum {
54d6b92ffaSHans Petter Selasky MLX5_RCV_DBR = 0,
55d6b92ffaSHans Petter Selasky MLX5_SND_DBR = 1,
56d6b92ffaSHans Petter Selasky };
57d6b92ffaSHans Petter Selasky
58d6b92ffaSHans Petter Selasky enum mlx5dv_context_comp_mask {
59d6b92ffaSHans Petter Selasky MLX5DV_CONTEXT_MASK_CQE_COMPRESION = 1 << 0,
60d6b92ffaSHans Petter Selasky MLX5DV_CONTEXT_MASK_RESERVED = 1 << 1,
61d6b92ffaSHans Petter Selasky };
62d6b92ffaSHans Petter Selasky
63d6b92ffaSHans Petter Selasky struct mlx5dv_cqe_comp_caps {
64d6b92ffaSHans Petter Selasky uint32_t max_num;
65d6b92ffaSHans Petter Selasky uint32_t supported_format; /* enum mlx5dv_cqe_comp_res_format */
66d6b92ffaSHans Petter Selasky };
67d6b92ffaSHans Petter Selasky
68d6b92ffaSHans Petter Selasky /*
69d6b92ffaSHans Petter Selasky * Direct verbs device-specific attributes
70d6b92ffaSHans Petter Selasky */
71d6b92ffaSHans Petter Selasky struct mlx5dv_context {
72d6b92ffaSHans Petter Selasky uint8_t version;
73d6b92ffaSHans Petter Selasky uint64_t flags;
74d6b92ffaSHans Petter Selasky uint64_t comp_mask;
75d6b92ffaSHans Petter Selasky struct mlx5dv_cqe_comp_caps cqe_comp_caps;
76d6b92ffaSHans Petter Selasky };
77d6b92ffaSHans Petter Selasky
78d6b92ffaSHans Petter Selasky enum mlx5dv_context_flags {
79d6b92ffaSHans Petter Selasky /*
80d6b92ffaSHans Petter Selasky * This flag indicates if CQE version 0 or 1 is needed.
81d6b92ffaSHans Petter Selasky */
82d6b92ffaSHans Petter Selasky MLX5DV_CONTEXT_FLAGS_CQE_V1 = (1 << 0),
83d6b92ffaSHans Petter Selasky MLX5DV_CONTEXT_FLAGS_MPW = (1 << 1),
84d6b92ffaSHans Petter Selasky };
85d6b92ffaSHans Petter Selasky
86d6b92ffaSHans Petter Selasky enum mlx5dv_cq_init_attr_mask {
87d6b92ffaSHans Petter Selasky MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE = 1 << 0,
88d6b92ffaSHans Petter Selasky MLX5DV_CQ_INIT_ATTR_MASK_RESERVED = 1 << 1,
89d6b92ffaSHans Petter Selasky };
90d6b92ffaSHans Petter Selasky
91d6b92ffaSHans Petter Selasky struct mlx5dv_cq_init_attr {
92d6b92ffaSHans Petter Selasky uint64_t comp_mask; /* Use enum mlx5dv_cq_init_attr_mask */
93d6b92ffaSHans Petter Selasky uint8_t cqe_comp_res_format; /* Use enum mlx5dv_cqe_comp_res_format */
94d6b92ffaSHans Petter Selasky };
95d6b92ffaSHans Petter Selasky
96d6b92ffaSHans Petter Selasky struct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context,
97d6b92ffaSHans Petter Selasky struct ibv_cq_init_attr_ex *cq_attr,
98d6b92ffaSHans Petter Selasky struct mlx5dv_cq_init_attr *mlx5_cq_attr);
99d6b92ffaSHans Petter Selasky /*
100d6b92ffaSHans Petter Selasky * Most device capabilities are exported by ibv_query_device(...),
101d6b92ffaSHans Petter Selasky * but there is HW device-specific information which is important
102d6b92ffaSHans Petter Selasky * for data-path, but isn't provided.
103d6b92ffaSHans Petter Selasky *
104d6b92ffaSHans Petter Selasky * Return 0 on success.
105d6b92ffaSHans Petter Selasky */
106d6b92ffaSHans Petter Selasky int mlx5dv_query_device(struct ibv_context *ctx_in,
107d6b92ffaSHans Petter Selasky struct mlx5dv_context *attrs_out);
108d6b92ffaSHans Petter Selasky
109d6b92ffaSHans Petter Selasky struct mlx5dv_qp {
110d6b92ffaSHans Petter Selasky uint32_t *dbrec;
111d6b92ffaSHans Petter Selasky struct {
112d6b92ffaSHans Petter Selasky void *buf;
113d6b92ffaSHans Petter Selasky uint32_t wqe_cnt;
114d6b92ffaSHans Petter Selasky uint32_t stride;
115d6b92ffaSHans Petter Selasky } sq;
116d6b92ffaSHans Petter Selasky struct {
117d6b92ffaSHans Petter Selasky void *buf;
118d6b92ffaSHans Petter Selasky uint32_t wqe_cnt;
119d6b92ffaSHans Petter Selasky uint32_t stride;
120d6b92ffaSHans Petter Selasky } rq;
121d6b92ffaSHans Petter Selasky struct {
122d6b92ffaSHans Petter Selasky void *reg;
123d6b92ffaSHans Petter Selasky uint32_t size;
124d6b92ffaSHans Petter Selasky } bf;
125d6b92ffaSHans Petter Selasky uint64_t comp_mask;
126d6b92ffaSHans Petter Selasky };
127d6b92ffaSHans Petter Selasky
128d6b92ffaSHans Petter Selasky struct mlx5dv_cq {
129d6b92ffaSHans Petter Selasky void *buf;
130d6b92ffaSHans Petter Selasky uint32_t *dbrec;
131d6b92ffaSHans Petter Selasky uint32_t cqe_cnt;
132d6b92ffaSHans Petter Selasky uint32_t cqe_size;
133d6b92ffaSHans Petter Selasky void *uar;
134d6b92ffaSHans Petter Selasky uint32_t cqn;
135d6b92ffaSHans Petter Selasky uint64_t comp_mask;
136d6b92ffaSHans Petter Selasky };
137d6b92ffaSHans Petter Selasky
138d6b92ffaSHans Petter Selasky struct mlx5dv_srq {
139d6b92ffaSHans Petter Selasky void *buf;
140d6b92ffaSHans Petter Selasky uint32_t *dbrec;
141d6b92ffaSHans Petter Selasky uint32_t stride;
142d6b92ffaSHans Petter Selasky uint32_t head;
143d6b92ffaSHans Petter Selasky uint32_t tail;
144d6b92ffaSHans Petter Selasky uint64_t comp_mask;
145d6b92ffaSHans Petter Selasky };
146d6b92ffaSHans Petter Selasky
147d6b92ffaSHans Petter Selasky struct mlx5dv_rwq {
148d6b92ffaSHans Petter Selasky void *buf;
149d6b92ffaSHans Petter Selasky uint32_t *dbrec;
150d6b92ffaSHans Petter Selasky uint32_t wqe_cnt;
151d6b92ffaSHans Petter Selasky uint32_t stride;
152d6b92ffaSHans Petter Selasky uint64_t comp_mask;
153d6b92ffaSHans Petter Selasky };
154d6b92ffaSHans Petter Selasky
155d6b92ffaSHans Petter Selasky struct mlx5dv_obj {
156d6b92ffaSHans Petter Selasky struct {
157d6b92ffaSHans Petter Selasky struct ibv_qp *in;
158d6b92ffaSHans Petter Selasky struct mlx5dv_qp *out;
159d6b92ffaSHans Petter Selasky } qp;
160d6b92ffaSHans Petter Selasky struct {
161d6b92ffaSHans Petter Selasky struct ibv_cq *in;
162d6b92ffaSHans Petter Selasky struct mlx5dv_cq *out;
163d6b92ffaSHans Petter Selasky } cq;
164d6b92ffaSHans Petter Selasky struct {
165d6b92ffaSHans Petter Selasky struct ibv_srq *in;
166d6b92ffaSHans Petter Selasky struct mlx5dv_srq *out;
167d6b92ffaSHans Petter Selasky } srq;
168d6b92ffaSHans Petter Selasky struct {
169d6b92ffaSHans Petter Selasky struct ibv_wq *in;
170d6b92ffaSHans Petter Selasky struct mlx5dv_rwq *out;
171d6b92ffaSHans Petter Selasky } rwq;
172d6b92ffaSHans Petter Selasky };
173d6b92ffaSHans Petter Selasky
174d6b92ffaSHans Petter Selasky enum mlx5dv_obj_type {
175d6b92ffaSHans Petter Selasky MLX5DV_OBJ_QP = 1 << 0,
176d6b92ffaSHans Petter Selasky MLX5DV_OBJ_CQ = 1 << 1,
177d6b92ffaSHans Petter Selasky MLX5DV_OBJ_SRQ = 1 << 2,
178d6b92ffaSHans Petter Selasky MLX5DV_OBJ_RWQ = 1 << 3,
179d6b92ffaSHans Petter Selasky };
180d6b92ffaSHans Petter Selasky
181d6b92ffaSHans Petter Selasky /*
182d6b92ffaSHans Petter Selasky * This function will initialize mlx5dv_xxx structs based on supplied type.
183d6b92ffaSHans Petter Selasky * The information for initialization is taken from ibv_xx structs supplied
184d6b92ffaSHans Petter Selasky * as part of input.
185d6b92ffaSHans Petter Selasky *
186d6b92ffaSHans Petter Selasky * Request information of CQ marks its owned by DV for all consumer index
187d6b92ffaSHans Petter Selasky * related actions.
188d6b92ffaSHans Petter Selasky *
189d6b92ffaSHans Petter Selasky * The initialization type can be combination of several types together.
190d6b92ffaSHans Petter Selasky *
191d6b92ffaSHans Petter Selasky * Return: 0 in case of success.
192d6b92ffaSHans Petter Selasky */
193d6b92ffaSHans Petter Selasky int mlx5dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type);
194d6b92ffaSHans Petter Selasky
195d6b92ffaSHans Petter Selasky enum {
196d6b92ffaSHans Petter Selasky MLX5_OPCODE_NOP = 0x00,
197d6b92ffaSHans Petter Selasky MLX5_OPCODE_SEND_INVAL = 0x01,
198d6b92ffaSHans Petter Selasky MLX5_OPCODE_RDMA_WRITE = 0x08,
199d6b92ffaSHans Petter Selasky MLX5_OPCODE_RDMA_WRITE_IMM = 0x09,
200d6b92ffaSHans Petter Selasky MLX5_OPCODE_SEND = 0x0a,
201d6b92ffaSHans Petter Selasky MLX5_OPCODE_SEND_IMM = 0x0b,
202d6b92ffaSHans Petter Selasky MLX5_OPCODE_TSO = 0x0e,
203d6b92ffaSHans Petter Selasky MLX5_OPCODE_RDMA_READ = 0x10,
204d6b92ffaSHans Petter Selasky MLX5_OPCODE_ATOMIC_CS = 0x11,
205d6b92ffaSHans Petter Selasky MLX5_OPCODE_ATOMIC_FA = 0x12,
206d6b92ffaSHans Petter Selasky MLX5_OPCODE_ATOMIC_MASKED_CS = 0x14,
207d6b92ffaSHans Petter Selasky MLX5_OPCODE_ATOMIC_MASKED_FA = 0x15,
208d6b92ffaSHans Petter Selasky MLX5_OPCODE_FMR = 0x19,
209d6b92ffaSHans Petter Selasky MLX5_OPCODE_LOCAL_INVAL = 0x1b,
210d6b92ffaSHans Petter Selasky MLX5_OPCODE_CONFIG_CMD = 0x1f,
211d6b92ffaSHans Petter Selasky MLX5_OPCODE_UMR = 0x25,
212d6b92ffaSHans Petter Selasky };
213d6b92ffaSHans Petter Selasky
214d6b92ffaSHans Petter Selasky /*
215d6b92ffaSHans Petter Selasky * CQE related part
216d6b92ffaSHans Petter Selasky */
217d6b92ffaSHans Petter Selasky
218d6b92ffaSHans Petter Selasky enum {
219d6b92ffaSHans Petter Selasky MLX5_INLINE_SCATTER_32 = 0x4,
220d6b92ffaSHans Petter Selasky MLX5_INLINE_SCATTER_64 = 0x8,
221d6b92ffaSHans Petter Selasky };
222d6b92ffaSHans Petter Selasky
223d6b92ffaSHans Petter Selasky enum {
224d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR = 0x01,
225d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR = 0x02,
226d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_LOCAL_PROT_ERR = 0x04,
227d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_WR_FLUSH_ERR = 0x05,
228d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_MW_BIND_ERR = 0x06,
229d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_BAD_RESP_ERR = 0x10,
230d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR = 0x11,
231d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12,
232d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR = 0x13,
233d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_REMOTE_OP_ERR = 0x14,
234d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR = 0x15,
235d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR = 0x16,
236d6b92ffaSHans Petter Selasky MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR = 0x22,
237d6b92ffaSHans Petter Selasky };
238d6b92ffaSHans Petter Selasky
239d6b92ffaSHans Petter Selasky enum {
240d6b92ffaSHans Petter Selasky MLX5_CQE_L2_OK = 1 << 0,
241d6b92ffaSHans Petter Selasky MLX5_CQE_L3_OK = 1 << 1,
242d6b92ffaSHans Petter Selasky MLX5_CQE_L4_OK = 1 << 2,
243d6b92ffaSHans Petter Selasky };
244d6b92ffaSHans Petter Selasky
245d6b92ffaSHans Petter Selasky enum {
246d6b92ffaSHans Petter Selasky MLX5_CQE_L3_HDR_TYPE_NONE = 0x0,
247d6b92ffaSHans Petter Selasky MLX5_CQE_L3_HDR_TYPE_IPV6 = 0x1,
248d6b92ffaSHans Petter Selasky MLX5_CQE_L3_HDR_TYPE_IPV4 = 0x2,
249d6b92ffaSHans Petter Selasky };
250d6b92ffaSHans Petter Selasky
251d6b92ffaSHans Petter Selasky enum {
252d6b92ffaSHans Petter Selasky MLX5_CQE_OWNER_MASK = 1,
253d6b92ffaSHans Petter Selasky MLX5_CQE_REQ = 0,
254d6b92ffaSHans Petter Selasky MLX5_CQE_RESP_WR_IMM = 1,
255d6b92ffaSHans Petter Selasky MLX5_CQE_RESP_SEND = 2,
256d6b92ffaSHans Petter Selasky MLX5_CQE_RESP_SEND_IMM = 3,
257d6b92ffaSHans Petter Selasky MLX5_CQE_RESP_SEND_INV = 4,
258d6b92ffaSHans Petter Selasky MLX5_CQE_RESIZE_CQ = 5,
259d6b92ffaSHans Petter Selasky MLX5_CQE_REQ_ERR = 13,
260d6b92ffaSHans Petter Selasky MLX5_CQE_RESP_ERR = 14,
261d6b92ffaSHans Petter Selasky MLX5_CQE_INVALID = 15,
262d6b92ffaSHans Petter Selasky };
263d6b92ffaSHans Petter Selasky
264d6b92ffaSHans Petter Selasky enum {
265d6b92ffaSHans Petter Selasky MLX5_CQ_DOORBELL = 0x20
266d6b92ffaSHans Petter Selasky };
267d6b92ffaSHans Petter Selasky
268d6b92ffaSHans Petter Selasky enum {
269d6b92ffaSHans Petter Selasky MLX5_CQ_DB_REQ_NOT_SOL = 1 << 24,
270d6b92ffaSHans Petter Selasky MLX5_CQ_DB_REQ_NOT = 0 << 24,
271d6b92ffaSHans Petter Selasky };
272d6b92ffaSHans Petter Selasky
273d6b92ffaSHans Petter Selasky struct mlx5_err_cqe {
274d6b92ffaSHans Petter Selasky uint8_t rsvd0[32];
275d6b92ffaSHans Petter Selasky uint32_t srqn;
276d6b92ffaSHans Petter Selasky uint8_t rsvd1[18];
277d6b92ffaSHans Petter Selasky uint8_t vendor_err_synd;
278d6b92ffaSHans Petter Selasky uint8_t syndrome;
279d6b92ffaSHans Petter Selasky uint32_t s_wqe_opcode_qpn;
280d6b92ffaSHans Petter Selasky uint16_t wqe_counter;
281d6b92ffaSHans Petter Selasky uint8_t signature;
282d6b92ffaSHans Petter Selasky uint8_t op_own;
283d6b92ffaSHans Petter Selasky };
284d6b92ffaSHans Petter Selasky
285d6b92ffaSHans Petter Selasky struct mlx5_cqe64 {
286d6b92ffaSHans Petter Selasky uint8_t rsvd0[17];
287d6b92ffaSHans Petter Selasky uint8_t ml_path;
288d6b92ffaSHans Petter Selasky uint8_t rsvd20[4];
289d6b92ffaSHans Petter Selasky uint16_t slid;
290d6b92ffaSHans Petter Selasky uint32_t flags_rqpn;
291d6b92ffaSHans Petter Selasky uint8_t hds_ip_ext;
292d6b92ffaSHans Petter Selasky uint8_t l4_hdr_type_etc;
293d6b92ffaSHans Petter Selasky uint16_t vlan_info;
294d6b92ffaSHans Petter Selasky uint32_t srqn_uidx;
295d6b92ffaSHans Petter Selasky uint32_t imm_inval_pkey;
296d6b92ffaSHans Petter Selasky uint8_t rsvd40[4];
297d6b92ffaSHans Petter Selasky uint32_t byte_cnt;
298d6b92ffaSHans Petter Selasky __be64 timestamp;
299d6b92ffaSHans Petter Selasky uint32_t sop_drop_qpn;
300d6b92ffaSHans Petter Selasky uint16_t wqe_counter;
301d6b92ffaSHans Petter Selasky uint8_t signature;
302d6b92ffaSHans Petter Selasky uint8_t op_own;
303d6b92ffaSHans Petter Selasky };
304d6b92ffaSHans Petter Selasky
305d6b92ffaSHans Petter Selasky enum mlx5dv_cqe_comp_res_format {
306d6b92ffaSHans Petter Selasky MLX5DV_CQE_RES_FORMAT_HASH = 1 << 0,
307d6b92ffaSHans Petter Selasky MLX5DV_CQE_RES_FORMAT_CSUM = 1 << 1,
308d6b92ffaSHans Petter Selasky MLX5DV_CQE_RES_FORMAT_RESERVED = 1 << 2,
309d6b92ffaSHans Petter Selasky };
310d6b92ffaSHans Petter Selasky
311d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_get_cqe_owner(struct mlx5_cqe64 * cqe)312d6b92ffaSHans Petter Selasky uint8_t mlx5dv_get_cqe_owner(struct mlx5_cqe64 *cqe)
313d6b92ffaSHans Petter Selasky {
314d6b92ffaSHans Petter Selasky return cqe->op_own & 0x1;
315d6b92ffaSHans Petter Selasky }
316d6b92ffaSHans Petter Selasky
317d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_set_cqe_owner(struct mlx5_cqe64 * cqe,uint8_t val)318d6b92ffaSHans Petter Selasky void mlx5dv_set_cqe_owner(struct mlx5_cqe64 *cqe, uint8_t val)
319d6b92ffaSHans Petter Selasky {
320d6b92ffaSHans Petter Selasky cqe->op_own = (val & 0x1) | (cqe->op_own & ~0x1);
321d6b92ffaSHans Petter Selasky }
322d6b92ffaSHans Petter Selasky
323d6b92ffaSHans Petter Selasky /* Solicited event */
324d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_get_cqe_se(struct mlx5_cqe64 * cqe)325d6b92ffaSHans Petter Selasky uint8_t mlx5dv_get_cqe_se(struct mlx5_cqe64 *cqe)
326d6b92ffaSHans Petter Selasky {
327d6b92ffaSHans Petter Selasky return (cqe->op_own >> 1) & 0x1;
328d6b92ffaSHans Petter Selasky }
329d6b92ffaSHans Petter Selasky
330d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_get_cqe_format(struct mlx5_cqe64 * cqe)331d6b92ffaSHans Petter Selasky uint8_t mlx5dv_get_cqe_format(struct mlx5_cqe64 *cqe)
332d6b92ffaSHans Petter Selasky {
333d6b92ffaSHans Petter Selasky return (cqe->op_own >> 2) & 0x3;
334d6b92ffaSHans Petter Selasky }
335d6b92ffaSHans Petter Selasky
336d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_get_cqe_opcode(struct mlx5_cqe64 * cqe)337d6b92ffaSHans Petter Selasky uint8_t mlx5dv_get_cqe_opcode(struct mlx5_cqe64 *cqe)
338d6b92ffaSHans Petter Selasky {
339d6b92ffaSHans Petter Selasky return cqe->op_own >> 4;
340d6b92ffaSHans Petter Selasky }
341d6b92ffaSHans Petter Selasky
342d6b92ffaSHans Petter Selasky /*
343d6b92ffaSHans Petter Selasky * WQE related part
344d6b92ffaSHans Petter Selasky */
345d6b92ffaSHans Petter Selasky enum {
346d6b92ffaSHans Petter Selasky MLX5_INVALID_LKEY = 0x100,
347d6b92ffaSHans Petter Selasky };
348d6b92ffaSHans Petter Selasky
349d6b92ffaSHans Petter Selasky enum {
350d6b92ffaSHans Petter Selasky MLX5_EXTENDED_UD_AV = 0x80000000,
351d6b92ffaSHans Petter Selasky };
352d6b92ffaSHans Petter Selasky
353d6b92ffaSHans Petter Selasky enum {
354d6b92ffaSHans Petter Selasky MLX5_WQE_CTRL_CQ_UPDATE = 2 << 2,
355d6b92ffaSHans Petter Selasky MLX5_WQE_CTRL_SOLICITED = 1 << 1,
356d6b92ffaSHans Petter Selasky MLX5_WQE_CTRL_FENCE = 4 << 5,
357d6b92ffaSHans Petter Selasky MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE = 1 << 5,
358d6b92ffaSHans Petter Selasky };
359d6b92ffaSHans Petter Selasky
360d6b92ffaSHans Petter Selasky enum {
361d6b92ffaSHans Petter Selasky MLX5_SEND_WQE_BB = 64,
362d6b92ffaSHans Petter Selasky MLX5_SEND_WQE_SHIFT = 6,
363d6b92ffaSHans Petter Selasky };
364d6b92ffaSHans Petter Selasky
365d6b92ffaSHans Petter Selasky enum {
366d6b92ffaSHans Petter Selasky MLX5_INLINE_SEG = 0x80000000,
367d6b92ffaSHans Petter Selasky };
368d6b92ffaSHans Petter Selasky
369d6b92ffaSHans Petter Selasky enum {
370d6b92ffaSHans Petter Selasky MLX5_ETH_WQE_L3_CSUM = (1 << 6),
371d6b92ffaSHans Petter Selasky MLX5_ETH_WQE_L4_CSUM = (1 << 7),
372d6b92ffaSHans Petter Selasky };
373d6b92ffaSHans Petter Selasky
374d6b92ffaSHans Petter Selasky struct mlx5_wqe_srq_next_seg {
375d6b92ffaSHans Petter Selasky uint8_t rsvd0[2];
376d6b92ffaSHans Petter Selasky uint16_t next_wqe_index;
377d6b92ffaSHans Petter Selasky uint8_t signature;
378d6b92ffaSHans Petter Selasky uint8_t rsvd1[11];
379d6b92ffaSHans Petter Selasky };
380d6b92ffaSHans Petter Selasky
381d6b92ffaSHans Petter Selasky struct mlx5_wqe_data_seg {
382d6b92ffaSHans Petter Selasky uint32_t byte_count;
383d6b92ffaSHans Petter Selasky uint32_t lkey;
384d6b92ffaSHans Petter Selasky uint64_t addr;
385d6b92ffaSHans Petter Selasky };
386d6b92ffaSHans Petter Selasky
387d6b92ffaSHans Petter Selasky struct mlx5_wqe_ctrl_seg {
388d6b92ffaSHans Petter Selasky uint32_t opmod_idx_opcode;
389d6b92ffaSHans Petter Selasky uint32_t qpn_ds;
390d6b92ffaSHans Petter Selasky uint8_t signature;
391d6b92ffaSHans Petter Selasky uint8_t rsvd[2];
392d6b92ffaSHans Petter Selasky uint8_t fm_ce_se;
393d6b92ffaSHans Petter Selasky uint32_t imm;
394d6b92ffaSHans Petter Selasky };
395d6b92ffaSHans Petter Selasky
396d6b92ffaSHans Petter Selasky struct mlx5_wqe_av {
397d6b92ffaSHans Petter Selasky union {
398d6b92ffaSHans Petter Selasky struct {
399d6b92ffaSHans Petter Selasky uint32_t qkey;
400d6b92ffaSHans Petter Selasky uint32_t reserved;
401d6b92ffaSHans Petter Selasky } qkey;
402d6b92ffaSHans Petter Selasky uint64_t dc_key;
403d6b92ffaSHans Petter Selasky } key;
404d6b92ffaSHans Petter Selasky uint32_t dqp_dct;
405d6b92ffaSHans Petter Selasky uint8_t stat_rate_sl;
406d6b92ffaSHans Petter Selasky uint8_t fl_mlid;
407d6b92ffaSHans Petter Selasky uint16_t rlid;
408d6b92ffaSHans Petter Selasky uint8_t reserved0[4];
409d6b92ffaSHans Petter Selasky uint8_t rmac[6];
410d6b92ffaSHans Petter Selasky uint8_t tclass;
411d6b92ffaSHans Petter Selasky uint8_t hop_limit;
412d6b92ffaSHans Petter Selasky uint32_t grh_gid_fl;
413d6b92ffaSHans Petter Selasky uint8_t rgid[16];
414d6b92ffaSHans Petter Selasky };
415d6b92ffaSHans Petter Selasky
416d6b92ffaSHans Petter Selasky struct mlx5_wqe_datagram_seg {
417d6b92ffaSHans Petter Selasky struct mlx5_wqe_av av;
418d6b92ffaSHans Petter Selasky };
419d6b92ffaSHans Petter Selasky
420d6b92ffaSHans Petter Selasky struct mlx5_wqe_raddr_seg {
421d6b92ffaSHans Petter Selasky uint64_t raddr;
422d6b92ffaSHans Petter Selasky uint32_t rkey;
423d6b92ffaSHans Petter Selasky uint32_t reserved;
424d6b92ffaSHans Petter Selasky };
425d6b92ffaSHans Petter Selasky
426d6b92ffaSHans Petter Selasky struct mlx5_wqe_atomic_seg {
427d6b92ffaSHans Petter Selasky uint64_t swap_add;
428d6b92ffaSHans Petter Selasky uint64_t compare;
429d6b92ffaSHans Petter Selasky };
430d6b92ffaSHans Petter Selasky
431d6b92ffaSHans Petter Selasky struct mlx5_wqe_inl_data_seg {
432d6b92ffaSHans Petter Selasky uint32_t byte_count;
433d6b92ffaSHans Petter Selasky };
434d6b92ffaSHans Petter Selasky
435d6b92ffaSHans Petter Selasky struct mlx5_wqe_eth_seg {
436d6b92ffaSHans Petter Selasky uint32_t rsvd0;
437d6b92ffaSHans Petter Selasky uint8_t cs_flags;
438d6b92ffaSHans Petter Selasky uint8_t rsvd1;
439d6b92ffaSHans Petter Selasky uint16_t mss;
440d6b92ffaSHans Petter Selasky uint32_t rsvd2;
441d6b92ffaSHans Petter Selasky uint16_t inline_hdr_sz;
442d6b92ffaSHans Petter Selasky uint8_t inline_hdr_start[2];
443d6b92ffaSHans Petter Selasky uint8_t inline_hdr[16];
444d6b92ffaSHans Petter Selasky };
445d6b92ffaSHans Petter Selasky
446d6b92ffaSHans Petter Selasky /*
447d6b92ffaSHans Petter Selasky * Control segment - contains some control information for the current WQE.
448d6b92ffaSHans Petter Selasky *
449d6b92ffaSHans Petter Selasky * Output:
450d6b92ffaSHans Petter Selasky * seg - control segment to be filled
451d6b92ffaSHans Petter Selasky * Input:
452d6b92ffaSHans Petter Selasky * pi - WQEBB number of the first block of this WQE.
453d6b92ffaSHans Petter Selasky * This number should wrap at 0xffff, regardless of
454d6b92ffaSHans Petter Selasky * size of the WQ.
455d6b92ffaSHans Petter Selasky * opcode - Opcode of this WQE. Encodes the type of operation
456d6b92ffaSHans Petter Selasky * to be executed on the QP.
457d6b92ffaSHans Petter Selasky * opmod - Opcode modifier.
458d6b92ffaSHans Petter Selasky * qp_num - QP/SQ number this WQE is posted to.
459d6b92ffaSHans Petter Selasky * fm_ce_se - FM (fence mode), CE (completion and event mode)
460d6b92ffaSHans Petter Selasky * and SE (solicited event).
461d6b92ffaSHans Petter Selasky * ds - WQE size in octowords (16-byte units). DS accounts for all
462d6b92ffaSHans Petter Selasky * the segments in the WQE as summarized in WQE construction.
463d6b92ffaSHans Petter Selasky * signature - WQE signature.
464d6b92ffaSHans Petter Selasky * imm - Immediate data/Invalidation key/UMR mkey.
465d6b92ffaSHans Petter Selasky */
466d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_set_ctrl_seg(struct mlx5_wqe_ctrl_seg * seg,uint16_t pi,uint8_t opcode,uint8_t opmod,uint32_t qp_num,uint8_t fm_ce_se,uint8_t ds,uint8_t signature,uint32_t imm)467d6b92ffaSHans Petter Selasky void mlx5dv_set_ctrl_seg(struct mlx5_wqe_ctrl_seg *seg, uint16_t pi,
468d6b92ffaSHans Petter Selasky uint8_t opcode, uint8_t opmod, uint32_t qp_num,
469d6b92ffaSHans Petter Selasky uint8_t fm_ce_se, uint8_t ds,
470d6b92ffaSHans Petter Selasky uint8_t signature, uint32_t imm)
471d6b92ffaSHans Petter Selasky {
472d6b92ffaSHans Petter Selasky seg->opmod_idx_opcode = htobe32(((uint32_t)opmod << 24) | ((uint32_t)pi << 8) | opcode);
473d6b92ffaSHans Petter Selasky seg->qpn_ds = htobe32((qp_num << 8) | ds);
474d6b92ffaSHans Petter Selasky seg->fm_ce_se = fm_ce_se;
475d6b92ffaSHans Petter Selasky seg->signature = signature;
476d6b92ffaSHans Petter Selasky /*
477d6b92ffaSHans Petter Selasky * The caller should prepare "imm" in advance based on WR opcode.
478d6b92ffaSHans Petter Selasky * For IBV_WR_SEND_WITH_IMM and IBV_WR_RDMA_WRITE_WITH_IMM,
479d6b92ffaSHans Petter Selasky * the "imm" should be assigned as is.
480d6b92ffaSHans Petter Selasky * For the IBV_WR_SEND_WITH_INV, it should be htobe32(imm).
481d6b92ffaSHans Petter Selasky */
482d6b92ffaSHans Petter Selasky seg->imm = imm;
483d6b92ffaSHans Petter Selasky }
484d6b92ffaSHans Petter Selasky
485d6b92ffaSHans Petter Selasky /* x86 optimized version of mlx5dv_set_ctrl_seg()
486d6b92ffaSHans Petter Selasky *
487d6b92ffaSHans Petter Selasky * This is useful when doing calculations on large data sets
488d6b92ffaSHans Petter Selasky * for parallel calculations.
489d6b92ffaSHans Petter Selasky *
490d6b92ffaSHans Petter Selasky * It doesn't suit for serialized algorithms.
491d6b92ffaSHans Petter Selasky */
492d6b92ffaSHans Petter Selasky #if defined(__SSE3__)
493d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_x86_set_ctrl_seg(struct mlx5_wqe_ctrl_seg * seg,uint16_t pi,uint8_t opcode,uint8_t opmod,uint32_t qp_num,uint8_t fm_ce_se,uint8_t ds,uint8_t signature,uint32_t imm)494d6b92ffaSHans Petter Selasky void mlx5dv_x86_set_ctrl_seg(struct mlx5_wqe_ctrl_seg *seg, uint16_t pi,
495d6b92ffaSHans Petter Selasky uint8_t opcode, uint8_t opmod, uint32_t qp_num,
496d6b92ffaSHans Petter Selasky uint8_t fm_ce_se, uint8_t ds,
497d6b92ffaSHans Petter Selasky uint8_t signature, uint32_t imm)
498d6b92ffaSHans Petter Selasky {
499d6b92ffaSHans Petter Selasky __m128i val = _mm_set_epi32(imm, qp_num, (ds << 16) | pi,
500d6b92ffaSHans Petter Selasky (signature << 24) | (opcode << 16) | (opmod << 8) | fm_ce_se);
501d6b92ffaSHans Petter Selasky __m128i mask = _mm_set_epi8(15, 14, 13, 12, /* immediate */
502d6b92ffaSHans Petter Selasky 0, /* signal/fence_mode */
503d6b92ffaSHans Petter Selasky 0x80, 0x80, /* reserved */
504d6b92ffaSHans Petter Selasky 3, /* signature */
505d6b92ffaSHans Petter Selasky 6, /* data size */
506d6b92ffaSHans Petter Selasky 8, 9, 10, /* QP num */
507d6b92ffaSHans Petter Selasky 2, /* opcode */
508d6b92ffaSHans Petter Selasky 4, 5, /* sw_pi in BE */
509d6b92ffaSHans Petter Selasky 1 /* opmod */
510d6b92ffaSHans Petter Selasky );
511d6b92ffaSHans Petter Selasky *(__m128i *) seg = _mm_shuffle_epi8(val, mask);
512d6b92ffaSHans Petter Selasky }
513d6b92ffaSHans Petter Selasky #endif /* defined(__SSE3__) */
514d6b92ffaSHans Petter Selasky
515d6b92ffaSHans Petter Selasky /*
516d6b92ffaSHans Petter Selasky * Datagram Segment - contains address information required in order
517d6b92ffaSHans Petter Selasky * to form a datagram message.
518d6b92ffaSHans Petter Selasky *
519d6b92ffaSHans Petter Selasky * Output:
520d6b92ffaSHans Petter Selasky * seg - datagram segment to be filled.
521d6b92ffaSHans Petter Selasky * Input:
522d6b92ffaSHans Petter Selasky * key - Q_key/access key.
523d6b92ffaSHans Petter Selasky * dqp_dct - Destination QP number for UD and DCT for DC.
524d6b92ffaSHans Petter Selasky * ext - Address vector extension.
525d6b92ffaSHans Petter Selasky * stat_rate_sl - Maximum static rate control, SL/ethernet priority.
526d6b92ffaSHans Petter Selasky * fl_mlid - Force loopback and source LID for IB.
527d6b92ffaSHans Petter Selasky * rlid - Remote LID
528d6b92ffaSHans Petter Selasky * rmac - Remote MAC
529d6b92ffaSHans Petter Selasky * tclass - GRH tclass/IPv6 tclass/IPv4 ToS
530d6b92ffaSHans Petter Selasky * hop_limit - GRH hop limit/IPv6 hop limit/IPv4 TTL
531d6b92ffaSHans Petter Selasky * grh_gid_fi - GRH, source GID address and IPv6 flow label.
532d6b92ffaSHans Petter Selasky * rgid - Remote GID/IP address.
533d6b92ffaSHans Petter Selasky */
534d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_set_dgram_seg(struct mlx5_wqe_datagram_seg * seg,uint64_t key,uint32_t dqp_dct,uint8_t ext,uint8_t stat_rate_sl,uint8_t fl_mlid,uint16_t rlid,uint8_t * rmac,uint8_t tclass,uint8_t hop_limit,uint32_t grh_gid_fi,uint8_t * rgid)535d6b92ffaSHans Petter Selasky void mlx5dv_set_dgram_seg(struct mlx5_wqe_datagram_seg *seg,
536d6b92ffaSHans Petter Selasky uint64_t key, uint32_t dqp_dct,
537d6b92ffaSHans Petter Selasky uint8_t ext, uint8_t stat_rate_sl,
538d6b92ffaSHans Petter Selasky uint8_t fl_mlid, uint16_t rlid,
539d6b92ffaSHans Petter Selasky uint8_t *rmac, uint8_t tclass,
540d6b92ffaSHans Petter Selasky uint8_t hop_limit, uint32_t grh_gid_fi,
541d6b92ffaSHans Petter Selasky uint8_t *rgid)
542d6b92ffaSHans Petter Selasky {
543d6b92ffaSHans Petter Selasky
544d6b92ffaSHans Petter Selasky /* Always put 64 bits, in q_key, the reserved part will be 0 */
545d6b92ffaSHans Petter Selasky seg->av.key.dc_key = htobe64(key);
546d6b92ffaSHans Petter Selasky seg->av.dqp_dct = htobe32(((uint32_t)ext << 31) | dqp_dct);
547d6b92ffaSHans Petter Selasky seg->av.stat_rate_sl = stat_rate_sl;
548d6b92ffaSHans Petter Selasky seg->av.fl_mlid = fl_mlid;
549d6b92ffaSHans Petter Selasky seg->av.rlid = htobe16(rlid);
550d6b92ffaSHans Petter Selasky memcpy(seg->av.rmac, rmac, 6);
551d6b92ffaSHans Petter Selasky seg->av.tclass = tclass;
552d6b92ffaSHans Petter Selasky seg->av.hop_limit = hop_limit;
553d6b92ffaSHans Petter Selasky seg->av.grh_gid_fl = htobe32(grh_gid_fi);
554d6b92ffaSHans Petter Selasky memcpy(seg->av.rgid, rgid, 16);
555d6b92ffaSHans Petter Selasky }
556d6b92ffaSHans Petter Selasky
557d6b92ffaSHans Petter Selasky /*
558d6b92ffaSHans Petter Selasky * Data Segments - contain pointers and a byte count for the scatter/gather list.
559d6b92ffaSHans Petter Selasky * They can optionally contain data, which will save a memory read access for
560d6b92ffaSHans Petter Selasky * gather Work Requests.
561d6b92ffaSHans Petter Selasky */
562d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_set_data_seg(struct mlx5_wqe_data_seg * seg,uint32_t length,uint32_t lkey,uintptr_t address)563d6b92ffaSHans Petter Selasky void mlx5dv_set_data_seg(struct mlx5_wqe_data_seg *seg,
564d6b92ffaSHans Petter Selasky uint32_t length, uint32_t lkey,
565d6b92ffaSHans Petter Selasky uintptr_t address)
566d6b92ffaSHans Petter Selasky {
567d6b92ffaSHans Petter Selasky seg->byte_count = htobe32(length);
568d6b92ffaSHans Petter Selasky seg->lkey = htobe32(lkey);
569d6b92ffaSHans Petter Selasky seg->addr = htobe64(address);
570d6b92ffaSHans Petter Selasky }
571d6b92ffaSHans Petter Selasky /*
572d6b92ffaSHans Petter Selasky * x86 optimized version of mlx5dv_set_data_seg()
573d6b92ffaSHans Petter Selasky *
574d6b92ffaSHans Petter Selasky * This is useful when doing calculations on large data sets
575d6b92ffaSHans Petter Selasky * for parallel calculations.
576d6b92ffaSHans Petter Selasky *
577d6b92ffaSHans Petter Selasky * It doesn't suit for serialized algorithms.
578d6b92ffaSHans Petter Selasky */
579d6b92ffaSHans Petter Selasky #if defined(__SSE3__)
580d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_x86_set_data_seg(struct mlx5_wqe_data_seg * seg,uint32_t length,uint32_t lkey,uintptr_t address)581d6b92ffaSHans Petter Selasky void mlx5dv_x86_set_data_seg(struct mlx5_wqe_data_seg *seg,
582d6b92ffaSHans Petter Selasky uint32_t length, uint32_t lkey,
583d6b92ffaSHans Petter Selasky uintptr_t address)
584d6b92ffaSHans Petter Selasky {
585d6b92ffaSHans Petter Selasky __m128i val = _mm_set_epi32((uint32_t)address, (uint32_t)(address >> 32), lkey, length);
586d6b92ffaSHans Petter Selasky __m128i mask = _mm_set_epi8(12, 13, 14, 15, /* local address low */
587d6b92ffaSHans Petter Selasky 8, 9, 10, 11, /* local address high */
588d6b92ffaSHans Petter Selasky 4, 5, 6, 7, /* l_key */
589d6b92ffaSHans Petter Selasky 0, 1, 2, 3 /* byte count */
590d6b92ffaSHans Petter Selasky );
591d6b92ffaSHans Petter Selasky *(__m128i *) seg = _mm_shuffle_epi8(val, mask);
592d6b92ffaSHans Petter Selasky }
593d6b92ffaSHans Petter Selasky #endif /* defined(__SSE3__) */
594d6b92ffaSHans Petter Selasky
595d6b92ffaSHans Petter Selasky /*
596d6b92ffaSHans Petter Selasky * Eth Segment - contains packet headers and information for stateless L2, L3, L4 offloading.
597d6b92ffaSHans Petter Selasky *
598d6b92ffaSHans Petter Selasky * Output:
599d6b92ffaSHans Petter Selasky * seg - Eth segment to be filled.
600d6b92ffaSHans Petter Selasky * Input:
601d6b92ffaSHans Petter Selasky * cs_flags - l3cs/l3cs_inner/l4cs/l4cs_inner.
602d6b92ffaSHans Petter Selasky * mss - Maximum segment size. For TSO WQEs, the number of bytes
603d6b92ffaSHans Petter Selasky * in the TCP payload to be transmitted in each packet. Must
604d6b92ffaSHans Petter Selasky * be 0 on non TSO WQEs.
605d6b92ffaSHans Petter Selasky * inline_hdr_sz - Length of the inlined packet headers.
606d6b92ffaSHans Petter Selasky * inline_hdr_start - Inlined packet header.
607d6b92ffaSHans Petter Selasky */
608d6b92ffaSHans Petter Selasky static MLX5DV_ALWAYS_INLINE
mlx5dv_set_eth_seg(struct mlx5_wqe_eth_seg * seg,uint8_t cs_flags,uint16_t mss,uint16_t inline_hdr_sz,uint8_t * inline_hdr_start)609d6b92ffaSHans Petter Selasky void mlx5dv_set_eth_seg(struct mlx5_wqe_eth_seg *seg, uint8_t cs_flags,
610d6b92ffaSHans Petter Selasky uint16_t mss, uint16_t inline_hdr_sz,
611d6b92ffaSHans Petter Selasky uint8_t *inline_hdr_start)
612d6b92ffaSHans Petter Selasky {
613d6b92ffaSHans Petter Selasky seg->cs_flags = cs_flags;
614d6b92ffaSHans Petter Selasky seg->mss = htobe16(mss);
615d6b92ffaSHans Petter Selasky seg->inline_hdr_sz = htobe16(inline_hdr_sz);
616d6b92ffaSHans Petter Selasky memcpy(seg->inline_hdr_start, inline_hdr_start, inline_hdr_sz);
617d6b92ffaSHans Petter Selasky }
618d6b92ffaSHans Petter Selasky #endif /* _MLX5DV_H_ */
619