1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2015 - 2023 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #ifndef IRDMA_TYPE_H 36 #define IRDMA_TYPE_H 37 38 #include "osdep.h" 39 40 #include "irdma.h" 41 #include "irdma_user.h" 42 #include "irdma_hmc.h" 43 #include "irdma_uda.h" 44 #include "irdma_ws.h" 45 enum irdma_debug_flag { 46 IRDMA_DEBUG_NONE = 0x00000000, 47 IRDMA_DEBUG_ERR = 0x00000001, 48 IRDMA_DEBUG_INIT = 0x00000002, 49 IRDMA_DEBUG_DEV = 0x00000004, 50 IRDMA_DEBUG_CM = 0x00000008, 51 IRDMA_DEBUG_VERBS = 0x00000010, 52 IRDMA_DEBUG_PUDA = 0x00000020, 53 IRDMA_DEBUG_ILQ = 0x00000040, 54 IRDMA_DEBUG_IEQ = 0x00000080, 55 IRDMA_DEBUG_QP = 0x00000100, 56 IRDMA_DEBUG_CQ = 0x00000200, 57 IRDMA_DEBUG_MR = 0x00000400, 58 IRDMA_DEBUG_PBLE = 0x00000800, 59 IRDMA_DEBUG_WQE = 0x00001000, 60 IRDMA_DEBUG_AEQ = 0x00002000, 61 IRDMA_DEBUG_CQP = 0x00004000, 62 IRDMA_DEBUG_HMC = 0x00008000, 63 IRDMA_DEBUG_USER = 0x00010000, 64 IRDMA_DEBUG_VIRT = 0x00020000, 65 IRDMA_DEBUG_DCB = 0x00040000, 66 IRDMA_DEBUG_CQE = 0x00800000, 67 IRDMA_DEBUG_CLNT = 0x01000000, 68 IRDMA_DEBUG_WS = 0x02000000, 69 IRDMA_DEBUG_STATS = 0x04000000, 70 IRDMA_DEBUG_ALL = 0xFFFFFFFF, 71 }; 72 73 enum irdma_page_size { 74 IRDMA_PAGE_SIZE_4K = 0, 75 IRDMA_PAGE_SIZE_2M, 76 IRDMA_PAGE_SIZE_1G, 77 }; 78 79 enum irdma_hdrct_flags { 80 DDP_LEN_FLAG = 0x80, 81 DDP_HDR_FLAG = 0x40, 82 RDMA_HDR_FLAG = 0x20, 83 }; 84 85 enum irdma_term_layers { 86 LAYER_RDMA = 0, 87 LAYER_DDP = 1, 88 LAYER_MPA = 2, 89 }; 90 91 enum irdma_term_error_types { 92 RDMAP_REMOTE_PROT = 1, 93 RDMAP_REMOTE_OP = 2, 94 DDP_CATASTROPHIC = 0, 95 DDP_TAGGED_BUF = 1, 96 DDP_UNTAGGED_BUF = 2, 97 DDP_LLP = 3, 98 }; 99 100 enum irdma_term_rdma_errors { 101 RDMAP_INV_STAG = 0x00, 102 RDMAP_INV_BOUNDS = 0x01, 103 RDMAP_ACCESS = 0x02, 104 RDMAP_UNASSOC_STAG = 0x03, 105 RDMAP_TO_WRAP = 0x04, 106 RDMAP_INV_RDMAP_VER = 0x05, 107 RDMAP_UNEXPECTED_OP = 0x06, 108 RDMAP_CATASTROPHIC_LOCAL = 0x07, 109 RDMAP_CATASTROPHIC_GLOBAL = 0x08, 110 RDMAP_CANT_INV_STAG = 0x09, 111 RDMAP_UNSPECIFIED = 0xff, 112 }; 113 114 enum irdma_term_ddp_errors { 115 DDP_CATASTROPHIC_LOCAL = 0x00, 116 DDP_TAGGED_INV_STAG = 0x00, 117 DDP_TAGGED_BOUNDS = 0x01, 118 DDP_TAGGED_UNASSOC_STAG = 0x02, 119 DDP_TAGGED_TO_WRAP = 0x03, 120 DDP_TAGGED_INV_DDP_VER = 0x04, 121 DDP_UNTAGGED_INV_QN = 0x01, 122 DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02, 123 DDP_UNTAGGED_INV_MSN_RANGE = 0x03, 124 DDP_UNTAGGED_INV_MO = 0x04, 125 DDP_UNTAGGED_INV_TOO_LONG = 0x05, 126 DDP_UNTAGGED_INV_DDP_VER = 0x06, 127 }; 128 129 enum irdma_term_mpa_errors { 130 MPA_CLOSED = 0x01, 131 MPA_CRC = 0x02, 132 MPA_MARKER = 0x03, 133 MPA_REQ_RSP = 0x04, 134 }; 135 136 enum irdma_hw_stats_index { 137 /* gen1 - 32-bit */ 138 IRDMA_HW_STAT_INDEX_IP4RXDISCARD = 0, 139 IRDMA_HW_STAT_INDEX_IP4RXTRUNC = 1, 140 IRDMA_HW_STAT_INDEX_IP4TXNOROUTE = 2, 141 IRDMA_HW_STAT_INDEX_IP6RXDISCARD = 3, 142 IRDMA_HW_STAT_INDEX_IP6RXTRUNC = 4, 143 IRDMA_HW_STAT_INDEX_IP6TXNOROUTE = 5, 144 IRDMA_HW_STAT_INDEX_TCPRTXSEG = 6, 145 IRDMA_HW_STAT_INDEX_TCPRXOPTERR = 7, 146 IRDMA_HW_STAT_INDEX_TCPRXPROTOERR = 8, 147 IRDMA_HW_STAT_INDEX_RXVLANERR = 9, 148 /* gen1 - 64-bit */ 149 IRDMA_HW_STAT_INDEX_IP4RXOCTS = 10, 150 IRDMA_HW_STAT_INDEX_IP4RXPKTS = 11, 151 IRDMA_HW_STAT_INDEX_IP4RXFRAGS = 12, 152 IRDMA_HW_STAT_INDEX_IP4RXMCPKTS = 13, 153 IRDMA_HW_STAT_INDEX_IP4TXOCTS = 14, 154 IRDMA_HW_STAT_INDEX_IP4TXPKTS = 15, 155 IRDMA_HW_STAT_INDEX_IP4TXFRAGS = 16, 156 IRDMA_HW_STAT_INDEX_IP4TXMCPKTS = 17, 157 IRDMA_HW_STAT_INDEX_IP6RXOCTS = 18, 158 IRDMA_HW_STAT_INDEX_IP6RXPKTS = 19, 159 IRDMA_HW_STAT_INDEX_IP6RXFRAGS = 20, 160 IRDMA_HW_STAT_INDEX_IP6RXMCPKTS = 21, 161 IRDMA_HW_STAT_INDEX_IP6TXOCTS = 22, 162 IRDMA_HW_STAT_INDEX_IP6TXPKTS = 23, 163 IRDMA_HW_STAT_INDEX_IP6TXFRAGS = 24, 164 IRDMA_HW_STAT_INDEX_IP6TXMCPKTS = 25, 165 IRDMA_HW_STAT_INDEX_TCPRXSEGS = 26, 166 IRDMA_HW_STAT_INDEX_TCPTXSEG = 27, 167 IRDMA_HW_STAT_INDEX_RDMARXRDS = 28, 168 IRDMA_HW_STAT_INDEX_RDMARXSNDS = 29, 169 IRDMA_HW_STAT_INDEX_RDMARXWRS = 30, 170 IRDMA_HW_STAT_INDEX_RDMATXRDS = 31, 171 IRDMA_HW_STAT_INDEX_RDMATXSNDS = 32, 172 IRDMA_HW_STAT_INDEX_RDMATXWRS = 33, 173 IRDMA_HW_STAT_INDEX_RDMAVBND = 34, 174 IRDMA_HW_STAT_INDEX_RDMAVINV = 35, 175 IRDMA_HW_STAT_INDEX_IP4RXMCOCTS = 36, 176 IRDMA_HW_STAT_INDEX_IP4TXMCOCTS = 37, 177 IRDMA_HW_STAT_INDEX_IP6RXMCOCTS = 38, 178 IRDMA_HW_STAT_INDEX_IP6TXMCOCTS = 39, 179 IRDMA_HW_STAT_INDEX_UDPRXPKTS = 40, 180 IRDMA_HW_STAT_INDEX_UDPTXPKTS = 41, 181 IRDMA_HW_STAT_INDEX_MAX_GEN_1 = 42, /* Must be same value as next entry */ 182 183 /* gen2 - 64-bit */ 184 IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 42, 185 186 /* gen2 - 32-bit */ 187 IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED = 43, 188 IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED = 44, 189 IRDMA_HW_STAT_INDEX_TXNPCNPSENT = 45, 190 IRDMA_HW_STAT_INDEX_MAX_GEN_2 = 46, 191 }; 192 193 #define IRDMA_MIN_FEATURES 2 194 195 enum irdma_feature_type { 196 IRDMA_FEATURE_FW_INFO = 0, 197 IRDMA_HW_VERSION_INFO = 1, 198 IRDMA_QSETS_MAX = 26, 199 IRDMA_MAX_FEATURES, /* Must be last entry */ 200 }; 201 202 enum irdma_sched_prio_type { 203 IRDMA_PRIO_WEIGHTED_RR = 1, 204 IRDMA_PRIO_STRICT = 2, 205 IRDMA_PRIO_WEIGHTED_STRICT = 3, 206 }; 207 208 enum irdma_vm_vf_type { 209 IRDMA_VF_TYPE = 0, 210 IRDMA_VM_TYPE, 211 IRDMA_PF_TYPE, 212 }; 213 214 enum irdma_cqp_hmc_profile { 215 IRDMA_HMC_PROFILE_DEFAULT = 1, 216 IRDMA_HMC_PROFILE_FAVOR_VF = 2, 217 IRDMA_HMC_PROFILE_EQUAL = 3, 218 }; 219 220 enum irdma_quad_entry_type { 221 IRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1, 222 IRDMA_QHASH_TYPE_TCP_SYN, 223 IRDMA_QHASH_TYPE_UDP_UNICAST, 224 IRDMA_QHASH_TYPE_UDP_MCAST, 225 IRDMA_QHASH_TYPE_ROCE_MCAST, 226 IRDMA_QHASH_TYPE_ROCEV2_HW, 227 }; 228 229 enum irdma_quad_hash_manage_type { 230 IRDMA_QHASH_MANAGE_TYPE_DELETE = 0, 231 IRDMA_QHASH_MANAGE_TYPE_ADD, 232 IRDMA_QHASH_MANAGE_TYPE_MODIFY, 233 }; 234 235 enum irdma_syn_rst_handling { 236 IRDMA_SYN_RST_HANDLING_HW_TCP_SECURE = 0, 237 IRDMA_SYN_RST_HANDLING_HW_TCP, 238 IRDMA_SYN_RST_HANDLING_FW_TCP_SECURE, 239 IRDMA_SYN_RST_HANDLING_FW_TCP, 240 }; 241 242 enum irdma_queue_type { 243 IRDMA_QUEUE_TYPE_SQ_RQ = 0, 244 IRDMA_QUEUE_TYPE_CQP, 245 }; 246 247 struct irdma_sc_dev; 248 struct irdma_vsi_pestat; 249 250 struct irdma_dcqcn_cc_params { 251 u8 cc_cfg_valid; 252 u8 min_dec_factor; 253 u8 min_rate; 254 u8 dcqcn_f; 255 u16 rai_factor; 256 u16 hai_factor; 257 u16 dcqcn_t; 258 u32 dcqcn_b; 259 u32 rreduce_mperiod; 260 }; 261 262 struct irdma_cqp_init_info { 263 u64 cqp_compl_ctx; 264 u64 host_ctx_pa; 265 u64 sq_pa; 266 struct irdma_sc_dev *dev; 267 struct irdma_cqp_quanta *sq; 268 struct irdma_dcqcn_cc_params dcqcn_params; 269 __le64 *host_ctx; 270 u64 *scratch_array; 271 u32 sq_size; 272 u16 hw_maj_ver; 273 u16 hw_min_ver; 274 u8 struct_ver; 275 u8 hmc_profile; 276 u8 ena_vf_count; 277 u8 ceqs_per_vf; 278 bool en_datacenter_tcp:1; 279 bool disable_packed:1; 280 bool rocev2_rto_policy:1; 281 bool en_rem_endpoint_trk:1; 282 enum irdma_protocol_used protocol_used; 283 }; 284 285 struct irdma_terminate_hdr { 286 u8 layer_etype; 287 u8 error_code; 288 u8 hdrct; 289 u8 rsvd; 290 }; 291 292 struct irdma_cqp_sq_wqe { 293 __le64 buf[IRDMA_CQP_WQE_SIZE]; 294 }; 295 296 struct irdma_sc_aeqe { 297 __le64 buf[IRDMA_AEQE_SIZE]; 298 }; 299 300 struct irdma_ceqe { 301 __le64 buf[IRDMA_CEQE_SIZE]; 302 }; 303 304 struct irdma_cqp_ctx { 305 __le64 buf[IRDMA_CQP_CTX_SIZE]; 306 }; 307 308 struct irdma_cq_shadow_area { 309 __le64 buf[IRDMA_SHADOW_AREA_SIZE]; 310 }; 311 312 struct irdma_dev_hw_stats { 313 u64 stats_val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)]; 314 }; 315 316 struct irdma_gather_stats { 317 u64 val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)]; 318 }; 319 320 struct irdma_hw_stat_map { 321 u16 byteoff; 322 u8 bitoff; 323 u64 bitmask; 324 }; 325 326 struct irdma_stats_gather_info { 327 bool use_hmc_fcn_index:1; 328 bool use_stats_inst:1; 329 u16 hmc_fcn_index; 330 u16 stats_inst_index; 331 struct irdma_dma_mem stats_buff_mem; 332 void *gather_stats_va; 333 void *last_gather_stats_va; 334 }; 335 336 struct irdma_vsi_pestat { 337 struct irdma_hw *hw; 338 struct irdma_dev_hw_stats hw_stats; 339 struct irdma_stats_gather_info gather_info; 340 struct OS_TIMER stats_timer; 341 struct irdma_sc_vsi *vsi; 342 spinlock_t lock; /* rdma stats lock */ 343 }; 344 345 struct irdma_hw { 346 u8 IOMEM *hw_addr; 347 u8 IOMEM *priv_hw_addr; 348 void *dev_context; 349 struct irdma_hmc_info hmc; 350 }; 351 352 struct irdma_pfpdu { 353 struct list_head rxlist; 354 u32 rcv_nxt; 355 u32 fps; 356 u32 max_fpdu_data; 357 u32 nextseqnum; 358 u32 rcv_start_seq; 359 bool mode:1; 360 bool mpa_crc_err:1; 361 u8 marker_len; 362 u64 total_ieq_bufs; 363 u64 fpdu_processed; 364 u64 bad_seq_num; 365 u64 crc_err; 366 u64 no_tx_bufs; 367 u64 tx_err; 368 u64 out_of_order; 369 u64 pmode_count; 370 struct irdma_sc_ah *ah; 371 struct irdma_puda_buf *ah_buf; 372 spinlock_t lock; /* fpdu processing lock */ 373 struct irdma_puda_buf *lastrcv_buf; 374 }; 375 376 struct irdma_sc_pd { 377 struct irdma_sc_dev *dev; 378 u32 pd_id; 379 int abi_ver; 380 }; 381 382 struct irdma_cqp_quanta { 383 __le64 elem[IRDMA_CQP_WQE_SIZE]; 384 }; 385 386 struct irdma_sc_cqp { 387 u32 size; 388 u64 sq_pa; 389 u64 host_ctx_pa; 390 void *back_cqp; 391 struct irdma_sc_dev *dev; 392 int (*process_cqp_sds)(struct irdma_sc_dev *dev, 393 struct irdma_update_sds_info *info); 394 struct irdma_dma_mem sdbuf; 395 struct irdma_ring sq_ring; 396 struct irdma_cqp_quanta *sq_base; 397 struct irdma_dcqcn_cc_params dcqcn_params; 398 __le64 *host_ctx; 399 u64 *scratch_array; 400 u64 requested_ops; 401 atomic64_t completed_ops; 402 u32 cqp_id; 403 u32 sq_size; 404 u32 hw_sq_size; 405 u16 hw_maj_ver; 406 u16 hw_min_ver; 407 u8 struct_ver; 408 u8 polarity; 409 u8 hmc_profile; 410 u8 ena_vf_count; 411 u8 timeout_count; 412 u8 ceqs_per_vf; 413 bool en_datacenter_tcp:1; 414 bool disable_packed:1; 415 bool rocev2_rto_policy:1; 416 bool en_rem_endpoint_trk:1; 417 enum irdma_protocol_used protocol_used; 418 }; 419 420 struct irdma_sc_aeq { 421 u32 size; 422 u64 aeq_elem_pa; 423 struct irdma_sc_dev *dev; 424 struct irdma_sc_aeqe *aeqe_base; 425 void *pbl_list; 426 u32 elem_cnt; 427 struct irdma_ring aeq_ring; 428 u8 pbl_chunk_size; 429 u32 first_pm_pbl_idx; 430 u32 msix_idx; 431 u8 polarity; 432 bool virtual_map:1; 433 }; 434 435 struct irdma_sc_ceq { 436 u32 size; 437 u64 ceq_elem_pa; 438 struct irdma_sc_dev *dev; 439 struct irdma_ceqe *ceqe_base; 440 void *pbl_list; 441 u32 ceq_id; 442 u32 elem_cnt; 443 struct irdma_ring ceq_ring; 444 u8 pbl_chunk_size; 445 u8 tph_val; 446 u32 first_pm_pbl_idx; 447 u8 polarity; 448 struct irdma_sc_vsi *vsi; 449 struct irdma_sc_cq **reg_cq; 450 u32 reg_cq_size; 451 spinlock_t req_cq_lock; /* protect access to reg_cq array */ 452 bool virtual_map:1; 453 bool tph_en:1; 454 bool itr_no_expire:1; 455 }; 456 457 struct irdma_sc_cq { 458 struct irdma_cq_uk cq_uk; 459 u64 cq_pa; 460 u64 shadow_area_pa; 461 struct irdma_sc_dev *dev; 462 struct irdma_sc_vsi *vsi; 463 void *pbl_list; 464 void *back_cq; 465 u32 ceq_id; 466 u32 shadow_read_threshold; 467 u8 pbl_chunk_size; 468 u8 cq_type; 469 u8 tph_val; 470 u32 first_pm_pbl_idx; 471 bool ceqe_mask:1; 472 bool virtual_map:1; 473 bool check_overflow:1; 474 bool ceq_id_valid:1; 475 bool tph_en; 476 }; 477 478 struct irdma_sc_qp { 479 struct irdma_qp_uk qp_uk; 480 u64 sq_pa; 481 u64 rq_pa; 482 u64 hw_host_ctx_pa; 483 u64 shadow_area_pa; 484 u64 q2_pa; 485 struct irdma_sc_dev *dev; 486 struct irdma_sc_vsi *vsi; 487 struct irdma_sc_pd *pd; 488 __le64 *hw_host_ctx; 489 void *llp_stream_handle; 490 struct irdma_pfpdu pfpdu; 491 u32 ieq_qp; 492 u8 *q2_buf; 493 u64 qp_compl_ctx; 494 u32 push_idx; 495 u16 qs_handle; 496 u16 push_offset; 497 u8 flush_wqes_count; 498 u8 sq_tph_val; 499 u8 rq_tph_val; 500 u8 qp_state; 501 u8 hw_sq_size; 502 u8 hw_rq_size; 503 u8 src_mac_addr_idx; 504 bool on_qoslist:1; 505 bool ieq_pass_thru:1; 506 bool sq_tph_en:1; 507 bool rq_tph_en:1; 508 bool rcv_tph_en:1; 509 bool xmit_tph_en:1; 510 bool virtual_map:1; 511 bool flush_sq:1; 512 bool flush_rq:1; 513 bool sq_flush_code:1; 514 bool rq_flush_code:1; 515 enum irdma_flush_opcode flush_code; 516 enum irdma_qp_event_type event_type; 517 u8 term_flags; 518 u8 user_pri; 519 struct list_head list; 520 }; 521 522 struct irdma_stats_inst_info { 523 bool use_hmc_fcn_index; 524 u16 hmc_fn_id; 525 u16 stats_idx; 526 }; 527 528 struct irdma_up_info { 529 u8 map[8]; 530 u8 cnp_up_override; 531 u16 hmc_fcn_idx; 532 bool use_vlan:1; 533 bool use_cnp_up_override:1; 534 }; 535 536 #define IRDMA_MAX_WS_NODES 0x3FF 537 #define IRDMA_WS_NODE_INVALID 0xFFFF 538 539 struct irdma_ws_node_info { 540 u16 id; 541 u16 vsi; 542 u16 parent_id; 543 u16 qs_handle; 544 bool type_leaf:1; 545 bool enable:1; 546 u8 prio_type; 547 u8 tc; 548 u8 weight; 549 }; 550 551 struct irdma_hmc_fpm_misc { 552 u32 max_ceqs; 553 u32 max_sds; 554 u32 xf_block_size; 555 u32 q1_block_size; 556 u32 ht_multiplier; 557 u32 timer_bucket; 558 u32 rrf_block_size; 559 u32 ooiscf_block_size; 560 }; 561 562 #define IRDMA_LEAF_DEFAULT_REL_BW 64 563 #define IRDMA_PARENT_DEFAULT_REL_BW 1 564 565 struct irdma_qos { 566 struct list_head qplist; 567 struct mutex qos_mutex; /* protect QoS attributes per QoS level */ 568 u32 l2_sched_node_id; 569 u16 qs_handle; 570 u8 traffic_class; 571 u8 rel_bw; 572 u8 prio_type; 573 bool valid; 574 }; 575 576 struct irdma_config_check { 577 bool config_ok:1; 578 bool lfc_set:1; 579 bool pfc_set:1; 580 u8 traffic_class; 581 u16 qs_handle; 582 }; 583 584 #define IRDMA_INVALID_STATS_IDX 0xff 585 struct irdma_sc_vsi { 586 u16 vsi_idx; 587 struct irdma_sc_dev *dev; 588 void *back_vsi; 589 u32 ilq_count; 590 struct irdma_virt_mem ilq_mem; 591 struct irdma_puda_rsrc *ilq; 592 u32 ieq_count; 593 struct irdma_virt_mem ieq_mem; 594 struct irdma_puda_rsrc *ieq; 595 u32 exception_lan_q; 596 u16 mtu; 597 enum irdma_vm_vf_type vm_vf_type; 598 bool stats_inst_alloc:1; 599 bool tc_change_pending:1; 600 bool mtu_change_pending:1; 601 struct irdma_vsi_pestat *pestat; 602 atomic_t qp_suspend_reqs; 603 int (*register_qset)(struct irdma_sc_vsi *vsi, 604 struct irdma_ws_node *tc_node); 605 void (*unregister_qset)(struct irdma_sc_vsi *vsi, 606 struct irdma_ws_node *tc_node); 607 struct irdma_config_check cfg_check[IRDMA_MAX_USER_PRIORITY]; 608 bool tc_print_warning[IRDMA_MAX_TRAFFIC_CLASS]; 609 u8 qos_rel_bw; 610 u8 qos_prio_type; 611 u16 stats_idx; 612 u8 dscp_map[IRDMA_DSCP_NUM_VAL]; 613 struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY]; 614 bool dscp_mode:1; 615 }; 616 617 struct irdma_sc_dev { 618 struct list_head cqp_cmd_head; /* head of the CQP command list */ 619 spinlock_t cqp_lock; /* protect CQP list access */ 620 struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT]; 621 u64 fpm_query_buf_pa; 622 u64 fpm_commit_buf_pa; 623 __le64 *fpm_query_buf; 624 __le64 *fpm_commit_buf; 625 struct irdma_hw *hw; 626 u8 IOMEM *db_addr; 627 u32 IOMEM *wqe_alloc_db; 628 u32 IOMEM *cq_arm_db; 629 u32 IOMEM *aeq_alloc_db; 630 u32 IOMEM *cqp_db; 631 u32 IOMEM *cq_ack_db; 632 u32 IOMEM *hw_regs[IRDMA_MAX_REGS]; 633 u32 ceq_itr; /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */ 634 u64 hw_masks[IRDMA_MAX_MASKS]; 635 u8 hw_shifts[IRDMA_MAX_SHIFTS]; 636 const struct irdma_hw_stat_map *hw_stats_map; 637 u64 feature_info[IRDMA_MAX_FEATURES]; 638 u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS]; 639 struct irdma_hw_attrs hw_attrs; 640 struct irdma_hmc_info *hmc_info; 641 struct irdma_sc_cqp *cqp; 642 struct irdma_sc_aeq *aeq; 643 struct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT]; 644 struct irdma_sc_cq *ccq; 645 const struct irdma_irq_ops *irq_ops; 646 struct irdma_hmc_fpm_misc hmc_fpm_misc; 647 struct irdma_ws_node *ws_tree_root; 648 struct mutex ws_mutex; /* ws tree mutex */ 649 u32 debug_mask; 650 u16 num_vfs; 651 u16 hmc_fn_id; 652 u8 vf_id; 653 bool vchnl_up:1; 654 bool ceq_valid:1; 655 u8 pci_rev; 656 int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri); 657 void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri); 658 void (*ws_reset)(struct irdma_sc_vsi *vsi); 659 }; 660 661 struct irdma_modify_cq_info { 662 u64 cq_pa; 663 struct irdma_cqe *cq_base; 664 u32 cq_size; 665 u32 shadow_read_threshold; 666 u8 pbl_chunk_size; 667 u32 first_pm_pbl_idx; 668 bool virtual_map:1; 669 bool check_overflow; 670 bool cq_resize:1; 671 }; 672 673 struct irdma_create_qp_info { 674 bool ord_valid:1; 675 bool tcp_ctx_valid:1; 676 bool cq_num_valid:1; 677 bool arp_cache_idx_valid:1; 678 bool mac_valid:1; 679 bool force_lpb; 680 u8 next_iwarp_state; 681 }; 682 683 struct irdma_modify_qp_info { 684 u64 rx_win0; 685 u64 rx_win1; 686 u16 new_mss; 687 u8 next_iwarp_state; 688 u8 curr_iwarp_state; 689 u8 termlen; 690 bool ord_valid:1; 691 bool tcp_ctx_valid:1; 692 bool udp_ctx_valid:1; 693 bool cq_num_valid:1; 694 bool arp_cache_idx_valid:1; 695 bool reset_tcp_conn:1; 696 bool remove_hash_idx:1; 697 bool dont_send_term:1; 698 bool dont_send_fin:1; 699 bool cached_var_valid:1; 700 bool mss_change:1; 701 bool force_lpb:1; 702 bool mac_valid:1; 703 }; 704 705 struct irdma_ccq_cqe_info { 706 struct irdma_sc_cqp *cqp; 707 u64 scratch; 708 u32 op_ret_val; 709 u16 maj_err_code; 710 u16 min_err_code; 711 u8 op_code; 712 bool error; 713 }; 714 715 struct irdma_qos_tc_info { 716 u64 tc_ctx; 717 u8 rel_bw; 718 u8 prio_type; 719 u8 egress_virt_up; 720 u8 ingress_virt_up; 721 }; 722 723 struct irdma_l2params { 724 struct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY]; 725 u32 num_apps; 726 u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY]; 727 u16 mtu; 728 u8 up2tc[IRDMA_MAX_USER_PRIORITY]; 729 u8 dscp_map[IRDMA_DSCP_NUM_VAL]; 730 u8 num_tc; 731 u8 vsi_rel_bw; 732 u8 vsi_prio_type; 733 bool mtu_changed:1; 734 bool tc_changed:1; 735 bool dscp_mode:1; 736 }; 737 738 struct irdma_vsi_init_info { 739 struct irdma_sc_dev *dev; 740 void *back_vsi; 741 struct irdma_l2params *params; 742 u16 exception_lan_q; 743 u16 pf_data_vsi_num; 744 enum irdma_vm_vf_type vm_vf_type; 745 int (*register_qset)(struct irdma_sc_vsi *vsi, 746 struct irdma_ws_node *tc_node); 747 void (*unregister_qset)(struct irdma_sc_vsi *vsi, 748 struct irdma_ws_node *tc_node); 749 }; 750 751 struct irdma_vsi_stats_info { 752 struct irdma_vsi_pestat *pestat; 753 u8 fcn_id; 754 bool alloc_stats_inst; 755 }; 756 757 struct irdma_device_init_info { 758 u64 fpm_query_buf_pa; 759 u64 fpm_commit_buf_pa; 760 __le64 *fpm_query_buf; 761 __le64 *fpm_commit_buf; 762 struct irdma_hw *hw; 763 void IOMEM *bar0; 764 u16 max_vfs; 765 u16 hmc_fn_id; 766 u32 debug_mask; 767 }; 768 769 struct irdma_ceq_init_info { 770 u64 ceqe_pa; 771 struct irdma_sc_dev *dev; 772 u64 *ceqe_base; 773 void *pbl_list; 774 u32 elem_cnt; 775 u32 ceq_id; 776 bool virtual_map:1; 777 bool tph_en:1; 778 bool itr_no_expire:1; 779 u8 pbl_chunk_size; 780 u8 tph_val; 781 u32 first_pm_pbl_idx; 782 struct irdma_sc_vsi *vsi; 783 struct irdma_sc_cq **reg_cq; 784 }; 785 786 struct irdma_aeq_init_info { 787 u64 aeq_elem_pa; 788 struct irdma_sc_dev *dev; 789 u32 *aeqe_base; 790 void *pbl_list; 791 u32 elem_cnt; 792 bool virtual_map; 793 u8 pbl_chunk_size; 794 u32 first_pm_pbl_idx; 795 u32 msix_idx; 796 }; 797 798 struct irdma_ccq_init_info { 799 u64 cq_pa; 800 u64 shadow_area_pa; 801 struct irdma_sc_dev *dev; 802 struct irdma_cqe *cq_base; 803 __le64 *shadow_area; 804 void *pbl_list; 805 u32 num_elem; 806 u32 ceq_id; 807 u32 shadow_read_threshold; 808 bool ceqe_mask:1; 809 bool ceq_id_valid:1; 810 bool avoid_mem_cflct:1; 811 bool virtual_map:1; 812 bool tph_en:1; 813 u8 tph_val; 814 u8 pbl_chunk_size; 815 u32 first_pm_pbl_idx; 816 struct irdma_sc_vsi *vsi; 817 }; 818 819 struct irdma_udp_offload_info { 820 bool ipv4:1; 821 bool insert_vlan_tag:1; 822 u8 ttl; 823 u8 tos; 824 u16 src_port; 825 u16 dst_port; 826 u32 dest_ip_addr[4]; 827 u32 snd_mss; 828 u16 vlan_tag; 829 u16 arp_idx; 830 u32 flow_label; 831 u8 udp_state; 832 u32 psn_nxt; 833 u32 lsn; 834 u32 epsn; 835 u32 psn_max; 836 u32 psn_una; 837 u32 local_ipaddr[4]; 838 u32 cwnd; 839 u8 rexmit_thresh; 840 u8 rnr_nak_thresh; 841 }; 842 843 struct irdma_roce_offload_info { 844 u16 p_key; 845 u32 err_rq_idx; 846 u32 qkey; 847 u32 dest_qp; 848 u8 roce_tver; 849 u8 ack_credits; 850 u8 err_rq_idx_valid; 851 u32 pd_id; 852 u16 ord_size; 853 u16 ird_size; 854 bool is_qp1:1; 855 bool udprivcq_en:1; 856 bool dcqcn_en:1; 857 bool rcv_no_icrc:1; 858 bool wr_rdresp_en:1; 859 bool bind_en:1; 860 bool fast_reg_en:1; 861 bool priv_mode_en:1; 862 bool rd_en:1; 863 bool timely_en:1; 864 bool dctcp_en:1; 865 bool fw_cc_enable:1; 866 bool use_stats_inst:1; 867 u16 t_high; 868 u16 t_low; 869 u8 last_byte_sent; 870 u8 mac_addr[ETHER_ADDR_LEN]; 871 u8 rtomin; 872 }; 873 874 struct irdma_iwarp_offload_info { 875 u16 rcv_mark_offset; 876 u16 snd_mark_offset; 877 u8 ddp_ver; 878 u8 rdmap_ver; 879 u8 iwarp_mode; 880 u32 err_rq_idx; 881 u32 pd_id; 882 u16 ord_size; 883 u16 ird_size; 884 bool ib_rd_en:1; 885 bool align_hdrs:1; 886 bool rcv_no_mpa_crc:1; 887 bool err_rq_idx_valid:1; 888 bool snd_mark_en:1; 889 bool rcv_mark_en:1; 890 bool wr_rdresp_en:1; 891 bool bind_en:1; 892 bool fast_reg_en:1; 893 bool priv_mode_en:1; 894 bool rd_en:1; 895 bool timely_en:1; 896 bool use_stats_inst:1; 897 bool ecn_en:1; 898 bool dctcp_en:1; 899 u16 t_high; 900 u16 t_low; 901 u8 last_byte_sent; 902 u8 mac_addr[ETHER_ADDR_LEN]; 903 u8 rtomin; 904 }; 905 906 struct irdma_tcp_offload_info { 907 bool ipv4:1; 908 bool no_nagle:1; 909 bool insert_vlan_tag:1; 910 bool time_stamp:1; 911 bool drop_ooo_seg:1; 912 bool avoid_stretch_ack:1; 913 bool wscale:1; 914 bool ignore_tcp_opt:1; 915 bool ignore_tcp_uns_opt:1; 916 u8 cwnd_inc_limit; 917 u8 dup_ack_thresh; 918 u8 ttl; 919 u8 src_mac_addr_idx; 920 u8 tos; 921 u16 src_port; 922 u16 dst_port; 923 u32 dest_ip_addr[4]; 924 //u32 dest_ip_addr0; 925 //u32 dest_ip_addr1; 926 //u32 dest_ip_addr2; 927 //u32 dest_ip_addr3; 928 u32 snd_mss; 929 u16 syn_rst_handling; 930 u16 vlan_tag; 931 u16 arp_idx; 932 u32 flow_label; 933 u8 tcp_state; 934 u8 snd_wscale; 935 u8 rcv_wscale; 936 u32 time_stamp_recent; 937 u32 time_stamp_age; 938 u32 snd_nxt; 939 u32 snd_wnd; 940 u32 rcv_nxt; 941 u32 rcv_wnd; 942 u32 snd_max; 943 u32 snd_una; 944 u32 srtt; 945 u32 rtt_var; 946 u32 ss_thresh; 947 u32 cwnd; 948 u32 snd_wl1; 949 u32 snd_wl2; 950 u32 max_snd_window; 951 u8 rexmit_thresh; 952 u32 local_ipaddr[4]; 953 }; 954 955 struct irdma_qp_host_ctx_info { 956 u64 qp_compl_ctx; 957 union { 958 struct irdma_tcp_offload_info *tcp_info; 959 struct irdma_udp_offload_info *udp_info; 960 }; 961 union { 962 struct irdma_iwarp_offload_info *iwarp_info; 963 struct irdma_roce_offload_info *roce_info; 964 }; 965 u32 send_cq_num; 966 u32 rcv_cq_num; 967 u32 rem_endpoint_idx; 968 u16 stats_idx; 969 bool tcp_info_valid:1; 970 bool iwarp_info_valid:1; 971 bool stats_idx_valid:1; 972 u8 user_pri; 973 }; 974 975 struct irdma_aeqe_info { 976 u64 compl_ctx; 977 u32 qp_cq_id; 978 u32 wqe_idx; 979 u16 ae_id; 980 u8 tcp_state; 981 u8 iwarp_state; 982 bool qp:1; 983 bool cq:1; 984 bool sq:1; 985 bool rq:1; 986 bool in_rdrsp_wr:1; 987 bool out_rdrsp:1; 988 bool aeqe_overflow:1; 989 /* This flag is used to determine if we should pass the rq tail 990 * in the QP context for FW/HW. It is set when ae_src is rq for GEN1/GEN2 991 * And additionally set for inbound atomic, read and write for GEN3 992 */ 993 bool err_rq_idx_valid:1; 994 u8 q2_data_written; 995 u8 ae_src; 996 }; 997 998 struct irdma_allocate_stag_info { 999 u64 total_len; 1000 u64 first_pm_pbl_idx; 1001 u32 chunk_size; 1002 u32 stag_idx; 1003 u32 page_size; 1004 u32 pd_id; 1005 u16 access_rights; 1006 bool remote_access:1; 1007 bool use_hmc_fcn_index:1; 1008 bool use_pf_rid:1; 1009 bool all_memory:1; 1010 u16 hmc_fcn_index; 1011 }; 1012 1013 struct irdma_mw_alloc_info { 1014 u32 mw_stag_index; 1015 u32 page_size; 1016 u32 pd_id; 1017 bool remote_access:1; 1018 bool mw_wide:1; 1019 bool mw1_bind_dont_vldt_key:1; 1020 }; 1021 1022 struct irdma_reg_ns_stag_info { 1023 u64 reg_addr_pa; 1024 u64 va; 1025 u64 total_len; 1026 u32 page_size; 1027 u32 chunk_size; 1028 u32 first_pm_pbl_index; 1029 enum irdma_addressing_type addr_type; 1030 irdma_stag_index stag_idx; 1031 u16 access_rights; 1032 u32 pd_id; 1033 irdma_stag_key stag_key; 1034 bool use_hmc_fcn_index:1; 1035 u16 hmc_fcn_index; 1036 bool use_pf_rid:1; 1037 bool all_memory:1; 1038 }; 1039 1040 struct irdma_fast_reg_stag_info { 1041 u64 wr_id; 1042 u64 reg_addr_pa; 1043 u64 fbo; 1044 void *va; 1045 u64 total_len; 1046 u32 page_size; 1047 u32 chunk_size; 1048 u32 first_pm_pbl_index; 1049 enum irdma_addressing_type addr_type; 1050 irdma_stag_index stag_idx; 1051 u16 access_rights; 1052 u32 pd_id; 1053 irdma_stag_key stag_key; 1054 bool local_fence:1; 1055 bool read_fence:1; 1056 bool signaled:1; 1057 bool push_wqe:1; 1058 bool use_hmc_fcn_index:1; 1059 u16 hmc_fcn_index; 1060 bool use_pf_rid:1; 1061 bool defer_flag:1; 1062 }; 1063 1064 struct irdma_dealloc_stag_info { 1065 u32 stag_idx; 1066 u32 pd_id; 1067 bool mr:1; 1068 bool dealloc_pbl:1; 1069 }; 1070 1071 struct irdma_register_shared_stag { 1072 u64 va; 1073 enum irdma_addressing_type addr_type; 1074 irdma_stag_index new_stag_idx; 1075 irdma_stag_index parent_stag_idx; 1076 u32 access_rights; 1077 u32 pd_id; 1078 u32 page_size; 1079 irdma_stag_key new_stag_key; 1080 }; 1081 1082 struct irdma_qp_init_info { 1083 struct irdma_qp_uk_init_info qp_uk_init_info; 1084 struct irdma_sc_pd *pd; 1085 struct irdma_sc_vsi *vsi; 1086 __le64 *host_ctx; 1087 u8 *q2; 1088 u64 sq_pa; 1089 u64 rq_pa; 1090 u64 host_ctx_pa; 1091 u64 q2_pa; 1092 u64 shadow_area_pa; 1093 u8 sq_tph_val; 1094 u8 rq_tph_val; 1095 bool sq_tph_en:1; 1096 bool rq_tph_en:1; 1097 bool rcv_tph_en:1; 1098 bool xmit_tph_en:1; 1099 bool virtual_map:1; 1100 }; 1101 1102 struct irdma_cq_init_info { 1103 struct irdma_sc_dev *dev; 1104 u64 cq_base_pa; 1105 u64 shadow_area_pa; 1106 u32 ceq_id; 1107 u32 shadow_read_threshold; 1108 u8 pbl_chunk_size; 1109 u32 first_pm_pbl_idx; 1110 bool virtual_map:1; 1111 bool ceqe_mask:1; 1112 bool ceq_id_valid:1; 1113 bool tph_en:1; 1114 u8 tph_val; 1115 u8 type; 1116 struct irdma_cq_uk_init_info cq_uk_init_info; 1117 struct irdma_sc_vsi *vsi; 1118 }; 1119 1120 struct irdma_upload_context_info { 1121 u64 buf_pa; 1122 u32 qp_id; 1123 u8 qp_type; 1124 bool freeze_qp:1; 1125 bool raw_format:1; 1126 }; 1127 1128 struct irdma_local_mac_entry_info { 1129 u8 mac_addr[6]; 1130 u16 entry_idx; 1131 }; 1132 1133 struct irdma_add_arp_cache_entry_info { 1134 u8 mac_addr[ETHER_ADDR_LEN]; 1135 u32 reach_max; 1136 u16 arp_index; 1137 bool permanent; 1138 }; 1139 1140 struct irdma_apbvt_info { 1141 u16 port; 1142 bool add; 1143 }; 1144 1145 struct irdma_qhash_table_info { 1146 struct irdma_sc_vsi *vsi; 1147 enum irdma_quad_hash_manage_type manage; 1148 enum irdma_quad_entry_type entry_type; 1149 bool vlan_valid:1; 1150 bool ipv4_valid:1; 1151 u8 mac_addr[ETHER_ADDR_LEN]; 1152 u16 vlan_id; 1153 u8 user_pri; 1154 u32 qp_num; 1155 u32 dest_ip[4]; 1156 u32 src_ip[4]; 1157 u16 dest_port; 1158 u16 src_port; 1159 }; 1160 1161 struct irdma_cqp_manage_push_page_info { 1162 u32 push_idx; 1163 u16 qs_handle; 1164 u8 free_page; 1165 u8 push_page_type; 1166 }; 1167 1168 struct irdma_qp_flush_info { 1169 u16 sq_minor_code; 1170 u16 sq_major_code; 1171 u16 rq_minor_code; 1172 u16 rq_major_code; 1173 u16 ae_code; 1174 u8 ae_src; 1175 bool sq:1; 1176 bool rq:1; 1177 bool userflushcode:1; 1178 bool generate_ae:1; 1179 }; 1180 1181 struct irdma_gen_ae_info { 1182 u16 ae_code; 1183 u8 ae_src; 1184 }; 1185 1186 struct irdma_cqp_timeout { 1187 u64 compl_cqp_cmds; 1188 u32 count; 1189 }; 1190 1191 struct irdma_irq_ops { 1192 void (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx, bool enable); 1193 void (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx, 1194 bool enable); 1195 void (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx); 1196 void (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx); 1197 }; 1198 1199 void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq); 1200 int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch, 1201 bool check_overflow, bool post_sq); 1202 int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq); 1203 int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq, 1204 struct irdma_ccq_cqe_info *info); 1205 int irdma_sc_ccq_init(struct irdma_sc_cq *ccq, 1206 struct irdma_ccq_init_info *info); 1207 1208 int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch); 1209 int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq); 1210 1211 int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq); 1212 int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq, 1213 struct irdma_ceq_init_info *info); 1214 void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq); 1215 void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq); 1216 1217 int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq, 1218 struct irdma_aeq_init_info *info); 1219 int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq, 1220 struct irdma_aeqe_info *info); 1221 void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count); 1222 1223 void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id, 1224 int abi_ver); 1225 void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable); 1226 void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout, 1227 struct irdma_sc_dev *dev); 1228 int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err); 1229 int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp, bool free_hwcqp); 1230 int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp, 1231 struct irdma_cqp_init_info *info); 1232 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp); 1233 int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode, 1234 struct irdma_ccq_cqe_info *cmpl_info); 1235 int irdma_sc_qp_create(struct irdma_sc_qp *qp, 1236 struct irdma_create_qp_info *info, u64 scratch, 1237 bool post_sq); 1238 int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch, 1239 bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq); 1240 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, 1241 struct irdma_qp_flush_info *info, u64 scratch, 1242 bool post_sq); 1243 int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info); 1244 int irdma_sc_qp_modify(struct irdma_sc_qp *qp, 1245 struct irdma_modify_qp_info *info, u64 scratch, 1246 bool post_sq); 1247 void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size, 1248 irdma_stag stag); 1249 void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read); 1250 void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx, 1251 struct irdma_qp_host_ctx_info *info); 1252 void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx, 1253 struct irdma_qp_host_ctx_info *info); 1254 int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq); 1255 int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info); 1256 void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info); 1257 int irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch, bool post_sq); 1258 int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch, 1259 u16 hmc_fn_id, bool post_sq, 1260 bool poll_registers); 1261 1262 void sc_vsi_update_stats(struct irdma_sc_vsi *vsi); 1263 struct cqp_info { 1264 union { 1265 struct { 1266 struct irdma_sc_qp *qp; 1267 struct irdma_create_qp_info info; 1268 u64 scratch; 1269 } qp_create; 1270 1271 struct { 1272 struct irdma_sc_qp *qp; 1273 struct irdma_modify_qp_info info; 1274 u64 scratch; 1275 } qp_modify; 1276 1277 struct { 1278 struct irdma_sc_qp *qp; 1279 u64 scratch; 1280 bool remove_hash_idx; 1281 bool ignore_mw_bnd; 1282 } qp_destroy; 1283 1284 struct { 1285 struct irdma_sc_cq *cq; 1286 u64 scratch; 1287 bool check_overflow; 1288 } cq_create; 1289 1290 struct { 1291 struct irdma_sc_cq *cq; 1292 struct irdma_modify_cq_info info; 1293 u64 scratch; 1294 } cq_modify; 1295 1296 struct { 1297 struct irdma_sc_cq *cq; 1298 u64 scratch; 1299 } cq_destroy; 1300 1301 struct { 1302 struct irdma_sc_dev *dev; 1303 struct irdma_allocate_stag_info info; 1304 u64 scratch; 1305 } alloc_stag; 1306 1307 struct { 1308 struct irdma_sc_dev *dev; 1309 struct irdma_mw_alloc_info info; 1310 u64 scratch; 1311 } mw_alloc; 1312 1313 struct { 1314 struct irdma_sc_dev *dev; 1315 struct irdma_reg_ns_stag_info info; 1316 u64 scratch; 1317 } mr_reg_non_shared; 1318 1319 struct { 1320 struct irdma_sc_dev *dev; 1321 struct irdma_dealloc_stag_info info; 1322 u64 scratch; 1323 } dealloc_stag; 1324 1325 struct { 1326 struct irdma_sc_cqp *cqp; 1327 struct irdma_add_arp_cache_entry_info info; 1328 u64 scratch; 1329 } add_arp_cache_entry; 1330 1331 struct { 1332 struct irdma_sc_cqp *cqp; 1333 u64 scratch; 1334 u16 arp_index; 1335 } del_arp_cache_entry; 1336 1337 struct { 1338 struct irdma_sc_cqp *cqp; 1339 struct irdma_local_mac_entry_info info; 1340 u64 scratch; 1341 } add_local_mac_entry; 1342 1343 struct { 1344 struct irdma_sc_cqp *cqp; 1345 u64 scratch; 1346 u8 entry_idx; 1347 u8 ignore_ref_count; 1348 } del_local_mac_entry; 1349 1350 struct { 1351 struct irdma_sc_cqp *cqp; 1352 u64 scratch; 1353 } alloc_local_mac_entry; 1354 1355 struct { 1356 struct irdma_sc_cqp *cqp; 1357 struct irdma_cqp_manage_push_page_info info; 1358 u64 scratch; 1359 } manage_push_page; 1360 1361 struct { 1362 struct irdma_sc_dev *dev; 1363 struct irdma_upload_context_info info; 1364 u64 scratch; 1365 } qp_upload_context; 1366 1367 struct { 1368 struct irdma_sc_dev *dev; 1369 struct irdma_hmc_fcn_info info; 1370 u64 scratch; 1371 } manage_hmc_pm; 1372 1373 struct { 1374 struct irdma_sc_ceq *ceq; 1375 u64 scratch; 1376 } ceq_create; 1377 1378 struct { 1379 struct irdma_sc_ceq *ceq; 1380 u64 scratch; 1381 } ceq_destroy; 1382 1383 struct { 1384 struct irdma_sc_aeq *aeq; 1385 u64 scratch; 1386 } aeq_create; 1387 1388 struct { 1389 struct irdma_sc_aeq *aeq; 1390 u64 scratch; 1391 } aeq_destroy; 1392 1393 struct { 1394 struct irdma_sc_qp *qp; 1395 struct irdma_qp_flush_info info; 1396 u64 scratch; 1397 } qp_flush_wqes; 1398 1399 struct { 1400 struct irdma_sc_qp *qp; 1401 struct irdma_gen_ae_info info; 1402 u64 scratch; 1403 } gen_ae; 1404 1405 struct { 1406 struct irdma_sc_cqp *cqp; 1407 void *fpm_val_va; 1408 u64 fpm_val_pa; 1409 u16 hmc_fn_id; 1410 u64 scratch; 1411 } query_fpm_val; 1412 1413 struct { 1414 struct irdma_sc_cqp *cqp; 1415 void *fpm_val_va; 1416 u64 fpm_val_pa; 1417 u16 hmc_fn_id; 1418 u64 scratch; 1419 } commit_fpm_val; 1420 1421 struct { 1422 struct irdma_sc_cqp *cqp; 1423 struct irdma_apbvt_info info; 1424 u64 scratch; 1425 } manage_apbvt_entry; 1426 1427 struct { 1428 struct irdma_sc_cqp *cqp; 1429 struct irdma_qhash_table_info info; 1430 u64 scratch; 1431 } manage_qhash_table_entry; 1432 1433 struct { 1434 struct irdma_sc_dev *dev; 1435 struct irdma_update_sds_info info; 1436 u64 scratch; 1437 } update_pe_sds; 1438 1439 struct { 1440 struct irdma_sc_cqp *cqp; 1441 struct irdma_sc_qp *qp; 1442 u64 scratch; 1443 } suspend_resume; 1444 1445 struct { 1446 struct irdma_sc_cqp *cqp; 1447 struct irdma_ah_info info; 1448 u64 scratch; 1449 } ah_create; 1450 1451 struct { 1452 struct irdma_sc_cqp *cqp; 1453 struct irdma_ah_info info; 1454 u64 scratch; 1455 } ah_destroy; 1456 1457 struct { 1458 struct irdma_sc_cqp *cqp; 1459 struct irdma_mcast_grp_info info; 1460 u64 scratch; 1461 } mc_create; 1462 1463 struct { 1464 struct irdma_sc_cqp *cqp; 1465 struct irdma_mcast_grp_info info; 1466 u64 scratch; 1467 } mc_destroy; 1468 1469 struct { 1470 struct irdma_sc_cqp *cqp; 1471 struct irdma_mcast_grp_info info; 1472 u64 scratch; 1473 } mc_modify; 1474 1475 struct { 1476 struct irdma_sc_cqp *cqp; 1477 struct irdma_stats_inst_info info; 1478 u64 scratch; 1479 } stats_manage; 1480 1481 struct { 1482 struct irdma_sc_cqp *cqp; 1483 struct irdma_stats_gather_info info; 1484 u64 scratch; 1485 } stats_gather; 1486 1487 struct { 1488 struct irdma_sc_cqp *cqp; 1489 struct irdma_ws_node_info info; 1490 u64 scratch; 1491 } ws_node; 1492 1493 struct { 1494 struct irdma_sc_cqp *cqp; 1495 struct irdma_up_info info; 1496 u64 scratch; 1497 } up_map; 1498 1499 struct { 1500 struct irdma_sc_cqp *cqp; 1501 struct irdma_dma_mem query_buff_mem; 1502 u64 scratch; 1503 } query_rdma; 1504 } u; 1505 }; 1506 1507 struct cqp_cmds_info { 1508 struct list_head cqp_cmd_entry; 1509 u8 cqp_cmd; 1510 u8 post_sq; 1511 struct cqp_info in; 1512 }; 1513 1514 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch, 1515 u32 *wqe_idx); 1516 1517 /** 1518 * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq 1519 * @cqp: struct for cqp hw 1520 * @scratch: private data for CQP WQE 1521 */ 1522 static inline __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch) 1523 { 1524 u32 wqe_idx; 1525 1526 return irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx); 1527 } 1528 #endif /* IRDMA_TYPE_H */ 1529