1 /* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */ 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #ifndef IRDMA_TYPE_H 4 #define IRDMA_TYPE_H 5 #include "osdep.h" 6 #include "irdma.h" 7 #include "user.h" 8 #include "hmc.h" 9 #include "uda.h" 10 #include "ws.h" 11 #define IRDMA_DEBUG_ERR "ERR" 12 #define IRDMA_DEBUG_INIT "INIT" 13 #define IRDMA_DEBUG_DEV "DEV" 14 #define IRDMA_DEBUG_CM "CM" 15 #define IRDMA_DEBUG_VERBS "VERBS" 16 #define IRDMA_DEBUG_PUDA "PUDA" 17 #define IRDMA_DEBUG_ILQ "ILQ" 18 #define IRDMA_DEBUG_IEQ "IEQ" 19 #define IRDMA_DEBUG_QP "QP" 20 #define IRDMA_DEBUG_CQ "CQ" 21 #define IRDMA_DEBUG_MR "MR" 22 #define IRDMA_DEBUG_PBLE "PBLE" 23 #define IRDMA_DEBUG_WQE "WQE" 24 #define IRDMA_DEBUG_AEQ "AEQ" 25 #define IRDMA_DEBUG_CQP "CQP" 26 #define IRDMA_DEBUG_HMC "HMC" 27 #define IRDMA_DEBUG_USER "USER" 28 #define IRDMA_DEBUG_VIRT "VIRT" 29 #define IRDMA_DEBUG_DCB "DCB" 30 #define IRDMA_DEBUG_CQE "CQE" 31 #define IRDMA_DEBUG_CLNT "CLNT" 32 #define IRDMA_DEBUG_WS "WS" 33 #define IRDMA_DEBUG_STATS "STATS" 34 35 enum irdma_page_size { 36 IRDMA_PAGE_SIZE_4K = 0, 37 IRDMA_PAGE_SIZE_2M, 38 IRDMA_PAGE_SIZE_1G, 39 }; 40 41 enum irdma_hdrct_flags { 42 DDP_LEN_FLAG = 0x80, 43 DDP_HDR_FLAG = 0x40, 44 RDMA_HDR_FLAG = 0x20, 45 }; 46 47 enum irdma_term_layers { 48 LAYER_RDMA = 0, 49 LAYER_DDP = 1, 50 LAYER_MPA = 2, 51 }; 52 53 enum irdma_term_error_types { 54 RDMAP_REMOTE_PROT = 1, 55 RDMAP_REMOTE_OP = 2, 56 DDP_CATASTROPHIC = 0, 57 DDP_TAGGED_BUF = 1, 58 DDP_UNTAGGED_BUF = 2, 59 DDP_LLP = 3, 60 }; 61 62 enum irdma_term_rdma_errors { 63 RDMAP_INV_STAG = 0x00, 64 RDMAP_INV_BOUNDS = 0x01, 65 RDMAP_ACCESS = 0x02, 66 RDMAP_UNASSOC_STAG = 0x03, 67 RDMAP_TO_WRAP = 0x04, 68 RDMAP_INV_RDMAP_VER = 0x05, 69 RDMAP_UNEXPECTED_OP = 0x06, 70 RDMAP_CATASTROPHIC_LOCAL = 0x07, 71 RDMAP_CATASTROPHIC_GLOBAL = 0x08, 72 RDMAP_CANT_INV_STAG = 0x09, 73 RDMAP_UNSPECIFIED = 0xff, 74 }; 75 76 enum irdma_term_ddp_errors { 77 DDP_CATASTROPHIC_LOCAL = 0x00, 78 DDP_TAGGED_INV_STAG = 0x00, 79 DDP_TAGGED_BOUNDS = 0x01, 80 DDP_TAGGED_UNASSOC_STAG = 0x02, 81 DDP_TAGGED_TO_WRAP = 0x03, 82 DDP_TAGGED_INV_DDP_VER = 0x04, 83 DDP_UNTAGGED_INV_QN = 0x01, 84 DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02, 85 DDP_UNTAGGED_INV_MSN_RANGE = 0x03, 86 DDP_UNTAGGED_INV_MO = 0x04, 87 DDP_UNTAGGED_INV_TOO_LONG = 0x05, 88 DDP_UNTAGGED_INV_DDP_VER = 0x06, 89 }; 90 91 enum irdma_term_mpa_errors { 92 MPA_CLOSED = 0x01, 93 MPA_CRC = 0x02, 94 MPA_MARKER = 0x03, 95 MPA_REQ_RSP = 0x04, 96 }; 97 98 enum irdma_qp_event_type { 99 IRDMA_QP_EVENT_CATASTROPHIC, 100 IRDMA_QP_EVENT_ACCESS_ERR, 101 IRDMA_QP_EVENT_REQ_ERR, 102 }; 103 104 enum irdma_hw_stats_index { 105 /* gen1 - 32-bit */ 106 IRDMA_HW_STAT_INDEX_IP4RXDISCARD = 0, 107 IRDMA_HW_STAT_INDEX_IP4RXTRUNC = 1, 108 IRDMA_HW_STAT_INDEX_IP4TXNOROUTE = 2, 109 IRDMA_HW_STAT_INDEX_IP6RXDISCARD = 3, 110 IRDMA_HW_STAT_INDEX_IP6RXTRUNC = 4, 111 IRDMA_HW_STAT_INDEX_IP6TXNOROUTE = 5, 112 IRDMA_HW_STAT_INDEX_TCPRTXSEG = 6, 113 IRDMA_HW_STAT_INDEX_TCPRXOPTERR = 7, 114 IRDMA_HW_STAT_INDEX_TCPRXPROTOERR = 8, 115 IRDMA_HW_STAT_INDEX_RXVLANERR = 9, 116 /* gen1 - 64-bit */ 117 IRDMA_HW_STAT_INDEX_IP4RXOCTS = 10, 118 IRDMA_HW_STAT_INDEX_IP4RXPKTS = 11, 119 IRDMA_HW_STAT_INDEX_IP4RXFRAGS = 12, 120 IRDMA_HW_STAT_INDEX_IP4RXMCPKTS = 13, 121 IRDMA_HW_STAT_INDEX_IP4TXOCTS = 14, 122 IRDMA_HW_STAT_INDEX_IP4TXPKTS = 15, 123 IRDMA_HW_STAT_INDEX_IP4TXFRAGS = 16, 124 IRDMA_HW_STAT_INDEX_IP4TXMCPKTS = 17, 125 IRDMA_HW_STAT_INDEX_IP6RXOCTS = 18, 126 IRDMA_HW_STAT_INDEX_IP6RXPKTS = 19, 127 IRDMA_HW_STAT_INDEX_IP6RXFRAGS = 20, 128 IRDMA_HW_STAT_INDEX_IP6RXMCPKTS = 21, 129 IRDMA_HW_STAT_INDEX_IP6TXOCTS = 22, 130 IRDMA_HW_STAT_INDEX_IP6TXPKTS = 23, 131 IRDMA_HW_STAT_INDEX_IP6TXFRAGS = 24, 132 IRDMA_HW_STAT_INDEX_IP6TXMCPKTS = 25, 133 IRDMA_HW_STAT_INDEX_TCPRXSEGS = 26, 134 IRDMA_HW_STAT_INDEX_TCPTXSEG = 27, 135 IRDMA_HW_STAT_INDEX_RDMARXRDS = 28, 136 IRDMA_HW_STAT_INDEX_RDMARXSNDS = 29, 137 IRDMA_HW_STAT_INDEX_RDMARXWRS = 30, 138 IRDMA_HW_STAT_INDEX_RDMATXRDS = 31, 139 IRDMA_HW_STAT_INDEX_RDMATXSNDS = 32, 140 IRDMA_HW_STAT_INDEX_RDMATXWRS = 33, 141 IRDMA_HW_STAT_INDEX_RDMAVBND = 34, 142 IRDMA_HW_STAT_INDEX_RDMAVINV = 35, 143 IRDMA_HW_STAT_INDEX_IP4RXMCOCTS = 36, 144 IRDMA_HW_STAT_INDEX_IP4TXMCOCTS = 37, 145 IRDMA_HW_STAT_INDEX_IP6RXMCOCTS = 38, 146 IRDMA_HW_STAT_INDEX_IP6TXMCOCTS = 39, 147 IRDMA_HW_STAT_INDEX_UDPRXPKTS = 40, 148 IRDMA_HW_STAT_INDEX_UDPTXPKTS = 41, 149 IRDMA_HW_STAT_INDEX_MAX_GEN_1 = 42, /* Must be same value as next entry */ 150 /* gen2 - 64-bit */ 151 IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS = 42, 152 /* gen2 - 32-bit */ 153 IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED = 43, 154 IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED = 44, 155 IRDMA_HW_STAT_INDEX_TXNPCNPSENT = 45, 156 IRDMA_HW_STAT_INDEX_MAX_GEN_2 = 46, 157 }; 158 159 enum irdma_feature_type { 160 IRDMA_FEATURE_FW_INFO = 0, 161 IRDMA_HW_VERSION_INFO = 1, 162 IRDMA_QSETS_MAX = 26, 163 IRDMA_MAX_FEATURES, /* Must be last entry */ 164 }; 165 166 enum irdma_sched_prio_type { 167 IRDMA_PRIO_WEIGHTED_RR = 1, 168 IRDMA_PRIO_STRICT = 2, 169 IRDMA_PRIO_WEIGHTED_STRICT = 3, 170 }; 171 172 enum irdma_vm_vf_type { 173 IRDMA_VF_TYPE = 0, 174 IRDMA_VM_TYPE, 175 IRDMA_PF_TYPE, 176 }; 177 178 enum irdma_cqp_hmc_profile { 179 IRDMA_HMC_PROFILE_DEFAULT = 1, 180 IRDMA_HMC_PROFILE_FAVOR_VF = 2, 181 IRDMA_HMC_PROFILE_EQUAL = 3, 182 }; 183 184 enum irdma_quad_entry_type { 185 IRDMA_QHASH_TYPE_TCP_ESTABLISHED = 1, 186 IRDMA_QHASH_TYPE_TCP_SYN, 187 IRDMA_QHASH_TYPE_UDP_UNICAST, 188 IRDMA_QHASH_TYPE_UDP_MCAST, 189 IRDMA_QHASH_TYPE_ROCE_MCAST, 190 IRDMA_QHASH_TYPE_ROCEV2_HW, 191 }; 192 193 enum irdma_quad_hash_manage_type { 194 IRDMA_QHASH_MANAGE_TYPE_DELETE = 0, 195 IRDMA_QHASH_MANAGE_TYPE_ADD, 196 IRDMA_QHASH_MANAGE_TYPE_MODIFY, 197 }; 198 199 enum irdma_syn_rst_handling { 200 IRDMA_SYN_RST_HANDLING_HW_TCP_SECURE = 0, 201 IRDMA_SYN_RST_HANDLING_HW_TCP, 202 IRDMA_SYN_RST_HANDLING_FW_TCP_SECURE, 203 IRDMA_SYN_RST_HANDLING_FW_TCP, 204 }; 205 206 enum irdma_queue_type { 207 IRDMA_QUEUE_TYPE_SQ_RQ = 0, 208 IRDMA_QUEUE_TYPE_CQP, 209 }; 210 211 struct irdma_sc_dev; 212 struct irdma_vsi_pestat; 213 214 struct irdma_dcqcn_cc_params { 215 u8 cc_cfg_valid; 216 u8 min_dec_factor; 217 u8 min_rate; 218 u8 dcqcn_f; 219 u16 rai_factor; 220 u16 hai_factor; 221 u16 dcqcn_t; 222 u32 dcqcn_b; 223 u32 rreduce_mperiod; 224 }; 225 226 struct irdma_cqp_init_info { 227 u64 cqp_compl_ctx; 228 u64 host_ctx_pa; 229 u64 sq_pa; 230 struct irdma_sc_dev *dev; 231 struct irdma_cqp_quanta *sq; 232 struct irdma_dcqcn_cc_params dcqcn_params; 233 __le64 *host_ctx; 234 u64 *scratch_array; 235 u32 sq_size; 236 u16 hw_maj_ver; 237 u16 hw_min_ver; 238 u8 struct_ver; 239 u8 hmc_profile; 240 u8 ena_vf_count; 241 u8 ceqs_per_vf; 242 bool en_datacenter_tcp:1; 243 bool disable_packed:1; 244 bool rocev2_rto_policy:1; 245 enum irdma_protocol_used protocol_used; 246 }; 247 248 struct irdma_terminate_hdr { 249 u8 layer_etype; 250 u8 error_code; 251 u8 hdrct; 252 u8 rsvd; 253 }; 254 255 struct irdma_cqp_sq_wqe { 256 __le64 buf[IRDMA_CQP_WQE_SIZE]; 257 }; 258 259 struct irdma_sc_aeqe { 260 __le64 buf[IRDMA_AEQE_SIZE]; 261 }; 262 263 struct irdma_ceqe { 264 __le64 buf[IRDMA_CEQE_SIZE]; 265 }; 266 267 struct irdma_cqp_ctx { 268 __le64 buf[IRDMA_CQP_CTX_SIZE]; 269 }; 270 271 struct irdma_cq_shadow_area { 272 __le64 buf[IRDMA_SHADOW_AREA_SIZE]; 273 }; 274 275 struct irdma_dev_hw_stats_offsets { 276 u32 stats_offset[IRDMA_HW_STAT_INDEX_MAX_GEN_1]; 277 }; 278 279 struct irdma_dev_hw_stats { 280 u64 stats_val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)]; 281 }; 282 283 struct irdma_gather_stats { 284 u64 val[IRDMA_GATHER_STATS_BUF_SIZE / sizeof(u64)]; 285 }; 286 287 struct irdma_hw_stat_map { 288 u16 byteoff; 289 u8 bitoff; 290 u64 bitmask; 291 }; 292 293 struct irdma_stats_gather_info { 294 bool use_hmc_fcn_index:1; 295 bool use_stats_inst:1; 296 u8 hmc_fcn_index; 297 u8 stats_inst_index; 298 struct irdma_dma_mem stats_buff_mem; 299 void *gather_stats_va; 300 void *last_gather_stats_va; 301 }; 302 303 struct irdma_vsi_pestat { 304 struct irdma_hw *hw; 305 struct irdma_dev_hw_stats hw_stats; 306 struct irdma_stats_gather_info gather_info; 307 struct timer_list stats_timer; 308 struct irdma_sc_vsi *vsi; 309 struct irdma_dev_hw_stats last_hw_stats; 310 spinlock_t lock; /* rdma stats lock */ 311 }; 312 313 struct irdma_hw { 314 u8 __iomem *hw_addr; 315 u8 __iomem *priv_hw_addr; 316 struct device *device; 317 struct irdma_hmc_info hmc; 318 }; 319 320 struct irdma_pfpdu { 321 struct list_head rxlist; 322 u32 rcv_nxt; 323 u32 fps; 324 u32 max_fpdu_data; 325 u32 nextseqnum; 326 u32 rcv_start_seq; 327 bool mode:1; 328 bool mpa_crc_err:1; 329 u8 marker_len; 330 u64 total_ieq_bufs; 331 u64 fpdu_processed; 332 u64 bad_seq_num; 333 u64 crc_err; 334 u64 no_tx_bufs; 335 u64 tx_err; 336 u64 out_of_order; 337 u64 pmode_count; 338 struct irdma_sc_ah *ah; 339 struct irdma_puda_buf *ah_buf; 340 spinlock_t lock; /* fpdu processing lock */ 341 struct irdma_puda_buf *lastrcv_buf; 342 }; 343 344 struct irdma_sc_pd { 345 struct irdma_sc_dev *dev; 346 u32 pd_id; 347 int abi_ver; 348 }; 349 350 struct irdma_cqp_quanta { 351 __le64 elem[IRDMA_CQP_WQE_SIZE]; 352 }; 353 354 struct irdma_sc_cqp { 355 u32 size; 356 u64 sq_pa; 357 u64 host_ctx_pa; 358 void *back_cqp; 359 struct irdma_sc_dev *dev; 360 int (*process_cqp_sds)(struct irdma_sc_dev *dev, 361 struct irdma_update_sds_info *info); 362 struct irdma_dma_mem sdbuf; 363 struct irdma_ring sq_ring; 364 struct irdma_cqp_quanta *sq_base; 365 struct irdma_dcqcn_cc_params dcqcn_params; 366 __le64 *host_ctx; 367 u64 *scratch_array; 368 u64 requested_ops; 369 atomic64_t completed_ops; 370 u32 cqp_id; 371 u32 sq_size; 372 u32 hw_sq_size; 373 u16 hw_maj_ver; 374 u16 hw_min_ver; 375 u8 struct_ver; 376 u8 polarity; 377 u8 hmc_profile; 378 u8 ena_vf_count; 379 u8 timeout_count; 380 u8 ceqs_per_vf; 381 bool en_datacenter_tcp:1; 382 bool disable_packed:1; 383 bool rocev2_rto_policy:1; 384 enum irdma_protocol_used protocol_used; 385 }; 386 387 struct irdma_sc_aeq { 388 u32 size; 389 u64 aeq_elem_pa; 390 struct irdma_sc_dev *dev; 391 struct irdma_sc_aeqe *aeqe_base; 392 void *pbl_list; 393 u32 elem_cnt; 394 struct irdma_ring aeq_ring; 395 u8 pbl_chunk_size; 396 u32 first_pm_pbl_idx; 397 u32 msix_idx; 398 u8 polarity; 399 bool virtual_map:1; 400 }; 401 402 struct irdma_sc_ceq { 403 u32 size; 404 u64 ceq_elem_pa; 405 struct irdma_sc_dev *dev; 406 struct irdma_ceqe *ceqe_base; 407 void *pbl_list; 408 u32 ceq_id; 409 u32 elem_cnt; 410 struct irdma_ring ceq_ring; 411 u8 pbl_chunk_size; 412 u8 tph_val; 413 u32 first_pm_pbl_idx; 414 u8 polarity; 415 struct irdma_sc_vsi *vsi; 416 struct irdma_sc_cq **reg_cq; 417 u32 reg_cq_size; 418 spinlock_t req_cq_lock; /* protect access to reg_cq array */ 419 bool virtual_map:1; 420 bool tph_en:1; 421 bool itr_no_expire:1; 422 }; 423 424 struct irdma_sc_cq { 425 struct irdma_cq_uk cq_uk; 426 u64 cq_pa; 427 u64 shadow_area_pa; 428 struct irdma_sc_dev *dev; 429 struct irdma_sc_vsi *vsi; 430 void *pbl_list; 431 void *back_cq; 432 u32 ceq_id; 433 u32 shadow_read_threshold; 434 u8 pbl_chunk_size; 435 u8 cq_type; 436 u8 tph_val; 437 u32 first_pm_pbl_idx; 438 bool ceqe_mask:1; 439 bool virtual_map:1; 440 bool check_overflow:1; 441 bool ceq_id_valid:1; 442 bool tph_en; 443 }; 444 445 struct irdma_sc_qp { 446 struct irdma_qp_uk qp_uk; 447 u64 sq_pa; 448 u64 rq_pa; 449 u64 hw_host_ctx_pa; 450 u64 shadow_area_pa; 451 u64 q2_pa; 452 struct irdma_sc_dev *dev; 453 struct irdma_sc_vsi *vsi; 454 struct irdma_sc_pd *pd; 455 __le64 *hw_host_ctx; 456 void *llp_stream_handle; 457 struct irdma_pfpdu pfpdu; 458 u32 ieq_qp; 459 u8 *q2_buf; 460 u64 qp_compl_ctx; 461 u32 push_idx; 462 u16 qs_handle; 463 u16 push_offset; 464 u8 flush_wqes_count; 465 u8 sq_tph_val; 466 u8 rq_tph_val; 467 u8 qp_state; 468 u8 hw_sq_size; 469 u8 hw_rq_size; 470 u8 src_mac_addr_idx; 471 bool on_qoslist:1; 472 bool ieq_pass_thru:1; 473 bool sq_tph_en:1; 474 bool rq_tph_en:1; 475 bool rcv_tph_en:1; 476 bool xmit_tph_en:1; 477 bool virtual_map:1; 478 bool flush_sq:1; 479 bool flush_rq:1; 480 bool sq_flush_code:1; 481 bool rq_flush_code:1; 482 enum irdma_flush_opcode flush_code; 483 enum irdma_qp_event_type event_type; 484 u8 term_flags; 485 u8 user_pri; 486 struct list_head list; 487 }; 488 489 struct irdma_stats_inst_info { 490 bool use_hmc_fcn_index; 491 u8 hmc_fn_id; 492 u8 stats_idx; 493 }; 494 495 struct irdma_up_info { 496 u8 map[8]; 497 u8 cnp_up_override; 498 u8 hmc_fcn_idx; 499 bool use_vlan:1; 500 bool use_cnp_up_override:1; 501 }; 502 503 #define IRDMA_MAX_WS_NODES 0x3FF 504 #define IRDMA_WS_NODE_INVALID 0xFFFF 505 506 struct irdma_ws_node_info { 507 u16 id; 508 u16 vsi; 509 u16 parent_id; 510 u16 qs_handle; 511 bool type_leaf:1; 512 bool enable:1; 513 u8 prio_type; 514 u8 tc; 515 u8 weight; 516 }; 517 518 struct irdma_hmc_fpm_misc { 519 u32 max_ceqs; 520 u32 max_sds; 521 u32 xf_block_size; 522 u32 q1_block_size; 523 u32 ht_multiplier; 524 u32 timer_bucket; 525 u32 rrf_block_size; 526 u32 ooiscf_block_size; 527 }; 528 529 #define IRDMA_LEAF_DEFAULT_REL_BW 64 530 #define IRDMA_PARENT_DEFAULT_REL_BW 1 531 532 struct irdma_qos { 533 struct list_head qplist; 534 struct mutex qos_mutex; /* protect QoS attributes per QoS level */ 535 u64 lan_qos_handle; 536 u32 l2_sched_node_id; 537 u16 qs_handle; 538 u8 traffic_class; 539 u8 rel_bw; 540 u8 prio_type; 541 bool valid; 542 }; 543 544 #define IRDMA_INVALID_STATS_IDX 0xff 545 struct irdma_sc_vsi { 546 u16 vsi_idx; 547 struct irdma_sc_dev *dev; 548 void *back_vsi; 549 u32 ilq_count; 550 struct irdma_virt_mem ilq_mem; 551 struct irdma_puda_rsrc *ilq; 552 u32 ieq_count; 553 struct irdma_virt_mem ieq_mem; 554 struct irdma_puda_rsrc *ieq; 555 u32 exception_lan_q; 556 u16 mtu; 557 u16 vm_id; 558 enum irdma_vm_vf_type vm_vf_type; 559 bool stats_inst_alloc:1; 560 bool tc_change_pending:1; 561 struct irdma_vsi_pestat *pestat; 562 atomic_t qp_suspend_reqs; 563 int (*register_qset)(struct irdma_sc_vsi *vsi, 564 struct irdma_ws_node *tc_node); 565 void (*unregister_qset)(struct irdma_sc_vsi *vsi, 566 struct irdma_ws_node *tc_node); 567 u8 qos_rel_bw; 568 u8 qos_prio_type; 569 u8 stats_idx; 570 u8 dscp_map[IIDC_MAX_DSCP_MAPPING]; 571 struct irdma_qos qos[IRDMA_MAX_USER_PRIORITY]; 572 u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1]; 573 bool dscp_mode:1; 574 }; 575 576 struct irdma_sc_dev { 577 struct list_head cqp_cmd_head; /* head of the CQP command list */ 578 spinlock_t cqp_lock; /* protect CQP list access */ 579 bool stats_idx_array[IRDMA_MAX_STATS_COUNT_GEN_1]; 580 struct irdma_dma_mem vf_fpm_query_buf[IRDMA_MAX_PE_ENA_VF_COUNT]; 581 u64 fpm_query_buf_pa; 582 u64 fpm_commit_buf_pa; 583 __le64 *fpm_query_buf; 584 __le64 *fpm_commit_buf; 585 struct irdma_hw *hw; 586 u8 __iomem *db_addr; 587 u32 __iomem *wqe_alloc_db; 588 u32 __iomem *cq_arm_db; 589 u32 __iomem *aeq_alloc_db; 590 u32 __iomem *cqp_db; 591 u32 __iomem *cq_ack_db; 592 u32 __iomem *ceq_itr_mask_db; 593 u32 __iomem *aeq_itr_mask_db; 594 u32 __iomem *hw_regs[IRDMA_MAX_REGS]; 595 u32 ceq_itr; /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */ 596 u64 hw_masks[IRDMA_MAX_MASKS]; 597 u64 hw_shifts[IRDMA_MAX_SHIFTS]; 598 const struct irdma_hw_stat_map *hw_stats_map; 599 u64 hw_stats_regs[IRDMA_HW_STAT_INDEX_MAX_GEN_1]; 600 u64 feature_info[IRDMA_MAX_FEATURES]; 601 u64 cqp_cmd_stats[IRDMA_MAX_CQP_OPS]; 602 struct irdma_hw_attrs hw_attrs; 603 struct irdma_hmc_info *hmc_info; 604 struct irdma_sc_cqp *cqp; 605 struct irdma_sc_aeq *aeq; 606 struct irdma_sc_ceq *ceq[IRDMA_CEQ_MAX_COUNT]; 607 struct irdma_sc_cq *ccq; 608 const struct irdma_irq_ops *irq_ops; 609 struct irdma_hmc_fpm_misc hmc_fpm_misc; 610 struct irdma_ws_node *ws_tree_root; 611 struct mutex ws_mutex; /* ws tree mutex */ 612 u16 num_vfs; 613 u8 hmc_fn_id; 614 u8 vf_id; 615 bool vchnl_up:1; 616 bool ceq_valid:1; 617 u8 pci_rev; 618 int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri); 619 void (*ws_remove)(struct irdma_sc_vsi *vsi, u8 user_pri); 620 void (*ws_reset)(struct irdma_sc_vsi *vsi); 621 }; 622 623 struct irdma_modify_cq_info { 624 u64 cq_pa; 625 struct irdma_cqe *cq_base; 626 u32 cq_size; 627 u32 shadow_read_threshold; 628 u8 pbl_chunk_size; 629 u32 first_pm_pbl_idx; 630 bool virtual_map:1; 631 bool check_overflow; 632 bool cq_resize:1; 633 }; 634 635 struct irdma_create_qp_info { 636 bool ord_valid:1; 637 bool tcp_ctx_valid:1; 638 bool cq_num_valid:1; 639 bool arp_cache_idx_valid:1; 640 bool mac_valid:1; 641 bool force_lpb; 642 u8 next_iwarp_state; 643 }; 644 645 struct irdma_modify_qp_info { 646 u64 rx_win0; 647 u64 rx_win1; 648 u16 new_mss; 649 u8 next_iwarp_state; 650 u8 curr_iwarp_state; 651 u8 termlen; 652 bool ord_valid:1; 653 bool tcp_ctx_valid:1; 654 bool udp_ctx_valid:1; 655 bool cq_num_valid:1; 656 bool arp_cache_idx_valid:1; 657 bool reset_tcp_conn:1; 658 bool remove_hash_idx:1; 659 bool dont_send_term:1; 660 bool dont_send_fin:1; 661 bool cached_var_valid:1; 662 bool mss_change:1; 663 bool force_lpb:1; 664 bool mac_valid:1; 665 }; 666 667 struct irdma_ccq_cqe_info { 668 struct irdma_sc_cqp *cqp; 669 u64 scratch; 670 u32 op_ret_val; 671 u16 maj_err_code; 672 u16 min_err_code; 673 u8 op_code; 674 bool error; 675 }; 676 677 struct irdma_dcb_app_info { 678 u8 priority; 679 u8 selector; 680 u16 prot_id; 681 }; 682 683 struct irdma_qos_tc_info { 684 u64 tc_ctx; 685 u8 rel_bw; 686 u8 prio_type; 687 u8 egress_virt_up; 688 u8 ingress_virt_up; 689 }; 690 691 struct irdma_l2params { 692 struct irdma_qos_tc_info tc_info[IRDMA_MAX_USER_PRIORITY]; 693 struct irdma_dcb_app_info apps[IRDMA_MAX_APPS]; 694 u32 num_apps; 695 u16 qs_handle_list[IRDMA_MAX_USER_PRIORITY]; 696 u16 mtu; 697 u8 up2tc[IRDMA_MAX_USER_PRIORITY]; 698 u8 dscp_map[IIDC_MAX_DSCP_MAPPING]; 699 u8 num_tc; 700 u8 vsi_rel_bw; 701 u8 vsi_prio_type; 702 bool mtu_changed:1; 703 bool tc_changed:1; 704 bool dscp_mode:1; 705 }; 706 707 struct irdma_vsi_init_info { 708 struct irdma_sc_dev *dev; 709 void *back_vsi; 710 struct irdma_l2params *params; 711 u16 exception_lan_q; 712 u16 pf_data_vsi_num; 713 enum irdma_vm_vf_type vm_vf_type; 714 u16 vm_id; 715 int (*register_qset)(struct irdma_sc_vsi *vsi, 716 struct irdma_ws_node *tc_node); 717 void (*unregister_qset)(struct irdma_sc_vsi *vsi, 718 struct irdma_ws_node *tc_node); 719 }; 720 721 struct irdma_vsi_stats_info { 722 struct irdma_vsi_pestat *pestat; 723 u8 fcn_id; 724 bool alloc_stats_inst; 725 }; 726 727 struct irdma_device_init_info { 728 u64 fpm_query_buf_pa; 729 u64 fpm_commit_buf_pa; 730 __le64 *fpm_query_buf; 731 __le64 *fpm_commit_buf; 732 struct irdma_hw *hw; 733 void __iomem *bar0; 734 u8 hmc_fn_id; 735 }; 736 737 struct irdma_ceq_init_info { 738 u64 ceqe_pa; 739 struct irdma_sc_dev *dev; 740 u64 *ceqe_base; 741 void *pbl_list; 742 u32 elem_cnt; 743 u32 ceq_id; 744 bool virtual_map:1; 745 bool tph_en:1; 746 bool itr_no_expire:1; 747 u8 pbl_chunk_size; 748 u8 tph_val; 749 u32 first_pm_pbl_idx; 750 struct irdma_sc_vsi *vsi; 751 struct irdma_sc_cq **reg_cq; 752 u32 reg_cq_idx; 753 }; 754 755 struct irdma_aeq_init_info { 756 u64 aeq_elem_pa; 757 struct irdma_sc_dev *dev; 758 u32 *aeqe_base; 759 void *pbl_list; 760 u32 elem_cnt; 761 bool virtual_map; 762 u8 pbl_chunk_size; 763 u32 first_pm_pbl_idx; 764 u32 msix_idx; 765 }; 766 767 struct irdma_ccq_init_info { 768 u64 cq_pa; 769 u64 shadow_area_pa; 770 struct irdma_sc_dev *dev; 771 struct irdma_cqe *cq_base; 772 __le64 *shadow_area; 773 void *pbl_list; 774 u32 num_elem; 775 u32 ceq_id; 776 u32 shadow_read_threshold; 777 bool ceqe_mask:1; 778 bool ceq_id_valid:1; 779 bool avoid_mem_cflct:1; 780 bool virtual_map:1; 781 bool tph_en:1; 782 u8 tph_val; 783 u8 pbl_chunk_size; 784 u32 first_pm_pbl_idx; 785 struct irdma_sc_vsi *vsi; 786 }; 787 788 struct irdma_udp_offload_info { 789 bool ipv4:1; 790 bool insert_vlan_tag:1; 791 u8 ttl; 792 u8 tos; 793 u16 src_port; 794 u16 dst_port; 795 u32 dest_ip_addr[4]; 796 u32 snd_mss; 797 u16 vlan_tag; 798 u16 arp_idx; 799 u32 flow_label; 800 u8 udp_state; 801 u32 psn_nxt; 802 u32 lsn; 803 u32 epsn; 804 u32 psn_max; 805 u32 psn_una; 806 u32 local_ipaddr[4]; 807 u32 cwnd; 808 u8 rexmit_thresh; 809 u8 rnr_nak_thresh; 810 }; 811 812 struct irdma_roce_offload_info { 813 u16 p_key; 814 u16 err_rq_idx; 815 u32 qkey; 816 u32 dest_qp; 817 u8 roce_tver; 818 u8 ack_credits; 819 u8 err_rq_idx_valid; 820 u32 pd_id; 821 u16 ord_size; 822 u16 ird_size; 823 bool is_qp1:1; 824 bool udprivcq_en:1; 825 bool dcqcn_en:1; 826 bool rcv_no_icrc:1; 827 bool wr_rdresp_en:1; 828 bool bind_en:1; 829 bool fast_reg_en:1; 830 bool priv_mode_en:1; 831 bool rd_en:1; 832 bool timely_en:1; 833 bool dctcp_en:1; 834 bool fw_cc_enable:1; 835 bool use_stats_inst:1; 836 u16 t_high; 837 u16 t_low; 838 u8 last_byte_sent; 839 u8 mac_addr[ETH_ALEN]; 840 u8 rtomin; 841 }; 842 843 struct irdma_iwarp_offload_info { 844 u16 rcv_mark_offset; 845 u16 snd_mark_offset; 846 u8 ddp_ver; 847 u8 rdmap_ver; 848 u8 iwarp_mode; 849 u16 err_rq_idx; 850 u32 pd_id; 851 u16 ord_size; 852 u16 ird_size; 853 bool ib_rd_en:1; 854 bool align_hdrs:1; 855 bool rcv_no_mpa_crc:1; 856 bool err_rq_idx_valid:1; 857 bool snd_mark_en:1; 858 bool rcv_mark_en:1; 859 bool wr_rdresp_en:1; 860 bool bind_en:1; 861 bool fast_reg_en:1; 862 bool priv_mode_en:1; 863 bool rd_en:1; 864 bool timely_en:1; 865 bool use_stats_inst:1; 866 bool ecn_en:1; 867 bool dctcp_en:1; 868 u16 t_high; 869 u16 t_low; 870 u8 last_byte_sent; 871 u8 mac_addr[ETH_ALEN]; 872 u8 rtomin; 873 }; 874 875 struct irdma_tcp_offload_info { 876 bool ipv4:1; 877 bool no_nagle:1; 878 bool insert_vlan_tag:1; 879 bool time_stamp:1; 880 bool drop_ooo_seg:1; 881 bool avoid_stretch_ack:1; 882 bool wscale:1; 883 bool ignore_tcp_opt:1; 884 bool ignore_tcp_uns_opt:1; 885 u8 cwnd_inc_limit; 886 u8 dup_ack_thresh; 887 u8 ttl; 888 u8 src_mac_addr_idx; 889 u8 tos; 890 u16 src_port; 891 u16 dst_port; 892 u32 dest_ip_addr[4]; 893 //u32 dest_ip_addr0; 894 //u32 dest_ip_addr1; 895 //u32 dest_ip_addr2; 896 //u32 dest_ip_addr3; 897 u32 snd_mss; 898 u16 syn_rst_handling; 899 u16 vlan_tag; 900 u16 arp_idx; 901 u32 flow_label; 902 u8 tcp_state; 903 u8 snd_wscale; 904 u8 rcv_wscale; 905 u32 time_stamp_recent; 906 u32 time_stamp_age; 907 u32 snd_nxt; 908 u32 snd_wnd; 909 u32 rcv_nxt; 910 u32 rcv_wnd; 911 u32 snd_max; 912 u32 snd_una; 913 u32 srtt; 914 u32 rtt_var; 915 u32 ss_thresh; 916 u32 cwnd; 917 u32 snd_wl1; 918 u32 snd_wl2; 919 u32 max_snd_window; 920 u8 rexmit_thresh; 921 u32 local_ipaddr[4]; 922 }; 923 924 struct irdma_qp_host_ctx_info { 925 u64 qp_compl_ctx; 926 union { 927 struct irdma_tcp_offload_info *tcp_info; 928 struct irdma_udp_offload_info *udp_info; 929 }; 930 union { 931 struct irdma_iwarp_offload_info *iwarp_info; 932 struct irdma_roce_offload_info *roce_info; 933 }; 934 u32 send_cq_num; 935 u32 rcv_cq_num; 936 u32 rem_endpoint_idx; 937 u8 stats_idx; 938 bool srq_valid:1; 939 bool tcp_info_valid:1; 940 bool iwarp_info_valid:1; 941 bool stats_idx_valid:1; 942 u8 user_pri; 943 }; 944 945 struct irdma_aeqe_info { 946 u64 compl_ctx; 947 u32 qp_cq_id; 948 u16 ae_id; 949 u16 wqe_idx; 950 u8 tcp_state; 951 u8 iwarp_state; 952 bool qp:1; 953 bool cq:1; 954 bool sq:1; 955 bool rq:1; 956 bool in_rdrsp_wr:1; 957 bool out_rdrsp:1; 958 bool aeqe_overflow:1; 959 u8 q2_data_written; 960 u8 ae_src; 961 }; 962 963 struct irdma_allocate_stag_info { 964 u64 total_len; 965 u64 first_pm_pbl_idx; 966 u32 chunk_size; 967 u32 stag_idx; 968 u32 page_size; 969 u32 pd_id; 970 u16 access_rights; 971 bool remote_access:1; 972 bool use_hmc_fcn_index:1; 973 bool use_pf_rid:1; 974 bool all_memory:1; 975 u8 hmc_fcn_index; 976 }; 977 978 struct irdma_mw_alloc_info { 979 u32 mw_stag_index; 980 u32 page_size; 981 u32 pd_id; 982 bool remote_access:1; 983 bool mw_wide:1; 984 bool mw1_bind_dont_vldt_key:1; 985 }; 986 987 struct irdma_reg_ns_stag_info { 988 u64 reg_addr_pa; 989 u64 va; 990 u64 total_len; 991 u32 page_size; 992 u32 chunk_size; 993 u32 first_pm_pbl_index; 994 enum irdma_addressing_type addr_type; 995 irdma_stag_index stag_idx; 996 u16 access_rights; 997 u32 pd_id; 998 irdma_stag_key stag_key; 999 bool use_hmc_fcn_index:1; 1000 u8 hmc_fcn_index; 1001 bool use_pf_rid:1; 1002 bool all_memory:1; 1003 }; 1004 1005 struct irdma_fast_reg_stag_info { 1006 u64 wr_id; 1007 u64 reg_addr_pa; 1008 u64 fbo; 1009 void *va; 1010 u64 total_len; 1011 u32 page_size; 1012 u32 chunk_size; 1013 u32 first_pm_pbl_index; 1014 enum irdma_addressing_type addr_type; 1015 irdma_stag_index stag_idx; 1016 u16 access_rights; 1017 u32 pd_id; 1018 irdma_stag_key stag_key; 1019 bool local_fence:1; 1020 bool read_fence:1; 1021 bool signaled:1; 1022 bool use_hmc_fcn_index:1; 1023 u8 hmc_fcn_index; 1024 bool use_pf_rid:1; 1025 bool defer_flag:1; 1026 }; 1027 1028 struct irdma_dealloc_stag_info { 1029 u32 stag_idx; 1030 u32 pd_id; 1031 bool mr:1; 1032 bool dealloc_pbl:1; 1033 }; 1034 1035 struct irdma_register_shared_stag { 1036 u64 va; 1037 enum irdma_addressing_type addr_type; 1038 irdma_stag_index new_stag_idx; 1039 irdma_stag_index parent_stag_idx; 1040 u32 access_rights; 1041 u32 pd_id; 1042 u32 page_size; 1043 irdma_stag_key new_stag_key; 1044 }; 1045 1046 struct irdma_qp_init_info { 1047 struct irdma_qp_uk_init_info qp_uk_init_info; 1048 struct irdma_sc_pd *pd; 1049 struct irdma_sc_vsi *vsi; 1050 __le64 *host_ctx; 1051 u8 *q2; 1052 u64 sq_pa; 1053 u64 rq_pa; 1054 u64 host_ctx_pa; 1055 u64 q2_pa; 1056 u64 shadow_area_pa; 1057 u8 sq_tph_val; 1058 u8 rq_tph_val; 1059 bool sq_tph_en:1; 1060 bool rq_tph_en:1; 1061 bool rcv_tph_en:1; 1062 bool xmit_tph_en:1; 1063 bool virtual_map:1; 1064 }; 1065 1066 struct irdma_cq_init_info { 1067 struct irdma_sc_dev *dev; 1068 u64 cq_base_pa; 1069 u64 shadow_area_pa; 1070 u32 ceq_id; 1071 u32 shadow_read_threshold; 1072 u8 pbl_chunk_size; 1073 u32 first_pm_pbl_idx; 1074 bool virtual_map:1; 1075 bool ceqe_mask:1; 1076 bool ceq_id_valid:1; 1077 bool tph_en:1; 1078 u8 tph_val; 1079 u8 type; 1080 struct irdma_cq_uk_init_info cq_uk_init_info; 1081 struct irdma_sc_vsi *vsi; 1082 }; 1083 1084 struct irdma_upload_context_info { 1085 u64 buf_pa; 1086 u32 qp_id; 1087 u8 qp_type; 1088 bool freeze_qp:1; 1089 bool raw_format:1; 1090 }; 1091 1092 struct irdma_local_mac_entry_info { 1093 u8 mac_addr[6]; 1094 u16 entry_idx; 1095 }; 1096 1097 struct irdma_add_arp_cache_entry_info { 1098 u8 mac_addr[ETH_ALEN]; 1099 u32 reach_max; 1100 u16 arp_index; 1101 bool permanent; 1102 }; 1103 1104 struct irdma_apbvt_info { 1105 u16 port; 1106 bool add; 1107 }; 1108 1109 struct irdma_qhash_table_info { 1110 struct irdma_sc_vsi *vsi; 1111 enum irdma_quad_hash_manage_type manage; 1112 enum irdma_quad_entry_type entry_type; 1113 bool vlan_valid:1; 1114 bool ipv4_valid:1; 1115 u8 mac_addr[ETH_ALEN]; 1116 u16 vlan_id; 1117 u8 user_pri; 1118 u32 qp_num; 1119 u32 dest_ip[4]; 1120 u32 src_ip[4]; 1121 u16 dest_port; 1122 u16 src_port; 1123 }; 1124 1125 struct irdma_cqp_manage_push_page_info { 1126 u32 push_idx; 1127 u16 qs_handle; 1128 u8 free_page; 1129 u8 push_page_type; 1130 }; 1131 1132 struct irdma_qp_flush_info { 1133 u16 sq_minor_code; 1134 u16 sq_major_code; 1135 u16 rq_minor_code; 1136 u16 rq_major_code; 1137 u16 ae_code; 1138 u8 ae_src; 1139 bool sq:1; 1140 bool rq:1; 1141 bool userflushcode:1; 1142 bool generate_ae:1; 1143 }; 1144 1145 struct irdma_gen_ae_info { 1146 u16 ae_code; 1147 u8 ae_src; 1148 }; 1149 1150 struct irdma_cqp_timeout { 1151 u64 compl_cqp_cmds; 1152 u32 count; 1153 }; 1154 1155 struct irdma_irq_ops { 1156 void (*irdma_cfg_aeq)(struct irdma_sc_dev *dev, u32 idx, bool enable); 1157 void (*irdma_cfg_ceq)(struct irdma_sc_dev *dev, u32 ceq_id, u32 idx, 1158 bool enable); 1159 void (*irdma_dis_irq)(struct irdma_sc_dev *dev, u32 idx); 1160 void (*irdma_en_irq)(struct irdma_sc_dev *dev, u32 idx); 1161 }; 1162 1163 void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq); 1164 int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch, 1165 bool check_overflow, bool post_sq); 1166 int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq); 1167 int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq, 1168 struct irdma_ccq_cqe_info *info); 1169 int irdma_sc_ccq_init(struct irdma_sc_cq *ccq, 1170 struct irdma_ccq_init_info *info); 1171 1172 int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch); 1173 int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq); 1174 1175 int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq); 1176 int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq, 1177 struct irdma_ceq_init_info *info); 1178 void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq); 1179 void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq); 1180 1181 int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq, 1182 struct irdma_aeq_init_info *info); 1183 int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq, 1184 struct irdma_aeqe_info *info); 1185 void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count); 1186 1187 void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id, 1188 int abi_ver); 1189 void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable); 1190 void irdma_check_cqp_progress(struct irdma_cqp_timeout *cqp_timeout, 1191 struct irdma_sc_dev *dev); 1192 int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err); 1193 int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp); 1194 int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp, 1195 struct irdma_cqp_init_info *info); 1196 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp); 1197 int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 opcode, 1198 struct irdma_ccq_cqe_info *cmpl_info); 1199 int irdma_sc_fast_register(struct irdma_sc_qp *qp, 1200 struct irdma_fast_reg_stag_info *info, bool post_sq); 1201 int irdma_sc_qp_create(struct irdma_sc_qp *qp, 1202 struct irdma_create_qp_info *info, u64 scratch, 1203 bool post_sq); 1204 int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch, 1205 bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq); 1206 int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, 1207 struct irdma_qp_flush_info *info, u64 scratch, 1208 bool post_sq); 1209 int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info); 1210 int irdma_sc_qp_modify(struct irdma_sc_qp *qp, 1211 struct irdma_modify_qp_info *info, u64 scratch, 1212 bool post_sq); 1213 void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size, 1214 irdma_stag stag); 1215 1216 void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read); 1217 void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx, 1218 struct irdma_qp_host_ctx_info *info); 1219 void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx, 1220 struct irdma_qp_host_ctx_info *info); 1221 int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq); 1222 int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info); 1223 void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info); 1224 int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch, 1225 u8 hmc_fn_id, bool post_sq, 1226 bool poll_registers); 1227 1228 void sc_vsi_update_stats(struct irdma_sc_vsi *vsi); 1229 struct cqp_info { 1230 union { 1231 struct { 1232 struct irdma_sc_qp *qp; 1233 struct irdma_create_qp_info info; 1234 u64 scratch; 1235 } qp_create; 1236 1237 struct { 1238 struct irdma_sc_qp *qp; 1239 struct irdma_modify_qp_info info; 1240 u64 scratch; 1241 } qp_modify; 1242 1243 struct { 1244 struct irdma_sc_qp *qp; 1245 u64 scratch; 1246 bool remove_hash_idx; 1247 bool ignore_mw_bnd; 1248 } qp_destroy; 1249 1250 struct { 1251 struct irdma_sc_cq *cq; 1252 u64 scratch; 1253 bool check_overflow; 1254 } cq_create; 1255 1256 struct { 1257 struct irdma_sc_cq *cq; 1258 struct irdma_modify_cq_info info; 1259 u64 scratch; 1260 } cq_modify; 1261 1262 struct { 1263 struct irdma_sc_cq *cq; 1264 u64 scratch; 1265 } cq_destroy; 1266 1267 struct { 1268 struct irdma_sc_dev *dev; 1269 struct irdma_allocate_stag_info info; 1270 u64 scratch; 1271 } alloc_stag; 1272 1273 struct { 1274 struct irdma_sc_dev *dev; 1275 struct irdma_mw_alloc_info info; 1276 u64 scratch; 1277 } mw_alloc; 1278 1279 struct { 1280 struct irdma_sc_dev *dev; 1281 struct irdma_reg_ns_stag_info info; 1282 u64 scratch; 1283 } mr_reg_non_shared; 1284 1285 struct { 1286 struct irdma_sc_dev *dev; 1287 struct irdma_dealloc_stag_info info; 1288 u64 scratch; 1289 } dealloc_stag; 1290 1291 struct { 1292 struct irdma_sc_cqp *cqp; 1293 struct irdma_add_arp_cache_entry_info info; 1294 u64 scratch; 1295 } add_arp_cache_entry; 1296 1297 struct { 1298 struct irdma_sc_cqp *cqp; 1299 u64 scratch; 1300 u16 arp_index; 1301 } del_arp_cache_entry; 1302 1303 struct { 1304 struct irdma_sc_cqp *cqp; 1305 struct irdma_local_mac_entry_info info; 1306 u64 scratch; 1307 } add_local_mac_entry; 1308 1309 struct { 1310 struct irdma_sc_cqp *cqp; 1311 u64 scratch; 1312 u8 entry_idx; 1313 u8 ignore_ref_count; 1314 } del_local_mac_entry; 1315 1316 struct { 1317 struct irdma_sc_cqp *cqp; 1318 u64 scratch; 1319 } alloc_local_mac_entry; 1320 1321 struct { 1322 struct irdma_sc_cqp *cqp; 1323 struct irdma_cqp_manage_push_page_info info; 1324 u64 scratch; 1325 } manage_push_page; 1326 1327 struct { 1328 struct irdma_sc_dev *dev; 1329 struct irdma_upload_context_info info; 1330 u64 scratch; 1331 } qp_upload_context; 1332 1333 struct { 1334 struct irdma_sc_dev *dev; 1335 struct irdma_hmc_fcn_info info; 1336 u64 scratch; 1337 } manage_hmc_pm; 1338 1339 struct { 1340 struct irdma_sc_ceq *ceq; 1341 u64 scratch; 1342 } ceq_create; 1343 1344 struct { 1345 struct irdma_sc_ceq *ceq; 1346 u64 scratch; 1347 } ceq_destroy; 1348 1349 struct { 1350 struct irdma_sc_aeq *aeq; 1351 u64 scratch; 1352 } aeq_create; 1353 1354 struct { 1355 struct irdma_sc_aeq *aeq; 1356 u64 scratch; 1357 } aeq_destroy; 1358 1359 struct { 1360 struct irdma_sc_qp *qp; 1361 struct irdma_qp_flush_info info; 1362 u64 scratch; 1363 } qp_flush_wqes; 1364 1365 struct { 1366 struct irdma_sc_qp *qp; 1367 struct irdma_gen_ae_info info; 1368 u64 scratch; 1369 } gen_ae; 1370 1371 struct { 1372 struct irdma_sc_cqp *cqp; 1373 void *fpm_val_va; 1374 u64 fpm_val_pa; 1375 u8 hmc_fn_id; 1376 u64 scratch; 1377 } query_fpm_val; 1378 1379 struct { 1380 struct irdma_sc_cqp *cqp; 1381 void *fpm_val_va; 1382 u64 fpm_val_pa; 1383 u8 hmc_fn_id; 1384 u64 scratch; 1385 } commit_fpm_val; 1386 1387 struct { 1388 struct irdma_sc_cqp *cqp; 1389 struct irdma_apbvt_info info; 1390 u64 scratch; 1391 } manage_apbvt_entry; 1392 1393 struct { 1394 struct irdma_sc_cqp *cqp; 1395 struct irdma_qhash_table_info info; 1396 u64 scratch; 1397 } manage_qhash_table_entry; 1398 1399 struct { 1400 struct irdma_sc_dev *dev; 1401 struct irdma_update_sds_info info; 1402 u64 scratch; 1403 } update_pe_sds; 1404 1405 struct { 1406 struct irdma_sc_cqp *cqp; 1407 struct irdma_sc_qp *qp; 1408 u64 scratch; 1409 } suspend_resume; 1410 1411 struct { 1412 struct irdma_sc_cqp *cqp; 1413 struct irdma_ah_info info; 1414 u64 scratch; 1415 } ah_create; 1416 1417 struct { 1418 struct irdma_sc_cqp *cqp; 1419 struct irdma_ah_info info; 1420 u64 scratch; 1421 } ah_destroy; 1422 1423 struct { 1424 struct irdma_sc_cqp *cqp; 1425 struct irdma_mcast_grp_info info; 1426 u64 scratch; 1427 } mc_create; 1428 1429 struct { 1430 struct irdma_sc_cqp *cqp; 1431 struct irdma_mcast_grp_info info; 1432 u64 scratch; 1433 } mc_destroy; 1434 1435 struct { 1436 struct irdma_sc_cqp *cqp; 1437 struct irdma_mcast_grp_info info; 1438 u64 scratch; 1439 } mc_modify; 1440 1441 struct { 1442 struct irdma_sc_cqp *cqp; 1443 struct irdma_stats_inst_info info; 1444 u64 scratch; 1445 } stats_manage; 1446 1447 struct { 1448 struct irdma_sc_cqp *cqp; 1449 struct irdma_stats_gather_info info; 1450 u64 scratch; 1451 } stats_gather; 1452 1453 struct { 1454 struct irdma_sc_cqp *cqp; 1455 struct irdma_ws_node_info info; 1456 u64 scratch; 1457 } ws_node; 1458 1459 struct { 1460 struct irdma_sc_cqp *cqp; 1461 struct irdma_up_info info; 1462 u64 scratch; 1463 } up_map; 1464 1465 struct { 1466 struct irdma_sc_cqp *cqp; 1467 struct irdma_dma_mem query_buff_mem; 1468 u64 scratch; 1469 } query_rdma; 1470 } u; 1471 }; 1472 1473 struct cqp_cmds_info { 1474 struct list_head cqp_cmd_entry; 1475 u8 cqp_cmd; 1476 u8 post_sq; 1477 struct cqp_info in; 1478 }; 1479 1480 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch, 1481 u32 *wqe_idx); 1482 1483 /** 1484 * irdma_sc_cqp_get_next_send_wqe - get next wqe on cqp sq 1485 * @cqp: struct for cqp hw 1486 * @scratch: private data for CQP WQE 1487 */ 1488 static inline __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch) 1489 { 1490 u32 wqe_idx; 1491 1492 return irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx); 1493 } 1494 #endif /* IRDMA_TYPE_H */ 1495