1 /* $OpenBSD: if_mcx.c,v 1.72 2020/08/21 04:47:52 jmatthew Exp $ */ 2 3 /* 4 * Copyright (c) 2017 David Gwynne <dlg@openbsd.org> 5 * Copyright (c) 2019 Jonathan Matthew <jmatthew@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include "bpfilter.h" 21 #include "vlan.h" 22 #include "kstat.h" 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/sockio.h> 27 #include <sys/mbuf.h> 28 #include <sys/kernel.h> 29 #include <sys/socket.h> 30 #include <sys/device.h> 31 #include <sys/pool.h> 32 #include <sys/queue.h> 33 #include <sys/timeout.h> 34 #include <sys/task.h> 35 #include <sys/atomic.h> 36 37 #include <machine/bus.h> 38 #include <machine/intr.h> 39 40 #include <net/if.h> 41 #include <net/if_dl.h> 42 #include <net/if_media.h> 43 #include <net/toeplitz.h> 44 45 #if NBPFILTER > 0 46 #include <net/bpf.h> 47 #endif 48 49 #if NKSTAT > 0 50 #include <sys/kstat.h> 51 #endif 52 53 #include <netinet/in.h> 54 #include <netinet/if_ether.h> 55 56 #include <dev/pci/pcireg.h> 57 #include <dev/pci/pcivar.h> 58 #include <dev/pci/pcidevs.h> 59 60 #define BUS_DMASYNC_PRERW (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE) 61 #define BUS_DMASYNC_POSTRW (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE) 62 63 #define MCX_HCA_BAR PCI_MAPREG_START /* BAR 0 */ 64 65 #define MCX_FW_VER 0x0000 66 #define MCX_FW_VER_MAJOR(_v) ((_v) & 0xffff) 67 #define MCX_FW_VER_MINOR(_v) ((_v) >> 16) 68 #define MCX_CMDIF_FW_SUBVER 0x0004 69 #define MCX_FW_VER_SUBMINOR(_v) ((_v) & 0xffff) 70 #define MCX_CMDIF(_v) ((_v) >> 16) 71 72 #define MCX_ISSI 1 /* as per the PRM */ 73 #define MCX_CMD_IF_SUPPORTED 5 74 75 #define MCX_HARDMTU 9500 76 77 #define MCX_PAGE_SHIFT 12 78 #define MCX_PAGE_SIZE (1 << MCX_PAGE_SHIFT) 79 80 /* queue sizes */ 81 #define MCX_LOG_EQ_SIZE 7 82 #define MCX_LOG_CQ_SIZE 12 83 #define MCX_LOG_RQ_SIZE 10 84 #define MCX_LOG_SQ_SIZE 11 85 86 #define MCX_MAX_QUEUES 1 87 88 /* completion event moderation - about 10khz, or 90% of the cq */ 89 #define MCX_CQ_MOD_PERIOD 50 90 #define MCX_CQ_MOD_COUNTER \ 91 (((1 << (MCX_LOG_CQ_SIZE - 1)) * 9) / 10) 92 93 #define MCX_LOG_SQ_ENTRY_SIZE 6 94 #define MCX_SQ_ENTRY_MAX_SLOTS 4 95 #define MCX_SQ_SEGS_PER_SLOT \ 96 (sizeof(struct mcx_sq_entry) / sizeof(struct mcx_sq_entry_seg)) 97 #define MCX_SQ_MAX_SEGMENTS \ 98 1 + ((MCX_SQ_ENTRY_MAX_SLOTS-1) * MCX_SQ_SEGS_PER_SLOT) 99 100 #define MCX_LOG_FLOW_TABLE_SIZE 5 101 #define MCX_NUM_STATIC_FLOWS 4 /* promisc, allmulti, ucast, bcast */ 102 #define MCX_NUM_MCAST_FLOWS \ 103 ((1 << MCX_LOG_FLOW_TABLE_SIZE) - MCX_NUM_STATIC_FLOWS) 104 105 #define MCX_SQ_INLINE_SIZE 18 106 CTASSERT(ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN == MCX_SQ_INLINE_SIZE); 107 108 /* doorbell offsets */ 109 #define MCX_DOORBELL_AREA_SIZE MCX_PAGE_SIZE 110 111 #define MCX_CQ_DOORBELL_BASE 0 112 #define MCX_CQ_DOORBELL_STRIDE 64 113 114 #define MCX_WQ_DOORBELL_BASE MCX_PAGE_SIZE/2 115 #define MCX_WQ_DOORBELL_STRIDE 64 116 /* make sure the doorbells fit */ 117 CTASSERT(MCX_MAX_QUEUES * MCX_CQ_DOORBELL_STRIDE < MCX_WQ_DOORBELL_BASE); 118 CTASSERT(MCX_MAX_QUEUES * MCX_WQ_DOORBELL_STRIDE < 119 MCX_DOORBELL_AREA_SIZE - MCX_WQ_DOORBELL_BASE); 120 121 #define MCX_WQ_DOORBELL_MASK 0xffff 122 123 /* uar registers */ 124 #define MCX_UAR_CQ_DOORBELL 0x20 125 #define MCX_UAR_EQ_DOORBELL_ARM 0x40 126 #define MCX_UAR_EQ_DOORBELL 0x48 127 #define MCX_UAR_BF 0x800 128 129 #define MCX_CMDQ_ADDR_HI 0x0010 130 #define MCX_CMDQ_ADDR_LO 0x0014 131 #define MCX_CMDQ_ADDR_NMASK 0xfff 132 #define MCX_CMDQ_LOG_SIZE(_v) ((_v) >> 4 & 0xf) 133 #define MCX_CMDQ_LOG_STRIDE(_v) ((_v) >> 0 & 0xf) 134 #define MCX_CMDQ_INTERFACE_MASK (0x3 << 8) 135 #define MCX_CMDQ_INTERFACE_FULL_DRIVER (0x0 << 8) 136 #define MCX_CMDQ_INTERFACE_DISABLED (0x1 << 8) 137 138 #define MCX_CMDQ_DOORBELL 0x0018 139 140 #define MCX_STATE 0x01fc 141 #define MCX_STATE_MASK (1 << 31) 142 #define MCX_STATE_INITIALIZING (1 << 31) 143 #define MCX_STATE_READY (0 << 31) 144 #define MCX_STATE_INTERFACE_MASK (0x3 << 24) 145 #define MCX_STATE_INTERFACE_FULL_DRIVER (0x0 << 24) 146 #define MCX_STATE_INTERFACE_DISABLED (0x1 << 24) 147 148 #define MCX_INTERNAL_TIMER 0x1000 149 #define MCX_INTERNAL_TIMER_H 0x1000 150 #define MCX_INTERNAL_TIMER_L 0x1004 151 152 #define MCX_CLEAR_INT 0x100c 153 154 #define MCX_REG_OP_WRITE 0 155 #define MCX_REG_OP_READ 1 156 157 #define MCX_REG_PMLP 0x5002 158 #define MCX_REG_PMTU 0x5003 159 #define MCX_REG_PTYS 0x5004 160 #define MCX_REG_PAOS 0x5006 161 #define MCX_REG_PFCC 0x5007 162 #define MCX_REG_PPCNT 0x5008 163 #define MCX_REG_MTCAP 0x9009 /* mgmt temp capabilities */ 164 #define MCX_REG_MTMP 0x900a /* mgmt temp */ 165 #define MCX_REG_MCIA 0x9014 166 167 #define MCX_ETHER_CAP_SGMII 0 168 #define MCX_ETHER_CAP_1000_KX 1 169 #define MCX_ETHER_CAP_10G_CX4 2 170 #define MCX_ETHER_CAP_10G_KX4 3 171 #define MCX_ETHER_CAP_10G_KR 4 172 #define MCX_ETHER_CAP_40G_CR4 6 173 #define MCX_ETHER_CAP_40G_KR4 7 174 #define MCX_ETHER_CAP_10G_CR 12 175 #define MCX_ETHER_CAP_10G_SR 13 176 #define MCX_ETHER_CAP_10G_LR 14 177 #define MCX_ETHER_CAP_40G_SR4 15 178 #define MCX_ETHER_CAP_40G_LR4 16 179 #define MCX_ETHER_CAP_50G_SR2 18 180 #define MCX_ETHER_CAP_100G_CR4 20 181 #define MCX_ETHER_CAP_100G_SR4 21 182 #define MCX_ETHER_CAP_100G_KR4 22 183 #define MCX_ETHER_CAP_25G_CR 27 184 #define MCX_ETHER_CAP_25G_KR 28 185 #define MCX_ETHER_CAP_25G_SR 29 186 #define MCX_ETHER_CAP_50G_CR2 30 187 #define MCX_ETHER_CAP_50G_KR2 31 188 189 #define MCX_MAX_CQE 32 190 191 #define MCX_CMD_QUERY_HCA_CAP 0x100 192 #define MCX_CMD_QUERY_ADAPTER 0x101 193 #define MCX_CMD_INIT_HCA 0x102 194 #define MCX_CMD_TEARDOWN_HCA 0x103 195 #define MCX_CMD_ENABLE_HCA 0x104 196 #define MCX_CMD_DISABLE_HCA 0x105 197 #define MCX_CMD_QUERY_PAGES 0x107 198 #define MCX_CMD_MANAGE_PAGES 0x108 199 #define MCX_CMD_SET_HCA_CAP 0x109 200 #define MCX_CMD_QUERY_ISSI 0x10a 201 #define MCX_CMD_SET_ISSI 0x10b 202 #define MCX_CMD_SET_DRIVER_VERSION 0x10d 203 #define MCX_CMD_QUERY_SPECIAL_CONTEXTS 0x203 204 #define MCX_CMD_CREATE_EQ 0x301 205 #define MCX_CMD_DESTROY_EQ 0x302 206 #define MCX_CMD_QUERY_EQ 0x303 207 #define MCX_CMD_CREATE_CQ 0x400 208 #define MCX_CMD_DESTROY_CQ 0x401 209 #define MCX_CMD_QUERY_CQ 0x402 210 #define MCX_CMD_QUERY_NIC_VPORT_CONTEXT 0x754 211 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT \ 212 0x755 213 #define MCX_CMD_QUERY_VPORT_COUNTERS 0x770 214 #define MCX_CMD_ALLOC_PD 0x800 215 #define MCX_CMD_ALLOC_UAR 0x802 216 #define MCX_CMD_ACCESS_REG 0x805 217 #define MCX_CMD_ALLOC_TRANSPORT_DOMAIN 0x816 218 #define MCX_CMD_CREATE_TIR 0x900 219 #define MCX_CMD_DESTROY_TIR 0x902 220 #define MCX_CMD_CREATE_SQ 0x904 221 #define MCX_CMD_MODIFY_SQ 0x905 222 #define MCX_CMD_DESTROY_SQ 0x906 223 #define MCX_CMD_QUERY_SQ 0x907 224 #define MCX_CMD_CREATE_RQ 0x908 225 #define MCX_CMD_MODIFY_RQ 0x909 226 #define MCX_CMD_DESTROY_RQ 0x90a 227 #define MCX_CMD_QUERY_RQ 0x90b 228 #define MCX_CMD_CREATE_TIS 0x912 229 #define MCX_CMD_DESTROY_TIS 0x914 230 #define MCX_CMD_CREATE_RQT 0x916 231 #define MCX_CMD_DESTROY_RQT 0x918 232 #define MCX_CMD_SET_FLOW_TABLE_ROOT 0x92f 233 #define MCX_CMD_CREATE_FLOW_TABLE 0x930 234 #define MCX_CMD_DESTROY_FLOW_TABLE 0x931 235 #define MCX_CMD_QUERY_FLOW_TABLE 0x932 236 #define MCX_CMD_CREATE_FLOW_GROUP 0x933 237 #define MCX_CMD_DESTROY_FLOW_GROUP 0x934 238 #define MCX_CMD_QUERY_FLOW_GROUP 0x935 239 #define MCX_CMD_SET_FLOW_TABLE_ENTRY 0x936 240 #define MCX_CMD_QUERY_FLOW_TABLE_ENTRY 0x937 241 #define MCX_CMD_DELETE_FLOW_TABLE_ENTRY 0x938 242 #define MCX_CMD_ALLOC_FLOW_COUNTER 0x939 243 #define MCX_CMD_QUERY_FLOW_COUNTER 0x93b 244 245 #define MCX_QUEUE_STATE_RST 0 246 #define MCX_QUEUE_STATE_RDY 1 247 #define MCX_QUEUE_STATE_ERR 3 248 249 #define MCX_FLOW_TABLE_TYPE_RX 0 250 #define MCX_FLOW_TABLE_TYPE_TX 1 251 252 #define MCX_CMDQ_INLINE_DATASIZE 16 253 254 struct mcx_cmdq_entry { 255 uint8_t cq_type; 256 #define MCX_CMDQ_TYPE_PCIE 0x7 257 uint8_t cq_reserved0[3]; 258 259 uint32_t cq_input_length; 260 uint64_t cq_input_ptr; 261 uint8_t cq_input_data[MCX_CMDQ_INLINE_DATASIZE]; 262 263 uint8_t cq_output_data[MCX_CMDQ_INLINE_DATASIZE]; 264 uint64_t cq_output_ptr; 265 uint32_t cq_output_length; 266 267 uint8_t cq_token; 268 uint8_t cq_signature; 269 uint8_t cq_reserved1[1]; 270 uint8_t cq_status; 271 #define MCX_CQ_STATUS_SHIFT 1 272 #define MCX_CQ_STATUS_MASK (0x7f << MCX_CQ_STATUS_SHIFT) 273 #define MCX_CQ_STATUS_OK (0x00 << MCX_CQ_STATUS_SHIFT) 274 #define MCX_CQ_STATUS_INT_ERR (0x01 << MCX_CQ_STATUS_SHIFT) 275 #define MCX_CQ_STATUS_BAD_OPCODE (0x02 << MCX_CQ_STATUS_SHIFT) 276 #define MCX_CQ_STATUS_BAD_PARAM (0x03 << MCX_CQ_STATUS_SHIFT) 277 #define MCX_CQ_STATUS_BAD_SYS_STATE (0x04 << MCX_CQ_STATUS_SHIFT) 278 #define MCX_CQ_STATUS_BAD_RESOURCE (0x05 << MCX_CQ_STATUS_SHIFT) 279 #define MCX_CQ_STATUS_RESOURCE_BUSY (0x06 << MCX_CQ_STATUS_SHIFT) 280 #define MCX_CQ_STATUS_EXCEED_LIM (0x08 << MCX_CQ_STATUS_SHIFT) 281 #define MCX_CQ_STATUS_BAD_RES_STATE (0x09 << MCX_CQ_STATUS_SHIFT) 282 #define MCX_CQ_STATUS_BAD_INDEX (0x0a << MCX_CQ_STATUS_SHIFT) 283 #define MCX_CQ_STATUS_NO_RESOURCES (0x0f << MCX_CQ_STATUS_SHIFT) 284 #define MCX_CQ_STATUS_BAD_INPUT_LEN (0x50 << MCX_CQ_STATUS_SHIFT) 285 #define MCX_CQ_STATUS_BAD_OUTPUT_LEN (0x51 << MCX_CQ_STATUS_SHIFT) 286 #define MCX_CQ_STATUS_BAD_RESOURCE_STATE \ 287 (0x10 << MCX_CQ_STATUS_SHIFT) 288 #define MCX_CQ_STATUS_BAD_SIZE (0x40 << MCX_CQ_STATUS_SHIFT) 289 #define MCX_CQ_STATUS_OWN_MASK 0x1 290 #define MCX_CQ_STATUS_OWN_SW 0x0 291 #define MCX_CQ_STATUS_OWN_HW 0x1 292 } __packed __aligned(8); 293 294 #define MCX_CMDQ_MAILBOX_DATASIZE 512 295 296 struct mcx_cmdq_mailbox { 297 uint8_t mb_data[MCX_CMDQ_MAILBOX_DATASIZE]; 298 uint8_t mb_reserved0[48]; 299 uint64_t mb_next_ptr; 300 uint32_t mb_block_number; 301 uint8_t mb_reserved1[1]; 302 uint8_t mb_token; 303 uint8_t mb_ctrl_signature; 304 uint8_t mb_signature; 305 } __packed __aligned(8); 306 307 #define MCX_CMDQ_MAILBOX_ALIGN (1 << 10) 308 #define MCX_CMDQ_MAILBOX_SIZE roundup(sizeof(struct mcx_cmdq_mailbox), \ 309 MCX_CMDQ_MAILBOX_ALIGN) 310 /* 311 * command mailbox structres 312 */ 313 314 struct mcx_cmd_enable_hca_in { 315 uint16_t cmd_opcode; 316 uint8_t cmd_reserved0[4]; 317 uint16_t cmd_op_mod; 318 uint8_t cmd_reserved1[2]; 319 uint16_t cmd_function_id; 320 uint8_t cmd_reserved2[4]; 321 } __packed __aligned(4); 322 323 struct mcx_cmd_enable_hca_out { 324 uint8_t cmd_status; 325 uint8_t cmd_reserved0[3]; 326 uint32_t cmd_syndrome; 327 uint8_t cmd_reserved1[4]; 328 } __packed __aligned(4); 329 330 struct mcx_cmd_init_hca_in { 331 uint16_t cmd_opcode; 332 uint8_t cmd_reserved0[4]; 333 uint16_t cmd_op_mod; 334 uint8_t cmd_reserved1[8]; 335 } __packed __aligned(4); 336 337 struct mcx_cmd_init_hca_out { 338 uint8_t cmd_status; 339 uint8_t cmd_reserved0[3]; 340 uint32_t cmd_syndrome; 341 uint8_t cmd_reserved1[8]; 342 } __packed __aligned(4); 343 344 struct mcx_cmd_teardown_hca_in { 345 uint16_t cmd_opcode; 346 uint8_t cmd_reserved0[4]; 347 uint16_t cmd_op_mod; 348 uint8_t cmd_reserved1[2]; 349 #define MCX_CMD_TEARDOWN_HCA_GRACEFUL 0x0 350 #define MCX_CMD_TEARDOWN_HCA_PANIC 0x1 351 uint16_t cmd_profile; 352 uint8_t cmd_reserved2[4]; 353 } __packed __aligned(4); 354 355 struct mcx_cmd_teardown_hca_out { 356 uint8_t cmd_status; 357 uint8_t cmd_reserved0[3]; 358 uint32_t cmd_syndrome; 359 uint8_t cmd_reserved1[8]; 360 } __packed __aligned(4); 361 362 struct mcx_cmd_access_reg_in { 363 uint16_t cmd_opcode; 364 uint8_t cmd_reserved0[4]; 365 uint16_t cmd_op_mod; 366 uint8_t cmd_reserved1[2]; 367 uint16_t cmd_register_id; 368 uint32_t cmd_argument; 369 } __packed __aligned(4); 370 371 struct mcx_cmd_access_reg_out { 372 uint8_t cmd_status; 373 uint8_t cmd_reserved0[3]; 374 uint32_t cmd_syndrome; 375 uint8_t cmd_reserved1[8]; 376 } __packed __aligned(4); 377 378 struct mcx_reg_pmtu { 379 uint8_t rp_reserved1; 380 uint8_t rp_local_port; 381 uint8_t rp_reserved2[2]; 382 uint16_t rp_max_mtu; 383 uint8_t rp_reserved3[2]; 384 uint16_t rp_admin_mtu; 385 uint8_t rp_reserved4[2]; 386 uint16_t rp_oper_mtu; 387 uint8_t rp_reserved5[2]; 388 } __packed __aligned(4); 389 390 struct mcx_reg_ptys { 391 uint8_t rp_reserved1; 392 uint8_t rp_local_port; 393 uint8_t rp_reserved2; 394 uint8_t rp_proto_mask; 395 #define MCX_REG_PTYS_PROTO_MASK_ETH (1 << 2) 396 uint8_t rp_reserved3[8]; 397 uint32_t rp_eth_proto_cap; 398 uint8_t rp_reserved4[8]; 399 uint32_t rp_eth_proto_admin; 400 uint8_t rp_reserved5[8]; 401 uint32_t rp_eth_proto_oper; 402 uint8_t rp_reserved6[24]; 403 } __packed __aligned(4); 404 405 struct mcx_reg_paos { 406 uint8_t rp_reserved1; 407 uint8_t rp_local_port; 408 uint8_t rp_admin_status; 409 #define MCX_REG_PAOS_ADMIN_STATUS_UP 1 410 #define MCX_REG_PAOS_ADMIN_STATUS_DOWN 2 411 #define MCX_REG_PAOS_ADMIN_STATUS_UP_ONCE 3 412 #define MCX_REG_PAOS_ADMIN_STATUS_DISABLED 4 413 uint8_t rp_oper_status; 414 #define MCX_REG_PAOS_OPER_STATUS_UP 1 415 #define MCX_REG_PAOS_OPER_STATUS_DOWN 2 416 #define MCX_REG_PAOS_OPER_STATUS_FAILED 4 417 uint8_t rp_admin_state_update; 418 #define MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN (1 << 7) 419 uint8_t rp_reserved2[11]; 420 } __packed __aligned(4); 421 422 struct mcx_reg_pfcc { 423 uint8_t rp_reserved1; 424 uint8_t rp_local_port; 425 uint8_t rp_reserved2[3]; 426 uint8_t rp_prio_mask_tx; 427 uint8_t rp_reserved3; 428 uint8_t rp_prio_mask_rx; 429 uint8_t rp_pptx_aptx; 430 uint8_t rp_pfctx; 431 uint8_t rp_fctx_dis; 432 uint8_t rp_reserved4; 433 uint8_t rp_pprx_aprx; 434 uint8_t rp_pfcrx; 435 uint8_t rp_reserved5[2]; 436 uint16_t rp_dev_stall_min; 437 uint16_t rp_dev_stall_crit; 438 uint8_t rp_reserved6[12]; 439 } __packed __aligned(4); 440 441 #define MCX_PMLP_MODULE_NUM_MASK 0xff 442 struct mcx_reg_pmlp { 443 uint8_t rp_rxtx; 444 uint8_t rp_local_port; 445 uint8_t rp_reserved0; 446 uint8_t rp_width; 447 uint32_t rp_lane0_mapping; 448 uint32_t rp_lane1_mapping; 449 uint32_t rp_lane2_mapping; 450 uint32_t rp_lane3_mapping; 451 uint8_t rp_reserved1[44]; 452 } __packed __aligned(4); 453 454 struct mcx_reg_ppcnt { 455 uint8_t ppcnt_swid; 456 uint8_t ppcnt_local_port; 457 uint8_t ppcnt_pnat; 458 uint8_t ppcnt_grp; 459 #define MCX_REG_PPCNT_GRP_IEEE8023 0x00 460 #define MCX_REG_PPCNT_GRP_RFC2863 0x01 461 #define MCX_REG_PPCNT_GRP_RFC2819 0x02 462 #define MCX_REG_PPCNT_GRP_RFC3635 0x03 463 #define MCX_REG_PPCNT_GRP_PER_PRIO 0x10 464 #define MCX_REG_PPCNT_GRP_PER_TC 0x11 465 #define MCX_REG_PPCNT_GRP_PER_RX_BUFFER 0x11 466 467 uint8_t ppcnt_clr; 468 uint8_t ppcnt_reserved1[2]; 469 uint8_t ppcnt_prio_tc; 470 #define MCX_REG_PPCNT_CLR (1 << 7) 471 472 uint8_t ppcnt_counter_set[248]; 473 } __packed __aligned(8); 474 CTASSERT(sizeof(struct mcx_reg_ppcnt) == 256); 475 CTASSERT((offsetof(struct mcx_reg_ppcnt, ppcnt_counter_set) % 476 sizeof(uint64_t)) == 0); 477 478 enum mcx_ppcnt_ieee8023 { 479 frames_transmitted_ok, 480 frames_received_ok, 481 frame_check_sequence_errors, 482 alignment_errors, 483 octets_transmitted_ok, 484 octets_received_ok, 485 multicast_frames_xmitted_ok, 486 broadcast_frames_xmitted_ok, 487 multicast_frames_received_ok, 488 broadcast_frames_received_ok, 489 in_range_length_errors, 490 out_of_range_length_field, 491 frame_too_long_errors, 492 symbol_error_during_carrier, 493 mac_control_frames_transmitted, 494 mac_control_frames_received, 495 unsupported_opcodes_received, 496 pause_mac_ctrl_frames_received, 497 pause_mac_ctrl_frames_transmitted, 498 499 mcx_ppcnt_ieee8023_count 500 }; 501 CTASSERT(mcx_ppcnt_ieee8023_count * sizeof(uint64_t) == 0x98); 502 503 enum mcx_ppcnt_rfc2863 { 504 in_octets, 505 in_ucast_pkts, 506 in_discards, 507 in_errors, 508 in_unknown_protos, 509 out_octets, 510 out_ucast_pkts, 511 out_discards, 512 out_errors, 513 in_multicast_pkts, 514 in_broadcast_pkts, 515 out_multicast_pkts, 516 out_broadcast_pkts, 517 518 mcx_ppcnt_rfc2863_count 519 }; 520 CTASSERT(mcx_ppcnt_rfc2863_count * sizeof(uint64_t) == 0x68); 521 522 enum mcx_ppcnt_rfc2819 { 523 drop_events, 524 octets, 525 pkts, 526 broadcast_pkts, 527 multicast_pkts, 528 crc_align_errors, 529 undersize_pkts, 530 oversize_pkts, 531 fragments, 532 jabbers, 533 collisions, 534 pkts64octets, 535 pkts65to127octets, 536 pkts128to255octets, 537 pkts256to511octets, 538 pkts512to1023octets, 539 pkts1024to1518octets, 540 pkts1519to2047octets, 541 pkts2048to4095octets, 542 pkts4096to8191octets, 543 pkts8192to10239octets, 544 545 mcx_ppcnt_rfc2819_count 546 }; 547 CTASSERT((mcx_ppcnt_rfc2819_count * sizeof(uint64_t)) == 0xa8); 548 549 enum mcx_ppcnt_rfc3635 { 550 dot3stats_alignment_errors, 551 dot3stats_fcs_errors, 552 dot3stats_single_collision_frames, 553 dot3stats_multiple_collision_frames, 554 dot3stats_sqe_test_errors, 555 dot3stats_deferred_transmissions, 556 dot3stats_late_collisions, 557 dot3stats_excessive_collisions, 558 dot3stats_internal_mac_transmit_errors, 559 dot3stats_carrier_sense_errors, 560 dot3stats_frame_too_longs, 561 dot3stats_internal_mac_receive_errors, 562 dot3stats_symbol_errors, 563 dot3control_in_unknown_opcodes, 564 dot3in_pause_frames, 565 dot3out_pause_frames, 566 567 mcx_ppcnt_rfc3635_count 568 }; 569 CTASSERT((mcx_ppcnt_rfc3635_count * sizeof(uint64_t)) == 0x80); 570 571 struct mcx_reg_mtcap { 572 uint8_t _reserved1[3]; 573 uint8_t mtcap_sensor_count; 574 uint8_t _reserved2[4]; 575 576 uint64_t mtcap_sensor_map; 577 }; 578 579 struct mcx_reg_mtmp { 580 uint8_t _reserved1[2]; 581 uint16_t mtmp_sensor_index; 582 583 uint8_t _reserved2[2]; 584 uint16_t mtmp_temperature; 585 586 uint16_t mtmp_mte_mtr; 587 #define MCX_REG_MTMP_MTE (1 << 15) 588 #define MCX_REG_MTMP_MTR (1 << 14) 589 uint16_t mtmp_max_temperature; 590 591 uint16_t mtmp_tee; 592 #define MCX_REG_MTMP_TEE_NOPE (0 << 14) 593 #define MCX_REG_MTMP_TEE_GENERATE (1 << 14) 594 #define MCX_REG_MTMP_TEE_GENERATE_ONE (2 << 14) 595 uint16_t mtmp_temperature_threshold_hi; 596 597 uint8_t _reserved3[2]; 598 uint16_t mtmp_temperature_threshold_lo; 599 600 uint8_t _reserved4[4]; 601 602 uint8_t mtmp_sensor_name[8]; 603 }; 604 CTASSERT(sizeof(struct mcx_reg_mtmp) == 0x20); 605 CTASSERT(offsetof(struct mcx_reg_mtmp, mtmp_sensor_name) == 0x18); 606 607 #define MCX_MCIA_EEPROM_BYTES 32 608 struct mcx_reg_mcia { 609 uint8_t rm_l; 610 uint8_t rm_module; 611 uint8_t rm_reserved0; 612 uint8_t rm_status; 613 uint8_t rm_i2c_addr; 614 uint8_t rm_page_num; 615 uint16_t rm_dev_addr; 616 uint16_t rm_reserved1; 617 uint16_t rm_size; 618 uint32_t rm_reserved2; 619 uint8_t rm_data[48]; 620 } __packed __aligned(4); 621 622 struct mcx_cmd_query_issi_in { 623 uint16_t cmd_opcode; 624 uint8_t cmd_reserved0[4]; 625 uint16_t cmd_op_mod; 626 uint8_t cmd_reserved1[8]; 627 } __packed __aligned(4); 628 629 struct mcx_cmd_query_issi_il_out { 630 uint8_t cmd_status; 631 uint8_t cmd_reserved0[3]; 632 uint32_t cmd_syndrome; 633 uint8_t cmd_reserved1[2]; 634 uint16_t cmd_current_issi; 635 uint8_t cmd_reserved2[4]; 636 } __packed __aligned(4); 637 638 CTASSERT(sizeof(struct mcx_cmd_query_issi_il_out) == MCX_CMDQ_INLINE_DATASIZE); 639 640 struct mcx_cmd_query_issi_mb_out { 641 uint8_t cmd_reserved2[16]; 642 uint8_t cmd_supported_issi[80]; /* very big endian */ 643 } __packed __aligned(4); 644 645 CTASSERT(sizeof(struct mcx_cmd_query_issi_mb_out) <= MCX_CMDQ_MAILBOX_DATASIZE); 646 647 struct mcx_cmd_set_issi_in { 648 uint16_t cmd_opcode; 649 uint8_t cmd_reserved0[4]; 650 uint16_t cmd_op_mod; 651 uint8_t cmd_reserved1[2]; 652 uint16_t cmd_current_issi; 653 uint8_t cmd_reserved2[4]; 654 } __packed __aligned(4); 655 656 CTASSERT(sizeof(struct mcx_cmd_set_issi_in) <= MCX_CMDQ_INLINE_DATASIZE); 657 658 struct mcx_cmd_set_issi_out { 659 uint8_t cmd_status; 660 uint8_t cmd_reserved0[3]; 661 uint32_t cmd_syndrome; 662 uint8_t cmd_reserved1[8]; 663 } __packed __aligned(4); 664 665 CTASSERT(sizeof(struct mcx_cmd_set_issi_out) <= MCX_CMDQ_INLINE_DATASIZE); 666 667 struct mcx_cmd_query_pages_in { 668 uint16_t cmd_opcode; 669 uint8_t cmd_reserved0[4]; 670 uint16_t cmd_op_mod; 671 #define MCX_CMD_QUERY_PAGES_BOOT 0x01 672 #define MCX_CMD_QUERY_PAGES_INIT 0x02 673 #define MCX_CMD_QUERY_PAGES_REGULAR 0x03 674 uint8_t cmd_reserved1[8]; 675 } __packed __aligned(4); 676 677 struct mcx_cmd_query_pages_out { 678 uint8_t cmd_status; 679 uint8_t cmd_reserved0[3]; 680 uint32_t cmd_syndrome; 681 uint8_t cmd_reserved1[2]; 682 uint16_t cmd_func_id; 683 int32_t cmd_num_pages; 684 } __packed __aligned(4); 685 686 struct mcx_cmd_manage_pages_in { 687 uint16_t cmd_opcode; 688 uint8_t cmd_reserved0[4]; 689 uint16_t cmd_op_mod; 690 #define MCX_CMD_MANAGE_PAGES_ALLOC_FAIL \ 691 0x00 692 #define MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS \ 693 0x01 694 #define MCX_CMD_MANAGE_PAGES_HCA_RETURN_PAGES \ 695 0x02 696 uint8_t cmd_reserved1[2]; 697 uint16_t cmd_func_id; 698 uint32_t cmd_input_num_entries; 699 } __packed __aligned(4); 700 701 CTASSERT(sizeof(struct mcx_cmd_manage_pages_in) == MCX_CMDQ_INLINE_DATASIZE); 702 703 struct mcx_cmd_manage_pages_out { 704 uint8_t cmd_status; 705 uint8_t cmd_reserved0[3]; 706 uint32_t cmd_syndrome; 707 uint32_t cmd_output_num_entries; 708 uint8_t cmd_reserved1[4]; 709 } __packed __aligned(4); 710 711 CTASSERT(sizeof(struct mcx_cmd_manage_pages_out) == MCX_CMDQ_INLINE_DATASIZE); 712 713 struct mcx_cmd_query_hca_cap_in { 714 uint16_t cmd_opcode; 715 uint8_t cmd_reserved0[4]; 716 uint16_t cmd_op_mod; 717 #define MCX_CMD_QUERY_HCA_CAP_MAX (0x0 << 0) 718 #define MCX_CMD_QUERY_HCA_CAP_CURRENT (0x1 << 0) 719 #define MCX_CMD_QUERY_HCA_CAP_DEVICE (0x0 << 1) 720 #define MCX_CMD_QUERY_HCA_CAP_OFFLOAD (0x1 << 1) 721 #define MCX_CMD_QUERY_HCA_CAP_FLOW (0x7 << 1) 722 uint8_t cmd_reserved1[8]; 723 } __packed __aligned(4); 724 725 struct mcx_cmd_query_hca_cap_out { 726 uint8_t cmd_status; 727 uint8_t cmd_reserved0[3]; 728 uint32_t cmd_syndrome; 729 uint8_t cmd_reserved1[8]; 730 } __packed __aligned(4); 731 732 #define MCX_HCA_CAP_LEN 0x1000 733 #define MCX_HCA_CAP_NMAILBOXES \ 734 (MCX_HCA_CAP_LEN / MCX_CMDQ_MAILBOX_DATASIZE) 735 736 #if __GNUC_PREREQ__(4, 3) 737 #define __counter__ __COUNTER__ 738 #else 739 #define __counter__ __LINE__ 740 #endif 741 742 #define __token(_tok, _num) _tok##_num 743 #define _token(_tok, _num) __token(_tok, _num) 744 #define __reserved__ _token(__reserved, __counter__) 745 746 struct mcx_cap_device { 747 uint8_t reserved0[16]; 748 749 uint8_t log_max_srq_sz; 750 uint8_t log_max_qp_sz; 751 uint8_t __reserved__[1]; 752 uint8_t log_max_qp; /* 5 bits */ 753 #define MCX_CAP_DEVICE_LOG_MAX_QP 0x1f 754 755 uint8_t __reserved__[1]; 756 uint8_t log_max_srq; /* 5 bits */ 757 #define MCX_CAP_DEVICE_LOG_MAX_SRQ 0x1f 758 uint8_t __reserved__[2]; 759 760 uint8_t __reserved__[1]; 761 uint8_t log_max_cq_sz; 762 uint8_t __reserved__[1]; 763 uint8_t log_max_cq; /* 5 bits */ 764 #define MCX_CAP_DEVICE_LOG_MAX_CQ 0x1f 765 766 uint8_t log_max_eq_sz; 767 uint8_t log_max_mkey; /* 6 bits */ 768 #define MCX_CAP_DEVICE_LOG_MAX_MKEY 0x3f 769 uint8_t __reserved__[1]; 770 uint8_t log_max_eq; /* 4 bits */ 771 #define MCX_CAP_DEVICE_LOG_MAX_EQ 0x0f 772 773 uint8_t max_indirection; 774 uint8_t log_max_mrw_sz; /* 7 bits */ 775 #define MCX_CAP_DEVICE_LOG_MAX_MRW_SZ 0x7f 776 uint8_t teardown_log_max_msf_list_size; 777 #define MCX_CAP_DEVICE_FORCE_TEARDOWN 0x80 778 #define MCX_CAP_DEVICE_LOG_MAX_MSF_LIST_SIZE \ 779 0x3f 780 uint8_t log_max_klm_list_size; /* 6 bits */ 781 #define MCX_CAP_DEVICE_LOG_MAX_KLM_LIST_SIZE \ 782 0x3f 783 784 uint8_t __reserved__[1]; 785 uint8_t log_max_ra_req_dc; /* 6 bits */ 786 #define MCX_CAP_DEVICE_LOG_MAX_REQ_DC 0x3f 787 uint8_t __reserved__[1]; 788 uint8_t log_max_ra_res_dc; /* 6 bits */ 789 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_DC \ 790 0x3f 791 792 uint8_t __reserved__[1]; 793 uint8_t log_max_ra_req_qp; /* 6 bits */ 794 #define MCX_CAP_DEVICE_LOG_MAX_RA_REQ_QP \ 795 0x3f 796 uint8_t __reserved__[1]; 797 uint8_t log_max_ra_res_qp; /* 6 bits */ 798 #define MCX_CAP_DEVICE_LOG_MAX_RA_RES_QP \ 799 0x3f 800 801 uint8_t flags1; 802 #define MCX_CAP_DEVICE_END_PAD 0x80 803 #define MCX_CAP_DEVICE_CC_QUERY_ALLOWED 0x40 804 #define MCX_CAP_DEVICE_CC_MODIFY_ALLOWED \ 805 0x20 806 #define MCX_CAP_DEVICE_START_PAD 0x10 807 #define MCX_CAP_DEVICE_128BYTE_CACHELINE \ 808 0x08 809 uint8_t __reserved__[1]; 810 uint16_t gid_table_size; 811 812 uint16_t flags2; 813 #define MCX_CAP_DEVICE_OUT_OF_SEQ_CNT 0x8000 814 #define MCX_CAP_DEVICE_VPORT_COUNTERS 0x4000 815 #define MCX_CAP_DEVICE_RETRANSMISSION_Q_COUNTERS \ 816 0x2000 817 #define MCX_CAP_DEVICE_DEBUG 0x1000 818 #define MCX_CAP_DEVICE_MODIFY_RQ_COUNTERS_SET_ID \ 819 0x8000 820 #define MCX_CAP_DEVICE_RQ_DELAY_DROP 0x4000 821 #define MCX_CAP_DEVICe_MAX_QP_CNT_MASK 0x03ff 822 uint16_t pkey_table_size; 823 824 uint8_t flags3; 825 #define MCX_CAP_DEVICE_VPORT_GROUP_MANAGER \ 826 0x80 827 #define MCX_CAP_DEVICE_VHCA_GROUP_MANAGER \ 828 0x40 829 #define MCX_CAP_DEVICE_IB_VIRTUAL 0x20 830 #define MCX_CAP_DEVICE_ETH_VIRTUAL 0x10 831 #define MCX_CAP_DEVICE_ETS 0x04 832 #define MCX_CAP_DEVICE_NIC_FLOW_TABLE 0x02 833 #define MCX_CAP_DEVICE_ESWITCH_FLOW_TABLE \ 834 0x01 835 uint8_t local_ca_ack_delay; /* 5 bits */ 836 #define MCX_CAP_DEVICE_LOCAL_CA_ACK_DELAY \ 837 0x1f 838 uint8_t port_type; 839 #define MCX_CAP_DEVICE_PORT_MODULE_EVENT \ 840 0x80 841 #define MCX_CAP_DEVICE_PORT_TYPE 0x03 842 uint8_t num_ports; 843 844 uint8_t snapshot_log_max_msg; 845 #define MCX_CAP_DEVICE_SNAPSHOT 0x80 846 #define MCX_CAP_DEVICE_LOG_MAX_MSG 0x1f 847 uint8_t max_tc; /* 4 bits */ 848 #define MCX_CAP_DEVICE_MAX_TC 0x0f 849 uint8_t flags4; 850 #define MCX_CAP_DEVICE_TEMP_WARN_EVENT 0x80 851 #define MCX_CAP_DEVICE_DCBX 0x40 852 #define MCX_CAP_DEVICE_ROL_S 0x02 853 #define MCX_CAP_DEVICE_ROL_G 0x01 854 uint8_t wol; 855 #define MCX_CAP_DEVICE_WOL_S 0x40 856 #define MCX_CAP_DEVICE_WOL_G 0x20 857 #define MCX_CAP_DEVICE_WOL_A 0x10 858 #define MCX_CAP_DEVICE_WOL_B 0x08 859 #define MCX_CAP_DEVICE_WOL_M 0x04 860 #define MCX_CAP_DEVICE_WOL_U 0x02 861 #define MCX_CAP_DEVICE_WOL_P 0x01 862 863 uint16_t stat_rate_support; 864 uint8_t __reserved__[1]; 865 uint8_t cqe_version; /* 4 bits */ 866 #define MCX_CAP_DEVICE_CQE_VERSION 0x0f 867 868 uint32_t flags5; 869 #define MCX_CAP_DEVICE_COMPACT_ADDRESS_VECTOR \ 870 0x80000000 871 #define MCX_CAP_DEVICE_STRIDING_RQ 0x40000000 872 #define MCX_CAP_DEVICE_IPOIP_ENHANCED_OFFLOADS \ 873 0x10000000 874 #define MCX_CAP_DEVICE_IPOIP_IPOIP_OFFLOADS \ 875 0x08000000 876 #define MCX_CAP_DEVICE_DC_CONNECT_CP 0x00040000 877 #define MCX_CAP_DEVICE_DC_CNAK_DRACE 0x00020000 878 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 879 #define MCX_CAP_DEVICE_DRAIN_SIGERR 0x00010000 880 #define MCX_CAP_DEVICE_CMDIF_CHECKSUM 0x0000c000 881 #define MCX_CAP_DEVICE_SIGERR_QCE 0x00002000 882 #define MCX_CAP_DEVICE_WQ_SIGNATURE 0x00000800 883 #define MCX_CAP_DEVICE_SCTR_DATA_CQE 0x00000400 884 #define MCX_CAP_DEVICE_SHO 0x00000100 885 #define MCX_CAP_DEVICE_TPH 0x00000080 886 #define MCX_CAP_DEVICE_RF 0x00000040 887 #define MCX_CAP_DEVICE_DCT 0x00000020 888 #define MCX_CAP_DEVICE_QOS 0x00000010 889 #define MCX_CAP_DEVICe_ETH_NET_OFFLOADS 0x00000008 890 #define MCX_CAP_DEVICE_ROCE 0x00000004 891 #define MCX_CAP_DEVICE_ATOMIC 0x00000002 892 893 uint32_t flags6; 894 #define MCX_CAP_DEVICE_CQ_OI 0x80000000 895 #define MCX_CAP_DEVICE_CQ_RESIZE 0x40000000 896 #define MCX_CAP_DEVICE_CQ_MODERATION 0x20000000 897 #define MCX_CAP_DEVICE_CQ_PERIOD_MODE_MODIFY \ 898 0x10000000 899 #define MCX_CAP_DEVICE_CQ_INVALIDATE 0x08000000 900 #define MCX_CAP_DEVICE_RESERVED_AT_255 0x04000000 901 #define MCX_CAP_DEVICE_CQ_EQ_REMAP 0x02000000 902 #define MCX_CAP_DEVICE_PG 0x01000000 903 #define MCX_CAP_DEVICE_BLOCK_LB_MC 0x00800000 904 #define MCX_CAP_DEVICE_EXPONENTIAL_BACKOFF \ 905 0x00400000 906 #define MCX_CAP_DEVICE_SCQE_BREAK_MODERATION \ 907 0x00200000 908 #define MCX_CAP_DEVICE_CQ_PERIOD_START_FROM_CQE \ 909 0x00100000 910 #define MCX_CAP_DEVICE_CD 0x00080000 911 #define MCX_CAP_DEVICE_ATM 0x00040000 912 #define MCX_CAP_DEVICE_APM 0x00020000 913 #define MCX_CAP_DEVICE_IMAICL 0x00010000 914 #define MCX_CAP_DEVICE_QKV 0x00000200 915 #define MCX_CAP_DEVICE_PKV 0x00000100 916 #define MCX_CAP_DEVICE_SET_DETH_SQPN 0x00000080 917 #define MCX_CAP_DEVICE_XRC 0x00000008 918 #define MCX_CAP_DEVICE_UD 0x00000004 919 #define MCX_CAP_DEVICE_UC 0x00000002 920 #define MCX_CAP_DEVICE_RC 0x00000001 921 922 uint8_t uar_flags; 923 #define MCX_CAP_DEVICE_UAR_4K 0x80 924 uint8_t uar_sz; /* 6 bits */ 925 #define MCX_CAP_DEVICE_UAR_SZ 0x3f 926 uint8_t __reserved__[1]; 927 uint8_t log_pg_sz; 928 929 uint8_t flags7; 930 #define MCX_CAP_DEVICE_BF 0x80 931 #define MCX_CAP_DEVICE_DRIVER_VERSION 0x40 932 #define MCX_CAP_DEVICE_PAD_TX_ETH_PACKET \ 933 0x20 934 uint8_t log_bf_reg_size; /* 5 bits */ 935 #define MCX_CAP_DEVICE_LOG_BF_REG_SIZE 0x1f 936 uint8_t __reserved__[2]; 937 938 uint16_t num_of_diagnostic_counters; 939 uint16_t max_wqe_sz_sq; 940 941 uint8_t __reserved__[2]; 942 uint16_t max_wqe_sz_rq; 943 944 uint8_t __reserved__[2]; 945 uint16_t max_wqe_sz_sq_dc; 946 947 uint32_t max_qp_mcg; /* 25 bits */ 948 #define MCX_CAP_DEVICE_MAX_QP_MCG 0x1ffffff 949 950 uint8_t __reserved__[3]; 951 uint8_t log_max_mcq; 952 953 uint8_t log_max_transport_domain; /* 5 bits */ 954 #define MCX_CAP_DEVICE_LOG_MAX_TRANSORT_DOMAIN \ 955 0x1f 956 uint8_t log_max_pd; /* 5 bits */ 957 #define MCX_CAP_DEVICE_LOG_MAX_PD 0x1f 958 uint8_t __reserved__[1]; 959 uint8_t log_max_xrcd; /* 5 bits */ 960 #define MCX_CAP_DEVICE_LOG_MAX_XRCD 0x1f 961 962 uint8_t __reserved__[2]; 963 uint16_t max_flow_counter; 964 965 uint8_t log_max_rq; /* 5 bits */ 966 #define MCX_CAP_DEVICE_LOG_MAX_RQ 0x1f 967 uint8_t log_max_sq; /* 5 bits */ 968 #define MCX_CAP_DEVICE_LOG_MAX_SQ 0x1f 969 uint8_t log_max_tir; /* 5 bits */ 970 #define MCX_CAP_DEVICE_LOG_MAX_TIR 0x1f 971 uint8_t log_max_tis; /* 5 bits */ 972 #define MCX_CAP_DEVICE_LOG_MAX_TIS 0x1f 973 974 uint8_t flags8; 975 #define MCX_CAP_DEVICE_BASIC_CYCLIC_RCV_WQE \ 976 0x80 977 #define MCX_CAP_DEVICE_LOG_MAX_RMP 0x1f 978 uint8_t log_max_rqt; /* 5 bits */ 979 #define MCX_CAP_DEVICE_LOG_MAX_RQT 0x1f 980 uint8_t log_max_rqt_size; /* 5 bits */ 981 #define MCX_CAP_DEVICE_LOG_MAX_RQT_SIZE 0x1f 982 uint8_t log_max_tis_per_sq; /* 5 bits */ 983 #define MCX_CAP_DEVICE_LOG_MAX_TIS_PER_SQ \ 984 0x1f 985 } __packed __aligned(8); 986 987 CTASSERT(offsetof(struct mcx_cap_device, max_indirection) == 0x20); 988 CTASSERT(offsetof(struct mcx_cap_device, flags1) == 0x2c); 989 CTASSERT(offsetof(struct mcx_cap_device, flags2) == 0x30); 990 CTASSERT(offsetof(struct mcx_cap_device, snapshot_log_max_msg) == 0x38); 991 CTASSERT(offsetof(struct mcx_cap_device, flags5) == 0x40); 992 CTASSERT(offsetof(struct mcx_cap_device, flags7) == 0x4c); 993 CTASSERT(sizeof(struct mcx_cap_device) <= MCX_CMDQ_MAILBOX_DATASIZE); 994 995 struct mcx_cmd_set_driver_version_in { 996 uint16_t cmd_opcode; 997 uint8_t cmd_reserved0[4]; 998 uint16_t cmd_op_mod; 999 uint8_t cmd_reserved1[8]; 1000 } __packed __aligned(4); 1001 1002 struct mcx_cmd_set_driver_version_out { 1003 uint8_t cmd_status; 1004 uint8_t cmd_reserved0[3]; 1005 uint32_t cmd_syndrome; 1006 uint8_t cmd_reserved1[8]; 1007 } __packed __aligned(4); 1008 1009 struct mcx_cmd_set_driver_version { 1010 uint8_t cmd_driver_version[64]; 1011 } __packed __aligned(8); 1012 1013 struct mcx_cmd_modify_nic_vport_context_in { 1014 uint16_t cmd_opcode; 1015 uint8_t cmd_reserved0[4]; 1016 uint16_t cmd_op_mod; 1017 uint8_t cmd_reserved1[4]; 1018 uint32_t cmd_field_select; 1019 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_ADDR 0x04 1020 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC 0x10 1021 #define MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU 0x40 1022 } __packed __aligned(4); 1023 1024 struct mcx_cmd_modify_nic_vport_context_out { 1025 uint8_t cmd_status; 1026 uint8_t cmd_reserved0[3]; 1027 uint32_t cmd_syndrome; 1028 uint8_t cmd_reserved1[8]; 1029 } __packed __aligned(4); 1030 1031 struct mcx_cmd_query_nic_vport_context_in { 1032 uint16_t cmd_opcode; 1033 uint8_t cmd_reserved0[4]; 1034 uint16_t cmd_op_mod; 1035 uint8_t cmd_reserved1[4]; 1036 uint8_t cmd_allowed_list_type; 1037 uint8_t cmd_reserved2[3]; 1038 } __packed __aligned(4); 1039 1040 struct mcx_cmd_query_nic_vport_context_out { 1041 uint8_t cmd_status; 1042 uint8_t cmd_reserved0[3]; 1043 uint32_t cmd_syndrome; 1044 uint8_t cmd_reserved1[8]; 1045 } __packed __aligned(4); 1046 1047 struct mcx_nic_vport_ctx { 1048 uint32_t vp_min_wqe_inline_mode; 1049 uint8_t vp_reserved0[32]; 1050 uint32_t vp_mtu; 1051 uint8_t vp_reserved1[200]; 1052 uint16_t vp_flags; 1053 #define MCX_NIC_VPORT_CTX_LIST_UC_MAC (0) 1054 #define MCX_NIC_VPORT_CTX_LIST_MC_MAC (1 << 24) 1055 #define MCX_NIC_VPORT_CTX_LIST_VLAN (2 << 24) 1056 #define MCX_NIC_VPORT_CTX_PROMISC_ALL (1 << 13) 1057 #define MCX_NIC_VPORT_CTX_PROMISC_MCAST (1 << 14) 1058 #define MCX_NIC_VPORT_CTX_PROMISC_UCAST (1 << 15) 1059 uint16_t vp_allowed_list_size; 1060 uint64_t vp_perm_addr; 1061 uint8_t vp_reserved2[4]; 1062 /* allowed list follows */ 1063 } __packed __aligned(4); 1064 1065 struct mcx_counter { 1066 uint64_t packets; 1067 uint64_t octets; 1068 } __packed __aligned(4); 1069 1070 struct mcx_nic_vport_counters { 1071 struct mcx_counter rx_err; 1072 struct mcx_counter tx_err; 1073 uint8_t reserved0[64]; /* 0x30 */ 1074 struct mcx_counter rx_bcast; 1075 struct mcx_counter tx_bcast; 1076 struct mcx_counter rx_ucast; 1077 struct mcx_counter tx_ucast; 1078 struct mcx_counter rx_mcast; 1079 struct mcx_counter tx_mcast; 1080 uint8_t reserved1[0x210 - 0xd0]; 1081 } __packed __aligned(4); 1082 1083 struct mcx_cmd_query_vport_counters_in { 1084 uint16_t cmd_opcode; 1085 uint8_t cmd_reserved0[4]; 1086 uint16_t cmd_op_mod; 1087 uint8_t cmd_reserved1[8]; 1088 } __packed __aligned(4); 1089 1090 struct mcx_cmd_query_vport_counters_mb_in { 1091 uint8_t cmd_reserved0[8]; 1092 uint8_t cmd_clear; 1093 uint8_t cmd_reserved1[7]; 1094 } __packed __aligned(4); 1095 1096 struct mcx_cmd_query_vport_counters_out { 1097 uint8_t cmd_status; 1098 uint8_t cmd_reserved0[3]; 1099 uint32_t cmd_syndrome; 1100 uint8_t cmd_reserved1[8]; 1101 } __packed __aligned(4); 1102 1103 struct mcx_cmd_query_flow_counter_in { 1104 uint16_t cmd_opcode; 1105 uint8_t cmd_reserved0[4]; 1106 uint16_t cmd_op_mod; 1107 uint8_t cmd_reserved1[8]; 1108 } __packed __aligned(4); 1109 1110 struct mcx_cmd_query_flow_counter_mb_in { 1111 uint8_t cmd_reserved0[8]; 1112 uint8_t cmd_clear; 1113 uint8_t cmd_reserved1[5]; 1114 uint16_t cmd_flow_counter_id; 1115 } __packed __aligned(4); 1116 1117 struct mcx_cmd_query_flow_counter_out { 1118 uint8_t cmd_status; 1119 uint8_t cmd_reserved0[3]; 1120 uint32_t cmd_syndrome; 1121 uint8_t cmd_reserved1[8]; 1122 } __packed __aligned(4); 1123 1124 struct mcx_cmd_alloc_uar_in { 1125 uint16_t cmd_opcode; 1126 uint8_t cmd_reserved0[4]; 1127 uint16_t cmd_op_mod; 1128 uint8_t cmd_reserved1[8]; 1129 } __packed __aligned(4); 1130 1131 struct mcx_cmd_alloc_uar_out { 1132 uint8_t cmd_status; 1133 uint8_t cmd_reserved0[3]; 1134 uint32_t cmd_syndrome; 1135 uint32_t cmd_uar; 1136 uint8_t cmd_reserved1[4]; 1137 } __packed __aligned(4); 1138 1139 struct mcx_cmd_query_special_ctx_in { 1140 uint16_t cmd_opcode; 1141 uint8_t cmd_reserved0[4]; 1142 uint16_t cmd_op_mod; 1143 uint8_t cmd_reserved1[8]; 1144 } __packed __aligned(4); 1145 1146 struct mcx_cmd_query_special_ctx_out { 1147 uint8_t cmd_status; 1148 uint8_t cmd_reserved0[3]; 1149 uint32_t cmd_syndrome; 1150 uint8_t cmd_reserved1[4]; 1151 uint32_t cmd_resd_lkey; 1152 } __packed __aligned(4); 1153 1154 struct mcx_eq_ctx { 1155 uint32_t eq_status; 1156 #define MCX_EQ_CTX_STATE_SHIFT 8 1157 #define MCX_EQ_CTX_STATE_MASK (0xf << MCX_EQ_CTX_STATE_SHIFT) 1158 #define MCX_EQ_CTX_STATE_ARMED 0x9 1159 #define MCX_EQ_CTX_STATE_FIRED 0xa 1160 #define MCX_EQ_CTX_OI_SHIFT 17 1161 #define MCX_EQ_CTX_OI (1 << MCX_EQ_CTX_OI_SHIFT) 1162 #define MCX_EQ_CTX_EC_SHIFT 18 1163 #define MCX_EQ_CTX_EC (1 << MCX_EQ_CTX_EC_SHIFT) 1164 #define MCX_EQ_CTX_STATUS_SHIFT 28 1165 #define MCX_EQ_CTX_STATUS_MASK (0xf << MCX_EQ_CTX_STATUS_SHIFT) 1166 #define MCX_EQ_CTX_STATUS_OK 0x0 1167 #define MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE 0xa 1168 uint32_t eq_reserved1; 1169 uint32_t eq_page_offset; 1170 #define MCX_EQ_CTX_PAGE_OFFSET_SHIFT 5 1171 uint32_t eq_uar_size; 1172 #define MCX_EQ_CTX_UAR_PAGE_MASK 0xffffff 1173 #define MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT 24 1174 uint32_t eq_reserved2; 1175 uint8_t eq_reserved3[3]; 1176 uint8_t eq_intr; 1177 uint32_t eq_log_page_size; 1178 #define MCX_EQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1179 uint32_t eq_reserved4[3]; 1180 uint32_t eq_consumer_counter; 1181 uint32_t eq_producer_counter; 1182 #define MCX_EQ_CTX_COUNTER_MASK 0xffffff 1183 uint32_t eq_reserved5[4]; 1184 } __packed __aligned(4); 1185 1186 CTASSERT(sizeof(struct mcx_eq_ctx) == 64); 1187 1188 struct mcx_cmd_create_eq_in { 1189 uint16_t cmd_opcode; 1190 uint8_t cmd_reserved0[4]; 1191 uint16_t cmd_op_mod; 1192 uint8_t cmd_reserved1[8]; 1193 } __packed __aligned(4); 1194 1195 struct mcx_cmd_create_eq_mb_in { 1196 struct mcx_eq_ctx cmd_eq_ctx; 1197 uint8_t cmd_reserved0[8]; 1198 uint64_t cmd_event_bitmask; 1199 #define MCX_EVENT_TYPE_COMPLETION 0x00 1200 #define MCX_EVENT_TYPE_CQ_ERROR 0x04 1201 #define MCX_EVENT_TYPE_INTERNAL_ERROR 0x08 1202 #define MCX_EVENT_TYPE_PORT_CHANGE 0x09 1203 #define MCX_EVENT_TYPE_CMD_COMPLETION 0x0a 1204 #define MCX_EVENT_TYPE_PAGE_REQUEST 0x0b 1205 #define MCX_EVENT_TYPE_LAST_WQE 0x13 1206 uint8_t cmd_reserved1[176]; 1207 } __packed __aligned(4); 1208 1209 struct mcx_cmd_create_eq_out { 1210 uint8_t cmd_status; 1211 uint8_t cmd_reserved0[3]; 1212 uint32_t cmd_syndrome; 1213 uint32_t cmd_eqn; 1214 uint8_t cmd_reserved1[4]; 1215 } __packed __aligned(4); 1216 1217 struct mcx_cmd_query_eq_in { 1218 uint16_t cmd_opcode; 1219 uint8_t cmd_reserved0[4]; 1220 uint16_t cmd_op_mod; 1221 uint32_t cmd_eqn; 1222 uint8_t cmd_reserved1[4]; 1223 } __packed __aligned(4); 1224 1225 struct mcx_cmd_query_eq_out { 1226 uint8_t cmd_status; 1227 uint8_t cmd_reserved0[3]; 1228 uint32_t cmd_syndrome; 1229 uint8_t cmd_reserved1[8]; 1230 } __packed __aligned(4); 1231 1232 struct mcx_eq_entry { 1233 uint8_t eq_reserved1; 1234 uint8_t eq_event_type; 1235 uint8_t eq_reserved2; 1236 uint8_t eq_event_sub_type; 1237 1238 uint8_t eq_reserved3[28]; 1239 uint32_t eq_event_data[7]; 1240 uint8_t eq_reserved4[2]; 1241 uint8_t eq_signature; 1242 uint8_t eq_owner; 1243 #define MCX_EQ_ENTRY_OWNER_INIT 1 1244 } __packed __aligned(4); 1245 1246 CTASSERT(sizeof(struct mcx_eq_entry) == 64); 1247 1248 struct mcx_cmd_alloc_pd_in { 1249 uint16_t cmd_opcode; 1250 uint8_t cmd_reserved0[4]; 1251 uint16_t cmd_op_mod; 1252 uint8_t cmd_reserved1[8]; 1253 } __packed __aligned(4); 1254 1255 struct mcx_cmd_alloc_pd_out { 1256 uint8_t cmd_status; 1257 uint8_t cmd_reserved0[3]; 1258 uint32_t cmd_syndrome; 1259 uint32_t cmd_pd; 1260 uint8_t cmd_reserved1[4]; 1261 } __packed __aligned(4); 1262 1263 struct mcx_cmd_alloc_td_in { 1264 uint16_t cmd_opcode; 1265 uint8_t cmd_reserved0[4]; 1266 uint16_t cmd_op_mod; 1267 uint8_t cmd_reserved1[8]; 1268 } __packed __aligned(4); 1269 1270 struct mcx_cmd_alloc_td_out { 1271 uint8_t cmd_status; 1272 uint8_t cmd_reserved0[3]; 1273 uint32_t cmd_syndrome; 1274 uint32_t cmd_tdomain; 1275 uint8_t cmd_reserved1[4]; 1276 } __packed __aligned(4); 1277 1278 struct mcx_cmd_create_tir_in { 1279 uint16_t cmd_opcode; 1280 uint8_t cmd_reserved0[4]; 1281 uint16_t cmd_op_mod; 1282 uint8_t cmd_reserved1[8]; 1283 } __packed __aligned(4); 1284 1285 struct mcx_cmd_create_tir_mb_in { 1286 uint8_t cmd_reserved0[20]; 1287 uint32_t cmd_disp_type; 1288 #define MCX_TIR_CTX_DISP_TYPE_DIRECT 0 1289 #define MCX_TIR_CTX_DISP_TYPE_INDIRECT 1 1290 #define MCX_TIR_CTX_DISP_TYPE_SHIFT 28 1291 uint8_t cmd_reserved1[8]; 1292 uint32_t cmd_lro; 1293 uint8_t cmd_reserved2[8]; 1294 uint32_t cmd_inline_rqn; 1295 uint32_t cmd_indir_table; 1296 uint32_t cmd_tdomain; 1297 #define MCX_TIR_CTX_HASH_TOEPLITZ 2 1298 #define MCX_TIR_CTX_HASH_SHIFT 28 1299 uint8_t cmd_rx_hash_key[40]; 1300 uint32_t cmd_rx_hash_sel_outer; 1301 #define MCX_TIR_CTX_HASH_SEL_SRC_IP (1 << 0) 1302 #define MCX_TIR_CTX_HASH_SEL_DST_IP (1 << 1) 1303 #define MCX_TIR_CTX_HASH_SEL_SPORT (1 << 2) 1304 #define MCX_TIR_CTX_HASH_SEL_DPORT (1 << 3) 1305 #define MCX_TIR_CTX_HASH_SEL_IPV4 (0 << 31) 1306 #define MCX_TIR_CTX_HASH_SEL_IPV6 (1 << 31) 1307 #define MCX_TIR_CTX_HASH_SEL_TCP (0 << 30) 1308 #define MCX_TIR_CTX_HASH_SEL_UDP (1 << 30) 1309 uint32_t cmd_rx_hash_sel_inner; 1310 uint8_t cmd_reserved3[152]; 1311 } __packed __aligned(4); 1312 1313 struct mcx_cmd_create_tir_out { 1314 uint8_t cmd_status; 1315 uint8_t cmd_reserved0[3]; 1316 uint32_t cmd_syndrome; 1317 uint32_t cmd_tirn; 1318 uint8_t cmd_reserved1[4]; 1319 } __packed __aligned(4); 1320 1321 struct mcx_cmd_destroy_tir_in { 1322 uint16_t cmd_opcode; 1323 uint8_t cmd_reserved0[4]; 1324 uint16_t cmd_op_mod; 1325 uint32_t cmd_tirn; 1326 uint8_t cmd_reserved1[4]; 1327 } __packed __aligned(4); 1328 1329 struct mcx_cmd_destroy_tir_out { 1330 uint8_t cmd_status; 1331 uint8_t cmd_reserved0[3]; 1332 uint32_t cmd_syndrome; 1333 uint8_t cmd_reserved1[8]; 1334 } __packed __aligned(4); 1335 1336 struct mcx_cmd_create_tis_in { 1337 uint16_t cmd_opcode; 1338 uint8_t cmd_reserved0[4]; 1339 uint16_t cmd_op_mod; 1340 uint8_t cmd_reserved1[8]; 1341 } __packed __aligned(4); 1342 1343 struct mcx_cmd_create_tis_mb_in { 1344 uint8_t cmd_reserved[16]; 1345 uint32_t cmd_prio; 1346 uint8_t cmd_reserved1[32]; 1347 uint32_t cmd_tdomain; 1348 uint8_t cmd_reserved2[120]; 1349 } __packed __aligned(4); 1350 1351 struct mcx_cmd_create_tis_out { 1352 uint8_t cmd_status; 1353 uint8_t cmd_reserved0[3]; 1354 uint32_t cmd_syndrome; 1355 uint32_t cmd_tisn; 1356 uint8_t cmd_reserved1[4]; 1357 } __packed __aligned(4); 1358 1359 struct mcx_cmd_destroy_tis_in { 1360 uint16_t cmd_opcode; 1361 uint8_t cmd_reserved0[4]; 1362 uint16_t cmd_op_mod; 1363 uint32_t cmd_tisn; 1364 uint8_t cmd_reserved1[4]; 1365 } __packed __aligned(4); 1366 1367 struct mcx_cmd_destroy_tis_out { 1368 uint8_t cmd_status; 1369 uint8_t cmd_reserved0[3]; 1370 uint32_t cmd_syndrome; 1371 uint8_t cmd_reserved1[8]; 1372 } __packed __aligned(4); 1373 1374 struct mcx_cmd_create_rqt_in { 1375 uint16_t cmd_opcode; 1376 uint8_t cmd_reserved0[4]; 1377 uint16_t cmd_op_mod; 1378 uint8_t cmd_reserved1[8]; 1379 } __packed __aligned(4); 1380 1381 struct mcx_rqt_ctx { 1382 uint8_t cmd_reserved0[20]; 1383 uint16_t cmd_reserved1; 1384 uint16_t cmd_rqt_max_size; 1385 uint16_t cmd_reserved2; 1386 uint16_t cmd_rqt_actual_size; 1387 uint8_t cmd_reserved3[212]; 1388 } __packed __aligned(4); 1389 1390 struct mcx_cmd_create_rqt_mb_in { 1391 uint8_t cmd_reserved0[16]; 1392 struct mcx_rqt_ctx cmd_rqt; 1393 } __packed __aligned(4); 1394 1395 struct mcx_cmd_create_rqt_out { 1396 uint8_t cmd_status; 1397 uint8_t cmd_reserved0[3]; 1398 uint32_t cmd_syndrome; 1399 uint32_t cmd_rqtn; 1400 uint8_t cmd_reserved1[4]; 1401 } __packed __aligned(4); 1402 1403 struct mcx_cmd_destroy_rqt_in { 1404 uint16_t cmd_opcode; 1405 uint8_t cmd_reserved0[4]; 1406 uint16_t cmd_op_mod; 1407 uint32_t cmd_rqtn; 1408 uint8_t cmd_reserved1[4]; 1409 } __packed __aligned(4); 1410 1411 struct mcx_cmd_destroy_rqt_out { 1412 uint8_t cmd_status; 1413 uint8_t cmd_reserved0[3]; 1414 uint32_t cmd_syndrome; 1415 uint8_t cmd_reserved1[8]; 1416 } __packed __aligned(4); 1417 1418 struct mcx_cq_ctx { 1419 uint32_t cq_status; 1420 #define MCX_CQ_CTX_STATUS_SHIFT 28 1421 #define MCX_CQ_CTX_STATUS_MASK (0xf << MCX_CQ_CTX_STATUS_SHIFT) 1422 #define MCX_CQ_CTX_STATUS_OK 0x0 1423 #define MCX_CQ_CTX_STATUS_OVERFLOW 0x9 1424 #define MCX_CQ_CTX_STATUS_WRITE_FAIL 0xa 1425 #define MCX_CQ_CTX_STATE_SHIFT 8 1426 #define MCX_CQ_CTX_STATE_MASK (0xf << MCX_CQ_CTX_STATE_SHIFT) 1427 #define MCX_CQ_CTX_STATE_SOLICITED 0x6 1428 #define MCX_CQ_CTX_STATE_ARMED 0x9 1429 #define MCX_CQ_CTX_STATE_FIRED 0xa 1430 uint32_t cq_reserved1; 1431 uint32_t cq_page_offset; 1432 uint32_t cq_uar_size; 1433 #define MCX_CQ_CTX_UAR_PAGE_MASK 0xffffff 1434 #define MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT 24 1435 uint32_t cq_period_max_count; 1436 #define MCX_CQ_CTX_PERIOD_SHIFT 16 1437 uint32_t cq_eqn; 1438 uint32_t cq_log_page_size; 1439 #define MCX_CQ_CTX_LOG_PAGE_SIZE_SHIFT 24 1440 uint32_t cq_reserved2; 1441 uint32_t cq_last_notified; 1442 uint32_t cq_last_solicit; 1443 uint32_t cq_consumer_counter; 1444 uint32_t cq_producer_counter; 1445 uint8_t cq_reserved3[8]; 1446 uint64_t cq_doorbell; 1447 } __packed __aligned(4); 1448 1449 CTASSERT(sizeof(struct mcx_cq_ctx) == 64); 1450 1451 struct mcx_cmd_create_cq_in { 1452 uint16_t cmd_opcode; 1453 uint8_t cmd_reserved0[4]; 1454 uint16_t cmd_op_mod; 1455 uint8_t cmd_reserved1[8]; 1456 } __packed __aligned(4); 1457 1458 struct mcx_cmd_create_cq_mb_in { 1459 struct mcx_cq_ctx cmd_cq_ctx; 1460 uint8_t cmd_reserved1[192]; 1461 } __packed __aligned(4); 1462 1463 struct mcx_cmd_create_cq_out { 1464 uint8_t cmd_status; 1465 uint8_t cmd_reserved0[3]; 1466 uint32_t cmd_syndrome; 1467 uint32_t cmd_cqn; 1468 uint8_t cmd_reserved1[4]; 1469 } __packed __aligned(4); 1470 1471 struct mcx_cmd_destroy_cq_in { 1472 uint16_t cmd_opcode; 1473 uint8_t cmd_reserved0[4]; 1474 uint16_t cmd_op_mod; 1475 uint32_t cmd_cqn; 1476 uint8_t cmd_reserved1[4]; 1477 } __packed __aligned(4); 1478 1479 struct mcx_cmd_destroy_cq_out { 1480 uint8_t cmd_status; 1481 uint8_t cmd_reserved0[3]; 1482 uint32_t cmd_syndrome; 1483 uint8_t cmd_reserved1[8]; 1484 } __packed __aligned(4); 1485 1486 struct mcx_cmd_query_cq_in { 1487 uint16_t cmd_opcode; 1488 uint8_t cmd_reserved0[4]; 1489 uint16_t cmd_op_mod; 1490 uint32_t cmd_cqn; 1491 uint8_t cmd_reserved1[4]; 1492 } __packed __aligned(4); 1493 1494 struct mcx_cmd_query_cq_out { 1495 uint8_t cmd_status; 1496 uint8_t cmd_reserved0[3]; 1497 uint32_t cmd_syndrome; 1498 uint8_t cmd_reserved1[8]; 1499 } __packed __aligned(4); 1500 1501 struct mcx_cq_entry { 1502 uint32_t __reserved__; 1503 uint32_t cq_lro; 1504 uint32_t cq_lro_ack_seq_num; 1505 uint32_t cq_rx_hash; 1506 uint8_t cq_rx_hash_type; 1507 uint8_t cq_ml_path; 1508 uint16_t __reserved__; 1509 uint32_t cq_checksum; 1510 uint32_t __reserved__; 1511 uint32_t cq_flags; 1512 #define MCX_CQ_ENTRY_FLAGS_L4_OK (1 << 26) 1513 #define MCX_CQ_ENTRY_FLAGS_L3_OK (1 << 25) 1514 #define MCX_CQ_ENTRY_FLAGS_L2_OK (1 << 24) 1515 #define MCX_CQ_ENTRY_FLAGS_CV (1 << 16) 1516 #define MCX_CQ_ENTRY_FLAGS_VLAN_MASK (0xffff) 1517 1518 uint32_t cq_lro_srqn; 1519 uint32_t __reserved__[2]; 1520 uint32_t cq_byte_cnt; 1521 uint64_t cq_timestamp; 1522 uint8_t cq_rx_drops; 1523 uint8_t cq_flow_tag[3]; 1524 uint16_t cq_wqe_count; 1525 uint8_t cq_signature; 1526 uint8_t cq_opcode_owner; 1527 #define MCX_CQ_ENTRY_FLAG_OWNER (1 << 0) 1528 #define MCX_CQ_ENTRY_FLAG_SE (1 << 1) 1529 #define MCX_CQ_ENTRY_FORMAT_SHIFT 2 1530 #define MCX_CQ_ENTRY_OPCODE_SHIFT 4 1531 1532 #define MCX_CQ_ENTRY_FORMAT_NO_INLINE 0 1533 #define MCX_CQ_ENTRY_FORMAT_INLINE_32 1 1534 #define MCX_CQ_ENTRY_FORMAT_INLINE_64 2 1535 #define MCX_CQ_ENTRY_FORMAT_COMPRESSED 3 1536 1537 #define MCX_CQ_ENTRY_OPCODE_REQ 0 1538 #define MCX_CQ_ENTRY_OPCODE_SEND 2 1539 #define MCX_CQ_ENTRY_OPCODE_REQ_ERR 13 1540 #define MCX_CQ_ENTRY_OPCODE_SEND_ERR 14 1541 #define MCX_CQ_ENTRY_OPCODE_INVALID 15 1542 1543 } __packed __aligned(4); 1544 1545 CTASSERT(sizeof(struct mcx_cq_entry) == 64); 1546 1547 struct mcx_cq_doorbell { 1548 uint32_t db_update_ci; 1549 uint32_t db_arm_ci; 1550 #define MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT 28 1551 #define MCX_CQ_DOORBELL_ARM_CMD (1 << 24) 1552 #define MCX_CQ_DOORBELL_ARM_CI_MASK (0xffffff) 1553 } __packed __aligned(8); 1554 1555 struct mcx_wq_ctx { 1556 uint8_t wq_type; 1557 #define MCX_WQ_CTX_TYPE_CYCLIC (1 << 4) 1558 #define MCX_WQ_CTX_TYPE_SIGNATURE (1 << 3) 1559 uint8_t wq_reserved0[5]; 1560 uint16_t wq_lwm; 1561 uint32_t wq_pd; 1562 uint32_t wq_uar_page; 1563 uint64_t wq_doorbell; 1564 uint32_t wq_hw_counter; 1565 uint32_t wq_sw_counter; 1566 uint16_t wq_log_stride; 1567 uint8_t wq_log_page_sz; 1568 uint8_t wq_log_size; 1569 uint8_t wq_reserved1[156]; 1570 } __packed __aligned(4); 1571 1572 CTASSERT(sizeof(struct mcx_wq_ctx) == 0xC0); 1573 1574 struct mcx_sq_ctx { 1575 uint32_t sq_flags; 1576 #define MCX_SQ_CTX_RLKEY (1 << 31) 1577 #define MCX_SQ_CTX_FRE_SHIFT (1 << 29) 1578 #define MCX_SQ_CTX_FLUSH_IN_ERROR (1 << 28) 1579 #define MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT 24 1580 #define MCX_SQ_CTX_STATE_SHIFT 20 1581 #define MCX_SQ_CTX_STATE_MASK (0xf << 20) 1582 #define MCX_SQ_CTX_STATE_RST 0 1583 #define MCX_SQ_CTX_STATE_RDY 1 1584 #define MCX_SQ_CTX_STATE_ERR 3 1585 uint32_t sq_user_index; 1586 uint32_t sq_cqn; 1587 uint32_t sq_reserved1[5]; 1588 uint32_t sq_tis_lst_sz; 1589 #define MCX_SQ_CTX_TIS_LST_SZ_SHIFT 16 1590 uint32_t sq_reserved2[2]; 1591 uint32_t sq_tis_num; 1592 struct mcx_wq_ctx sq_wq; 1593 } __packed __aligned(4); 1594 1595 struct mcx_sq_entry_seg { 1596 uint32_t sqs_byte_count; 1597 uint32_t sqs_lkey; 1598 uint64_t sqs_addr; 1599 } __packed __aligned(4); 1600 1601 struct mcx_sq_entry { 1602 /* control segment */ 1603 uint32_t sqe_opcode_index; 1604 #define MCX_SQE_WQE_INDEX_SHIFT 8 1605 #define MCX_SQE_WQE_OPCODE_NOP 0x00 1606 #define MCX_SQE_WQE_OPCODE_SEND 0x0a 1607 uint32_t sqe_ds_sq_num; 1608 #define MCX_SQE_SQ_NUM_SHIFT 8 1609 uint32_t sqe_signature; 1610 #define MCX_SQE_SIGNATURE_SHIFT 24 1611 #define MCX_SQE_SOLICITED_EVENT 0x02 1612 #define MCX_SQE_CE_CQE_ON_ERR 0x00 1613 #define MCX_SQE_CE_CQE_FIRST_ERR 0x04 1614 #define MCX_SQE_CE_CQE_ALWAYS 0x08 1615 #define MCX_SQE_CE_CQE_SOLICIT 0x0C 1616 #define MCX_SQE_FM_NO_FENCE 0x00 1617 #define MCX_SQE_FM_SMALL_FENCE 0x40 1618 uint32_t sqe_mkey; 1619 1620 /* ethernet segment */ 1621 uint32_t sqe_reserved1; 1622 uint32_t sqe_mss_csum; 1623 #define MCX_SQE_L4_CSUM (1 << 31) 1624 #define MCX_SQE_L3_CSUM (1 << 30) 1625 uint32_t sqe_reserved2; 1626 uint16_t sqe_inline_header_size; 1627 uint16_t sqe_inline_headers[9]; 1628 1629 /* data segment */ 1630 struct mcx_sq_entry_seg sqe_segs[1]; 1631 } __packed __aligned(64); 1632 1633 CTASSERT(sizeof(struct mcx_sq_entry) == 64); 1634 1635 struct mcx_cmd_create_sq_in { 1636 uint16_t cmd_opcode; 1637 uint8_t cmd_reserved0[4]; 1638 uint16_t cmd_op_mod; 1639 uint8_t cmd_reserved1[8]; 1640 } __packed __aligned(4); 1641 1642 struct mcx_cmd_create_sq_out { 1643 uint8_t cmd_status; 1644 uint8_t cmd_reserved0[3]; 1645 uint32_t cmd_syndrome; 1646 uint32_t cmd_sqn; 1647 uint8_t cmd_reserved1[4]; 1648 } __packed __aligned(4); 1649 1650 struct mcx_cmd_modify_sq_in { 1651 uint16_t cmd_opcode; 1652 uint8_t cmd_reserved0[4]; 1653 uint16_t cmd_op_mod; 1654 uint32_t cmd_sq_state; 1655 uint8_t cmd_reserved1[4]; 1656 } __packed __aligned(4); 1657 1658 struct mcx_cmd_modify_sq_mb_in { 1659 uint32_t cmd_modify_hi; 1660 uint32_t cmd_modify_lo; 1661 uint8_t cmd_reserved0[8]; 1662 struct mcx_sq_ctx cmd_sq_ctx; 1663 } __packed __aligned(4); 1664 1665 struct mcx_cmd_modify_sq_out { 1666 uint8_t cmd_status; 1667 uint8_t cmd_reserved0[3]; 1668 uint32_t cmd_syndrome; 1669 uint8_t cmd_reserved1[8]; 1670 } __packed __aligned(4); 1671 1672 struct mcx_cmd_destroy_sq_in { 1673 uint16_t cmd_opcode; 1674 uint8_t cmd_reserved0[4]; 1675 uint16_t cmd_op_mod; 1676 uint32_t cmd_sqn; 1677 uint8_t cmd_reserved1[4]; 1678 } __packed __aligned(4); 1679 1680 struct mcx_cmd_destroy_sq_out { 1681 uint8_t cmd_status; 1682 uint8_t cmd_reserved0[3]; 1683 uint32_t cmd_syndrome; 1684 uint8_t cmd_reserved1[8]; 1685 } __packed __aligned(4); 1686 1687 1688 struct mcx_rq_ctx { 1689 uint32_t rq_flags; 1690 #define MCX_RQ_CTX_RLKEY (1 << 31) 1691 #define MCX_RQ_CTX_VLAN_STRIP_DIS (1 << 28) 1692 #define MCX_RQ_CTX_MEM_RQ_TYPE_SHIFT 24 1693 #define MCX_RQ_CTX_STATE_SHIFT 20 1694 #define MCX_RQ_CTX_STATE_MASK (0xf << 20) 1695 #define MCX_RQ_CTX_STATE_RST 0 1696 #define MCX_RQ_CTX_STATE_RDY 1 1697 #define MCX_RQ_CTX_STATE_ERR 3 1698 #define MCX_RQ_CTX_FLUSH_IN_ERROR (1 << 18) 1699 uint32_t rq_user_index; 1700 uint32_t rq_cqn; 1701 uint32_t rq_reserved1; 1702 uint32_t rq_rmpn; 1703 uint32_t rq_reserved2[7]; 1704 struct mcx_wq_ctx rq_wq; 1705 } __packed __aligned(4); 1706 1707 struct mcx_rq_entry { 1708 uint32_t rqe_byte_count; 1709 uint32_t rqe_lkey; 1710 uint64_t rqe_addr; 1711 } __packed __aligned(16); 1712 1713 struct mcx_cmd_create_rq_in { 1714 uint16_t cmd_opcode; 1715 uint8_t cmd_reserved0[4]; 1716 uint16_t cmd_op_mod; 1717 uint8_t cmd_reserved1[8]; 1718 } __packed __aligned(4); 1719 1720 struct mcx_cmd_create_rq_out { 1721 uint8_t cmd_status; 1722 uint8_t cmd_reserved0[3]; 1723 uint32_t cmd_syndrome; 1724 uint32_t cmd_rqn; 1725 uint8_t cmd_reserved1[4]; 1726 } __packed __aligned(4); 1727 1728 struct mcx_cmd_modify_rq_in { 1729 uint16_t cmd_opcode; 1730 uint8_t cmd_reserved0[4]; 1731 uint16_t cmd_op_mod; 1732 uint32_t cmd_rq_state; 1733 uint8_t cmd_reserved1[4]; 1734 } __packed __aligned(4); 1735 1736 struct mcx_cmd_modify_rq_mb_in { 1737 uint32_t cmd_modify_hi; 1738 uint32_t cmd_modify_lo; 1739 uint8_t cmd_reserved0[8]; 1740 struct mcx_rq_ctx cmd_rq_ctx; 1741 } __packed __aligned(4); 1742 1743 struct mcx_cmd_modify_rq_out { 1744 uint8_t cmd_status; 1745 uint8_t cmd_reserved0[3]; 1746 uint32_t cmd_syndrome; 1747 uint8_t cmd_reserved1[8]; 1748 } __packed __aligned(4); 1749 1750 struct mcx_cmd_destroy_rq_in { 1751 uint16_t cmd_opcode; 1752 uint8_t cmd_reserved0[4]; 1753 uint16_t cmd_op_mod; 1754 uint32_t cmd_rqn; 1755 uint8_t cmd_reserved1[4]; 1756 } __packed __aligned(4); 1757 1758 struct mcx_cmd_destroy_rq_out { 1759 uint8_t cmd_status; 1760 uint8_t cmd_reserved0[3]; 1761 uint32_t cmd_syndrome; 1762 uint8_t cmd_reserved1[8]; 1763 } __packed __aligned(4); 1764 1765 struct mcx_cmd_create_flow_table_in { 1766 uint16_t cmd_opcode; 1767 uint8_t cmd_reserved0[4]; 1768 uint16_t cmd_op_mod; 1769 uint8_t cmd_reserved1[8]; 1770 } __packed __aligned(4); 1771 1772 struct mcx_flow_table_ctx { 1773 uint8_t ft_miss_action; 1774 uint8_t ft_level; 1775 uint8_t ft_reserved0; 1776 uint8_t ft_log_size; 1777 uint32_t ft_table_miss_id; 1778 uint8_t ft_reserved1[28]; 1779 } __packed __aligned(4); 1780 1781 struct mcx_cmd_create_flow_table_mb_in { 1782 uint8_t cmd_table_type; 1783 uint8_t cmd_reserved0[7]; 1784 struct mcx_flow_table_ctx cmd_ctx; 1785 } __packed __aligned(4); 1786 1787 struct mcx_cmd_create_flow_table_out { 1788 uint8_t cmd_status; 1789 uint8_t cmd_reserved0[3]; 1790 uint32_t cmd_syndrome; 1791 uint32_t cmd_table_id; 1792 uint8_t cmd_reserved1[4]; 1793 } __packed __aligned(4); 1794 1795 struct mcx_cmd_destroy_flow_table_in { 1796 uint16_t cmd_opcode; 1797 uint8_t cmd_reserved0[4]; 1798 uint16_t cmd_op_mod; 1799 uint8_t cmd_reserved1[8]; 1800 } __packed __aligned(4); 1801 1802 struct mcx_cmd_destroy_flow_table_mb_in { 1803 uint8_t cmd_table_type; 1804 uint8_t cmd_reserved0[3]; 1805 uint32_t cmd_table_id; 1806 uint8_t cmd_reserved1[40]; 1807 } __packed __aligned(4); 1808 1809 struct mcx_cmd_destroy_flow_table_out { 1810 uint8_t cmd_status; 1811 uint8_t cmd_reserved0[3]; 1812 uint32_t cmd_syndrome; 1813 uint8_t cmd_reserved1[8]; 1814 } __packed __aligned(4); 1815 1816 struct mcx_cmd_set_flow_table_root_in { 1817 uint16_t cmd_opcode; 1818 uint8_t cmd_reserved0[4]; 1819 uint16_t cmd_op_mod; 1820 uint8_t cmd_reserved1[8]; 1821 } __packed __aligned(4); 1822 1823 struct mcx_cmd_set_flow_table_root_mb_in { 1824 uint8_t cmd_table_type; 1825 uint8_t cmd_reserved0[3]; 1826 uint32_t cmd_table_id; 1827 uint8_t cmd_reserved1[56]; 1828 } __packed __aligned(4); 1829 1830 struct mcx_cmd_set_flow_table_root_out { 1831 uint8_t cmd_status; 1832 uint8_t cmd_reserved0[3]; 1833 uint32_t cmd_syndrome; 1834 uint8_t cmd_reserved1[8]; 1835 } __packed __aligned(4); 1836 1837 struct mcx_flow_match { 1838 /* outer headers */ 1839 uint8_t mc_src_mac[6]; 1840 uint16_t mc_ethertype; 1841 uint8_t mc_dest_mac[6]; 1842 uint16_t mc_first_vlan; 1843 uint8_t mc_ip_proto; 1844 uint8_t mc_ip_dscp_ecn; 1845 uint8_t mc_vlan_flags; 1846 #define MCX_FLOW_MATCH_IP_FRAG (1 << 5) 1847 uint8_t mc_tcp_flags; 1848 uint16_t mc_tcp_sport; 1849 uint16_t mc_tcp_dport; 1850 uint32_t mc_reserved0; 1851 uint16_t mc_udp_sport; 1852 uint16_t mc_udp_dport; 1853 uint8_t mc_src_ip[16]; 1854 uint8_t mc_dest_ip[16]; 1855 1856 /* misc parameters */ 1857 uint8_t mc_reserved1[8]; 1858 uint16_t mc_second_vlan; 1859 uint8_t mc_reserved2[2]; 1860 uint8_t mc_second_vlan_flags; 1861 uint8_t mc_reserved3[15]; 1862 uint32_t mc_outer_ipv6_flow_label; 1863 uint8_t mc_reserved4[32]; 1864 1865 uint8_t mc_reserved[384]; 1866 } __packed __aligned(4); 1867 1868 CTASSERT(sizeof(struct mcx_flow_match) == 512); 1869 1870 struct mcx_cmd_create_flow_group_in { 1871 uint16_t cmd_opcode; 1872 uint8_t cmd_reserved0[4]; 1873 uint16_t cmd_op_mod; 1874 uint8_t cmd_reserved1[8]; 1875 } __packed __aligned(4); 1876 1877 struct mcx_cmd_create_flow_group_mb_in { 1878 uint8_t cmd_table_type; 1879 uint8_t cmd_reserved0[3]; 1880 uint32_t cmd_table_id; 1881 uint8_t cmd_reserved1[4]; 1882 uint32_t cmd_start_flow_index; 1883 uint8_t cmd_reserved2[4]; 1884 uint32_t cmd_end_flow_index; 1885 uint8_t cmd_reserved3[23]; 1886 uint8_t cmd_match_criteria_enable; 1887 #define MCX_CREATE_FLOW_GROUP_CRIT_OUTER (1 << 0) 1888 #define MCX_CREATE_FLOW_GROUP_CRIT_MISC (1 << 1) 1889 #define MCX_CREATE_FLOW_GROUP_CRIT_INNER (1 << 2) 1890 struct mcx_flow_match cmd_match_criteria; 1891 uint8_t cmd_reserved4[448]; 1892 } __packed __aligned(4); 1893 1894 struct mcx_cmd_create_flow_group_out { 1895 uint8_t cmd_status; 1896 uint8_t cmd_reserved0[3]; 1897 uint32_t cmd_syndrome; 1898 uint32_t cmd_group_id; 1899 uint8_t cmd_reserved1[4]; 1900 } __packed __aligned(4); 1901 1902 struct mcx_flow_ctx { 1903 uint8_t fc_reserved0[4]; 1904 uint32_t fc_group_id; 1905 uint32_t fc_flow_tag; 1906 uint32_t fc_action; 1907 #define MCX_FLOW_CONTEXT_ACTION_ALLOW (1 << 0) 1908 #define MCX_FLOW_CONTEXT_ACTION_DROP (1 << 1) 1909 #define MCX_FLOW_CONTEXT_ACTION_FORWARD (1 << 2) 1910 #define MCX_FLOW_CONTEXT_ACTION_COUNT (1 << 3) 1911 uint32_t fc_dest_list_size; 1912 uint32_t fc_counter_list_size; 1913 uint8_t fc_reserved1[40]; 1914 struct mcx_flow_match fc_match_value; 1915 uint8_t fc_reserved2[192]; 1916 } __packed __aligned(4); 1917 1918 #define MCX_FLOW_CONTEXT_DEST_TYPE_TABLE (1 << 24) 1919 #define MCX_FLOW_CONTEXT_DEST_TYPE_TIR (2 << 24) 1920 1921 struct mcx_cmd_destroy_flow_group_in { 1922 uint16_t cmd_opcode; 1923 uint8_t cmd_reserved0[4]; 1924 uint16_t cmd_op_mod; 1925 uint8_t cmd_reserved1[8]; 1926 } __packed __aligned(4); 1927 1928 struct mcx_cmd_destroy_flow_group_mb_in { 1929 uint8_t cmd_table_type; 1930 uint8_t cmd_reserved0[3]; 1931 uint32_t cmd_table_id; 1932 uint32_t cmd_group_id; 1933 uint8_t cmd_reserved1[36]; 1934 } __packed __aligned(4); 1935 1936 struct mcx_cmd_destroy_flow_group_out { 1937 uint8_t cmd_status; 1938 uint8_t cmd_reserved0[3]; 1939 uint32_t cmd_syndrome; 1940 uint8_t cmd_reserved1[8]; 1941 } __packed __aligned(4); 1942 1943 struct mcx_cmd_set_flow_table_entry_in { 1944 uint16_t cmd_opcode; 1945 uint8_t cmd_reserved0[4]; 1946 uint16_t cmd_op_mod; 1947 uint8_t cmd_reserved1[8]; 1948 } __packed __aligned(4); 1949 1950 struct mcx_cmd_set_flow_table_entry_mb_in { 1951 uint8_t cmd_table_type; 1952 uint8_t cmd_reserved0[3]; 1953 uint32_t cmd_table_id; 1954 uint32_t cmd_modify_enable_mask; 1955 uint8_t cmd_reserved1[4]; 1956 uint32_t cmd_flow_index; 1957 uint8_t cmd_reserved2[28]; 1958 struct mcx_flow_ctx cmd_flow_ctx; 1959 } __packed __aligned(4); 1960 1961 struct mcx_cmd_set_flow_table_entry_out { 1962 uint8_t cmd_status; 1963 uint8_t cmd_reserved0[3]; 1964 uint32_t cmd_syndrome; 1965 uint8_t cmd_reserved1[8]; 1966 } __packed __aligned(4); 1967 1968 struct mcx_cmd_query_flow_table_entry_in { 1969 uint16_t cmd_opcode; 1970 uint8_t cmd_reserved0[4]; 1971 uint16_t cmd_op_mod; 1972 uint8_t cmd_reserved1[8]; 1973 } __packed __aligned(4); 1974 1975 struct mcx_cmd_query_flow_table_entry_mb_in { 1976 uint8_t cmd_table_type; 1977 uint8_t cmd_reserved0[3]; 1978 uint32_t cmd_table_id; 1979 uint8_t cmd_reserved1[8]; 1980 uint32_t cmd_flow_index; 1981 uint8_t cmd_reserved2[28]; 1982 } __packed __aligned(4); 1983 1984 struct mcx_cmd_query_flow_table_entry_out { 1985 uint8_t cmd_status; 1986 uint8_t cmd_reserved0[3]; 1987 uint32_t cmd_syndrome; 1988 uint8_t cmd_reserved1[8]; 1989 } __packed __aligned(4); 1990 1991 struct mcx_cmd_query_flow_table_entry_mb_out { 1992 uint8_t cmd_reserved0[48]; 1993 struct mcx_flow_ctx cmd_flow_ctx; 1994 } __packed __aligned(4); 1995 1996 struct mcx_cmd_delete_flow_table_entry_in { 1997 uint16_t cmd_opcode; 1998 uint8_t cmd_reserved0[4]; 1999 uint16_t cmd_op_mod; 2000 uint8_t cmd_reserved1[8]; 2001 } __packed __aligned(4); 2002 2003 struct mcx_cmd_delete_flow_table_entry_mb_in { 2004 uint8_t cmd_table_type; 2005 uint8_t cmd_reserved0[3]; 2006 uint32_t cmd_table_id; 2007 uint8_t cmd_reserved1[8]; 2008 uint32_t cmd_flow_index; 2009 uint8_t cmd_reserved2[28]; 2010 } __packed __aligned(4); 2011 2012 struct mcx_cmd_delete_flow_table_entry_out { 2013 uint8_t cmd_status; 2014 uint8_t cmd_reserved0[3]; 2015 uint32_t cmd_syndrome; 2016 uint8_t cmd_reserved1[8]; 2017 } __packed __aligned(4); 2018 2019 struct mcx_cmd_query_flow_group_in { 2020 uint16_t cmd_opcode; 2021 uint8_t cmd_reserved0[4]; 2022 uint16_t cmd_op_mod; 2023 uint8_t cmd_reserved1[8]; 2024 } __packed __aligned(4); 2025 2026 struct mcx_cmd_query_flow_group_mb_in { 2027 uint8_t cmd_table_type; 2028 uint8_t cmd_reserved0[3]; 2029 uint32_t cmd_table_id; 2030 uint32_t cmd_group_id; 2031 uint8_t cmd_reserved1[36]; 2032 } __packed __aligned(4); 2033 2034 struct mcx_cmd_query_flow_group_out { 2035 uint8_t cmd_status; 2036 uint8_t cmd_reserved0[3]; 2037 uint32_t cmd_syndrome; 2038 uint8_t cmd_reserved1[8]; 2039 } __packed __aligned(4); 2040 2041 struct mcx_cmd_query_flow_group_mb_out { 2042 uint8_t cmd_reserved0[12]; 2043 uint32_t cmd_start_flow_index; 2044 uint8_t cmd_reserved1[4]; 2045 uint32_t cmd_end_flow_index; 2046 uint8_t cmd_reserved2[20]; 2047 uint32_t cmd_match_criteria_enable; 2048 uint8_t cmd_match_criteria[512]; 2049 uint8_t cmd_reserved4[448]; 2050 } __packed __aligned(4); 2051 2052 struct mcx_cmd_query_flow_table_in { 2053 uint16_t cmd_opcode; 2054 uint8_t cmd_reserved0[4]; 2055 uint16_t cmd_op_mod; 2056 uint8_t cmd_reserved1[8]; 2057 } __packed __aligned(4); 2058 2059 struct mcx_cmd_query_flow_table_mb_in { 2060 uint8_t cmd_table_type; 2061 uint8_t cmd_reserved0[3]; 2062 uint32_t cmd_table_id; 2063 uint8_t cmd_reserved1[40]; 2064 } __packed __aligned(4); 2065 2066 struct mcx_cmd_query_flow_table_out { 2067 uint8_t cmd_status; 2068 uint8_t cmd_reserved0[3]; 2069 uint32_t cmd_syndrome; 2070 uint8_t cmd_reserved1[8]; 2071 } __packed __aligned(4); 2072 2073 struct mcx_cmd_query_flow_table_mb_out { 2074 uint8_t cmd_reserved0[4]; 2075 struct mcx_flow_table_ctx cmd_ctx; 2076 } __packed __aligned(4); 2077 2078 struct mcx_cmd_alloc_flow_counter_in { 2079 uint16_t cmd_opcode; 2080 uint8_t cmd_reserved0[4]; 2081 uint16_t cmd_op_mod; 2082 uint8_t cmd_reserved1[8]; 2083 } __packed __aligned(4); 2084 2085 struct mcx_cmd_query_rq_in { 2086 uint16_t cmd_opcode; 2087 uint8_t cmd_reserved0[4]; 2088 uint16_t cmd_op_mod; 2089 uint32_t cmd_rqn; 2090 uint8_t cmd_reserved1[4]; 2091 } __packed __aligned(4); 2092 2093 struct mcx_cmd_query_rq_out { 2094 uint8_t cmd_status; 2095 uint8_t cmd_reserved0[3]; 2096 uint32_t cmd_syndrome; 2097 uint8_t cmd_reserved1[8]; 2098 } __packed __aligned(4); 2099 2100 struct mcx_cmd_query_rq_mb_out { 2101 uint8_t cmd_reserved0[16]; 2102 struct mcx_rq_ctx cmd_ctx; 2103 }; 2104 2105 struct mcx_cmd_query_sq_in { 2106 uint16_t cmd_opcode; 2107 uint8_t cmd_reserved0[4]; 2108 uint16_t cmd_op_mod; 2109 uint32_t cmd_sqn; 2110 uint8_t cmd_reserved1[4]; 2111 } __packed __aligned(4); 2112 2113 struct mcx_cmd_query_sq_out { 2114 uint8_t cmd_status; 2115 uint8_t cmd_reserved0[3]; 2116 uint32_t cmd_syndrome; 2117 uint8_t cmd_reserved1[8]; 2118 } __packed __aligned(4); 2119 2120 struct mcx_cmd_query_sq_mb_out { 2121 uint8_t cmd_reserved0[16]; 2122 struct mcx_sq_ctx cmd_ctx; 2123 }; 2124 2125 struct mcx_cmd_alloc_flow_counter_out { 2126 uint8_t cmd_status; 2127 uint8_t cmd_reserved0[3]; 2128 uint32_t cmd_syndrome; 2129 uint8_t cmd_reserved1[2]; 2130 uint16_t cmd_flow_counter_id; 2131 uint8_t cmd_reserved2[4]; 2132 } __packed __aligned(4); 2133 2134 struct mcx_wq_doorbell { 2135 uint32_t db_recv_counter; 2136 uint32_t db_send_counter; 2137 } __packed __aligned(8); 2138 2139 struct mcx_dmamem { 2140 bus_dmamap_t mxm_map; 2141 bus_dma_segment_t mxm_seg; 2142 int mxm_nsegs; 2143 size_t mxm_size; 2144 caddr_t mxm_kva; 2145 }; 2146 #define MCX_DMA_MAP(_mxm) ((_mxm)->mxm_map) 2147 #define MCX_DMA_DVA(_mxm) ((_mxm)->mxm_map->dm_segs[0].ds_addr) 2148 #define MCX_DMA_KVA(_mxm) ((void *)(_mxm)->mxm_kva) 2149 #define MCX_DMA_LEN(_mxm) ((_mxm)->mxm_size) 2150 2151 struct mcx_hwmem { 2152 bus_dmamap_t mhm_map; 2153 bus_dma_segment_t *mhm_segs; 2154 unsigned int mhm_seg_count; 2155 unsigned int mhm_npages; 2156 }; 2157 2158 struct mcx_slot { 2159 bus_dmamap_t ms_map; 2160 struct mbuf *ms_m; 2161 }; 2162 2163 struct mcx_eq { 2164 int eq_n; 2165 uint32_t eq_cons; 2166 struct mcx_dmamem eq_mem; 2167 }; 2168 2169 struct mcx_cq { 2170 int cq_n; 2171 struct mcx_dmamem cq_mem; 2172 uint32_t *cq_doorbell; 2173 uint32_t cq_cons; 2174 uint32_t cq_count; 2175 }; 2176 2177 struct mcx_calibration { 2178 uint64_t c_timestamp; /* previous mcx chip time */ 2179 uint64_t c_uptime; /* previous kernel nanouptime */ 2180 uint64_t c_tbase; /* mcx chip time */ 2181 uint64_t c_ubase; /* kernel nanouptime */ 2182 uint64_t c_tdiff; 2183 uint64_t c_udiff; 2184 }; 2185 2186 #define MCX_CALIBRATE_FIRST 2 2187 #define MCX_CALIBRATE_NORMAL 30 2188 2189 struct mcx_rx { 2190 struct mcx_softc *rx_softc; 2191 struct ifiqueue *rx_ifiq; 2192 2193 int rx_rqn; 2194 struct mcx_dmamem rx_rq_mem; 2195 struct mcx_slot *rx_slots; 2196 uint32_t *rx_doorbell; 2197 2198 uint32_t rx_prod; 2199 struct timeout rx_refill; 2200 struct if_rxring rx_rxr; 2201 } __aligned(64); 2202 2203 struct mcx_tx { 2204 struct mcx_softc *tx_softc; 2205 struct ifqueue *tx_ifq; 2206 2207 int tx_uar; 2208 int tx_sqn; 2209 struct mcx_dmamem tx_sq_mem; 2210 struct mcx_slot *tx_slots; 2211 uint32_t *tx_doorbell; 2212 int tx_bf_offset; 2213 2214 uint32_t tx_cons; 2215 uint32_t tx_prod; 2216 } __aligned(64); 2217 2218 struct mcx_queues { 2219 char q_name[16]; 2220 void *q_ihc; 2221 struct mcx_softc *q_sc; 2222 int q_uar; 2223 int q_index; 2224 struct mcx_rx q_rx; 2225 struct mcx_tx q_tx; 2226 struct mcx_cq q_cq; 2227 struct mcx_eq q_eq; 2228 #if NKSTAT > 0 2229 struct kstat *q_kstat; 2230 #endif 2231 }; 2232 2233 struct mcx_flow_group { 2234 int g_id; 2235 int g_table; 2236 int g_start; 2237 int g_size; 2238 }; 2239 2240 #define MCX_FLOW_GROUP_PROMISC 0 2241 #define MCX_FLOW_GROUP_ALLMULTI 1 2242 #define MCX_FLOW_GROUP_MAC 2 2243 #define MCX_FLOW_GROUP_RSS_L4 3 2244 #define MCX_FLOW_GROUP_RSS_L3 4 2245 #define MCX_FLOW_GROUP_RSS_NONE 5 2246 #define MCX_NUM_FLOW_GROUPS 6 2247 2248 #define MCX_HASH_SEL_L3 MCX_TIR_CTX_HASH_SEL_SRC_IP | \ 2249 MCX_TIR_CTX_HASH_SEL_DST_IP 2250 #define MCX_HASH_SEL_L4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_SPORT | \ 2251 MCX_TIR_CTX_HASH_SEL_DPORT 2252 2253 #define MCX_RSS_HASH_SEL_V4_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP |\ 2254 MCX_TIR_CTX_HASH_SEL_IPV4 2255 #define MCX_RSS_HASH_SEL_V6_TCP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_TCP | \ 2256 MCX_TIR_CTX_HASH_SEL_IPV6 2257 #define MCX_RSS_HASH_SEL_V4_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2258 MCX_TIR_CTX_HASH_SEL_IPV4 2259 #define MCX_RSS_HASH_SEL_V6_UDP MCX_HASH_SEL_L4 | MCX_TIR_CTX_HASH_SEL_UDP | \ 2260 MCX_TIR_CTX_HASH_SEL_IPV6 2261 #define MCX_RSS_HASH_SEL_V4 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV4 2262 #define MCX_RSS_HASH_SEL_V6 MCX_HASH_SEL_L3 | MCX_TIR_CTX_HASH_SEL_IPV6 2263 2264 /* 2265 * There are a few different pieces involved in configuring RSS. 2266 * A Receive Queue Table (RQT) is the indirection table that maps packets to 2267 * different rx queues based on a hash value. We only create one, because 2268 * we want to scatter any traffic we can apply RSS to across all our rx 2269 * queues. Anything else will only be delivered to the first rx queue, 2270 * which doesn't require an RQT. 2271 * 2272 * A Transport Interface Receive (TIR) delivers packets to either a single rx 2273 * queue or an RQT, and in the latter case, specifies the set of fields 2274 * hashed, the hash function, and the hash key. We need one of these for each 2275 * type of RSS traffic - v4 TCP, v6 TCP, v4 UDP, v6 UDP, other v4, other v6, 2276 * and one for non-RSS traffic. 2277 * 2278 * Flow tables hold flow table entries in sequence. The first entry that 2279 * matches a packet is applied, sending the packet to either another flow 2280 * table or a TIR. We use one flow table to select packets based on 2281 * destination MAC address, and a second to apply RSS. The entries in the 2282 * first table send matching packets to the second, and the entries in the 2283 * RSS table send packets to RSS TIRs if possible, or the non-RSS TIR. 2284 * 2285 * The flow table entry that delivers packets to an RSS TIR must include match 2286 * criteria that ensure packets delivered to the TIR include all the fields 2287 * that the TIR hashes on - so for a v4 TCP TIR, the flow table entry must 2288 * only accept v4 TCP packets. Accordingly, we need flow table entries for 2289 * each TIR. 2290 * 2291 * All of this is a lot more flexible than we need, and we can describe most 2292 * of the stuff we need with a simple array. 2293 * 2294 * An RSS config creates a TIR with hashing enabled on a set of fields, 2295 * pointing to either the first rx queue or the RQT containing all the rx 2296 * queues, and a flow table entry that matches on an ether type and 2297 * optionally an ip proto, that delivers packets to the TIR. 2298 */ 2299 static struct mcx_rss_rule { 2300 int hash_sel; 2301 int flow_group; 2302 int ethertype; 2303 int ip_proto; 2304 } mcx_rss_config[] = { 2305 /* udp and tcp for v4/v6 */ 2306 { MCX_RSS_HASH_SEL_V4_TCP, MCX_FLOW_GROUP_RSS_L4, 2307 ETHERTYPE_IP, IPPROTO_TCP }, 2308 { MCX_RSS_HASH_SEL_V6_TCP, MCX_FLOW_GROUP_RSS_L4, 2309 ETHERTYPE_IPV6, IPPROTO_TCP }, 2310 { MCX_RSS_HASH_SEL_V4_UDP, MCX_FLOW_GROUP_RSS_L4, 2311 ETHERTYPE_IP, IPPROTO_UDP }, 2312 { MCX_RSS_HASH_SEL_V6_UDP, MCX_FLOW_GROUP_RSS_L4, 2313 ETHERTYPE_IPV6, IPPROTO_UDP }, 2314 2315 /* other v4/v6 */ 2316 { MCX_RSS_HASH_SEL_V4, MCX_FLOW_GROUP_RSS_L3, 2317 ETHERTYPE_IP, 0 }, 2318 { MCX_RSS_HASH_SEL_V6, MCX_FLOW_GROUP_RSS_L3, 2319 ETHERTYPE_IPV6, 0 }, 2320 2321 /* non v4/v6 */ 2322 { 0, MCX_FLOW_GROUP_RSS_NONE, 0, 0 } 2323 }; 2324 2325 struct mcx_softc { 2326 struct device sc_dev; 2327 struct arpcom sc_ac; 2328 struct ifmedia sc_media; 2329 uint64_t sc_media_status; 2330 uint64_t sc_media_active; 2331 2332 pci_chipset_tag_t sc_pc; 2333 pci_intr_handle_t sc_ih; 2334 void *sc_ihc; 2335 pcitag_t sc_tag; 2336 2337 bus_dma_tag_t sc_dmat; 2338 bus_space_tag_t sc_memt; 2339 bus_space_handle_t sc_memh; 2340 bus_size_t sc_mems; 2341 2342 struct mcx_dmamem sc_cmdq_mem; 2343 unsigned int sc_cmdq_mask; 2344 unsigned int sc_cmdq_size; 2345 2346 unsigned int sc_cmdq_token; 2347 2348 struct mcx_hwmem sc_boot_pages; 2349 struct mcx_hwmem sc_init_pages; 2350 struct mcx_hwmem sc_regular_pages; 2351 2352 int sc_uar; 2353 int sc_pd; 2354 int sc_tdomain; 2355 uint32_t sc_lkey; 2356 int sc_tis; 2357 int sc_tir[nitems(mcx_rss_config)]; 2358 int sc_rqt; 2359 2360 struct mcx_dmamem sc_doorbell_mem; 2361 2362 struct mcx_eq sc_admin_eq; 2363 struct mcx_eq sc_queue_eq; 2364 2365 int sc_hardmtu; 2366 int sc_rxbufsz; 2367 2368 int sc_bf_size; 2369 int sc_max_rqt_size; 2370 2371 struct task sc_port_change; 2372 2373 int sc_mac_flow_table_id; 2374 int sc_rss_flow_table_id; 2375 struct mcx_flow_group sc_flow_group[MCX_NUM_FLOW_GROUPS]; 2376 int sc_promisc_flow_enabled; 2377 int sc_allmulti_flow_enabled; 2378 int sc_mcast_flow_base; 2379 int sc_extra_mcast; 2380 uint8_t sc_mcast_flows[MCX_NUM_MCAST_FLOWS][ETHER_ADDR_LEN]; 2381 2382 struct mcx_calibration sc_calibration[2]; 2383 unsigned int sc_calibration_gen; 2384 struct timeout sc_calibrate; 2385 2386 struct mcx_queues sc_queues[MCX_MAX_QUEUES]; 2387 unsigned int sc_nqueues; 2388 2389 #if NKSTAT > 0 2390 struct kstat *sc_kstat_ieee8023; 2391 struct kstat *sc_kstat_rfc2863; 2392 struct kstat *sc_kstat_rfc2819; 2393 struct kstat *sc_kstat_rfc3635; 2394 unsigned int sc_kstat_mtmp_count; 2395 struct kstat **sc_kstat_mtmp; 2396 #endif 2397 }; 2398 #define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname) 2399 2400 static int mcx_match(struct device *, void *, void *); 2401 static void mcx_attach(struct device *, struct device *, void *); 2402 2403 #if NKSTAT > 0 2404 static void mcx_kstat_attach(struct mcx_softc *); 2405 #endif 2406 2407 static int mcx_version(struct mcx_softc *); 2408 static int mcx_init_wait(struct mcx_softc *); 2409 static int mcx_enable_hca(struct mcx_softc *); 2410 static int mcx_teardown_hca(struct mcx_softc *, uint16_t); 2411 static int mcx_access_hca_reg(struct mcx_softc *, uint16_t, int, void *, 2412 int); 2413 static int mcx_issi(struct mcx_softc *); 2414 static int mcx_pages(struct mcx_softc *, struct mcx_hwmem *, uint16_t); 2415 static int mcx_hca_max_caps(struct mcx_softc *); 2416 static int mcx_hca_set_caps(struct mcx_softc *); 2417 static int mcx_init_hca(struct mcx_softc *); 2418 static int mcx_set_driver_version(struct mcx_softc *); 2419 static int mcx_iff(struct mcx_softc *); 2420 static int mcx_alloc_uar(struct mcx_softc *, int *); 2421 static int mcx_alloc_pd(struct mcx_softc *); 2422 static int mcx_alloc_tdomain(struct mcx_softc *); 2423 static int mcx_create_eq(struct mcx_softc *, struct mcx_eq *, int, 2424 uint64_t, int); 2425 static int mcx_query_nic_vport_context(struct mcx_softc *); 2426 static int mcx_query_special_contexts(struct mcx_softc *); 2427 static int mcx_set_port_mtu(struct mcx_softc *, int); 2428 static int mcx_create_cq(struct mcx_softc *, struct mcx_cq *, int, int, 2429 int); 2430 static int mcx_destroy_cq(struct mcx_softc *, struct mcx_cq *); 2431 static int mcx_create_sq(struct mcx_softc *, struct mcx_tx *, int, int, 2432 int); 2433 static int mcx_destroy_sq(struct mcx_softc *, struct mcx_tx *); 2434 static int mcx_ready_sq(struct mcx_softc *, struct mcx_tx *); 2435 static int mcx_create_rq(struct mcx_softc *, struct mcx_rx *, int, int); 2436 static int mcx_destroy_rq(struct mcx_softc *, struct mcx_rx *); 2437 static int mcx_ready_rq(struct mcx_softc *, struct mcx_rx *); 2438 static int mcx_create_tir_direct(struct mcx_softc *, struct mcx_rx *, 2439 int *); 2440 static int mcx_create_tir_indirect(struct mcx_softc *, int, uint32_t, 2441 int *); 2442 static int mcx_destroy_tir(struct mcx_softc *, int); 2443 static int mcx_create_tis(struct mcx_softc *, int *); 2444 static int mcx_destroy_tis(struct mcx_softc *, int); 2445 static int mcx_create_rqt(struct mcx_softc *, int, int *, int *); 2446 static int mcx_destroy_rqt(struct mcx_softc *, int); 2447 static int mcx_create_flow_table(struct mcx_softc *, int, int, int *); 2448 static int mcx_set_flow_table_root(struct mcx_softc *, int); 2449 static int mcx_destroy_flow_table(struct mcx_softc *, int); 2450 static int mcx_create_flow_group(struct mcx_softc *, int, int, int, 2451 int, int, struct mcx_flow_match *); 2452 static int mcx_destroy_flow_group(struct mcx_softc *, int); 2453 static int mcx_set_flow_table_entry_mac(struct mcx_softc *, int, int, 2454 uint8_t *, uint32_t); 2455 static int mcx_set_flow_table_entry_proto(struct mcx_softc *, int, int, 2456 int, int, uint32_t); 2457 static int mcx_delete_flow_table_entry(struct mcx_softc *, int, int); 2458 2459 #if NKSTAT > 0 2460 static int mcx_query_rq(struct mcx_softc *, struct mcx_rx *, struct mcx_rq_ctx *); 2461 static int mcx_query_sq(struct mcx_softc *, struct mcx_tx *, struct mcx_sq_ctx *); 2462 static int mcx_query_cq(struct mcx_softc *, struct mcx_cq *, struct mcx_cq_ctx *); 2463 static int mcx_query_eq(struct mcx_softc *, struct mcx_eq *, struct mcx_eq_ctx *); 2464 #endif 2465 2466 #if 0 2467 static int mcx_dump_flow_table(struct mcx_softc *, int); 2468 static int mcx_dump_flow_table_entry(struct mcx_softc *, int, int); 2469 static int mcx_dump_flow_group(struct mcx_softc *, int); 2470 #endif 2471 2472 2473 /* 2474 static void mcx_cmdq_dump(const struct mcx_cmdq_entry *); 2475 static void mcx_cmdq_mbox_dump(struct mcx_dmamem *, int); 2476 */ 2477 static void mcx_refill(void *); 2478 static int mcx_process_rx(struct mcx_softc *, struct mcx_rx *, 2479 struct mcx_cq_entry *, struct mbuf_list *, 2480 const struct mcx_calibration *); 2481 static void mcx_process_txeof(struct mcx_softc *, struct mcx_tx *, 2482 struct mcx_cq_entry *, int *); 2483 static void mcx_process_cq(struct mcx_softc *, struct mcx_queues *, 2484 struct mcx_cq *); 2485 2486 static void mcx_arm_cq(struct mcx_softc *, struct mcx_cq *, int); 2487 static void mcx_arm_eq(struct mcx_softc *, struct mcx_eq *, int); 2488 static int mcx_admin_intr(void *); 2489 static int mcx_cq_intr(void *); 2490 2491 static int mcx_up(struct mcx_softc *); 2492 static void mcx_down(struct mcx_softc *); 2493 static int mcx_ioctl(struct ifnet *, u_long, caddr_t); 2494 static int mcx_rxrinfo(struct mcx_softc *, struct if_rxrinfo *); 2495 static void mcx_start(struct ifqueue *); 2496 static void mcx_watchdog(struct ifnet *); 2497 static void mcx_media_add_types(struct mcx_softc *); 2498 static void mcx_media_status(struct ifnet *, struct ifmediareq *); 2499 static int mcx_media_change(struct ifnet *); 2500 static int mcx_get_sffpage(struct ifnet *, struct if_sffpage *); 2501 static void mcx_port_change(void *); 2502 2503 static void mcx_calibrate_first(struct mcx_softc *); 2504 static void mcx_calibrate(void *); 2505 2506 static inline uint32_t 2507 mcx_rd(struct mcx_softc *, bus_size_t); 2508 static inline void 2509 mcx_wr(struct mcx_softc *, bus_size_t, uint32_t); 2510 static inline void 2511 mcx_bar(struct mcx_softc *, bus_size_t, bus_size_t, int); 2512 2513 static uint64_t mcx_timer(struct mcx_softc *); 2514 2515 static int mcx_dmamem_alloc(struct mcx_softc *, struct mcx_dmamem *, 2516 bus_size_t, u_int align); 2517 static void mcx_dmamem_zero(struct mcx_dmamem *); 2518 static void mcx_dmamem_free(struct mcx_softc *, struct mcx_dmamem *); 2519 2520 static int mcx_hwmem_alloc(struct mcx_softc *, struct mcx_hwmem *, 2521 unsigned int); 2522 static void mcx_hwmem_free(struct mcx_softc *, struct mcx_hwmem *); 2523 2524 struct cfdriver mcx_cd = { 2525 NULL, 2526 "mcx", 2527 DV_IFNET, 2528 }; 2529 2530 struct cfattach mcx_ca = { 2531 sizeof(struct mcx_softc), 2532 mcx_match, 2533 mcx_attach, 2534 }; 2535 2536 static const struct pci_matchid mcx_devices[] = { 2537 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700 }, 2538 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27700VF }, 2539 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710 }, 2540 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27710VF }, 2541 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800 }, 2542 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT27800VF }, 2543 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800 }, 2544 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT28800VF }, 2545 { PCI_VENDOR_MELLANOX, PCI_PRODUCT_MELLANOX_MT2892 }, 2546 }; 2547 2548 struct mcx_eth_proto_capability { 2549 uint64_t cap_media; 2550 uint64_t cap_baudrate; 2551 }; 2552 2553 static const struct mcx_eth_proto_capability mcx_eth_cap_map[] = { 2554 [MCX_ETHER_CAP_SGMII] = { IFM_1000_SGMII, IF_Gbps(1) }, 2555 [MCX_ETHER_CAP_1000_KX] = { IFM_1000_KX, IF_Gbps(1) }, 2556 [MCX_ETHER_CAP_10G_CX4] = { IFM_10G_CX4, IF_Gbps(10) }, 2557 [MCX_ETHER_CAP_10G_KX4] = { IFM_10G_KX4, IF_Gbps(10) }, 2558 [MCX_ETHER_CAP_10G_KR] = { IFM_10G_KR, IF_Gbps(10) }, 2559 [MCX_ETHER_CAP_40G_CR4] = { IFM_40G_CR4, IF_Gbps(40) }, 2560 [MCX_ETHER_CAP_40G_KR4] = { IFM_40G_KR4, IF_Gbps(40) }, 2561 [MCX_ETHER_CAP_10G_CR] = { IFM_10G_SFP_CU, IF_Gbps(10) }, 2562 [MCX_ETHER_CAP_10G_SR] = { IFM_10G_SR, IF_Gbps(10) }, 2563 [MCX_ETHER_CAP_10G_LR] = { IFM_10G_LR, IF_Gbps(10) }, 2564 [MCX_ETHER_CAP_40G_SR4] = { IFM_40G_SR4, IF_Gbps(40) }, 2565 [MCX_ETHER_CAP_40G_LR4] = { IFM_40G_LR4, IF_Gbps(40) }, 2566 [MCX_ETHER_CAP_50G_SR2] = { 0 /*IFM_50G_SR2*/, IF_Gbps(50) }, 2567 [MCX_ETHER_CAP_100G_CR4] = { IFM_100G_CR4, IF_Gbps(100) }, 2568 [MCX_ETHER_CAP_100G_SR4] = { IFM_100G_SR4, IF_Gbps(100) }, 2569 [MCX_ETHER_CAP_100G_KR4] = { IFM_100G_KR4, IF_Gbps(100) }, 2570 [MCX_ETHER_CAP_25G_CR] = { IFM_25G_CR, IF_Gbps(25) }, 2571 [MCX_ETHER_CAP_25G_KR] = { IFM_25G_KR, IF_Gbps(25) }, 2572 [MCX_ETHER_CAP_25G_SR] = { IFM_25G_SR, IF_Gbps(25) }, 2573 [MCX_ETHER_CAP_50G_CR2] = { IFM_50G_CR2, IF_Gbps(50) }, 2574 [MCX_ETHER_CAP_50G_KR2] = { IFM_50G_KR2, IF_Gbps(50) }, 2575 }; 2576 2577 static int 2578 mcx_get_id(uint32_t val) 2579 { 2580 return betoh32(val) & 0x00ffffff; 2581 } 2582 2583 static int 2584 mcx_match(struct device *parent, void *match, void *aux) 2585 { 2586 return (pci_matchbyid(aux, mcx_devices, nitems(mcx_devices))); 2587 } 2588 2589 void 2590 mcx_attach(struct device *parent, struct device *self, void *aux) 2591 { 2592 struct mcx_softc *sc = (struct mcx_softc *)self; 2593 struct ifnet *ifp = &sc->sc_ac.ac_if; 2594 struct pci_attach_args *pa = aux; 2595 pcireg_t memtype; 2596 uint32_t r; 2597 unsigned int cq_stride; 2598 unsigned int cq_size; 2599 const char *intrstr; 2600 int i, msix; 2601 2602 sc->sc_pc = pa->pa_pc; 2603 sc->sc_tag = pa->pa_tag; 2604 sc->sc_dmat = pa->pa_dmat; 2605 2606 /* Map the PCI memory space */ 2607 memtype = pci_mapreg_type(sc->sc_pc, sc->sc_tag, MCX_HCA_BAR); 2608 if (pci_mapreg_map(pa, MCX_HCA_BAR, memtype, 2609 BUS_SPACE_MAP_PREFETCHABLE, &sc->sc_memt, &sc->sc_memh, 2610 NULL, &sc->sc_mems, 0)) { 2611 printf(": unable to map register memory\n"); 2612 return; 2613 } 2614 2615 if (mcx_version(sc) != 0) { 2616 /* error printed by mcx_version */ 2617 goto unmap; 2618 } 2619 2620 r = mcx_rd(sc, MCX_CMDQ_ADDR_LO); 2621 cq_stride = 1 << MCX_CMDQ_LOG_STRIDE(r); /* size of the entries */ 2622 cq_size = 1 << MCX_CMDQ_LOG_SIZE(r); /* number of entries */ 2623 if (cq_size > MCX_MAX_CQE) { 2624 printf(", command queue size overflow %u\n", cq_size); 2625 goto unmap; 2626 } 2627 if (cq_stride < sizeof(struct mcx_cmdq_entry)) { 2628 printf(", command queue entry size underflow %u\n", cq_stride); 2629 goto unmap; 2630 } 2631 if (cq_stride * cq_size > MCX_PAGE_SIZE) { 2632 printf(", command queue page overflow\n"); 2633 goto unmap; 2634 } 2635 2636 if (mcx_dmamem_alloc(sc, &sc->sc_doorbell_mem, MCX_DOORBELL_AREA_SIZE, 2637 MCX_PAGE_SIZE) != 0) { 2638 printf(", unable to allocate doorbell memory\n"); 2639 goto unmap; 2640 } 2641 2642 if (mcx_dmamem_alloc(sc, &sc->sc_cmdq_mem, MCX_PAGE_SIZE, 2643 MCX_PAGE_SIZE) != 0) { 2644 printf(", unable to allocate command queue\n"); 2645 goto dbfree; 2646 } 2647 2648 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2649 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint32_t), 2650 BUS_SPACE_BARRIER_WRITE); 2651 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem)); 2652 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint32_t), 2653 BUS_SPACE_BARRIER_WRITE); 2654 2655 if (mcx_init_wait(sc) != 0) { 2656 printf(", timeout waiting for init\n"); 2657 goto cqfree; 2658 } 2659 2660 sc->sc_cmdq_mask = cq_size - 1; 2661 sc->sc_cmdq_size = cq_stride; 2662 2663 if (mcx_enable_hca(sc) != 0) { 2664 /* error printed by mcx_enable_hca */ 2665 goto cqfree; 2666 } 2667 2668 if (mcx_issi(sc) != 0) { 2669 /* error printed by mcx_issi */ 2670 goto teardown; 2671 } 2672 2673 if (mcx_pages(sc, &sc->sc_boot_pages, 2674 htobe16(MCX_CMD_QUERY_PAGES_BOOT)) != 0) { 2675 /* error printed by mcx_pages */ 2676 goto teardown; 2677 } 2678 2679 if (mcx_hca_max_caps(sc) != 0) { 2680 /* error printed by mcx_hca_max_caps */ 2681 goto teardown; 2682 } 2683 2684 if (mcx_hca_set_caps(sc) != 0) { 2685 /* error printed by mcx_hca_set_caps */ 2686 goto teardown; 2687 } 2688 2689 if (mcx_pages(sc, &sc->sc_init_pages, 2690 htobe16(MCX_CMD_QUERY_PAGES_INIT)) != 0) { 2691 /* error printed by mcx_pages */ 2692 goto teardown; 2693 } 2694 2695 if (mcx_init_hca(sc) != 0) { 2696 /* error printed by mcx_init_hca */ 2697 goto teardown; 2698 } 2699 2700 if (mcx_pages(sc, &sc->sc_regular_pages, 2701 htobe16(MCX_CMD_QUERY_PAGES_REGULAR)) != 0) { 2702 /* error printed by mcx_pages */ 2703 goto teardown; 2704 } 2705 2706 /* apparently not necessary? */ 2707 if (mcx_set_driver_version(sc) != 0) { 2708 /* error printed by mcx_set_driver_version */ 2709 goto teardown; 2710 } 2711 2712 if (mcx_iff(sc) != 0) { /* modify nic vport context */ 2713 /* error printed by mcx_iff? */ 2714 goto teardown; 2715 } 2716 2717 if (mcx_alloc_uar(sc, &sc->sc_uar) != 0) { 2718 /* error printed by mcx_alloc_uar */ 2719 goto teardown; 2720 } 2721 2722 if (mcx_alloc_pd(sc) != 0) { 2723 /* error printed by mcx_alloc_pd */ 2724 goto teardown; 2725 } 2726 2727 if (mcx_alloc_tdomain(sc) != 0) { 2728 /* error printed by mcx_alloc_tdomain */ 2729 goto teardown; 2730 } 2731 2732 /* 2733 * PRM makes no mention of msi interrupts, just legacy and msi-x. 2734 * mellanox support tells me legacy interrupts are not supported, 2735 * so we're stuck with just msi-x. 2736 */ 2737 if (pci_intr_map_msix(pa, 0, &sc->sc_ih) != 0) { 2738 printf(": unable to map interrupt\n"); 2739 goto teardown; 2740 } 2741 intrstr = pci_intr_string(sc->sc_pc, sc->sc_ih); 2742 sc->sc_ihc = pci_intr_establish(sc->sc_pc, sc->sc_ih, 2743 IPL_NET | IPL_MPSAFE, mcx_admin_intr, sc, DEVNAME(sc)); 2744 if (sc->sc_ihc == NULL) { 2745 printf(": unable to establish interrupt"); 2746 if (intrstr != NULL) 2747 printf(" at %s", intrstr); 2748 printf("\n"); 2749 goto teardown; 2750 } 2751 2752 if (mcx_create_eq(sc, &sc->sc_admin_eq, sc->sc_uar, 2753 (1ull << MCX_EVENT_TYPE_INTERNAL_ERROR) | 2754 (1ull << MCX_EVENT_TYPE_PORT_CHANGE) | 2755 (1ull << MCX_EVENT_TYPE_CMD_COMPLETION) | 2756 (1ull << MCX_EVENT_TYPE_PAGE_REQUEST), 0) != 0) { 2757 /* error printed by mcx_create_eq */ 2758 goto teardown; 2759 } 2760 2761 if (mcx_query_nic_vport_context(sc) != 0) { 2762 /* error printed by mcx_query_nic_vport_context */ 2763 goto teardown; 2764 } 2765 2766 if (mcx_query_special_contexts(sc) != 0) { 2767 /* error printed by mcx_query_special_contexts */ 2768 goto teardown; 2769 } 2770 2771 if (mcx_set_port_mtu(sc, MCX_HARDMTU) != 0) { 2772 /* error printed by mcx_set_port_mtu */ 2773 goto teardown; 2774 } 2775 2776 printf(", %s, address %s\n", intrstr, 2777 ether_sprintf(sc->sc_ac.ac_enaddr)); 2778 2779 msix = pci_intr_msix_count(pa->pa_pc, pa->pa_tag); 2780 sc->sc_nqueues = 1; 2781 2782 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ); 2783 ifp->if_softc = sc; 2784 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX; 2785 ifp->if_xflags = IFXF_MPSAFE; 2786 ifp->if_ioctl = mcx_ioctl; 2787 ifp->if_qstart = mcx_start; 2788 ifp->if_watchdog = mcx_watchdog; 2789 ifp->if_hardmtu = sc->sc_hardmtu; 2790 ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_CSUM_IPv4 | 2791 IFCAP_CSUM_UDPv4 | IFCAP_CSUM_UDPv6 | IFCAP_CSUM_TCPv4 | 2792 IFCAP_CSUM_TCPv6; 2793 #if NVLAN > 0 2794 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 2795 #endif 2796 ifq_set_maxlen(&ifp->if_snd, 1024); 2797 2798 ifmedia_init(&sc->sc_media, IFM_IMASK, mcx_media_change, 2799 mcx_media_status); 2800 mcx_media_add_types(sc); 2801 ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL); 2802 ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO); 2803 2804 if_attach(ifp); 2805 ether_ifattach(ifp); 2806 2807 if_attach_iqueues(ifp, sc->sc_nqueues); 2808 if_attach_queues(ifp, sc->sc_nqueues); 2809 for (i = 0; i < sc->sc_nqueues; i++) { 2810 struct ifiqueue *ifiq = ifp->if_iqs[i]; 2811 struct ifqueue *ifq = ifp->if_ifqs[i]; 2812 struct mcx_queues *q = &sc->sc_queues[i]; 2813 struct mcx_rx *rx = &q->q_rx; 2814 struct mcx_tx *tx = &q->q_tx; 2815 pci_intr_handle_t ih; 2816 int vec; 2817 2818 vec = i + 1; 2819 q->q_sc = sc; 2820 q->q_index = i; 2821 2822 if (mcx_alloc_uar(sc, &q->q_uar) != 0) { 2823 printf("%s: unable to alloc uar %d\n", 2824 DEVNAME(sc), i); 2825 goto teardown; 2826 } 2827 2828 if (mcx_create_eq(sc, &q->q_eq, q->q_uar, 0, vec) != 0) { 2829 printf("%s: unable to create event queue %d\n", 2830 DEVNAME(sc), i); 2831 goto teardown; 2832 } 2833 2834 rx->rx_softc = sc; 2835 rx->rx_ifiq = ifiq; 2836 timeout_set(&rx->rx_refill, mcx_refill, rx); 2837 ifiq->ifiq_softc = rx; 2838 2839 tx->tx_softc = sc; 2840 tx->tx_ifq = ifq; 2841 ifq->ifq_softc = tx; 2842 2843 if (pci_intr_map_msix(pa, i + 1, &ih) != 0) { 2844 printf("%s: unable to map queue interrupt %d\n", 2845 DEVNAME(sc), i); 2846 goto teardown; 2847 } 2848 snprintf(q->q_name, sizeof(q->q_name), "%s:%d", 2849 DEVNAME(sc), i); 2850 q->q_ihc = pci_intr_establish(sc->sc_pc, ih, 2851 IPL_NET | IPL_MPSAFE, mcx_cq_intr, q, q->q_name); 2852 } 2853 2854 timeout_set(&sc->sc_calibrate, mcx_calibrate, sc); 2855 2856 task_set(&sc->sc_port_change, mcx_port_change, sc); 2857 mcx_port_change(sc); 2858 2859 sc->sc_mac_flow_table_id = -1; 2860 sc->sc_rss_flow_table_id = -1; 2861 sc->sc_rqt = -1; 2862 for (i = 0; i < MCX_NUM_FLOW_GROUPS; i++) { 2863 struct mcx_flow_group *mfg = &sc->sc_flow_group[i]; 2864 mfg->g_id = -1; 2865 mfg->g_table = -1; 2866 mfg->g_size = 0; 2867 mfg->g_start = 0; 2868 } 2869 sc->sc_extra_mcast = 0; 2870 memset(sc->sc_mcast_flows, 0, sizeof(sc->sc_mcast_flows)); 2871 2872 #if NKSTAT > 0 2873 mcx_kstat_attach(sc); 2874 #endif 2875 return; 2876 2877 teardown: 2878 mcx_teardown_hca(sc, htobe16(MCX_CMD_TEARDOWN_HCA_GRACEFUL)); 2879 /* error printed by mcx_teardown_hca, and we're already unwinding */ 2880 cqfree: 2881 mcx_wr(sc, MCX_CMDQ_ADDR_HI, MCX_DMA_DVA(&sc->sc_cmdq_mem) >> 32); 2882 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 2883 BUS_SPACE_BARRIER_WRITE); 2884 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_DMA_DVA(&sc->sc_cmdq_mem) | 2885 MCX_CMDQ_INTERFACE_DISABLED); 2886 mcx_bar(sc, MCX_CMDQ_ADDR_LO, sizeof(uint64_t), 2887 BUS_SPACE_BARRIER_WRITE); 2888 2889 mcx_wr(sc, MCX_CMDQ_ADDR_HI, 0); 2890 mcx_bar(sc, MCX_CMDQ_ADDR_HI, sizeof(uint64_t), 2891 BUS_SPACE_BARRIER_WRITE); 2892 mcx_wr(sc, MCX_CMDQ_ADDR_LO, MCX_CMDQ_INTERFACE_DISABLED); 2893 2894 mcx_dmamem_free(sc, &sc->sc_cmdq_mem); 2895 dbfree: 2896 mcx_dmamem_free(sc, &sc->sc_doorbell_mem); 2897 unmap: 2898 bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems); 2899 sc->sc_mems = 0; 2900 } 2901 2902 static int 2903 mcx_version(struct mcx_softc *sc) 2904 { 2905 uint32_t fw0, fw1; 2906 uint16_t cmdif; 2907 2908 fw0 = mcx_rd(sc, MCX_FW_VER); 2909 fw1 = mcx_rd(sc, MCX_CMDIF_FW_SUBVER); 2910 2911 printf(": FW %u.%u.%04u", MCX_FW_VER_MAJOR(fw0), 2912 MCX_FW_VER_MINOR(fw0), MCX_FW_VER_SUBMINOR(fw1)); 2913 2914 cmdif = MCX_CMDIF(fw1); 2915 if (cmdif != MCX_CMD_IF_SUPPORTED) { 2916 printf(", unsupported command interface %u\n", cmdif); 2917 return (-1); 2918 } 2919 2920 return (0); 2921 } 2922 2923 static int 2924 mcx_init_wait(struct mcx_softc *sc) 2925 { 2926 unsigned int i; 2927 uint32_t r; 2928 2929 for (i = 0; i < 2000; i++) { 2930 r = mcx_rd(sc, MCX_STATE); 2931 if ((r & MCX_STATE_MASK) == MCX_STATE_READY) 2932 return (0); 2933 2934 delay(1000); 2935 mcx_bar(sc, MCX_STATE, sizeof(uint32_t), 2936 BUS_SPACE_BARRIER_READ); 2937 } 2938 2939 return (-1); 2940 } 2941 2942 static uint8_t 2943 mcx_cmdq_poll(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 2944 unsigned int msec) 2945 { 2946 unsigned int i; 2947 2948 for (i = 0; i < msec; i++) { 2949 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 2950 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_POSTRW); 2951 2952 if ((cqe->cq_status & MCX_CQ_STATUS_OWN_MASK) == 2953 MCX_CQ_STATUS_OWN_SW) 2954 return (0); 2955 2956 delay(1000); 2957 } 2958 2959 return (ETIMEDOUT); 2960 } 2961 2962 static uint32_t 2963 mcx_mix_u64(uint32_t xor, uint64_t u64) 2964 { 2965 xor ^= u64 >> 32; 2966 xor ^= u64; 2967 2968 return (xor); 2969 } 2970 2971 static uint32_t 2972 mcx_mix_u32(uint32_t xor, uint32_t u32) 2973 { 2974 xor ^= u32; 2975 2976 return (xor); 2977 } 2978 2979 static uint32_t 2980 mcx_mix_u8(uint32_t xor, uint8_t u8) 2981 { 2982 xor ^= u8; 2983 2984 return (xor); 2985 } 2986 2987 static uint8_t 2988 mcx_mix_done(uint32_t xor) 2989 { 2990 xor ^= xor >> 16; 2991 xor ^= xor >> 8; 2992 2993 return (xor); 2994 } 2995 2996 static uint8_t 2997 mcx_xor(const void *buf, size_t len) 2998 { 2999 const uint32_t *dwords = buf; 3000 uint32_t xor = 0xff; 3001 size_t i; 3002 3003 len /= sizeof(*dwords); 3004 3005 for (i = 0; i < len; i++) 3006 xor ^= dwords[i]; 3007 3008 return (mcx_mix_done(xor)); 3009 } 3010 3011 static uint8_t 3012 mcx_cmdq_token(struct mcx_softc *sc) 3013 { 3014 uint8_t token; 3015 3016 do { 3017 token = ++sc->sc_cmdq_token; 3018 } while (token == 0); 3019 3020 return (token); 3021 } 3022 3023 static void 3024 mcx_cmdq_init(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3025 uint32_t ilen, uint32_t olen, uint8_t token) 3026 { 3027 memset(cqe, 0, sc->sc_cmdq_size); 3028 3029 cqe->cq_type = MCX_CMDQ_TYPE_PCIE; 3030 htobem32(&cqe->cq_input_length, ilen); 3031 htobem32(&cqe->cq_output_length, olen); 3032 cqe->cq_token = token; 3033 cqe->cq_status = MCX_CQ_STATUS_OWN_HW; 3034 } 3035 3036 static void 3037 mcx_cmdq_sign(struct mcx_cmdq_entry *cqe) 3038 { 3039 cqe->cq_signature = ~mcx_xor(cqe, sizeof(*cqe)); 3040 } 3041 3042 static int 3043 mcx_cmdq_verify(const struct mcx_cmdq_entry *cqe) 3044 { 3045 /* return (mcx_xor(cqe, sizeof(*cqe)) ? -1 : 0); */ 3046 return (0); 3047 } 3048 3049 static void * 3050 mcx_cmdq_in(struct mcx_cmdq_entry *cqe) 3051 { 3052 return (&cqe->cq_input_data); 3053 } 3054 3055 static void * 3056 mcx_cmdq_out(struct mcx_cmdq_entry *cqe) 3057 { 3058 return (&cqe->cq_output_data); 3059 } 3060 3061 static void 3062 mcx_cmdq_post(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3063 unsigned int slot) 3064 { 3065 mcx_cmdq_sign(cqe); 3066 3067 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(&sc->sc_cmdq_mem), 3068 0, MCX_DMA_LEN(&sc->sc_cmdq_mem), BUS_DMASYNC_PRERW); 3069 3070 mcx_wr(sc, MCX_CMDQ_DOORBELL, 1U << slot); 3071 } 3072 3073 static int 3074 mcx_enable_hca(struct mcx_softc *sc) 3075 { 3076 struct mcx_cmdq_entry *cqe; 3077 struct mcx_cmd_enable_hca_in *in; 3078 struct mcx_cmd_enable_hca_out *out; 3079 int error; 3080 uint8_t status; 3081 3082 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3083 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3084 3085 in = mcx_cmdq_in(cqe); 3086 in->cmd_opcode = htobe16(MCX_CMD_ENABLE_HCA); 3087 in->cmd_op_mod = htobe16(0); 3088 in->cmd_function_id = htobe16(0); 3089 3090 mcx_cmdq_post(sc, cqe, 0); 3091 3092 error = mcx_cmdq_poll(sc, cqe, 1000); 3093 if (error != 0) { 3094 printf(", hca enable timeout\n"); 3095 return (-1); 3096 } 3097 if (mcx_cmdq_verify(cqe) != 0) { 3098 printf(", hca enable command corrupt\n"); 3099 return (-1); 3100 } 3101 3102 status = cqe->cq_output_data[0]; 3103 if (status != MCX_CQ_STATUS_OK) { 3104 printf(", hca enable failed (%x)\n", status); 3105 return (-1); 3106 } 3107 3108 return (0); 3109 } 3110 3111 static int 3112 mcx_teardown_hca(struct mcx_softc *sc, uint16_t profile) 3113 { 3114 struct mcx_cmdq_entry *cqe; 3115 struct mcx_cmd_teardown_hca_in *in; 3116 struct mcx_cmd_teardown_hca_out *out; 3117 int error; 3118 uint8_t status; 3119 3120 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3121 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3122 3123 in = mcx_cmdq_in(cqe); 3124 in->cmd_opcode = htobe16(MCX_CMD_TEARDOWN_HCA); 3125 in->cmd_op_mod = htobe16(0); 3126 in->cmd_profile = profile; 3127 3128 mcx_cmdq_post(sc, cqe, 0); 3129 3130 error = mcx_cmdq_poll(sc, cqe, 1000); 3131 if (error != 0) { 3132 printf(", hca teardown timeout\n"); 3133 return (-1); 3134 } 3135 if (mcx_cmdq_verify(cqe) != 0) { 3136 printf(", hca teardown command corrupt\n"); 3137 return (-1); 3138 } 3139 3140 status = cqe->cq_output_data[0]; 3141 if (status != MCX_CQ_STATUS_OK) { 3142 printf(", hca teardown failed (%x)\n", status); 3143 return (-1); 3144 } 3145 3146 return (0); 3147 } 3148 3149 static int 3150 mcx_cmdq_mboxes_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 3151 unsigned int nmb, uint64_t *ptr, uint8_t token) 3152 { 3153 caddr_t kva; 3154 uint64_t dva; 3155 int i; 3156 int error; 3157 3158 error = mcx_dmamem_alloc(sc, mxm, 3159 nmb * MCX_CMDQ_MAILBOX_SIZE, MCX_CMDQ_MAILBOX_ALIGN); 3160 if (error != 0) 3161 return (error); 3162 3163 mcx_dmamem_zero(mxm); 3164 3165 dva = MCX_DMA_DVA(mxm); 3166 kva = MCX_DMA_KVA(mxm); 3167 for (i = 0; i < nmb; i++) { 3168 struct mcx_cmdq_mailbox *mbox = (struct mcx_cmdq_mailbox *)kva; 3169 3170 /* patch the cqe or mbox pointing at this one */ 3171 htobem64(ptr, dva); 3172 3173 /* fill in this mbox */ 3174 htobem32(&mbox->mb_block_number, i); 3175 mbox->mb_token = token; 3176 3177 /* move to the next one */ 3178 ptr = &mbox->mb_next_ptr; 3179 3180 dva += MCX_CMDQ_MAILBOX_SIZE; 3181 kva += MCX_CMDQ_MAILBOX_SIZE; 3182 } 3183 3184 return (0); 3185 } 3186 3187 static uint32_t 3188 mcx_cmdq_mbox_ctrl_sig(const struct mcx_cmdq_mailbox *mb) 3189 { 3190 uint32_t xor = 0xff; 3191 3192 /* only 3 fields get set, so mix them directly */ 3193 xor = mcx_mix_u64(xor, mb->mb_next_ptr); 3194 xor = mcx_mix_u32(xor, mb->mb_block_number); 3195 xor = mcx_mix_u8(xor, mb->mb_token); 3196 3197 return (mcx_mix_done(xor)); 3198 } 3199 3200 static void 3201 mcx_cmdq_mboxes_sign(struct mcx_dmamem *mxm, unsigned int nmb) 3202 { 3203 caddr_t kva; 3204 int i; 3205 3206 kva = MCX_DMA_KVA(mxm); 3207 3208 for (i = 0; i < nmb; i++) { 3209 struct mcx_cmdq_mailbox *mb = (struct mcx_cmdq_mailbox *)kva; 3210 uint8_t sig = mcx_cmdq_mbox_ctrl_sig(mb); 3211 mb->mb_ctrl_signature = sig; 3212 mb->mb_signature = sig ^ 3213 mcx_xor(mb->mb_data, sizeof(mb->mb_data)); 3214 3215 kva += MCX_CMDQ_MAILBOX_SIZE; 3216 } 3217 } 3218 3219 static void 3220 mcx_cmdq_mboxes_sync(struct mcx_softc *sc, struct mcx_dmamem *mxm, int ops) 3221 { 3222 bus_dmamap_sync(sc->sc_dmat, MCX_DMA_MAP(mxm), 3223 0, MCX_DMA_LEN(mxm), ops); 3224 } 3225 3226 static struct mcx_cmdq_mailbox * 3227 mcx_cq_mbox(struct mcx_dmamem *mxm, unsigned int i) 3228 { 3229 caddr_t kva; 3230 3231 kva = MCX_DMA_KVA(mxm); 3232 kva += i * MCX_CMDQ_MAILBOX_SIZE; 3233 3234 return ((struct mcx_cmdq_mailbox *)kva); 3235 } 3236 3237 static inline void * 3238 mcx_cq_mbox_data(struct mcx_cmdq_mailbox *mb) 3239 { 3240 return (&mb->mb_data); 3241 } 3242 3243 static void 3244 mcx_cmdq_mboxes_copyin(struct mcx_dmamem *mxm, unsigned int nmb, 3245 void *b, size_t len) 3246 { 3247 caddr_t buf = b; 3248 struct mcx_cmdq_mailbox *mb; 3249 int i; 3250 3251 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3252 for (i = 0; i < nmb; i++) { 3253 3254 memcpy(mb->mb_data, buf, min(sizeof(mb->mb_data), len)); 3255 3256 if (sizeof(mb->mb_data) >= len) 3257 break; 3258 3259 buf += sizeof(mb->mb_data); 3260 len -= sizeof(mb->mb_data); 3261 mb++; 3262 } 3263 } 3264 3265 static void 3266 mcx_cmdq_mboxes_pas(struct mcx_dmamem *mxm, int offset, int npages, 3267 struct mcx_dmamem *buf) 3268 { 3269 uint64_t *pas; 3270 int mbox, mbox_pages, i; 3271 3272 mbox = offset / MCX_CMDQ_MAILBOX_DATASIZE; 3273 offset %= MCX_CMDQ_MAILBOX_DATASIZE; 3274 3275 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3276 pas += (offset / sizeof(*pas)); 3277 mbox_pages = (MCX_CMDQ_MAILBOX_DATASIZE - offset) / sizeof(*pas); 3278 for (i = 0; i < npages; i++) { 3279 if (i == mbox_pages) { 3280 mbox++; 3281 pas = mcx_cq_mbox_data(mcx_cq_mbox(mxm, mbox)); 3282 mbox_pages += MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas); 3283 } 3284 *pas = htobe64(MCX_DMA_DVA(buf) + (i * MCX_PAGE_SIZE)); 3285 pas++; 3286 } 3287 } 3288 3289 static void 3290 mcx_cmdq_mboxes_copyout(struct mcx_dmamem *mxm, int nmb, void *b, size_t len) 3291 { 3292 caddr_t buf = b; 3293 struct mcx_cmdq_mailbox *mb; 3294 int i; 3295 3296 mb = (struct mcx_cmdq_mailbox *)MCX_DMA_KVA(mxm); 3297 for (i = 0; i < nmb; i++) { 3298 memcpy(buf, mb->mb_data, min(sizeof(mb->mb_data), len)); 3299 3300 if (sizeof(mb->mb_data) >= len) 3301 break; 3302 3303 buf += sizeof(mb->mb_data); 3304 len -= sizeof(mb->mb_data); 3305 mb++; 3306 } 3307 } 3308 3309 static void 3310 mcx_cq_mboxes_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 3311 { 3312 mcx_dmamem_free(sc, mxm); 3313 } 3314 3315 #if 0 3316 static void 3317 mcx_cmdq_dump(const struct mcx_cmdq_entry *cqe) 3318 { 3319 unsigned int i; 3320 3321 printf(" type %02x, ilen %u, iptr %016llx", cqe->cq_type, 3322 bemtoh32(&cqe->cq_input_length), bemtoh64(&cqe->cq_input_ptr)); 3323 3324 printf(", idata "); 3325 for (i = 0; i < sizeof(cqe->cq_input_data); i++) 3326 printf("%02x", cqe->cq_input_data[i]); 3327 3328 printf(", odata "); 3329 for (i = 0; i < sizeof(cqe->cq_output_data); i++) 3330 printf("%02x", cqe->cq_output_data[i]); 3331 3332 printf(", optr %016llx, olen %u, token %02x, sig %02x, status %02x", 3333 bemtoh64(&cqe->cq_output_ptr), bemtoh32(&cqe->cq_output_length), 3334 cqe->cq_token, cqe->cq_signature, cqe->cq_status); 3335 } 3336 3337 static void 3338 mcx_cmdq_mbox_dump(struct mcx_dmamem *mboxes, int num) 3339 { 3340 int i, j; 3341 uint8_t *d; 3342 3343 for (i = 0; i < num; i++) { 3344 struct mcx_cmdq_mailbox *mbox; 3345 mbox = mcx_cq_mbox(mboxes, i); 3346 3347 d = mcx_cq_mbox_data(mbox); 3348 for (j = 0; j < MCX_CMDQ_MAILBOX_DATASIZE; j++) { 3349 if (j != 0 && (j % 16 == 0)) 3350 printf("\n"); 3351 printf("%.2x ", d[j]); 3352 } 3353 } 3354 } 3355 #endif 3356 3357 static int 3358 mcx_access_hca_reg(struct mcx_softc *sc, uint16_t reg, int op, void *data, 3359 int len) 3360 { 3361 struct mcx_dmamem mxm; 3362 struct mcx_cmdq_entry *cqe; 3363 struct mcx_cmd_access_reg_in *in; 3364 struct mcx_cmd_access_reg_out *out; 3365 uint8_t token = mcx_cmdq_token(sc); 3366 int error, nmb; 3367 3368 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3369 mcx_cmdq_init(sc, cqe, sizeof(*in) + len, sizeof(*out) + len, 3370 token); 3371 3372 in = mcx_cmdq_in(cqe); 3373 in->cmd_opcode = htobe16(MCX_CMD_ACCESS_REG); 3374 in->cmd_op_mod = htobe16(op); 3375 in->cmd_register_id = htobe16(reg); 3376 3377 nmb = howmany(len, MCX_CMDQ_MAILBOX_DATASIZE); 3378 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3379 &cqe->cq_output_ptr, token) != 0) { 3380 printf(", unable to allocate access reg mailboxen\n"); 3381 return (-1); 3382 } 3383 cqe->cq_input_ptr = cqe->cq_output_ptr; 3384 mcx_cmdq_mboxes_copyin(&mxm, nmb, data, len); 3385 mcx_cmdq_mboxes_sign(&mxm, nmb); 3386 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3387 3388 mcx_cmdq_post(sc, cqe, 0); 3389 error = mcx_cmdq_poll(sc, cqe, 1000); 3390 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3391 3392 if (error != 0) { 3393 printf("%s: access reg (%s %x) timeout\n", DEVNAME(sc), 3394 (op == MCX_REG_OP_WRITE ? "write" : "read"), reg); 3395 goto free; 3396 } 3397 error = mcx_cmdq_verify(cqe); 3398 if (error != 0) { 3399 printf("%s: access reg (%s %x) reply corrupt\n", 3400 (op == MCX_REG_OP_WRITE ? "write" : "read"), DEVNAME(sc), 3401 reg); 3402 goto free; 3403 } 3404 3405 out = mcx_cmdq_out(cqe); 3406 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3407 printf("%s: access reg (%s %x) failed (%x, %.6x)\n", 3408 DEVNAME(sc), (op == MCX_REG_OP_WRITE ? "write" : "read"), 3409 reg, out->cmd_status, betoh32(out->cmd_syndrome)); 3410 error = -1; 3411 goto free; 3412 } 3413 3414 mcx_cmdq_mboxes_copyout(&mxm, nmb, data, len); 3415 free: 3416 mcx_dmamem_free(sc, &mxm); 3417 3418 return (error); 3419 } 3420 3421 static int 3422 mcx_set_issi(struct mcx_softc *sc, struct mcx_cmdq_entry *cqe, 3423 unsigned int slot) 3424 { 3425 struct mcx_cmd_set_issi_in *in; 3426 struct mcx_cmd_set_issi_out *out; 3427 uint8_t status; 3428 3429 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3430 3431 in = mcx_cmdq_in(cqe); 3432 in->cmd_opcode = htobe16(MCX_CMD_SET_ISSI); 3433 in->cmd_op_mod = htobe16(0); 3434 in->cmd_current_issi = htobe16(MCX_ISSI); 3435 3436 mcx_cmdq_post(sc, cqe, slot); 3437 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) 3438 return (-1); 3439 if (mcx_cmdq_verify(cqe) != 0) 3440 return (-1); 3441 3442 status = cqe->cq_output_data[0]; 3443 if (status != MCX_CQ_STATUS_OK) 3444 return (-1); 3445 3446 return (0); 3447 } 3448 3449 static int 3450 mcx_issi(struct mcx_softc *sc) 3451 { 3452 struct mcx_dmamem mxm; 3453 struct mcx_cmdq_entry *cqe; 3454 struct mcx_cmd_query_issi_in *in; 3455 struct mcx_cmd_query_issi_il_out *out; 3456 struct mcx_cmd_query_issi_mb_out *mb; 3457 uint8_t token = mcx_cmdq_token(sc); 3458 uint8_t status; 3459 int error; 3460 3461 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3462 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mb), token); 3463 3464 in = mcx_cmdq_in(cqe); 3465 in->cmd_opcode = htobe16(MCX_CMD_QUERY_ISSI); 3466 in->cmd_op_mod = htobe16(0); 3467 3468 CTASSERT(sizeof(*mb) <= MCX_CMDQ_MAILBOX_DATASIZE); 3469 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 3470 &cqe->cq_output_ptr, token) != 0) { 3471 printf(", unable to allocate query issi mailbox\n"); 3472 return (-1); 3473 } 3474 mcx_cmdq_mboxes_sign(&mxm, 1); 3475 3476 mcx_cmdq_post(sc, cqe, 0); 3477 error = mcx_cmdq_poll(sc, cqe, 1000); 3478 if (error != 0) { 3479 printf(", query issi timeout\n"); 3480 goto free; 3481 } 3482 error = mcx_cmdq_verify(cqe); 3483 if (error != 0) { 3484 printf(", query issi reply corrupt\n"); 3485 goto free; 3486 } 3487 3488 status = cqe->cq_output_data[0]; 3489 switch (status) { 3490 case MCX_CQ_STATUS_OK: 3491 break; 3492 case MCX_CQ_STATUS_BAD_OPCODE: 3493 /* use ISSI 0 */ 3494 goto free; 3495 default: 3496 printf(", query issi failed (%x)\n", status); 3497 error = -1; 3498 goto free; 3499 } 3500 3501 out = mcx_cmdq_out(cqe); 3502 if (out->cmd_current_issi == htobe16(MCX_ISSI)) { 3503 /* use ISSI 1 */ 3504 goto free; 3505 } 3506 3507 /* don't need to read cqe anymore, can be used for SET ISSI */ 3508 3509 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 3510 CTASSERT(MCX_ISSI < NBBY); 3511 /* XXX math is hard */ 3512 if (!ISSET(mb->cmd_supported_issi[79], 1 << MCX_ISSI)) { 3513 /* use ISSI 0 */ 3514 goto free; 3515 } 3516 3517 if (mcx_set_issi(sc, cqe, 0) != 0) { 3518 /* ignore the error, just use ISSI 0 */ 3519 } else { 3520 /* use ISSI 1 */ 3521 } 3522 3523 free: 3524 mcx_cq_mboxes_free(sc, &mxm); 3525 return (error); 3526 } 3527 3528 static int 3529 mcx_query_pages(struct mcx_softc *sc, uint16_t type, 3530 int32_t *npages, uint16_t *func_id) 3531 { 3532 struct mcx_cmdq_entry *cqe; 3533 struct mcx_cmd_query_pages_in *in; 3534 struct mcx_cmd_query_pages_out *out; 3535 3536 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3537 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3538 3539 in = mcx_cmdq_in(cqe); 3540 in->cmd_opcode = htobe16(MCX_CMD_QUERY_PAGES); 3541 in->cmd_op_mod = type; 3542 3543 mcx_cmdq_post(sc, cqe, 0); 3544 if (mcx_cmdq_poll(sc, cqe, 1000) != 0) { 3545 printf(", query pages timeout\n"); 3546 return (-1); 3547 } 3548 if (mcx_cmdq_verify(cqe) != 0) { 3549 printf(", query pages reply corrupt\n"); 3550 return (-1); 3551 } 3552 3553 out = mcx_cmdq_out(cqe); 3554 if (out->cmd_status != MCX_CQ_STATUS_OK) { 3555 printf(", query pages failed (%x)\n", out->cmd_status); 3556 return (-1); 3557 } 3558 3559 *func_id = out->cmd_func_id; 3560 *npages = bemtoh32(&out->cmd_num_pages); 3561 3562 return (0); 3563 } 3564 3565 struct bus_dma_iter { 3566 bus_dmamap_t i_map; 3567 bus_size_t i_offset; 3568 unsigned int i_index; 3569 }; 3570 3571 static void 3572 bus_dma_iter_init(struct bus_dma_iter *i, bus_dmamap_t map) 3573 { 3574 i->i_map = map; 3575 i->i_offset = 0; 3576 i->i_index = 0; 3577 } 3578 3579 static bus_addr_t 3580 bus_dma_iter_addr(struct bus_dma_iter *i) 3581 { 3582 return (i->i_map->dm_segs[i->i_index].ds_addr + i->i_offset); 3583 } 3584 3585 static void 3586 bus_dma_iter_add(struct bus_dma_iter *i, bus_size_t size) 3587 { 3588 bus_dma_segment_t *seg = i->i_map->dm_segs + i->i_index; 3589 bus_size_t diff; 3590 3591 do { 3592 diff = seg->ds_len - i->i_offset; 3593 if (size < diff) 3594 break; 3595 3596 size -= diff; 3597 3598 seg++; 3599 3600 i->i_offset = 0; 3601 i->i_index++; 3602 } while (size > 0); 3603 3604 i->i_offset += size; 3605 } 3606 3607 static int 3608 mcx_add_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t func_id) 3609 { 3610 struct mcx_dmamem mxm; 3611 struct mcx_cmdq_entry *cqe; 3612 struct mcx_cmd_manage_pages_in *in; 3613 struct mcx_cmd_manage_pages_out *out; 3614 unsigned int paslen, nmb, i, j, npages; 3615 struct bus_dma_iter iter; 3616 uint64_t *pas; 3617 uint8_t status; 3618 uint8_t token = mcx_cmdq_token(sc); 3619 int error; 3620 3621 npages = mhm->mhm_npages; 3622 3623 paslen = sizeof(*pas) * npages; 3624 nmb = howmany(paslen, MCX_CMDQ_MAILBOX_DATASIZE); 3625 3626 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3627 mcx_cmdq_init(sc, cqe, sizeof(*in) + paslen, sizeof(*out), token); 3628 3629 in = mcx_cmdq_in(cqe); 3630 in->cmd_opcode = htobe16(MCX_CMD_MANAGE_PAGES); 3631 in->cmd_op_mod = htobe16(MCX_CMD_MANAGE_PAGES_ALLOC_SUCCESS); 3632 in->cmd_func_id = func_id; 3633 htobem32(&in->cmd_input_num_entries, npages); 3634 3635 if (mcx_cmdq_mboxes_alloc(sc, &mxm, nmb, 3636 &cqe->cq_input_ptr, token) != 0) { 3637 printf(", unable to allocate manage pages mailboxen\n"); 3638 return (-1); 3639 } 3640 3641 bus_dma_iter_init(&iter, mhm->mhm_map); 3642 for (i = 0; i < nmb; i++) { 3643 unsigned int lim; 3644 3645 pas = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, i)); 3646 lim = min(MCX_CMDQ_MAILBOX_DATASIZE / sizeof(*pas), npages); 3647 3648 for (j = 0; j < lim; j++) { 3649 htobem64(&pas[j], bus_dma_iter_addr(&iter)); 3650 bus_dma_iter_add(&iter, MCX_PAGE_SIZE); 3651 } 3652 3653 npages -= lim; 3654 } 3655 3656 mcx_cmdq_mboxes_sign(&mxm, nmb); 3657 3658 mcx_cmdq_post(sc, cqe, 0); 3659 error = mcx_cmdq_poll(sc, cqe, 1000); 3660 if (error != 0) { 3661 printf(", manage pages timeout\n"); 3662 goto free; 3663 } 3664 error = mcx_cmdq_verify(cqe); 3665 if (error != 0) { 3666 printf(", manage pages reply corrupt\n"); 3667 goto free; 3668 } 3669 3670 status = cqe->cq_output_data[0]; 3671 if (status != MCX_CQ_STATUS_OK) { 3672 printf(", manage pages failed (%x)\n", status); 3673 error = -1; 3674 goto free; 3675 } 3676 3677 free: 3678 mcx_dmamem_free(sc, &mxm); 3679 3680 return (error); 3681 } 3682 3683 static int 3684 mcx_pages(struct mcx_softc *sc, struct mcx_hwmem *mhm, uint16_t type) 3685 { 3686 int32_t npages; 3687 uint16_t func_id; 3688 3689 if (mcx_query_pages(sc, type, &npages, &func_id) != 0) { 3690 /* error printed by mcx_query_pages */ 3691 return (-1); 3692 } 3693 3694 if (npages < 1) 3695 return (0); 3696 3697 if (mcx_hwmem_alloc(sc, mhm, npages) != 0) { 3698 printf(", unable to allocate hwmem\n"); 3699 return (-1); 3700 } 3701 3702 if (mcx_add_pages(sc, mhm, func_id) != 0) { 3703 printf(", unable to add hwmem\n"); 3704 goto free; 3705 } 3706 3707 return (0); 3708 3709 free: 3710 mcx_hwmem_free(sc, mhm); 3711 3712 return (-1); 3713 } 3714 3715 static int 3716 mcx_hca_max_caps(struct mcx_softc *sc) 3717 { 3718 struct mcx_dmamem mxm; 3719 struct mcx_cmdq_entry *cqe; 3720 struct mcx_cmd_query_hca_cap_in *in; 3721 struct mcx_cmd_query_hca_cap_out *out; 3722 struct mcx_cmdq_mailbox *mb; 3723 struct mcx_cap_device *hca; 3724 uint8_t status; 3725 uint8_t token = mcx_cmdq_token(sc); 3726 int error; 3727 3728 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3729 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3730 token); 3731 3732 in = mcx_cmdq_in(cqe); 3733 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3734 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_MAX | 3735 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3736 3737 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3738 &cqe->cq_output_ptr, token) != 0) { 3739 printf(", unable to allocate query hca caps mailboxen\n"); 3740 return (-1); 3741 } 3742 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3743 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3744 3745 mcx_cmdq_post(sc, cqe, 0); 3746 error = mcx_cmdq_poll(sc, cqe, 1000); 3747 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3748 3749 if (error != 0) { 3750 printf(", query hca caps timeout\n"); 3751 goto free; 3752 } 3753 error = mcx_cmdq_verify(cqe); 3754 if (error != 0) { 3755 printf(", query hca caps reply corrupt\n"); 3756 goto free; 3757 } 3758 3759 status = cqe->cq_output_data[0]; 3760 if (status != MCX_CQ_STATUS_OK) { 3761 printf(", query hca caps failed (%x)\n", status); 3762 error = -1; 3763 goto free; 3764 } 3765 3766 mb = mcx_cq_mbox(&mxm, 0); 3767 hca = mcx_cq_mbox_data(mb); 3768 3769 if (hca->log_pg_sz > PAGE_SHIFT) { 3770 printf(", minimum system page shift %u is too large\n", 3771 hca->log_pg_sz); 3772 error = -1; 3773 goto free; 3774 } 3775 /* 3776 * blueflame register is split into two buffers, and we must alternate 3777 * between the two of them. 3778 */ 3779 sc->sc_bf_size = (1 << hca->log_bf_reg_size) / 2; 3780 sc->sc_max_rqt_size = (1 << hca->log_max_rqt_size); 3781 3782 free: 3783 mcx_dmamem_free(sc, &mxm); 3784 3785 return (error); 3786 } 3787 3788 static int 3789 mcx_hca_set_caps(struct mcx_softc *sc) 3790 { 3791 struct mcx_dmamem mxm; 3792 struct mcx_cmdq_entry *cqe; 3793 struct mcx_cmd_query_hca_cap_in *in; 3794 struct mcx_cmd_query_hca_cap_out *out; 3795 struct mcx_cmdq_mailbox *mb; 3796 struct mcx_cap_device *hca; 3797 uint8_t status; 3798 uint8_t token = mcx_cmdq_token(sc); 3799 int error; 3800 3801 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3802 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + MCX_HCA_CAP_LEN, 3803 token); 3804 3805 in = mcx_cmdq_in(cqe); 3806 in->cmd_opcode = htobe16(MCX_CMD_QUERY_HCA_CAP); 3807 in->cmd_op_mod = htobe16(MCX_CMD_QUERY_HCA_CAP_CURRENT | 3808 MCX_CMD_QUERY_HCA_CAP_DEVICE); 3809 3810 if (mcx_cmdq_mboxes_alloc(sc, &mxm, MCX_HCA_CAP_NMAILBOXES, 3811 &cqe->cq_output_ptr, token) != 0) { 3812 printf(", unable to allocate manage pages mailboxen\n"); 3813 return (-1); 3814 } 3815 mcx_cmdq_mboxes_sign(&mxm, MCX_HCA_CAP_NMAILBOXES); 3816 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_PRERW); 3817 3818 mcx_cmdq_post(sc, cqe, 0); 3819 error = mcx_cmdq_poll(sc, cqe, 1000); 3820 mcx_cmdq_mboxes_sync(sc, &mxm, BUS_DMASYNC_POSTRW); 3821 3822 if (error != 0) { 3823 printf(", query hca caps timeout\n"); 3824 goto free; 3825 } 3826 error = mcx_cmdq_verify(cqe); 3827 if (error != 0) { 3828 printf(", query hca caps reply corrupt\n"); 3829 goto free; 3830 } 3831 3832 status = cqe->cq_output_data[0]; 3833 if (status != MCX_CQ_STATUS_OK) { 3834 printf(", query hca caps failed (%x)\n", status); 3835 error = -1; 3836 goto free; 3837 } 3838 3839 mb = mcx_cq_mbox(&mxm, 0); 3840 hca = mcx_cq_mbox_data(mb); 3841 3842 hca->log_pg_sz = PAGE_SHIFT; 3843 3844 free: 3845 mcx_dmamem_free(sc, &mxm); 3846 3847 return (error); 3848 } 3849 3850 3851 static int 3852 mcx_init_hca(struct mcx_softc *sc) 3853 { 3854 struct mcx_cmdq_entry *cqe; 3855 struct mcx_cmd_init_hca_in *in; 3856 struct mcx_cmd_init_hca_out *out; 3857 int error; 3858 uint8_t status; 3859 3860 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3861 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 3862 3863 in = mcx_cmdq_in(cqe); 3864 in->cmd_opcode = htobe16(MCX_CMD_INIT_HCA); 3865 in->cmd_op_mod = htobe16(0); 3866 3867 mcx_cmdq_post(sc, cqe, 0); 3868 3869 error = mcx_cmdq_poll(sc, cqe, 1000); 3870 if (error != 0) { 3871 printf(", hca init timeout\n"); 3872 return (-1); 3873 } 3874 if (mcx_cmdq_verify(cqe) != 0) { 3875 printf(", hca init command corrupt\n"); 3876 return (-1); 3877 } 3878 3879 status = cqe->cq_output_data[0]; 3880 if (status != MCX_CQ_STATUS_OK) { 3881 printf(", hca init failed (%x)\n", status); 3882 return (-1); 3883 } 3884 3885 return (0); 3886 } 3887 3888 static int 3889 mcx_set_driver_version(struct mcx_softc *sc) 3890 { 3891 struct mcx_dmamem mxm; 3892 struct mcx_cmdq_entry *cqe; 3893 struct mcx_cmd_set_driver_version_in *in; 3894 struct mcx_cmd_set_driver_version_out *out; 3895 int error; 3896 int token; 3897 uint8_t status; 3898 3899 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3900 token = mcx_cmdq_token(sc); 3901 mcx_cmdq_init(sc, cqe, sizeof(*in) + 3902 sizeof(struct mcx_cmd_set_driver_version), sizeof(*out), token); 3903 3904 in = mcx_cmdq_in(cqe); 3905 in->cmd_opcode = htobe16(MCX_CMD_SET_DRIVER_VERSION); 3906 in->cmd_op_mod = htobe16(0); 3907 3908 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 3909 &cqe->cq_input_ptr, token) != 0) { 3910 printf(", unable to allocate set driver version mailboxen\n"); 3911 return (-1); 3912 } 3913 strlcpy(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)), 3914 "OpenBSD,mcx,1.000.000000", MCX_CMDQ_MAILBOX_DATASIZE); 3915 3916 mcx_cmdq_mboxes_sign(&mxm, 1); 3917 mcx_cmdq_post(sc, cqe, 0); 3918 3919 error = mcx_cmdq_poll(sc, cqe, 1000); 3920 if (error != 0) { 3921 printf(", set driver version timeout\n"); 3922 goto free; 3923 } 3924 if (mcx_cmdq_verify(cqe) != 0) { 3925 printf(", set driver version command corrupt\n"); 3926 goto free; 3927 } 3928 3929 status = cqe->cq_output_data[0]; 3930 if (status != MCX_CQ_STATUS_OK) { 3931 printf(", set driver version failed (%x)\n", status); 3932 error = -1; 3933 goto free; 3934 } 3935 3936 free: 3937 mcx_dmamem_free(sc, &mxm); 3938 3939 return (error); 3940 } 3941 3942 static int 3943 mcx_iff(struct mcx_softc *sc) 3944 { 3945 struct ifnet *ifp = &sc->sc_ac.ac_if; 3946 struct mcx_dmamem mxm; 3947 struct mcx_cmdq_entry *cqe; 3948 struct mcx_cmd_modify_nic_vport_context_in *in; 3949 struct mcx_cmd_modify_nic_vport_context_out *out; 3950 struct mcx_nic_vport_ctx *ctx; 3951 int error; 3952 int token; 3953 int insize; 3954 uint32_t dest; 3955 3956 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 3957 sc->sc_rss_flow_table_id; 3958 3959 /* enable or disable the promisc flow */ 3960 if (ISSET(ifp->if_flags, IFF_PROMISC)) { 3961 if (sc->sc_promisc_flow_enabled == 0) { 3962 mcx_set_flow_table_entry_mac(sc, 3963 MCX_FLOW_GROUP_PROMISC, 0, NULL, dest); 3964 sc->sc_promisc_flow_enabled = 1; 3965 } 3966 } else if (sc->sc_promisc_flow_enabled != 0) { 3967 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 3968 sc->sc_promisc_flow_enabled = 0; 3969 } 3970 3971 /* enable or disable the all-multicast flow */ 3972 if (ISSET(ifp->if_flags, IFF_ALLMULTI)) { 3973 if (sc->sc_allmulti_flow_enabled == 0) { 3974 uint8_t mcast[ETHER_ADDR_LEN]; 3975 3976 memset(mcast, 0, sizeof(mcast)); 3977 mcast[0] = 0x01; 3978 mcx_set_flow_table_entry_mac(sc, 3979 MCX_FLOW_GROUP_ALLMULTI, 0, mcast, dest); 3980 sc->sc_allmulti_flow_enabled = 1; 3981 } 3982 } else if (sc->sc_allmulti_flow_enabled != 0) { 3983 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 3984 sc->sc_allmulti_flow_enabled = 0; 3985 } 3986 3987 insize = sizeof(struct mcx_nic_vport_ctx) + 240; 3988 3989 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 3990 token = mcx_cmdq_token(sc); 3991 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 3992 3993 in = mcx_cmdq_in(cqe); 3994 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_NIC_VPORT_CONTEXT); 3995 in->cmd_op_mod = htobe16(0); 3996 in->cmd_field_select = htobe32( 3997 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_PROMISC | 3998 MCX_CMD_MODIFY_NIC_VPORT_CONTEXT_FIELD_MTU); 3999 4000 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, &cqe->cq_input_ptr, token) != 0) { 4001 printf(", unable to allocate modify " 4002 "nic vport context mailboxen\n"); 4003 return (-1); 4004 } 4005 ctx = (struct mcx_nic_vport_ctx *) 4006 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 240); 4007 ctx->vp_mtu = htobe32(sc->sc_hardmtu); 4008 /* 4009 * always leave promisc-all enabled on the vport since we 4010 * can't give it a vlan list, and we're already doing multicast 4011 * filtering in the flow table. 4012 */ 4013 ctx->vp_flags = htobe16(MCX_NIC_VPORT_CTX_PROMISC_ALL); 4014 4015 mcx_cmdq_mboxes_sign(&mxm, 1); 4016 mcx_cmdq_post(sc, cqe, 0); 4017 4018 error = mcx_cmdq_poll(sc, cqe, 1000); 4019 if (error != 0) { 4020 printf(", modify nic vport context timeout\n"); 4021 goto free; 4022 } 4023 if (mcx_cmdq_verify(cqe) != 0) { 4024 printf(", modify nic vport context command corrupt\n"); 4025 goto free; 4026 } 4027 4028 out = mcx_cmdq_out(cqe); 4029 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4030 printf(", modify nic vport context failed (%x, %x)\n", 4031 out->cmd_status, betoh32(out->cmd_syndrome)); 4032 error = -1; 4033 goto free; 4034 } 4035 4036 free: 4037 mcx_dmamem_free(sc, &mxm); 4038 4039 return (error); 4040 } 4041 4042 static int 4043 mcx_alloc_uar(struct mcx_softc *sc, int *uar) 4044 { 4045 struct mcx_cmdq_entry *cqe; 4046 struct mcx_cmd_alloc_uar_in *in; 4047 struct mcx_cmd_alloc_uar_out *out; 4048 int error; 4049 4050 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4051 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4052 4053 in = mcx_cmdq_in(cqe); 4054 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_UAR); 4055 in->cmd_op_mod = htobe16(0); 4056 4057 mcx_cmdq_post(sc, cqe, 0); 4058 4059 error = mcx_cmdq_poll(sc, cqe, 1000); 4060 if (error != 0) { 4061 printf(", alloc uar timeout\n"); 4062 return (-1); 4063 } 4064 if (mcx_cmdq_verify(cqe) != 0) { 4065 printf(", alloc uar command corrupt\n"); 4066 return (-1); 4067 } 4068 4069 out = mcx_cmdq_out(cqe); 4070 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4071 printf(", alloc uar failed (%x)\n", out->cmd_status); 4072 return (-1); 4073 } 4074 4075 *uar = mcx_get_id(out->cmd_uar); 4076 return (0); 4077 } 4078 4079 static int 4080 mcx_create_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar, 4081 uint64_t events, int vector) 4082 { 4083 struct mcx_cmdq_entry *cqe; 4084 struct mcx_dmamem mxm; 4085 struct mcx_cmd_create_eq_in *in; 4086 struct mcx_cmd_create_eq_mb_in *mbin; 4087 struct mcx_cmd_create_eq_out *out; 4088 struct mcx_eq_entry *eqe; 4089 int error; 4090 uint64_t *pas; 4091 int insize, npages, paslen, i, token; 4092 4093 eq->eq_cons = 0; 4094 4095 npages = howmany((1 << MCX_LOG_EQ_SIZE) * sizeof(struct mcx_eq_entry), 4096 MCX_PAGE_SIZE); 4097 paslen = npages * sizeof(*pas); 4098 insize = sizeof(struct mcx_cmd_create_eq_mb_in) + paslen; 4099 4100 if (mcx_dmamem_alloc(sc, &eq->eq_mem, npages * MCX_PAGE_SIZE, 4101 MCX_PAGE_SIZE) != 0) { 4102 printf(", unable to allocate event queue memory\n"); 4103 return (-1); 4104 } 4105 4106 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 4107 for (i = 0; i < (1 << MCX_LOG_EQ_SIZE); i++) { 4108 eqe[i].eq_owner = MCX_EQ_ENTRY_OWNER_INIT; 4109 } 4110 4111 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4112 token = mcx_cmdq_token(sc); 4113 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4114 4115 in = mcx_cmdq_in(cqe); 4116 in->cmd_opcode = htobe16(MCX_CMD_CREATE_EQ); 4117 in->cmd_op_mod = htobe16(0); 4118 4119 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4120 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4121 &cqe->cq_input_ptr, token) != 0) { 4122 printf(", unable to allocate create eq mailboxen\n"); 4123 return (-1); 4124 } 4125 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4126 mbin->cmd_eq_ctx.eq_uar_size = htobe32( 4127 (MCX_LOG_EQ_SIZE << MCX_EQ_CTX_LOG_EQ_SIZE_SHIFT) | uar); 4128 mbin->cmd_eq_ctx.eq_intr = vector; 4129 mbin->cmd_event_bitmask = htobe64(events); 4130 4131 /* physical addresses follow the mailbox in data */ 4132 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &eq->eq_mem); 4133 mcx_cmdq_mboxes_sign(&mxm, howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE)); 4134 mcx_cmdq_post(sc, cqe, 0); 4135 4136 error = mcx_cmdq_poll(sc, cqe, 1000); 4137 if (error != 0) { 4138 printf(", create eq timeout\n"); 4139 goto free; 4140 } 4141 if (mcx_cmdq_verify(cqe) != 0) { 4142 printf(", create eq command corrupt\n"); 4143 goto free; 4144 } 4145 4146 out = mcx_cmdq_out(cqe); 4147 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4148 printf(", create eq failed (%x, %x)\n", out->cmd_status, 4149 betoh32(out->cmd_syndrome)); 4150 error = -1; 4151 goto free; 4152 } 4153 4154 eq->eq_n = mcx_get_id(out->cmd_eqn); 4155 mcx_arm_eq(sc, eq, uar); 4156 free: 4157 mcx_dmamem_free(sc, &mxm); 4158 return (error); 4159 } 4160 4161 static int 4162 mcx_alloc_pd(struct mcx_softc *sc) 4163 { 4164 struct mcx_cmdq_entry *cqe; 4165 struct mcx_cmd_alloc_pd_in *in; 4166 struct mcx_cmd_alloc_pd_out *out; 4167 int error; 4168 4169 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4170 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4171 4172 in = mcx_cmdq_in(cqe); 4173 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_PD); 4174 in->cmd_op_mod = htobe16(0); 4175 4176 mcx_cmdq_post(sc, cqe, 0); 4177 4178 error = mcx_cmdq_poll(sc, cqe, 1000); 4179 if (error != 0) { 4180 printf(", alloc pd timeout\n"); 4181 return (-1); 4182 } 4183 if (mcx_cmdq_verify(cqe) != 0) { 4184 printf(", alloc pd command corrupt\n"); 4185 return (-1); 4186 } 4187 4188 out = mcx_cmdq_out(cqe); 4189 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4190 printf(", alloc pd failed (%x)\n", out->cmd_status); 4191 return (-1); 4192 } 4193 4194 sc->sc_pd = mcx_get_id(out->cmd_pd); 4195 return (0); 4196 } 4197 4198 static int 4199 mcx_alloc_tdomain(struct mcx_softc *sc) 4200 { 4201 struct mcx_cmdq_entry *cqe; 4202 struct mcx_cmd_alloc_td_in *in; 4203 struct mcx_cmd_alloc_td_out *out; 4204 int error; 4205 4206 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4207 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4208 4209 in = mcx_cmdq_in(cqe); 4210 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_TRANSPORT_DOMAIN); 4211 in->cmd_op_mod = htobe16(0); 4212 4213 mcx_cmdq_post(sc, cqe, 0); 4214 4215 error = mcx_cmdq_poll(sc, cqe, 1000); 4216 if (error != 0) { 4217 printf(", alloc transport domain timeout\n"); 4218 return (-1); 4219 } 4220 if (mcx_cmdq_verify(cqe) != 0) { 4221 printf(", alloc transport domain command corrupt\n"); 4222 return (-1); 4223 } 4224 4225 out = mcx_cmdq_out(cqe); 4226 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4227 printf(", alloc transport domain failed (%x)\n", 4228 out->cmd_status); 4229 return (-1); 4230 } 4231 4232 sc->sc_tdomain = mcx_get_id(out->cmd_tdomain); 4233 return (0); 4234 } 4235 4236 static int 4237 mcx_query_nic_vport_context(struct mcx_softc *sc) 4238 { 4239 struct mcx_dmamem mxm; 4240 struct mcx_cmdq_entry *cqe; 4241 struct mcx_cmd_query_nic_vport_context_in *in; 4242 struct mcx_cmd_query_nic_vport_context_out *out; 4243 struct mcx_nic_vport_ctx *ctx; 4244 uint8_t *addr; 4245 int error, token, i; 4246 4247 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4248 token = mcx_cmdq_token(sc); 4249 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx), token); 4250 4251 in = mcx_cmdq_in(cqe); 4252 in->cmd_opcode = htobe16(MCX_CMD_QUERY_NIC_VPORT_CONTEXT); 4253 in->cmd_op_mod = htobe16(0); 4254 in->cmd_allowed_list_type = 0; 4255 4256 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4257 &cqe->cq_output_ptr, token) != 0) { 4258 printf(", unable to allocate " 4259 "query nic vport context mailboxen\n"); 4260 return (-1); 4261 } 4262 mcx_cmdq_mboxes_sign(&mxm, 1); 4263 mcx_cmdq_post(sc, cqe, 0); 4264 4265 error = mcx_cmdq_poll(sc, cqe, 1000); 4266 if (error != 0) { 4267 printf(", query nic vport context timeout\n"); 4268 goto free; 4269 } 4270 if (mcx_cmdq_verify(cqe) != 0) { 4271 printf(", query nic vport context command corrupt\n"); 4272 goto free; 4273 } 4274 4275 out = mcx_cmdq_out(cqe); 4276 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4277 printf(", query nic vport context failed (%x, %x)\n", 4278 out->cmd_status, betoh32(out->cmd_syndrome)); 4279 error = -1; 4280 goto free; 4281 } 4282 4283 ctx = (struct mcx_nic_vport_ctx *) 4284 mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4285 addr = (uint8_t *)&ctx->vp_perm_addr; 4286 for (i = 0; i < ETHER_ADDR_LEN; i++) { 4287 sc->sc_ac.ac_enaddr[i] = addr[i + 2]; 4288 } 4289 free: 4290 mcx_dmamem_free(sc, &mxm); 4291 4292 return (error); 4293 } 4294 4295 static int 4296 mcx_query_special_contexts(struct mcx_softc *sc) 4297 { 4298 struct mcx_cmdq_entry *cqe; 4299 struct mcx_cmd_query_special_ctx_in *in; 4300 struct mcx_cmd_query_special_ctx_out *out; 4301 int error; 4302 4303 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4304 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 4305 4306 in = mcx_cmdq_in(cqe); 4307 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SPECIAL_CONTEXTS); 4308 in->cmd_op_mod = htobe16(0); 4309 4310 mcx_cmdq_post(sc, cqe, 0); 4311 4312 error = mcx_cmdq_poll(sc, cqe, 1000); 4313 if (error != 0) { 4314 printf(", query special contexts timeout\n"); 4315 return (-1); 4316 } 4317 if (mcx_cmdq_verify(cqe) != 0) { 4318 printf(", query special contexts command corrupt\n"); 4319 return (-1); 4320 } 4321 4322 out = mcx_cmdq_out(cqe); 4323 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4324 printf(", query special contexts failed (%x)\n", 4325 out->cmd_status); 4326 return (-1); 4327 } 4328 4329 sc->sc_lkey = betoh32(out->cmd_resd_lkey); 4330 return (0); 4331 } 4332 4333 static int 4334 mcx_set_port_mtu(struct mcx_softc *sc, int mtu) 4335 { 4336 struct mcx_reg_pmtu pmtu; 4337 int error; 4338 4339 /* read max mtu */ 4340 memset(&pmtu, 0, sizeof(pmtu)); 4341 pmtu.rp_local_port = 1; 4342 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_READ, &pmtu, 4343 sizeof(pmtu)); 4344 if (error != 0) { 4345 printf(", unable to get port MTU\n"); 4346 return error; 4347 } 4348 4349 mtu = min(mtu, betoh16(pmtu.rp_max_mtu)); 4350 pmtu.rp_admin_mtu = htobe16(mtu); 4351 error = mcx_access_hca_reg(sc, MCX_REG_PMTU, MCX_REG_OP_WRITE, &pmtu, 4352 sizeof(pmtu)); 4353 if (error != 0) { 4354 printf(", unable to set port MTU\n"); 4355 return error; 4356 } 4357 4358 sc->sc_hardmtu = mtu; 4359 sc->sc_rxbufsz = roundup(mtu + ETHER_ALIGN, sizeof(long)); 4360 return 0; 4361 } 4362 4363 static int 4364 mcx_create_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar, int db, int eqn) 4365 { 4366 struct mcx_cmdq_entry *cmde; 4367 struct mcx_cq_entry *cqe; 4368 struct mcx_dmamem mxm; 4369 struct mcx_cmd_create_cq_in *in; 4370 struct mcx_cmd_create_cq_mb_in *mbin; 4371 struct mcx_cmd_create_cq_out *out; 4372 int error; 4373 uint64_t *pas; 4374 int insize, npages, paslen, i, token; 4375 4376 npages = howmany((1 << MCX_LOG_CQ_SIZE) * sizeof(struct mcx_cq_entry), 4377 MCX_PAGE_SIZE); 4378 paslen = npages * sizeof(*pas); 4379 insize = sizeof(struct mcx_cmd_create_cq_mb_in) + paslen; 4380 4381 if (mcx_dmamem_alloc(sc, &cq->cq_mem, npages * MCX_PAGE_SIZE, 4382 MCX_PAGE_SIZE) != 0) { 4383 printf("%s: unable to allocate completion queue memory\n", 4384 DEVNAME(sc)); 4385 return (-1); 4386 } 4387 cqe = MCX_DMA_KVA(&cq->cq_mem); 4388 for (i = 0; i < (1 << MCX_LOG_CQ_SIZE); i++) { 4389 cqe[i].cq_opcode_owner = MCX_CQ_ENTRY_FLAG_OWNER; 4390 } 4391 4392 cmde = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4393 token = mcx_cmdq_token(sc); 4394 mcx_cmdq_init(sc, cmde, sizeof(*in) + insize, sizeof(*out), token); 4395 4396 in = mcx_cmdq_in(cmde); 4397 in->cmd_opcode = htobe16(MCX_CMD_CREATE_CQ); 4398 in->cmd_op_mod = htobe16(0); 4399 4400 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4401 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4402 &cmde->cq_input_ptr, token) != 0) { 4403 printf("%s: unable to allocate create cq mailboxen\n", 4404 DEVNAME(sc)); 4405 error = -1; 4406 goto free; 4407 } 4408 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4409 mbin->cmd_cq_ctx.cq_uar_size = htobe32( 4410 (MCX_LOG_CQ_SIZE << MCX_CQ_CTX_LOG_CQ_SIZE_SHIFT) | uar); 4411 mbin->cmd_cq_ctx.cq_eqn = htobe32(eqn); 4412 mbin->cmd_cq_ctx.cq_period_max_count = htobe32( 4413 (MCX_CQ_MOD_PERIOD << MCX_CQ_CTX_PERIOD_SHIFT) | 4414 MCX_CQ_MOD_COUNTER); 4415 mbin->cmd_cq_ctx.cq_doorbell = htobe64( 4416 MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4417 MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db)); 4418 4419 /* physical addresses follow the mailbox in data */ 4420 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin), npages, &cq->cq_mem); 4421 mcx_cmdq_post(sc, cmde, 0); 4422 4423 error = mcx_cmdq_poll(sc, cmde, 1000); 4424 if (error != 0) { 4425 printf("%s: create cq timeout\n", DEVNAME(sc)); 4426 goto free; 4427 } 4428 if (mcx_cmdq_verify(cmde) != 0) { 4429 printf("%s: create cq command corrupt\n", DEVNAME(sc)); 4430 goto free; 4431 } 4432 4433 out = mcx_cmdq_out(cmde); 4434 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4435 printf("%s: create cq failed (%x, %x)\n", DEVNAME(sc), 4436 out->cmd_status, betoh32(out->cmd_syndrome)); 4437 error = -1; 4438 goto free; 4439 } 4440 4441 cq->cq_n = mcx_get_id(out->cmd_cqn); 4442 cq->cq_cons = 0; 4443 cq->cq_count = 0; 4444 cq->cq_doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem) + 4445 MCX_CQ_DOORBELL_BASE + (MCX_CQ_DOORBELL_STRIDE * db); 4446 mcx_arm_cq(sc, cq, uar); 4447 4448 free: 4449 mcx_dmamem_free(sc, &mxm); 4450 return (error); 4451 } 4452 4453 static int 4454 mcx_destroy_cq(struct mcx_softc *sc, struct mcx_cq *cq) 4455 { 4456 struct mcx_cmdq_entry *cqe; 4457 struct mcx_cmd_destroy_cq_in *in; 4458 struct mcx_cmd_destroy_cq_out *out; 4459 int error; 4460 int token; 4461 4462 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4463 token = mcx_cmdq_token(sc); 4464 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4465 4466 in = mcx_cmdq_in(cqe); 4467 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_CQ); 4468 in->cmd_op_mod = htobe16(0); 4469 in->cmd_cqn = htobe32(cq->cq_n); 4470 4471 mcx_cmdq_post(sc, cqe, 0); 4472 error = mcx_cmdq_poll(sc, cqe, 1000); 4473 if (error != 0) { 4474 printf("%s: destroy cq timeout\n", DEVNAME(sc)); 4475 return error; 4476 } 4477 if (mcx_cmdq_verify(cqe) != 0) { 4478 printf("%s: destroy cq command corrupt\n", DEVNAME(sc)); 4479 return error; 4480 } 4481 4482 out = mcx_cmdq_out(cqe); 4483 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4484 printf("%s: destroy cq failed (%x, %x)\n", DEVNAME(sc), 4485 out->cmd_status, betoh32(out->cmd_syndrome)); 4486 return -1; 4487 } 4488 4489 cq->cq_n = 0; 4490 mcx_dmamem_free(sc, &cq->cq_mem); 4491 cq->cq_cons = 0; 4492 cq->cq_count = 0; 4493 return 0; 4494 } 4495 4496 static int 4497 mcx_create_rq(struct mcx_softc *sc, struct mcx_rx *rx, int db, int cqn) 4498 { 4499 struct mcx_cmdq_entry *cqe; 4500 struct mcx_dmamem mxm; 4501 struct mcx_cmd_create_rq_in *in; 4502 struct mcx_cmd_create_rq_out *out; 4503 struct mcx_rq_ctx *mbin; 4504 int error; 4505 uint64_t *pas; 4506 uint32_t rq_flags; 4507 uint8_t *doorbell; 4508 int insize, npages, paslen, token; 4509 4510 npages = howmany((1 << MCX_LOG_RQ_SIZE) * sizeof(struct mcx_rq_entry), 4511 MCX_PAGE_SIZE); 4512 paslen = npages * sizeof(*pas); 4513 insize = 0x10 + sizeof(struct mcx_rq_ctx) + paslen; 4514 4515 if (mcx_dmamem_alloc(sc, &rx->rx_rq_mem, npages * MCX_PAGE_SIZE, 4516 MCX_PAGE_SIZE) != 0) { 4517 printf("%s: unable to allocate receive queue memory\n", 4518 DEVNAME(sc)); 4519 return (-1); 4520 } 4521 4522 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4523 token = mcx_cmdq_token(sc); 4524 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize, sizeof(*out), token); 4525 4526 in = mcx_cmdq_in(cqe); 4527 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQ); 4528 in->cmd_op_mod = htobe16(0); 4529 4530 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4531 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4532 &cqe->cq_input_ptr, token) != 0) { 4533 printf("%s: unable to allocate create rq mailboxen\n", 4534 DEVNAME(sc)); 4535 error = -1; 4536 goto free; 4537 } 4538 mbin = (struct mcx_rq_ctx *) 4539 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 4540 rq_flags = MCX_RQ_CTX_RLKEY; 4541 #if NVLAN == 0 4542 rq_flags |= MCX_RQ_CTX_VLAN_STRIP_DIS; 4543 #endif 4544 mbin->rq_flags = htobe32(rq_flags); 4545 mbin->rq_cqn = htobe32(cqn); 4546 mbin->rq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 4547 mbin->rq_wq.wq_pd = htobe32(sc->sc_pd); 4548 mbin->rq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4549 MCX_WQ_DOORBELL_BASE + (db * MCX_WQ_DOORBELL_STRIDE)); 4550 mbin->rq_wq.wq_log_stride = htobe16(4); 4551 mbin->rq_wq.wq_log_size = MCX_LOG_RQ_SIZE; 4552 4553 /* physical addresses follow the mailbox in data */ 4554 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, npages, &rx->rx_rq_mem); 4555 mcx_cmdq_post(sc, cqe, 0); 4556 4557 error = mcx_cmdq_poll(sc, cqe, 1000); 4558 if (error != 0) { 4559 printf("%s: create rq timeout\n", DEVNAME(sc)); 4560 goto free; 4561 } 4562 if (mcx_cmdq_verify(cqe) != 0) { 4563 printf("%s: create rq command corrupt\n", DEVNAME(sc)); 4564 goto free; 4565 } 4566 4567 out = mcx_cmdq_out(cqe); 4568 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4569 printf("%s: create rq failed (%x, %x)\n", DEVNAME(sc), 4570 out->cmd_status, betoh32(out->cmd_syndrome)); 4571 error = -1; 4572 goto free; 4573 } 4574 4575 rx->rx_rqn = mcx_get_id(out->cmd_rqn); 4576 4577 doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem); 4578 rx->rx_doorbell = (uint32_t *)(doorbell + MCX_WQ_DOORBELL_BASE + 4579 (db * MCX_WQ_DOORBELL_STRIDE)); 4580 4581 free: 4582 mcx_dmamem_free(sc, &mxm); 4583 return (error); 4584 } 4585 4586 static int 4587 mcx_ready_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4588 { 4589 struct mcx_cmdq_entry *cqe; 4590 struct mcx_dmamem mxm; 4591 struct mcx_cmd_modify_rq_in *in; 4592 struct mcx_cmd_modify_rq_mb_in *mbin; 4593 struct mcx_cmd_modify_rq_out *out; 4594 int error; 4595 int token; 4596 4597 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4598 token = mcx_cmdq_token(sc); 4599 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4600 sizeof(*out), token); 4601 4602 in = mcx_cmdq_in(cqe); 4603 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_RQ); 4604 in->cmd_op_mod = htobe16(0); 4605 in->cmd_rq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | rx->rx_rqn); 4606 4607 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4608 &cqe->cq_input_ptr, token) != 0) { 4609 printf("%s: unable to allocate modify rq mailbox\n", 4610 DEVNAME(sc)); 4611 return (-1); 4612 } 4613 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4614 mbin->cmd_rq_ctx.rq_flags = htobe32( 4615 MCX_QUEUE_STATE_RDY << MCX_RQ_CTX_STATE_SHIFT); 4616 4617 mcx_cmdq_mboxes_sign(&mxm, 1); 4618 mcx_cmdq_post(sc, cqe, 0); 4619 error = mcx_cmdq_poll(sc, cqe, 1000); 4620 if (error != 0) { 4621 printf("%s: modify rq timeout\n", DEVNAME(sc)); 4622 goto free; 4623 } 4624 if (mcx_cmdq_verify(cqe) != 0) { 4625 printf("%s: modify rq command corrupt\n", DEVNAME(sc)); 4626 goto free; 4627 } 4628 4629 out = mcx_cmdq_out(cqe); 4630 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4631 printf("%s: modify rq failed (%x, %x)\n", DEVNAME(sc), 4632 out->cmd_status, betoh32(out->cmd_syndrome)); 4633 error = -1; 4634 goto free; 4635 } 4636 4637 free: 4638 mcx_dmamem_free(sc, &mxm); 4639 return (error); 4640 } 4641 4642 static int 4643 mcx_destroy_rq(struct mcx_softc *sc, struct mcx_rx *rx) 4644 { 4645 struct mcx_cmdq_entry *cqe; 4646 struct mcx_cmd_destroy_rq_in *in; 4647 struct mcx_cmd_destroy_rq_out *out; 4648 int error; 4649 int token; 4650 4651 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4652 token = mcx_cmdq_token(sc); 4653 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4654 4655 in = mcx_cmdq_in(cqe); 4656 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQ); 4657 in->cmd_op_mod = htobe16(0); 4658 in->cmd_rqn = htobe32(rx->rx_rqn); 4659 4660 mcx_cmdq_post(sc, cqe, 0); 4661 error = mcx_cmdq_poll(sc, cqe, 1000); 4662 if (error != 0) { 4663 printf("%s: destroy rq timeout\n", DEVNAME(sc)); 4664 return error; 4665 } 4666 if (mcx_cmdq_verify(cqe) != 0) { 4667 printf("%s: destroy rq command corrupt\n", DEVNAME(sc)); 4668 return error; 4669 } 4670 4671 out = mcx_cmdq_out(cqe); 4672 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4673 printf("%s: destroy rq failed (%x, %x)\n", DEVNAME(sc), 4674 out->cmd_status, betoh32(out->cmd_syndrome)); 4675 return -1; 4676 } 4677 4678 rx->rx_rqn = 0; 4679 return 0; 4680 } 4681 4682 static int 4683 mcx_create_tir_direct(struct mcx_softc *sc, struct mcx_rx *rx, int *tirn) 4684 { 4685 struct mcx_cmdq_entry *cqe; 4686 struct mcx_dmamem mxm; 4687 struct mcx_cmd_create_tir_in *in; 4688 struct mcx_cmd_create_tir_mb_in *mbin; 4689 struct mcx_cmd_create_tir_out *out; 4690 int error; 4691 int token; 4692 4693 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4694 token = mcx_cmdq_token(sc); 4695 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4696 sizeof(*out), token); 4697 4698 in = mcx_cmdq_in(cqe); 4699 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 4700 in->cmd_op_mod = htobe16(0); 4701 4702 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4703 &cqe->cq_input_ptr, token) != 0) { 4704 printf("%s: unable to allocate create tir mailbox\n", 4705 DEVNAME(sc)); 4706 return (-1); 4707 } 4708 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4709 /* leave disp_type = 0, so packets get sent to the inline rqn */ 4710 mbin->cmd_inline_rqn = htobe32(rx->rx_rqn); 4711 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 4712 4713 mcx_cmdq_post(sc, cqe, 0); 4714 error = mcx_cmdq_poll(sc, cqe, 1000); 4715 if (error != 0) { 4716 printf("%s: create tir timeout\n", DEVNAME(sc)); 4717 goto free; 4718 } 4719 if (mcx_cmdq_verify(cqe) != 0) { 4720 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 4721 goto free; 4722 } 4723 4724 out = mcx_cmdq_out(cqe); 4725 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4726 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 4727 out->cmd_status, betoh32(out->cmd_syndrome)); 4728 error = -1; 4729 goto free; 4730 } 4731 4732 *tirn = mcx_get_id(out->cmd_tirn); 4733 free: 4734 mcx_dmamem_free(sc, &mxm); 4735 return (error); 4736 } 4737 4738 static int 4739 mcx_create_tir_indirect(struct mcx_softc *sc, int rqtn, uint32_t hash_sel, 4740 int *tirn) 4741 { 4742 struct mcx_cmdq_entry *cqe; 4743 struct mcx_dmamem mxm; 4744 struct mcx_cmd_create_tir_in *in; 4745 struct mcx_cmd_create_tir_mb_in *mbin; 4746 struct mcx_cmd_create_tir_out *out; 4747 int error; 4748 int token; 4749 4750 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4751 token = mcx_cmdq_token(sc); 4752 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4753 sizeof(*out), token); 4754 4755 in = mcx_cmdq_in(cqe); 4756 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIR); 4757 in->cmd_op_mod = htobe16(0); 4758 4759 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4760 &cqe->cq_input_ptr, token) != 0) { 4761 printf("%s: unable to allocate create tir mailbox\n", 4762 DEVNAME(sc)); 4763 return (-1); 4764 } 4765 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4766 mbin->cmd_disp_type = htobe32(MCX_TIR_CTX_DISP_TYPE_INDIRECT 4767 << MCX_TIR_CTX_DISP_TYPE_SHIFT); 4768 mbin->cmd_indir_table = htobe32(rqtn); 4769 mbin->cmd_tdomain = htobe32(sc->sc_tdomain | 4770 MCX_TIR_CTX_HASH_TOEPLITZ << MCX_TIR_CTX_HASH_SHIFT); 4771 mbin->cmd_rx_hash_sel_outer = htobe32(hash_sel); 4772 stoeplitz_to_key(&mbin->cmd_rx_hash_key, 4773 sizeof(mbin->cmd_rx_hash_key)); 4774 4775 mcx_cmdq_post(sc, cqe, 0); 4776 error = mcx_cmdq_poll(sc, cqe, 1000); 4777 if (error != 0) { 4778 printf("%s: create tir timeout\n", DEVNAME(sc)); 4779 goto free; 4780 } 4781 if (mcx_cmdq_verify(cqe) != 0) { 4782 printf("%s: create tir command corrupt\n", DEVNAME(sc)); 4783 goto free; 4784 } 4785 4786 out = mcx_cmdq_out(cqe); 4787 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4788 printf("%s: create tir failed (%x, %x)\n", DEVNAME(sc), 4789 out->cmd_status, betoh32(out->cmd_syndrome)); 4790 error = -1; 4791 goto free; 4792 } 4793 4794 *tirn = mcx_get_id(out->cmd_tirn); 4795 free: 4796 mcx_dmamem_free(sc, &mxm); 4797 return (error); 4798 } 4799 4800 static int 4801 mcx_destroy_tir(struct mcx_softc *sc, int tirn) 4802 { 4803 struct mcx_cmdq_entry *cqe; 4804 struct mcx_cmd_destroy_tir_in *in; 4805 struct mcx_cmd_destroy_tir_out *out; 4806 int error; 4807 int token; 4808 4809 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4810 token = mcx_cmdq_token(sc); 4811 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4812 4813 in = mcx_cmdq_in(cqe); 4814 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIR); 4815 in->cmd_op_mod = htobe16(0); 4816 in->cmd_tirn = htobe32(tirn); 4817 4818 mcx_cmdq_post(sc, cqe, 0); 4819 error = mcx_cmdq_poll(sc, cqe, 1000); 4820 if (error != 0) { 4821 printf("%s: destroy tir timeout\n", DEVNAME(sc)); 4822 return error; 4823 } 4824 if (mcx_cmdq_verify(cqe) != 0) { 4825 printf("%s: destroy tir command corrupt\n", DEVNAME(sc)); 4826 return error; 4827 } 4828 4829 out = mcx_cmdq_out(cqe); 4830 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4831 printf("%s: destroy tir failed (%x, %x)\n", DEVNAME(sc), 4832 out->cmd_status, betoh32(out->cmd_syndrome)); 4833 return -1; 4834 } 4835 4836 return (0); 4837 } 4838 4839 static int 4840 mcx_create_sq(struct mcx_softc *sc, struct mcx_tx *tx, int uar, int db, 4841 int cqn) 4842 { 4843 struct mcx_cmdq_entry *cqe; 4844 struct mcx_dmamem mxm; 4845 struct mcx_cmd_create_sq_in *in; 4846 struct mcx_sq_ctx *mbin; 4847 struct mcx_cmd_create_sq_out *out; 4848 int error; 4849 uint64_t *pas; 4850 uint8_t *doorbell; 4851 int insize, npages, paslen, token; 4852 4853 npages = howmany((1 << MCX_LOG_SQ_SIZE) * sizeof(struct mcx_sq_entry), 4854 MCX_PAGE_SIZE); 4855 paslen = npages * sizeof(*pas); 4856 insize = sizeof(struct mcx_sq_ctx) + paslen; 4857 4858 if (mcx_dmamem_alloc(sc, &tx->tx_sq_mem, npages * MCX_PAGE_SIZE, 4859 MCX_PAGE_SIZE) != 0) { 4860 printf("%s: unable to allocate send queue memory\n", 4861 DEVNAME(sc)); 4862 return (-1); 4863 } 4864 4865 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4866 token = mcx_cmdq_token(sc); 4867 mcx_cmdq_init(sc, cqe, sizeof(*in) + insize + paslen, sizeof(*out), 4868 token); 4869 4870 in = mcx_cmdq_in(cqe); 4871 in->cmd_opcode = htobe16(MCX_CMD_CREATE_SQ); 4872 in->cmd_op_mod = htobe16(0); 4873 4874 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 4875 howmany(insize, MCX_CMDQ_MAILBOX_DATASIZE), 4876 &cqe->cq_input_ptr, token) != 0) { 4877 printf("%s: unable to allocate create sq mailboxen\n", 4878 DEVNAME(sc)); 4879 error = -1; 4880 goto free; 4881 } 4882 mbin = (struct mcx_sq_ctx *) 4883 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))) + 0x10); 4884 mbin->sq_flags = htobe32(MCX_SQ_CTX_RLKEY | 4885 (1 << MCX_SQ_CTX_MIN_WQE_INLINE_SHIFT)); 4886 mbin->sq_cqn = htobe32(cqn); 4887 mbin->sq_tis_lst_sz = htobe32(1 << MCX_SQ_CTX_TIS_LST_SZ_SHIFT); 4888 mbin->sq_tis_num = htobe32(sc->sc_tis); 4889 mbin->sq_wq.wq_type = MCX_WQ_CTX_TYPE_CYCLIC; 4890 mbin->sq_wq.wq_pd = htobe32(sc->sc_pd); 4891 mbin->sq_wq.wq_uar_page = htobe32(uar); 4892 mbin->sq_wq.wq_doorbell = htobe64(MCX_DMA_DVA(&sc->sc_doorbell_mem) + 4893 MCX_WQ_DOORBELL_BASE + (db * MCX_WQ_DOORBELL_STRIDE)); 4894 mbin->sq_wq.wq_log_stride = htobe16(MCX_LOG_SQ_ENTRY_SIZE); 4895 mbin->sq_wq.wq_log_size = MCX_LOG_SQ_SIZE; 4896 4897 /* physical addresses follow the mailbox in data */ 4898 mcx_cmdq_mboxes_pas(&mxm, sizeof(*mbin) + 0x10, 4899 npages, &tx->tx_sq_mem); 4900 mcx_cmdq_post(sc, cqe, 0); 4901 4902 error = mcx_cmdq_poll(sc, cqe, 1000); 4903 if (error != 0) { 4904 printf("%s: create sq timeout\n", DEVNAME(sc)); 4905 goto free; 4906 } 4907 if (mcx_cmdq_verify(cqe) != 0) { 4908 printf("%s: create sq command corrupt\n", DEVNAME(sc)); 4909 goto free; 4910 } 4911 4912 out = mcx_cmdq_out(cqe); 4913 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4914 printf("%s: create sq failed (%x, %x)\n", DEVNAME(sc), 4915 out->cmd_status, betoh32(out->cmd_syndrome)); 4916 error = -1; 4917 goto free; 4918 } 4919 4920 tx->tx_uar = uar; 4921 tx->tx_sqn = mcx_get_id(out->cmd_sqn); 4922 4923 doorbell = MCX_DMA_KVA(&sc->sc_doorbell_mem); 4924 tx->tx_doorbell = (uint32_t *)(doorbell + MCX_WQ_DOORBELL_BASE + 4925 (db * MCX_WQ_DOORBELL_STRIDE) + 4); 4926 free: 4927 mcx_dmamem_free(sc, &mxm); 4928 return (error); 4929 } 4930 4931 static int 4932 mcx_destroy_sq(struct mcx_softc *sc, struct mcx_tx *tx) 4933 { 4934 struct mcx_cmdq_entry *cqe; 4935 struct mcx_cmd_destroy_sq_in *in; 4936 struct mcx_cmd_destroy_sq_out *out; 4937 int error; 4938 int token; 4939 4940 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4941 token = mcx_cmdq_token(sc); 4942 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 4943 4944 in = mcx_cmdq_in(cqe); 4945 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_SQ); 4946 in->cmd_op_mod = htobe16(0); 4947 in->cmd_sqn = htobe32(tx->tx_sqn); 4948 4949 mcx_cmdq_post(sc, cqe, 0); 4950 error = mcx_cmdq_poll(sc, cqe, 1000); 4951 if (error != 0) { 4952 printf("%s: destroy sq timeout\n", DEVNAME(sc)); 4953 return error; 4954 } 4955 if (mcx_cmdq_verify(cqe) != 0) { 4956 printf("%s: destroy sq command corrupt\n", DEVNAME(sc)); 4957 return error; 4958 } 4959 4960 out = mcx_cmdq_out(cqe); 4961 if (out->cmd_status != MCX_CQ_STATUS_OK) { 4962 printf("%s: destroy sq failed (%x, %x)\n", DEVNAME(sc), 4963 out->cmd_status, betoh32(out->cmd_syndrome)); 4964 return -1; 4965 } 4966 4967 tx->tx_sqn = 0; 4968 return 0; 4969 } 4970 4971 static int 4972 mcx_ready_sq(struct mcx_softc *sc, struct mcx_tx *tx) 4973 { 4974 struct mcx_cmdq_entry *cqe; 4975 struct mcx_dmamem mxm; 4976 struct mcx_cmd_modify_sq_in *in; 4977 struct mcx_cmd_modify_sq_mb_in *mbin; 4978 struct mcx_cmd_modify_sq_out *out; 4979 int error; 4980 int token; 4981 4982 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 4983 token = mcx_cmdq_token(sc); 4984 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 4985 sizeof(*out), token); 4986 4987 in = mcx_cmdq_in(cqe); 4988 in->cmd_opcode = htobe16(MCX_CMD_MODIFY_SQ); 4989 in->cmd_op_mod = htobe16(0); 4990 in->cmd_sq_state = htobe32((MCX_QUEUE_STATE_RST << 28) | tx->tx_sqn); 4991 4992 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 4993 &cqe->cq_input_ptr, token) != 0) { 4994 printf("%s: unable to allocate modify sq mailbox\n", 4995 DEVNAME(sc)); 4996 return (-1); 4997 } 4998 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 4999 mbin->cmd_sq_ctx.sq_flags = htobe32( 5000 MCX_QUEUE_STATE_RDY << MCX_SQ_CTX_STATE_SHIFT); 5001 5002 mcx_cmdq_mboxes_sign(&mxm, 1); 5003 mcx_cmdq_post(sc, cqe, 0); 5004 error = mcx_cmdq_poll(sc, cqe, 1000); 5005 if (error != 0) { 5006 printf("%s: modify sq timeout\n", DEVNAME(sc)); 5007 goto free; 5008 } 5009 if (mcx_cmdq_verify(cqe) != 0) { 5010 printf("%s: modify sq command corrupt\n", DEVNAME(sc)); 5011 goto free; 5012 } 5013 5014 out = mcx_cmdq_out(cqe); 5015 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5016 printf("%s: modify sq failed (%x, %x)\n", DEVNAME(sc), 5017 out->cmd_status, betoh32(out->cmd_syndrome)); 5018 error = -1; 5019 goto free; 5020 } 5021 5022 free: 5023 mcx_dmamem_free(sc, &mxm); 5024 return (error); 5025 } 5026 5027 static int 5028 mcx_create_tis(struct mcx_softc *sc, int *tis) 5029 { 5030 struct mcx_cmdq_entry *cqe; 5031 struct mcx_dmamem mxm; 5032 struct mcx_cmd_create_tis_in *in; 5033 struct mcx_cmd_create_tis_mb_in *mbin; 5034 struct mcx_cmd_create_tis_out *out; 5035 int error; 5036 int token; 5037 5038 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5039 token = mcx_cmdq_token(sc); 5040 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5041 sizeof(*out), token); 5042 5043 in = mcx_cmdq_in(cqe); 5044 in->cmd_opcode = htobe16(MCX_CMD_CREATE_TIS); 5045 in->cmd_op_mod = htobe16(0); 5046 5047 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5048 &cqe->cq_input_ptr, token) != 0) { 5049 printf("%s: unable to allocate create tis mailbox\n", 5050 DEVNAME(sc)); 5051 return (-1); 5052 } 5053 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5054 mbin->cmd_tdomain = htobe32(sc->sc_tdomain); 5055 5056 mcx_cmdq_mboxes_sign(&mxm, 1); 5057 mcx_cmdq_post(sc, cqe, 0); 5058 error = mcx_cmdq_poll(sc, cqe, 1000); 5059 if (error != 0) { 5060 printf("%s: create tis timeout\n", DEVNAME(sc)); 5061 goto free; 5062 } 5063 if (mcx_cmdq_verify(cqe) != 0) { 5064 printf("%s: create tis command corrupt\n", DEVNAME(sc)); 5065 goto free; 5066 } 5067 5068 out = mcx_cmdq_out(cqe); 5069 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5070 printf("%s: create tis failed (%x, %x)\n", DEVNAME(sc), 5071 out->cmd_status, betoh32(out->cmd_syndrome)); 5072 error = -1; 5073 goto free; 5074 } 5075 5076 *tis = mcx_get_id(out->cmd_tisn); 5077 free: 5078 mcx_dmamem_free(sc, &mxm); 5079 return (error); 5080 } 5081 5082 static int 5083 mcx_destroy_tis(struct mcx_softc *sc, int tis) 5084 { 5085 struct mcx_cmdq_entry *cqe; 5086 struct mcx_cmd_destroy_tis_in *in; 5087 struct mcx_cmd_destroy_tis_out *out; 5088 int error; 5089 int token; 5090 5091 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5092 token = mcx_cmdq_token(sc); 5093 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5094 5095 in = mcx_cmdq_in(cqe); 5096 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_TIS); 5097 in->cmd_op_mod = htobe16(0); 5098 in->cmd_tisn = htobe32(tis); 5099 5100 mcx_cmdq_post(sc, cqe, 0); 5101 error = mcx_cmdq_poll(sc, cqe, 1000); 5102 if (error != 0) { 5103 printf("%s: destroy tis timeout\n", DEVNAME(sc)); 5104 return error; 5105 } 5106 if (mcx_cmdq_verify(cqe) != 0) { 5107 printf("%s: destroy tis command corrupt\n", DEVNAME(sc)); 5108 return error; 5109 } 5110 5111 out = mcx_cmdq_out(cqe); 5112 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5113 printf("%s: destroy tis failed (%x, %x)\n", DEVNAME(sc), 5114 out->cmd_status, betoh32(out->cmd_syndrome)); 5115 return -1; 5116 } 5117 5118 return 0; 5119 } 5120 5121 static int 5122 mcx_create_rqt(struct mcx_softc *sc, int size, int *rqns, int *rqt) 5123 { 5124 struct mcx_cmdq_entry *cqe; 5125 struct mcx_dmamem mxm; 5126 struct mcx_cmd_create_rqt_in *in; 5127 struct mcx_cmd_create_rqt_mb_in *mbin; 5128 struct mcx_cmd_create_rqt_out *out; 5129 struct mcx_rqt_ctx *rqt_ctx; 5130 int *rqtn; 5131 int error; 5132 int token; 5133 int i; 5134 5135 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5136 token = mcx_cmdq_token(sc); 5137 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + 5138 (size * sizeof(int)), sizeof(*out), token); 5139 5140 in = mcx_cmdq_in(cqe); 5141 in->cmd_opcode = htobe16(MCX_CMD_CREATE_RQT); 5142 in->cmd_op_mod = htobe16(0); 5143 5144 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5145 &cqe->cq_input_ptr, token) != 0) { 5146 printf("%s: unable to allocate create rqt mailbox\n", 5147 DEVNAME(sc)); 5148 return (-1); 5149 } 5150 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5151 rqt_ctx = &mbin->cmd_rqt; 5152 rqt_ctx->cmd_rqt_max_size = htobe16(sc->sc_max_rqt_size); 5153 rqt_ctx->cmd_rqt_actual_size = htobe16(size); 5154 5155 /* rqt list follows the rqt context */ 5156 rqtn = (int *)(rqt_ctx + 1); 5157 for (i = 0; i < size; i++) { 5158 rqtn[i] = htobe32(rqns[i]); 5159 } 5160 5161 mcx_cmdq_mboxes_sign(&mxm, 1); 5162 mcx_cmdq_post(sc, cqe, 0); 5163 error = mcx_cmdq_poll(sc, cqe, 1000); 5164 if (error != 0) { 5165 printf("%s: create rqt timeout\n", DEVNAME(sc)); 5166 goto free; 5167 } 5168 if (mcx_cmdq_verify(cqe) != 0) { 5169 printf("%s: create rqt command corrupt\n", DEVNAME(sc)); 5170 goto free; 5171 } 5172 5173 out = mcx_cmdq_out(cqe); 5174 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5175 printf("%s: create rqt failed (%x, %x)\n", DEVNAME(sc), 5176 out->cmd_status, betoh32(out->cmd_syndrome)); 5177 error = -1; 5178 goto free; 5179 } 5180 5181 *rqt = mcx_get_id(out->cmd_rqtn); 5182 return (0); 5183 free: 5184 mcx_dmamem_free(sc, &mxm); 5185 return (error); 5186 } 5187 5188 static int 5189 mcx_destroy_rqt(struct mcx_softc *sc, int rqt) 5190 { 5191 struct mcx_cmdq_entry *cqe; 5192 struct mcx_cmd_destroy_rqt_in *in; 5193 struct mcx_cmd_destroy_rqt_out *out; 5194 int error; 5195 int token; 5196 5197 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5198 token = mcx_cmdq_token(sc); 5199 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), token); 5200 5201 in = mcx_cmdq_in(cqe); 5202 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_RQT); 5203 in->cmd_op_mod = htobe16(0); 5204 in->cmd_rqtn = htobe32(rqt); 5205 5206 mcx_cmdq_post(sc, cqe, 0); 5207 error = mcx_cmdq_poll(sc, cqe, 1000); 5208 if (error != 0) { 5209 printf("%s: destroy rqt timeout\n", DEVNAME(sc)); 5210 return error; 5211 } 5212 if (mcx_cmdq_verify(cqe) != 0) { 5213 printf("%s: destroy rqt command corrupt\n", DEVNAME(sc)); 5214 return error; 5215 } 5216 5217 out = mcx_cmdq_out(cqe); 5218 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5219 printf("%s: destroy rqt failed (%x, %x)\n", DEVNAME(sc), 5220 out->cmd_status, betoh32(out->cmd_syndrome)); 5221 return -1; 5222 } 5223 5224 return 0; 5225 } 5226 5227 #if 0 5228 static int 5229 mcx_alloc_flow_counter(struct mcx_softc *sc, int i) 5230 { 5231 struct mcx_cmdq_entry *cqe; 5232 struct mcx_cmd_alloc_flow_counter_in *in; 5233 struct mcx_cmd_alloc_flow_counter_out *out; 5234 int error; 5235 5236 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5237 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out), mcx_cmdq_token(sc)); 5238 5239 in = mcx_cmdq_in(cqe); 5240 in->cmd_opcode = htobe16(MCX_CMD_ALLOC_FLOW_COUNTER); 5241 in->cmd_op_mod = htobe16(0); 5242 5243 mcx_cmdq_post(sc, cqe, 0); 5244 5245 error = mcx_cmdq_poll(sc, cqe, 1000); 5246 if (error != 0) { 5247 printf("%s: alloc flow counter timeout\n", DEVNAME(sc)); 5248 return (-1); 5249 } 5250 if (mcx_cmdq_verify(cqe) != 0) { 5251 printf("%s: alloc flow counter command corrupt\n", DEVNAME(sc)); 5252 return (-1); 5253 } 5254 5255 out = (struct mcx_cmd_alloc_flow_counter_out *)cqe->cq_output_data; 5256 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5257 printf("%s: alloc flow counter failed (%x)\n", DEVNAME(sc), 5258 out->cmd_status); 5259 return (-1); 5260 } 5261 5262 sc->sc_flow_counter_id[i] = betoh16(out->cmd_flow_counter_id); 5263 printf("flow counter id %d = %d\n", i, sc->sc_flow_counter_id[i]); 5264 5265 return (0); 5266 } 5267 #endif 5268 5269 static int 5270 mcx_create_flow_table(struct mcx_softc *sc, int log_size, int level, 5271 int *flow_table_id) 5272 { 5273 struct mcx_cmdq_entry *cqe; 5274 struct mcx_dmamem mxm; 5275 struct mcx_cmd_create_flow_table_in *in; 5276 struct mcx_cmd_create_flow_table_mb_in *mbin; 5277 struct mcx_cmd_create_flow_table_out *out; 5278 int error; 5279 int token; 5280 5281 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5282 token = mcx_cmdq_token(sc); 5283 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5284 sizeof(*out), token); 5285 5286 in = mcx_cmdq_in(cqe); 5287 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_TABLE); 5288 in->cmd_op_mod = htobe16(0); 5289 5290 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5291 &cqe->cq_input_ptr, token) != 0) { 5292 printf("%s: unable to allocate create flow table mailbox\n", 5293 DEVNAME(sc)); 5294 return (-1); 5295 } 5296 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5297 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5298 mbin->cmd_ctx.ft_log_size = log_size; 5299 mbin->cmd_ctx.ft_level = level; 5300 5301 mcx_cmdq_mboxes_sign(&mxm, 1); 5302 mcx_cmdq_post(sc, cqe, 0); 5303 error = mcx_cmdq_poll(sc, cqe, 1000); 5304 if (error != 0) { 5305 printf("%s: create flow table timeout\n", DEVNAME(sc)); 5306 goto free; 5307 } 5308 if (mcx_cmdq_verify(cqe) != 0) { 5309 printf("%s: create flow table command corrupt\n", DEVNAME(sc)); 5310 goto free; 5311 } 5312 5313 out = mcx_cmdq_out(cqe); 5314 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5315 printf("%s: create flow table failed (%x, %x)\n", DEVNAME(sc), 5316 out->cmd_status, betoh32(out->cmd_syndrome)); 5317 error = -1; 5318 goto free; 5319 } 5320 5321 *flow_table_id = mcx_get_id(out->cmd_table_id); 5322 free: 5323 mcx_dmamem_free(sc, &mxm); 5324 return (error); 5325 } 5326 5327 static int 5328 mcx_set_flow_table_root(struct mcx_softc *sc, int flow_table_id) 5329 { 5330 struct mcx_cmdq_entry *cqe; 5331 struct mcx_dmamem mxm; 5332 struct mcx_cmd_set_flow_table_root_in *in; 5333 struct mcx_cmd_set_flow_table_root_mb_in *mbin; 5334 struct mcx_cmd_set_flow_table_root_out *out; 5335 int error; 5336 int token; 5337 5338 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5339 token = mcx_cmdq_token(sc); 5340 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5341 sizeof(*out), token); 5342 5343 in = mcx_cmdq_in(cqe); 5344 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ROOT); 5345 in->cmd_op_mod = htobe16(0); 5346 5347 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5348 &cqe->cq_input_ptr, token) != 0) { 5349 printf("%s: unable to allocate set flow table root mailbox\n", 5350 DEVNAME(sc)); 5351 return (-1); 5352 } 5353 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5354 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5355 mbin->cmd_table_id = htobe32(flow_table_id); 5356 5357 mcx_cmdq_mboxes_sign(&mxm, 1); 5358 mcx_cmdq_post(sc, cqe, 0); 5359 error = mcx_cmdq_poll(sc, cqe, 1000); 5360 if (error != 0) { 5361 printf("%s: set flow table root timeout\n", DEVNAME(sc)); 5362 goto free; 5363 } 5364 if (mcx_cmdq_verify(cqe) != 0) { 5365 printf("%s: set flow table root command corrupt\n", 5366 DEVNAME(sc)); 5367 goto free; 5368 } 5369 5370 out = mcx_cmdq_out(cqe); 5371 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5372 printf("%s: set flow table root failed (%x, %x)\n", 5373 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5374 error = -1; 5375 goto free; 5376 } 5377 5378 free: 5379 mcx_dmamem_free(sc, &mxm); 5380 return (error); 5381 } 5382 5383 static int 5384 mcx_destroy_flow_table(struct mcx_softc *sc, int flow_table_id) 5385 { 5386 struct mcx_cmdq_entry *cqe; 5387 struct mcx_dmamem mxm; 5388 struct mcx_cmd_destroy_flow_table_in *in; 5389 struct mcx_cmd_destroy_flow_table_mb_in *mb; 5390 struct mcx_cmd_destroy_flow_table_out *out; 5391 int error; 5392 int token; 5393 5394 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5395 token = mcx_cmdq_token(sc); 5396 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5397 5398 in = mcx_cmdq_in(cqe); 5399 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_TABLE); 5400 in->cmd_op_mod = htobe16(0); 5401 5402 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 5403 &cqe->cq_input_ptr, token) != 0) { 5404 printf("%s: unable to allocate destroy flow table mailbox\n", 5405 DEVNAME(sc)); 5406 return (-1); 5407 } 5408 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5409 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5410 mb->cmd_table_id = htobe32(flow_table_id); 5411 5412 mcx_cmdq_mboxes_sign(&mxm, 1); 5413 mcx_cmdq_post(sc, cqe, 0); 5414 error = mcx_cmdq_poll(sc, cqe, 1000); 5415 if (error != 0) { 5416 printf("%s: destroy flow table timeout\n", DEVNAME(sc)); 5417 goto free; 5418 } 5419 if (mcx_cmdq_verify(cqe) != 0) { 5420 printf("%s: destroy flow table command corrupt\n", 5421 DEVNAME(sc)); 5422 goto free; 5423 } 5424 5425 out = mcx_cmdq_out(cqe); 5426 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5427 printf("%s: destroy flow table failed (%x, %x)\n", DEVNAME(sc), 5428 out->cmd_status, betoh32(out->cmd_syndrome)); 5429 error = -1; 5430 goto free; 5431 } 5432 5433 free: 5434 mcx_dmamem_free(sc, &mxm); 5435 return (error); 5436 } 5437 5438 5439 static int 5440 mcx_create_flow_group(struct mcx_softc *sc, int flow_table_id, int group, 5441 int start, int size, int match_enable, struct mcx_flow_match *match) 5442 { 5443 struct mcx_cmdq_entry *cqe; 5444 struct mcx_dmamem mxm; 5445 struct mcx_cmd_create_flow_group_in *in; 5446 struct mcx_cmd_create_flow_group_mb_in *mbin; 5447 struct mcx_cmd_create_flow_group_out *out; 5448 struct mcx_flow_group *mfg; 5449 int error; 5450 int token; 5451 5452 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5453 token = mcx_cmdq_token(sc); 5454 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 5455 token); 5456 5457 in = mcx_cmdq_in(cqe); 5458 in->cmd_opcode = htobe16(MCX_CMD_CREATE_FLOW_GROUP); 5459 in->cmd_op_mod = htobe16(0); 5460 5461 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5462 != 0) { 5463 printf("%s: unable to allocate create flow group mailbox\n", 5464 DEVNAME(sc)); 5465 return (-1); 5466 } 5467 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5468 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5469 mbin->cmd_table_id = htobe32(flow_table_id); 5470 mbin->cmd_start_flow_index = htobe32(start); 5471 mbin->cmd_end_flow_index = htobe32(start + (size - 1)); 5472 5473 mbin->cmd_match_criteria_enable = match_enable; 5474 memcpy(&mbin->cmd_match_criteria, match, sizeof(*match)); 5475 5476 mcx_cmdq_mboxes_sign(&mxm, 2); 5477 mcx_cmdq_post(sc, cqe, 0); 5478 error = mcx_cmdq_poll(sc, cqe, 1000); 5479 if (error != 0) { 5480 printf("%s: create flow group timeout\n", DEVNAME(sc)); 5481 goto free; 5482 } 5483 if (mcx_cmdq_verify(cqe) != 0) { 5484 printf("%s: create flow group command corrupt\n", DEVNAME(sc)); 5485 goto free; 5486 } 5487 5488 out = mcx_cmdq_out(cqe); 5489 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5490 printf("%s: create flow group failed (%x, %x)\n", DEVNAME(sc), 5491 out->cmd_status, betoh32(out->cmd_syndrome)); 5492 error = -1; 5493 goto free; 5494 } 5495 5496 mfg = &sc->sc_flow_group[group]; 5497 mfg->g_id = mcx_get_id(out->cmd_group_id); 5498 mfg->g_table = flow_table_id; 5499 mfg->g_start = start; 5500 mfg->g_size = size; 5501 5502 free: 5503 mcx_dmamem_free(sc, &mxm); 5504 return (error); 5505 } 5506 5507 static int 5508 mcx_destroy_flow_group(struct mcx_softc *sc, int group) 5509 { 5510 struct mcx_cmdq_entry *cqe; 5511 struct mcx_dmamem mxm; 5512 struct mcx_cmd_destroy_flow_group_in *in; 5513 struct mcx_cmd_destroy_flow_group_mb_in *mb; 5514 struct mcx_cmd_destroy_flow_group_out *out; 5515 struct mcx_flow_group *mfg; 5516 int error; 5517 int token; 5518 5519 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5520 token = mcx_cmdq_token(sc); 5521 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mb), sizeof(*out), token); 5522 5523 in = mcx_cmdq_in(cqe); 5524 in->cmd_opcode = htobe16(MCX_CMD_DESTROY_FLOW_GROUP); 5525 in->cmd_op_mod = htobe16(0); 5526 5527 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5528 &cqe->cq_input_ptr, token) != 0) { 5529 printf("%s: unable to allocate destroy flow group mailbox\n", 5530 DEVNAME(sc)); 5531 return (-1); 5532 } 5533 mb = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5534 mb->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5535 mfg = &sc->sc_flow_group[group]; 5536 mb->cmd_table_id = htobe32(mfg->g_table); 5537 mb->cmd_group_id = htobe32(mfg->g_id); 5538 5539 mcx_cmdq_mboxes_sign(&mxm, 2); 5540 mcx_cmdq_post(sc, cqe, 0); 5541 error = mcx_cmdq_poll(sc, cqe, 1000); 5542 if (error != 0) { 5543 printf("%s: destroy flow group timeout\n", DEVNAME(sc)); 5544 goto free; 5545 } 5546 if (mcx_cmdq_verify(cqe) != 0) { 5547 printf("%s: destroy flow group command corrupt\n", DEVNAME(sc)); 5548 goto free; 5549 } 5550 5551 out = mcx_cmdq_out(cqe); 5552 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5553 printf("%s: destroy flow group failed (%x, %x)\n", DEVNAME(sc), 5554 out->cmd_status, betoh32(out->cmd_syndrome)); 5555 error = -1; 5556 goto free; 5557 } 5558 5559 mfg->g_id = -1; 5560 mfg->g_table = -1; 5561 mfg->g_size = 0; 5562 mfg->g_start = 0; 5563 free: 5564 mcx_dmamem_free(sc, &mxm); 5565 return (error); 5566 } 5567 5568 static int 5569 mcx_set_flow_table_entry_mac(struct mcx_softc *sc, int group, int index, 5570 uint8_t *macaddr, uint32_t dest) 5571 { 5572 struct mcx_cmdq_entry *cqe; 5573 struct mcx_dmamem mxm; 5574 struct mcx_cmd_set_flow_table_entry_in *in; 5575 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5576 struct mcx_cmd_set_flow_table_entry_out *out; 5577 struct mcx_flow_group *mfg; 5578 uint32_t *pdest; 5579 int error; 5580 int token; 5581 5582 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5583 token = mcx_cmdq_token(sc); 5584 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5585 sizeof(*out), token); 5586 5587 in = mcx_cmdq_in(cqe); 5588 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5589 in->cmd_op_mod = htobe16(0); 5590 5591 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5592 != 0) { 5593 printf("%s: unable to allocate set flow table entry mailbox\n", 5594 DEVNAME(sc)); 5595 return (-1); 5596 } 5597 5598 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5599 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5600 5601 mfg = &sc->sc_flow_group[group]; 5602 mbin->cmd_table_id = htobe32(mfg->g_table); 5603 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5604 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5605 5606 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5607 pdest = (uint32_t *) 5608 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5609 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5610 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5611 *pdest = htobe32(dest); 5612 5613 /* the only thing we match on at the moment is the dest mac address */ 5614 if (macaddr != NULL) { 5615 memcpy(mbin->cmd_flow_ctx.fc_match_value.mc_dest_mac, macaddr, 5616 ETHER_ADDR_LEN); 5617 } 5618 5619 mcx_cmdq_mboxes_sign(&mxm, 2); 5620 mcx_cmdq_post(sc, cqe, 0); 5621 error = mcx_cmdq_poll(sc, cqe, 1000); 5622 if (error != 0) { 5623 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5624 goto free; 5625 } 5626 if (mcx_cmdq_verify(cqe) != 0) { 5627 printf("%s: set flow table entry command corrupt\n", 5628 DEVNAME(sc)); 5629 goto free; 5630 } 5631 5632 out = mcx_cmdq_out(cqe); 5633 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5634 printf("%s: set flow table entry failed (%x, %x)\n", 5635 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5636 error = -1; 5637 goto free; 5638 } 5639 5640 free: 5641 mcx_dmamem_free(sc, &mxm); 5642 return (error); 5643 } 5644 5645 static int 5646 mcx_set_flow_table_entry_proto(struct mcx_softc *sc, int group, int index, 5647 int ethertype, int ip_proto, uint32_t dest) 5648 { 5649 struct mcx_cmdq_entry *cqe; 5650 struct mcx_dmamem mxm; 5651 struct mcx_cmd_set_flow_table_entry_in *in; 5652 struct mcx_cmd_set_flow_table_entry_mb_in *mbin; 5653 struct mcx_cmd_set_flow_table_entry_out *out; 5654 struct mcx_flow_group *mfg; 5655 uint32_t *pdest; 5656 int error; 5657 int token; 5658 5659 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5660 token = mcx_cmdq_token(sc); 5661 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin) + sizeof(*pdest), 5662 sizeof(*out), token); 5663 5664 in = mcx_cmdq_in(cqe); 5665 in->cmd_opcode = htobe16(MCX_CMD_SET_FLOW_TABLE_ENTRY); 5666 in->cmd_op_mod = htobe16(0); 5667 5668 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, &cqe->cq_input_ptr, token) 5669 != 0) { 5670 printf("%s: unable to allocate set flow table entry mailbox\n", 5671 DEVNAME(sc)); 5672 return (-1); 5673 } 5674 5675 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5676 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5677 5678 mfg = &sc->sc_flow_group[group]; 5679 mbin->cmd_table_id = htobe32(mfg->g_table); 5680 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5681 mbin->cmd_flow_ctx.fc_group_id = htobe32(mfg->g_id); 5682 5683 /* flow context ends at offset 0x330, 0x130 into the second mbox */ 5684 pdest = (uint32_t *) 5685 (((char *)mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))) + 0x130); 5686 mbin->cmd_flow_ctx.fc_action = htobe32(MCX_FLOW_CONTEXT_ACTION_FORWARD); 5687 mbin->cmd_flow_ctx.fc_dest_list_size = htobe32(1); 5688 *pdest = htobe32(dest); 5689 5690 mbin->cmd_flow_ctx.fc_match_value.mc_ethertype = htobe16(ethertype); 5691 mbin->cmd_flow_ctx.fc_match_value.mc_ip_proto = ip_proto; 5692 5693 mcx_cmdq_mboxes_sign(&mxm, 2); 5694 mcx_cmdq_post(sc, cqe, 0); 5695 error = mcx_cmdq_poll(sc, cqe, 1000); 5696 if (error != 0) { 5697 printf("%s: set flow table entry timeout\n", DEVNAME(sc)); 5698 goto free; 5699 } 5700 if (mcx_cmdq_verify(cqe) != 0) { 5701 printf("%s: set flow table entry command corrupt\n", 5702 DEVNAME(sc)); 5703 goto free; 5704 } 5705 5706 out = mcx_cmdq_out(cqe); 5707 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5708 printf("%s: set flow table entry failed (%x, %x)\n", 5709 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5710 error = -1; 5711 goto free; 5712 } 5713 5714 free: 5715 mcx_dmamem_free(sc, &mxm); 5716 return (error); 5717 } 5718 5719 static int 5720 mcx_delete_flow_table_entry(struct mcx_softc *sc, int group, int index) 5721 { 5722 struct mcx_cmdq_entry *cqe; 5723 struct mcx_dmamem mxm; 5724 struct mcx_cmd_delete_flow_table_entry_in *in; 5725 struct mcx_cmd_delete_flow_table_entry_mb_in *mbin; 5726 struct mcx_cmd_delete_flow_table_entry_out *out; 5727 struct mcx_flow_group *mfg; 5728 int error; 5729 int token; 5730 5731 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5732 token = mcx_cmdq_token(sc); 5733 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out), 5734 token); 5735 5736 in = mcx_cmdq_in(cqe); 5737 in->cmd_opcode = htobe16(MCX_CMD_DELETE_FLOW_TABLE_ENTRY); 5738 in->cmd_op_mod = htobe16(0); 5739 5740 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5741 &cqe->cq_input_ptr, token) != 0) { 5742 printf("%s: unable to allocate " 5743 "delete flow table entry mailbox\n", DEVNAME(sc)); 5744 return (-1); 5745 } 5746 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5747 mbin->cmd_table_type = MCX_FLOW_TABLE_TYPE_RX; 5748 5749 mfg = &sc->sc_flow_group[group]; 5750 mbin->cmd_table_id = htobe32(mfg->g_table); 5751 mbin->cmd_flow_index = htobe32(mfg->g_start + index); 5752 5753 mcx_cmdq_mboxes_sign(&mxm, 2); 5754 mcx_cmdq_post(sc, cqe, 0); 5755 error = mcx_cmdq_poll(sc, cqe, 1000); 5756 if (error != 0) { 5757 printf("%s: delete flow table entry timeout\n", DEVNAME(sc)); 5758 goto free; 5759 } 5760 if (mcx_cmdq_verify(cqe) != 0) { 5761 printf("%s: delete flow table entry command corrupt\n", 5762 DEVNAME(sc)); 5763 goto free; 5764 } 5765 5766 out = mcx_cmdq_out(cqe); 5767 if (out->cmd_status != MCX_CQ_STATUS_OK) { 5768 printf("%s: delete flow table entry %d:%d failed (%x, %x)\n", 5769 DEVNAME(sc), group, index, out->cmd_status, 5770 betoh32(out->cmd_syndrome)); 5771 error = -1; 5772 goto free; 5773 } 5774 5775 free: 5776 mcx_dmamem_free(sc, &mxm); 5777 return (error); 5778 } 5779 5780 #if 0 5781 int 5782 mcx_dump_flow_table(struct mcx_softc *sc, int flow_table_id) 5783 { 5784 struct mcx_dmamem mxm; 5785 struct mcx_cmdq_entry *cqe; 5786 struct mcx_cmd_query_flow_table_in *in; 5787 struct mcx_cmd_query_flow_table_mb_in *mbin; 5788 struct mcx_cmd_query_flow_table_out *out; 5789 struct mcx_cmd_query_flow_table_mb_out *mbout; 5790 uint8_t token = mcx_cmdq_token(sc); 5791 int error; 5792 int i; 5793 uint8_t *dump; 5794 5795 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5796 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5797 sizeof(*out) + sizeof(*mbout) + 16, token); 5798 5799 in = mcx_cmdq_in(cqe); 5800 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE); 5801 in->cmd_op_mod = htobe16(0); 5802 5803 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5804 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE); 5805 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5806 &cqe->cq_output_ptr, token) != 0) { 5807 printf(", unable to allocate query flow table mailboxes\n"); 5808 return (-1); 5809 } 5810 cqe->cq_input_ptr = cqe->cq_output_ptr; 5811 5812 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5813 mbin->cmd_table_type = 0; 5814 mbin->cmd_table_id = htobe32(flow_table_id); 5815 5816 mcx_cmdq_mboxes_sign(&mxm, 1); 5817 5818 mcx_cmdq_post(sc, cqe, 0); 5819 error = mcx_cmdq_poll(sc, cqe, 1000); 5820 if (error != 0) { 5821 printf("%s: query flow table timeout\n", DEVNAME(sc)); 5822 goto free; 5823 } 5824 error = mcx_cmdq_verify(cqe); 5825 if (error != 0) { 5826 printf("%s: query flow table reply corrupt\n", DEVNAME(sc)); 5827 goto free; 5828 } 5829 5830 out = mcx_cmdq_out(cqe); 5831 switch (out->cmd_status) { 5832 case MCX_CQ_STATUS_OK: 5833 break; 5834 default: 5835 printf("%s: query flow table failed (%x/%x)\n", DEVNAME(sc), 5836 out->cmd_status, betoh32(out->cmd_syndrome)); 5837 error = -1; 5838 goto free; 5839 } 5840 5841 mbout = (struct mcx_cmd_query_flow_table_mb_out *) 5842 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5843 dump = (uint8_t *)mbout + 8; 5844 for (i = 0; i < sizeof(struct mcx_flow_table_ctx); i++) { 5845 printf("%.2x ", dump[i]); 5846 if (i % 16 == 15) 5847 printf("\n"); 5848 } 5849 free: 5850 mcx_cq_mboxes_free(sc, &mxm); 5851 return (error); 5852 } 5853 int 5854 mcx_dump_flow_table_entry(struct mcx_softc *sc, int flow_table_id, int index) 5855 { 5856 struct mcx_dmamem mxm; 5857 struct mcx_cmdq_entry *cqe; 5858 struct mcx_cmd_query_flow_table_entry_in *in; 5859 struct mcx_cmd_query_flow_table_entry_mb_in *mbin; 5860 struct mcx_cmd_query_flow_table_entry_out *out; 5861 struct mcx_cmd_query_flow_table_entry_mb_out *mbout; 5862 uint8_t token = mcx_cmdq_token(sc); 5863 int error; 5864 int i; 5865 uint8_t *dump; 5866 5867 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5868 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5869 sizeof(*out) + sizeof(*mbout) + 16, token); 5870 5871 in = mcx_cmdq_in(cqe); 5872 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_TABLE_ENTRY); 5873 in->cmd_op_mod = htobe16(0); 5874 5875 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5876 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 5877 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5878 &cqe->cq_output_ptr, token) != 0) { 5879 printf(", unable to allocate " 5880 "query flow table entry mailboxes\n"); 5881 return (-1); 5882 } 5883 cqe->cq_input_ptr = cqe->cq_output_ptr; 5884 5885 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5886 mbin->cmd_table_type = 0; 5887 mbin->cmd_table_id = htobe32(flow_table_id); 5888 mbin->cmd_flow_index = htobe32(index); 5889 5890 mcx_cmdq_mboxes_sign(&mxm, 1); 5891 5892 mcx_cmdq_post(sc, cqe, 0); 5893 error = mcx_cmdq_poll(sc, cqe, 1000); 5894 if (error != 0) { 5895 printf("%s: query flow table entry timeout\n", DEVNAME(sc)); 5896 goto free; 5897 } 5898 error = mcx_cmdq_verify(cqe); 5899 if (error != 0) { 5900 printf("%s: query flow table entry reply corrupt\n", 5901 DEVNAME(sc)); 5902 goto free; 5903 } 5904 5905 out = mcx_cmdq_out(cqe); 5906 switch (out->cmd_status) { 5907 case MCX_CQ_STATUS_OK: 5908 break; 5909 default: 5910 printf("%s: query flow table entry failed (%x/%x)\n", 5911 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 5912 error = -1; 5913 goto free; 5914 } 5915 5916 mbout = (struct mcx_cmd_query_flow_table_entry_mb_out *) 5917 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5918 dump = (uint8_t *)mbout; 5919 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 5920 printf("%.2x ", dump[i]); 5921 if (i % 16 == 15) 5922 printf("\n"); 5923 } 5924 5925 free: 5926 mcx_cq_mboxes_free(sc, &mxm); 5927 return (error); 5928 } 5929 5930 int 5931 mcx_dump_flow_group(struct mcx_softc *sc, int flow_table_id) 5932 { 5933 struct mcx_dmamem mxm; 5934 struct mcx_cmdq_entry *cqe; 5935 struct mcx_cmd_query_flow_group_in *in; 5936 struct mcx_cmd_query_flow_group_mb_in *mbin; 5937 struct mcx_cmd_query_flow_group_out *out; 5938 struct mcx_cmd_query_flow_group_mb_out *mbout; 5939 uint8_t token = mcx_cmdq_token(sc); 5940 int error; 5941 int i; 5942 uint8_t *dump; 5943 5944 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 5945 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 5946 sizeof(*out) + sizeof(*mbout) + 16, token); 5947 5948 in = mcx_cmdq_in(cqe); 5949 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_GROUP); 5950 in->cmd_op_mod = htobe16(0); 5951 5952 CTASSERT(sizeof(*mbin) <= MCX_CMDQ_MAILBOX_DATASIZE); 5953 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 5954 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 5955 &cqe->cq_output_ptr, token) != 0) { 5956 printf(", unable to allocate query flow group mailboxes\n"); 5957 return (-1); 5958 } 5959 cqe->cq_input_ptr = cqe->cq_output_ptr; 5960 5961 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 5962 mbin->cmd_table_type = 0; 5963 mbin->cmd_table_id = htobe32(flow_table_id); 5964 mbin->cmd_group_id = htobe32(sc->sc_flow_group_id); 5965 5966 mcx_cmdq_mboxes_sign(&mxm, 1); 5967 5968 mcx_cmdq_post(sc, cqe, 0); 5969 error = mcx_cmdq_poll(sc, cqe, 1000); 5970 if (error != 0) { 5971 printf("%s: query flow group timeout\n", DEVNAME(sc)); 5972 goto free; 5973 } 5974 error = mcx_cmdq_verify(cqe); 5975 if (error != 0) { 5976 printf("%s: query flow group reply corrupt\n", DEVNAME(sc)); 5977 goto free; 5978 } 5979 5980 out = mcx_cmdq_out(cqe); 5981 switch (out->cmd_status) { 5982 case MCX_CQ_STATUS_OK: 5983 break; 5984 default: 5985 printf("%s: query flow group failed (%x/%x)\n", DEVNAME(sc), 5986 out->cmd_status, betoh32(out->cmd_syndrome)); 5987 error = -1; 5988 goto free; 5989 } 5990 5991 mbout = (struct mcx_cmd_query_flow_group_mb_out *) 5992 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 5993 dump = (uint8_t *)mbout; 5994 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 5995 printf("%.2x ", dump[i]); 5996 if (i % 16 == 15) 5997 printf("\n"); 5998 } 5999 dump = (uint8_t *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 1))); 6000 for (i = 0; i < MCX_CMDQ_MAILBOX_DATASIZE; i++) { 6001 printf("%.2x ", dump[i]); 6002 if (i % 16 == 15) 6003 printf("\n"); 6004 } 6005 6006 free: 6007 mcx_cq_mboxes_free(sc, &mxm); 6008 return (error); 6009 } 6010 6011 static int 6012 mcx_dump_counters(struct mcx_softc *sc) 6013 { 6014 struct mcx_dmamem mxm; 6015 struct mcx_cmdq_entry *cqe; 6016 struct mcx_cmd_query_vport_counters_in *in; 6017 struct mcx_cmd_query_vport_counters_mb_in *mbin; 6018 struct mcx_cmd_query_vport_counters_out *out; 6019 struct mcx_nic_vport_counters *counters; 6020 int error, token; 6021 6022 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6023 token = mcx_cmdq_token(sc); 6024 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), 6025 sizeof(*out) + sizeof(*counters), token); 6026 6027 in = mcx_cmdq_in(cqe); 6028 in->cmd_opcode = htobe16(MCX_CMD_QUERY_VPORT_COUNTERS); 6029 in->cmd_op_mod = htobe16(0); 6030 6031 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6032 &cqe->cq_output_ptr, token) != 0) { 6033 printf(", unable to allocate " 6034 "query nic vport counters mailboxen\n"); 6035 return (-1); 6036 } 6037 cqe->cq_input_ptr = cqe->cq_output_ptr; 6038 6039 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6040 mbin->cmd_clear = 0x80; 6041 6042 mcx_cmdq_mboxes_sign(&mxm, 1); 6043 mcx_cmdq_post(sc, cqe, 0); 6044 6045 error = mcx_cmdq_poll(sc, cqe, 1000); 6046 if (error != 0) { 6047 printf("%s: query nic vport counters timeout\n", DEVNAME(sc)); 6048 goto free; 6049 } 6050 if (mcx_cmdq_verify(cqe) != 0) { 6051 printf("%s: query nic vport counters command corrupt\n", 6052 DEVNAME(sc)); 6053 goto free; 6054 } 6055 6056 out = mcx_cmdq_out(cqe); 6057 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6058 printf("%s: query nic vport counters failed (%x, %x)\n", 6059 DEVNAME(sc), out->cmd_status, betoh32(out->cmd_syndrome)); 6060 error = -1; 6061 goto free; 6062 } 6063 6064 counters = (struct mcx_nic_vport_counters *) 6065 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6066 if (counters->rx_bcast.packets + counters->tx_bcast.packets + 6067 counters->rx_ucast.packets + counters->tx_ucast.packets + 6068 counters->rx_err.packets + counters->tx_err.packets) 6069 printf("%s: err %llx/%llx uc %llx/%llx bc %llx/%llx\n", 6070 DEVNAME(sc), 6071 betoh64(counters->tx_err.packets), 6072 betoh64(counters->rx_err.packets), 6073 betoh64(counters->tx_ucast.packets), 6074 betoh64(counters->rx_ucast.packets), 6075 betoh64(counters->tx_bcast.packets), 6076 betoh64(counters->rx_bcast.packets)); 6077 free: 6078 mcx_dmamem_free(sc, &mxm); 6079 6080 return (error); 6081 } 6082 6083 static int 6084 mcx_dump_flow_counter(struct mcx_softc *sc, int index, const char *what) 6085 { 6086 struct mcx_dmamem mxm; 6087 struct mcx_cmdq_entry *cqe; 6088 struct mcx_cmd_query_flow_counter_in *in; 6089 struct mcx_cmd_query_flow_counter_mb_in *mbin; 6090 struct mcx_cmd_query_flow_counter_out *out; 6091 struct mcx_counter *counters; 6092 int error, token; 6093 6094 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6095 token = mcx_cmdq_token(sc); 6096 mcx_cmdq_init(sc, cqe, sizeof(*in) + sizeof(*mbin), sizeof(*out) + 6097 sizeof(*counters), token); 6098 6099 in = mcx_cmdq_in(cqe); 6100 in->cmd_opcode = htobe16(MCX_CMD_QUERY_FLOW_COUNTER); 6101 in->cmd_op_mod = htobe16(0); 6102 6103 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 1, 6104 &cqe->cq_output_ptr, token) != 0) { 6105 printf(", unable to allocate query flow counter mailboxen\n"); 6106 return (-1); 6107 } 6108 cqe->cq_input_ptr = cqe->cq_output_ptr; 6109 mbin = mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0)); 6110 mbin->cmd_flow_counter_id = htobe16(sc->sc_flow_counter_id[index]); 6111 mbin->cmd_clear = 0x80; 6112 6113 mcx_cmdq_mboxes_sign(&mxm, 1); 6114 mcx_cmdq_post(sc, cqe, 0); 6115 6116 error = mcx_cmdq_poll(sc, cqe, 1000); 6117 if (error != 0) { 6118 printf("%s: query flow counter timeout\n", DEVNAME(sc)); 6119 goto free; 6120 } 6121 if (mcx_cmdq_verify(cqe) != 0) { 6122 printf("%s: query flow counter command corrupt\n", DEVNAME(sc)); 6123 goto free; 6124 } 6125 6126 out = mcx_cmdq_out(cqe); 6127 if (out->cmd_status != MCX_CQ_STATUS_OK) { 6128 printf("%s: query flow counter failed (%x, %x)\n", DEVNAME(sc), 6129 out->cmd_status, betoh32(out->cmd_syndrome)); 6130 error = -1; 6131 goto free; 6132 } 6133 6134 counters = (struct mcx_counter *) 6135 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6136 if (counters->packets) 6137 printf("%s: %s inflow %llx\n", DEVNAME(sc), what, 6138 betoh64(counters->packets)); 6139 free: 6140 mcx_dmamem_free(sc, &mxm); 6141 6142 return (error); 6143 } 6144 6145 #endif 6146 6147 #if NKSTAT > 0 6148 6149 int 6150 mcx_query_rq(struct mcx_softc *sc, struct mcx_rx *rx, struct mcx_rq_ctx *rq_ctx) 6151 { 6152 struct mcx_dmamem mxm; 6153 struct mcx_cmdq_entry *cqe; 6154 struct mcx_cmd_query_rq_in *in; 6155 struct mcx_cmd_query_rq_out *out; 6156 struct mcx_cmd_query_rq_mb_out *mbout; 6157 uint8_t token = mcx_cmdq_token(sc); 6158 int error; 6159 6160 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6161 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6162 token); 6163 6164 in = mcx_cmdq_in(cqe); 6165 in->cmd_opcode = htobe16(MCX_CMD_QUERY_RQ); 6166 in->cmd_op_mod = htobe16(0); 6167 in->cmd_rqn = htobe32(rx->rx_rqn); 6168 6169 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6170 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6171 &cqe->cq_output_ptr, token) != 0) { 6172 printf("%s: unable to allocate query rq mailboxes\n", DEVNAME(sc)); 6173 return (-1); 6174 } 6175 6176 mcx_cmdq_mboxes_sign(&mxm, 1); 6177 6178 mcx_cmdq_post(sc, cqe, 0); 6179 error = mcx_cmdq_poll(sc, cqe, 1000); 6180 if (error != 0) { 6181 printf("%s: query rq timeout\n", DEVNAME(sc)); 6182 goto free; 6183 } 6184 error = mcx_cmdq_verify(cqe); 6185 if (error != 0) { 6186 printf("%s: query rq reply corrupt\n", DEVNAME(sc)); 6187 goto free; 6188 } 6189 6190 out = mcx_cmdq_out(cqe); 6191 switch (out->cmd_status) { 6192 case MCX_CQ_STATUS_OK: 6193 break; 6194 default: 6195 printf("%s: query rq failed (%x/%x)\n", DEVNAME(sc), 6196 out->cmd_status, betoh32(out->cmd_syndrome)); 6197 error = -1; 6198 goto free; 6199 } 6200 6201 mbout = (struct mcx_cmd_query_rq_mb_out *) 6202 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6203 memcpy(rq_ctx, &mbout->cmd_ctx, sizeof(*rq_ctx)); 6204 6205 free: 6206 mcx_cq_mboxes_free(sc, &mxm); 6207 return (error); 6208 } 6209 6210 int 6211 mcx_query_sq(struct mcx_softc *sc, struct mcx_tx *tx, struct mcx_sq_ctx *sq_ctx) 6212 { 6213 struct mcx_dmamem mxm; 6214 struct mcx_cmdq_entry *cqe; 6215 struct mcx_cmd_query_sq_in *in; 6216 struct mcx_cmd_query_sq_out *out; 6217 struct mcx_cmd_query_sq_mb_out *mbout; 6218 uint8_t token = mcx_cmdq_token(sc); 6219 int error; 6220 6221 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6222 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*mbout) + 16, 6223 token); 6224 6225 in = mcx_cmdq_in(cqe); 6226 in->cmd_opcode = htobe16(MCX_CMD_QUERY_SQ); 6227 in->cmd_op_mod = htobe16(0); 6228 in->cmd_sqn = htobe32(tx->tx_sqn); 6229 6230 CTASSERT(sizeof(*mbout) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6231 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6232 &cqe->cq_output_ptr, token) != 0) { 6233 printf("%s: unable to allocate query sq mailboxes\n", DEVNAME(sc)); 6234 return (-1); 6235 } 6236 6237 mcx_cmdq_mboxes_sign(&mxm, 1); 6238 6239 mcx_cmdq_post(sc, cqe, 0); 6240 error = mcx_cmdq_poll(sc, cqe, 1000); 6241 if (error != 0) { 6242 printf("%s: query sq timeout\n", DEVNAME(sc)); 6243 goto free; 6244 } 6245 error = mcx_cmdq_verify(cqe); 6246 if (error != 0) { 6247 printf("%s: query sq reply corrupt\n", DEVNAME(sc)); 6248 goto free; 6249 } 6250 6251 out = mcx_cmdq_out(cqe); 6252 switch (out->cmd_status) { 6253 case MCX_CQ_STATUS_OK: 6254 break; 6255 default: 6256 printf("%s: query sq failed (%x/%x)\n", DEVNAME(sc), 6257 out->cmd_status, betoh32(out->cmd_syndrome)); 6258 error = -1; 6259 goto free; 6260 } 6261 6262 mbout = (struct mcx_cmd_query_sq_mb_out *) 6263 (mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6264 memcpy(sq_ctx, &mbout->cmd_ctx, sizeof(*sq_ctx)); 6265 6266 free: 6267 mcx_cq_mboxes_free(sc, &mxm); 6268 return (error); 6269 } 6270 6271 int 6272 mcx_query_cq(struct mcx_softc *sc, struct mcx_cq *cq, struct mcx_cq_ctx *cq_ctx) 6273 { 6274 struct mcx_dmamem mxm; 6275 struct mcx_cmdq_entry *cqe; 6276 struct mcx_cmd_query_cq_in *in; 6277 struct mcx_cmd_query_cq_out *out; 6278 struct mcx_cq_ctx *ctx; 6279 uint8_t token = mcx_cmdq_token(sc); 6280 int error; 6281 6282 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6283 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16, 6284 token); 6285 6286 in = mcx_cmdq_in(cqe); 6287 in->cmd_opcode = htobe16(MCX_CMD_QUERY_CQ); 6288 in->cmd_op_mod = htobe16(0); 6289 in->cmd_cqn = htobe32(cq->cq_n); 6290 6291 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6292 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6293 &cqe->cq_output_ptr, token) != 0) { 6294 printf("%s: unable to allocate query cq mailboxes\n", DEVNAME(sc)); 6295 return (-1); 6296 } 6297 6298 mcx_cmdq_mboxes_sign(&mxm, 1); 6299 6300 mcx_cmdq_post(sc, cqe, 0); 6301 error = mcx_cmdq_poll(sc, cqe, 1000); 6302 if (error != 0) { 6303 printf("%s: query cq timeout\n", DEVNAME(sc)); 6304 goto free; 6305 } 6306 error = mcx_cmdq_verify(cqe); 6307 if (error != 0) { 6308 printf("%s: query cq reply corrupt\n", DEVNAME(sc)); 6309 goto free; 6310 } 6311 6312 out = mcx_cmdq_out(cqe); 6313 switch (out->cmd_status) { 6314 case MCX_CQ_STATUS_OK: 6315 break; 6316 default: 6317 printf("%s: query cq failed (%x/%x)\n", DEVNAME(sc), 6318 out->cmd_status, betoh32(out->cmd_syndrome)); 6319 error = -1; 6320 goto free; 6321 } 6322 6323 ctx = (struct mcx_cq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6324 memcpy(cq_ctx, ctx, sizeof(*cq_ctx)); 6325 free: 6326 mcx_cq_mboxes_free(sc, &mxm); 6327 return (error); 6328 } 6329 6330 int 6331 mcx_query_eq(struct mcx_softc *sc, struct mcx_eq *eq, struct mcx_eq_ctx *eq_ctx) 6332 { 6333 struct mcx_dmamem mxm; 6334 struct mcx_cmdq_entry *cqe; 6335 struct mcx_cmd_query_eq_in *in; 6336 struct mcx_cmd_query_eq_out *out; 6337 struct mcx_eq_ctx *ctx; 6338 uint8_t token = mcx_cmdq_token(sc); 6339 int error; 6340 6341 cqe = MCX_DMA_KVA(&sc->sc_cmdq_mem); 6342 mcx_cmdq_init(sc, cqe, sizeof(*in), sizeof(*out) + sizeof(*ctx) + 16, 6343 token); 6344 6345 in = mcx_cmdq_in(cqe); 6346 in->cmd_opcode = htobe16(MCX_CMD_QUERY_EQ); 6347 in->cmd_op_mod = htobe16(0); 6348 in->cmd_eqn = htobe32(eq->eq_n); 6349 6350 CTASSERT(sizeof(*ctx) <= MCX_CMDQ_MAILBOX_DATASIZE*2); 6351 if (mcx_cmdq_mboxes_alloc(sc, &mxm, 2, 6352 &cqe->cq_output_ptr, token) != 0) { 6353 printf("%s: unable to allocate query eq mailboxes\n", DEVNAME(sc)); 6354 return (-1); 6355 } 6356 6357 mcx_cmdq_mboxes_sign(&mxm, 1); 6358 6359 mcx_cmdq_post(sc, cqe, 0); 6360 error = mcx_cmdq_poll(sc, cqe, 1000); 6361 if (error != 0) { 6362 printf("%s: query eq timeout\n", DEVNAME(sc)); 6363 goto free; 6364 } 6365 error = mcx_cmdq_verify(cqe); 6366 if (error != 0) { 6367 printf("%s: query eq reply corrupt\n", DEVNAME(sc)); 6368 goto free; 6369 } 6370 6371 out = mcx_cmdq_out(cqe); 6372 switch (out->cmd_status) { 6373 case MCX_CQ_STATUS_OK: 6374 break; 6375 default: 6376 printf("%s: query eq failed (%x/%x)\n", DEVNAME(sc), 6377 out->cmd_status, betoh32(out->cmd_syndrome)); 6378 error = -1; 6379 goto free; 6380 } 6381 6382 ctx = (struct mcx_eq_ctx *)(mcx_cq_mbox_data(mcx_cq_mbox(&mxm, 0))); 6383 memcpy(eq_ctx, ctx, sizeof(*eq_ctx)); 6384 free: 6385 mcx_cq_mboxes_free(sc, &mxm); 6386 return (error); 6387 } 6388 6389 #endif /* NKSTAT > 0 */ 6390 6391 6392 int 6393 mcx_rx_fill_slots(struct mcx_softc *sc, struct mcx_rx *rx, 6394 void *ring, struct mcx_slot *slots, uint *prod, uint nslots) 6395 { 6396 struct mcx_rq_entry *rqe; 6397 struct mcx_slot *ms; 6398 struct mbuf *m; 6399 uint slot, p, fills; 6400 6401 p = *prod; 6402 slot = (p % (1 << MCX_LOG_RQ_SIZE)); 6403 rqe = ring; 6404 for (fills = 0; fills < nslots; fills++) { 6405 ms = &slots[slot]; 6406 m = MCLGETI(NULL, M_DONTWAIT, NULL, sc->sc_rxbufsz); 6407 if (m == NULL) 6408 break; 6409 6410 m->m_data += (m->m_ext.ext_size - sc->sc_rxbufsz); 6411 m->m_data += ETHER_ALIGN; 6412 m->m_len = m->m_pkthdr.len = sc->sc_hardmtu; 6413 if (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 6414 BUS_DMA_NOWAIT) != 0) { 6415 m_freem(m); 6416 break; 6417 } 6418 ms->ms_m = m; 6419 6420 rqe[slot].rqe_byte_count = 6421 htobe32(ms->ms_map->dm_segs[0].ds_len); 6422 rqe[slot].rqe_addr = htobe64(ms->ms_map->dm_segs[0].ds_addr); 6423 rqe[slot].rqe_lkey = htobe32(sc->sc_lkey); 6424 6425 p++; 6426 slot++; 6427 if (slot == (1 << MCX_LOG_RQ_SIZE)) 6428 slot = 0; 6429 } 6430 6431 if (fills != 0) { 6432 *rx->rx_doorbell = htobe32(p & MCX_WQ_DOORBELL_MASK); 6433 /* barrier? */ 6434 } 6435 6436 *prod = p; 6437 6438 return (nslots - fills); 6439 } 6440 6441 int 6442 mcx_rx_fill(struct mcx_softc *sc, struct mcx_rx *rx) 6443 { 6444 u_int slots; 6445 6446 slots = if_rxr_get(&rx->rx_rxr, (1 << MCX_LOG_RQ_SIZE)); 6447 if (slots == 0) 6448 return (1); 6449 6450 slots = mcx_rx_fill_slots(sc, rx, MCX_DMA_KVA(&rx->rx_rq_mem), 6451 rx->rx_slots, &rx->rx_prod, slots); 6452 if_rxr_put(&rx->rx_rxr, slots); 6453 return (0); 6454 } 6455 6456 void 6457 mcx_refill(void *xrx) 6458 { 6459 struct mcx_rx *rx = xrx; 6460 struct mcx_softc *sc = rx->rx_softc; 6461 6462 mcx_rx_fill(sc, rx); 6463 6464 if (if_rxr_inuse(&rx->rx_rxr) == 0) 6465 timeout_add(&rx->rx_refill, 1); 6466 } 6467 6468 void 6469 mcx_process_txeof(struct mcx_softc *sc, struct mcx_tx *tx, 6470 struct mcx_cq_entry *cqe, int *txfree) 6471 { 6472 struct mcx_slot *ms; 6473 bus_dmamap_t map; 6474 int slot, slots; 6475 6476 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_SQ_SIZE); 6477 6478 ms = &tx->tx_slots[slot]; 6479 map = ms->ms_map; 6480 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 6481 BUS_DMASYNC_POSTWRITE); 6482 6483 slots = 1; 6484 if (map->dm_nsegs > 1) 6485 slots += (map->dm_nsegs+2) / MCX_SQ_SEGS_PER_SLOT; 6486 6487 (*txfree) += slots; 6488 bus_dmamap_unload(sc->sc_dmat, map); 6489 m_freem(ms->ms_m); 6490 ms->ms_m = NULL; 6491 } 6492 6493 static uint64_t 6494 mcx_uptime(void) 6495 { 6496 struct timespec ts; 6497 6498 nanouptime(&ts); 6499 6500 return ((uint64_t)ts.tv_sec * 1000000000 + (uint64_t)ts.tv_nsec); 6501 } 6502 6503 static void 6504 mcx_calibrate_first(struct mcx_softc *sc) 6505 { 6506 struct mcx_calibration *c = &sc->sc_calibration[0]; 6507 6508 sc->sc_calibration_gen = 0; 6509 6510 c->c_ubase = mcx_uptime(); 6511 c->c_tbase = mcx_timer(sc); 6512 c->c_tdiff = 0; 6513 6514 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_FIRST); 6515 } 6516 6517 #define MCX_TIMESTAMP_SHIFT 10 6518 6519 static void 6520 mcx_calibrate(void *arg) 6521 { 6522 struct mcx_softc *sc = arg; 6523 struct mcx_calibration *nc, *pc; 6524 unsigned int gen; 6525 6526 if (!ISSET(sc->sc_ac.ac_if.if_flags, IFF_RUNNING)) 6527 return; 6528 6529 timeout_add_sec(&sc->sc_calibrate, MCX_CALIBRATE_NORMAL); 6530 6531 gen = sc->sc_calibration_gen; 6532 pc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6533 gen++; 6534 nc = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6535 6536 nc->c_uptime = pc->c_ubase; 6537 nc->c_timestamp = pc->c_tbase; 6538 6539 nc->c_ubase = mcx_uptime(); 6540 nc->c_tbase = mcx_timer(sc); 6541 6542 nc->c_udiff = (nc->c_ubase - nc->c_uptime) >> MCX_TIMESTAMP_SHIFT; 6543 nc->c_tdiff = (nc->c_tbase - nc->c_timestamp) >> MCX_TIMESTAMP_SHIFT; 6544 6545 membar_producer(); 6546 sc->sc_calibration_gen = gen; 6547 } 6548 6549 static int 6550 mcx_process_rx(struct mcx_softc *sc, struct mcx_rx *rx, 6551 struct mcx_cq_entry *cqe, struct mbuf_list *ml, 6552 const struct mcx_calibration *c) 6553 { 6554 struct mcx_slot *ms; 6555 struct mbuf *m; 6556 uint32_t flags; 6557 int slot; 6558 6559 slot = betoh16(cqe->cq_wqe_count) % (1 << MCX_LOG_RQ_SIZE); 6560 6561 ms = &rx->rx_slots[slot]; 6562 bus_dmamap_sync(sc->sc_dmat, ms->ms_map, 0, ms->ms_map->dm_mapsize, 6563 BUS_DMASYNC_POSTREAD); 6564 bus_dmamap_unload(sc->sc_dmat, ms->ms_map); 6565 6566 m = ms->ms_m; 6567 ms->ms_m = NULL; 6568 6569 m->m_pkthdr.len = m->m_len = bemtoh32(&cqe->cq_byte_cnt); 6570 6571 if (cqe->cq_rx_hash_type) { 6572 m->m_pkthdr.ph_flowid = betoh32(cqe->cq_rx_hash); 6573 m->m_pkthdr.csum_flags |= M_FLOWID; 6574 } 6575 6576 flags = bemtoh32(&cqe->cq_flags); 6577 if (flags & MCX_CQ_ENTRY_FLAGS_L3_OK) 6578 m->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK; 6579 if (flags & MCX_CQ_ENTRY_FLAGS_L4_OK) 6580 m->m_pkthdr.csum_flags |= M_TCP_CSUM_IN_OK | 6581 M_UDP_CSUM_IN_OK; 6582 #if NVLAN > 0 6583 if (flags & MCX_CQ_ENTRY_FLAGS_CV) { 6584 m->m_pkthdr.ether_vtag = (flags & 6585 MCX_CQ_ENTRY_FLAGS_VLAN_MASK); 6586 m->m_flags |= M_VLANTAG; 6587 } 6588 #endif 6589 6590 if (c->c_tdiff) { 6591 uint64_t t = bemtoh64(&cqe->cq_timestamp) - c->c_timestamp; 6592 t *= c->c_udiff; 6593 t /= c->c_tdiff; 6594 6595 m->m_pkthdr.ph_timestamp = c->c_uptime + t; 6596 SET(m->m_pkthdr.csum_flags, M_TIMESTAMP); 6597 } 6598 6599 ml_enqueue(ml, m); 6600 6601 return (1); 6602 } 6603 6604 static struct mcx_cq_entry * 6605 mcx_next_cq_entry(struct mcx_softc *sc, struct mcx_cq *cq) 6606 { 6607 struct mcx_cq_entry *cqe; 6608 int next; 6609 6610 cqe = (struct mcx_cq_entry *)MCX_DMA_KVA(&cq->cq_mem); 6611 next = cq->cq_cons % (1 << MCX_LOG_CQ_SIZE); 6612 6613 if ((cqe[next].cq_opcode_owner & MCX_CQ_ENTRY_FLAG_OWNER) == 6614 ((cq->cq_cons >> MCX_LOG_CQ_SIZE) & 1)) { 6615 return (&cqe[next]); 6616 } 6617 6618 return (NULL); 6619 } 6620 6621 static void 6622 mcx_arm_cq(struct mcx_softc *sc, struct mcx_cq *cq, int uar) 6623 { 6624 bus_size_t offset; 6625 uint32_t val; 6626 uint64_t uval; 6627 6628 offset = (MCX_PAGE_SIZE * uar); 6629 val = ((cq->cq_count) & 3) << MCX_CQ_DOORBELL_ARM_CMD_SN_SHIFT; 6630 val |= (cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 6631 6632 cq->cq_doorbell[0] = htobe32(cq->cq_cons & MCX_CQ_DOORBELL_ARM_CI_MASK); 6633 cq->cq_doorbell[1] = htobe32(val); 6634 6635 uval = val; 6636 uval <<= 32; 6637 uval |= cq->cq_n; 6638 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, 6639 offset + MCX_UAR_CQ_DOORBELL, htobe64(uval)); 6640 mcx_bar(sc, offset + MCX_UAR_CQ_DOORBELL, sizeof(uint64_t), 6641 BUS_SPACE_BARRIER_WRITE); 6642 } 6643 6644 void 6645 mcx_process_cq(struct mcx_softc *sc, struct mcx_queues *q, struct mcx_cq *cq) 6646 { 6647 struct mcx_rx *rx = &q->q_rx; 6648 struct mcx_tx *tx = &q->q_tx; 6649 const struct mcx_calibration *c; 6650 unsigned int gen; 6651 struct mcx_cq_entry *cqe; 6652 uint8_t *cqp; 6653 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 6654 int rxfree, txfree; 6655 6656 gen = sc->sc_calibration_gen; 6657 membar_consumer(); 6658 c = &sc->sc_calibration[gen % nitems(sc->sc_calibration)]; 6659 6660 rxfree = 0; 6661 txfree = 0; 6662 while ((cqe = mcx_next_cq_entry(sc, cq))) { 6663 uint8_t opcode; 6664 opcode = (cqe->cq_opcode_owner >> MCX_CQ_ENTRY_OPCODE_SHIFT); 6665 switch (opcode) { 6666 case MCX_CQ_ENTRY_OPCODE_REQ: 6667 mcx_process_txeof(sc, tx, cqe, &txfree); 6668 break; 6669 case MCX_CQ_ENTRY_OPCODE_SEND: 6670 rxfree += mcx_process_rx(sc, rx, cqe, &ml, c); 6671 break; 6672 case MCX_CQ_ENTRY_OPCODE_REQ_ERR: 6673 case MCX_CQ_ENTRY_OPCODE_SEND_ERR: 6674 cqp = (uint8_t *)cqe; 6675 /* printf("%s: cq completion error: %x\n", 6676 DEVNAME(sc), cqp[0x37]); */ 6677 break; 6678 6679 default: 6680 /* printf("%s: cq completion opcode %x??\n", 6681 DEVNAME(sc), opcode); */ 6682 break; 6683 } 6684 6685 cq->cq_cons++; 6686 } 6687 6688 cq->cq_count++; 6689 mcx_arm_cq(sc, cq, q->q_uar); 6690 6691 if (rxfree > 0) { 6692 if_rxr_put(&rx->rx_rxr, rxfree); 6693 if (ifiq_input(rx->rx_ifiq, &ml)) 6694 if_rxr_livelocked(&rx->rx_rxr); 6695 6696 mcx_rx_fill(sc, rx); 6697 if (if_rxr_inuse(&rx->rx_rxr) == 0) 6698 timeout_add(&rx->rx_refill, 1); 6699 } 6700 if (txfree > 0) { 6701 tx->tx_cons += txfree; 6702 if (ifq_is_oactive(tx->tx_ifq)) 6703 ifq_restart(tx->tx_ifq); 6704 } 6705 } 6706 6707 6708 static void 6709 mcx_arm_eq(struct mcx_softc *sc, struct mcx_eq *eq, int uar) 6710 { 6711 bus_size_t offset; 6712 uint32_t val; 6713 6714 offset = (MCX_PAGE_SIZE * uar) + MCX_UAR_EQ_DOORBELL_ARM; 6715 val = (eq->eq_n << 24) | (eq->eq_cons & 0xffffff); 6716 6717 mcx_wr(sc, offset, val); 6718 /* barrier? */ 6719 } 6720 6721 static struct mcx_eq_entry * 6722 mcx_next_eq_entry(struct mcx_softc *sc, struct mcx_eq *eq) 6723 { 6724 struct mcx_eq_entry *eqe; 6725 int next; 6726 6727 eqe = (struct mcx_eq_entry *)MCX_DMA_KVA(&eq->eq_mem); 6728 next = eq->eq_cons % (1 << MCX_LOG_EQ_SIZE); 6729 if ((eqe[next].eq_owner & 1) == 6730 ((eq->eq_cons >> MCX_LOG_EQ_SIZE) & 1)) { 6731 eq->eq_cons++; 6732 return (&eqe[next]); 6733 } 6734 return (NULL); 6735 } 6736 6737 int 6738 mcx_admin_intr(void *xsc) 6739 { 6740 struct mcx_softc *sc = (struct mcx_softc *)xsc; 6741 struct mcx_eq_entry *eqe; 6742 6743 while ((eqe = mcx_next_eq_entry(sc, &sc->sc_admin_eq))) { 6744 switch (eqe->eq_event_type) { 6745 case MCX_EVENT_TYPE_LAST_WQE: 6746 /* printf("%s: last wqe reached?\n", DEVNAME(sc)); */ 6747 break; 6748 6749 case MCX_EVENT_TYPE_CQ_ERROR: 6750 /* printf("%s: cq error\n", DEVNAME(sc)); */ 6751 break; 6752 6753 case MCX_EVENT_TYPE_CMD_COMPLETION: 6754 /* wakeup probably */ 6755 break; 6756 6757 case MCX_EVENT_TYPE_PORT_CHANGE: 6758 task_add(systq, &sc->sc_port_change); 6759 break; 6760 6761 default: 6762 /* printf("%s: something happened\n", DEVNAME(sc)); */ 6763 break; 6764 } 6765 } 6766 mcx_arm_eq(sc, &sc->sc_admin_eq, sc->sc_uar); 6767 return (1); 6768 } 6769 6770 int 6771 mcx_cq_intr(void *xq) 6772 { 6773 struct mcx_queues *q = (struct mcx_queues *)xq; 6774 struct mcx_softc *sc = q->q_sc; 6775 struct mcx_eq_entry *eqe; 6776 int cqn; 6777 6778 while ((eqe = mcx_next_eq_entry(sc, &q->q_eq))) { 6779 switch (eqe->eq_event_type) { 6780 case MCX_EVENT_TYPE_COMPLETION: 6781 cqn = betoh32(eqe->eq_event_data[6]); 6782 if (cqn == q->q_cq.cq_n) 6783 mcx_process_cq(sc, q, &q->q_cq); 6784 break; 6785 } 6786 } 6787 6788 mcx_arm_eq(sc, &q->q_eq, q->q_uar); 6789 return (1); 6790 } 6791 6792 6793 static void 6794 mcx_free_slots(struct mcx_softc *sc, struct mcx_slot *slots, int allocated, 6795 int total) 6796 { 6797 struct mcx_slot *ms; 6798 6799 int i = allocated; 6800 while (i-- > 0) { 6801 ms = &slots[i]; 6802 bus_dmamap_destroy(sc->sc_dmat, ms->ms_map); 6803 if (ms->ms_m != NULL) 6804 m_freem(ms->ms_m); 6805 } 6806 free(slots, M_DEVBUF, total * sizeof(*ms)); 6807 } 6808 6809 static int 6810 mcx_queue_up(struct mcx_softc *sc, struct mcx_queues *q) 6811 { 6812 struct mcx_rx *rx; 6813 struct mcx_tx *tx; 6814 struct mcx_slot *ms; 6815 int i; 6816 6817 rx = &q->q_rx; 6818 rx->rx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_RQ_SIZE), 6819 M_DEVBUF, M_WAITOK | M_ZERO); 6820 if (rx->rx_slots == NULL) { 6821 printf("%s: failed to allocate rx slots\n", DEVNAME(sc)); 6822 return ENOMEM; 6823 } 6824 6825 for (i = 0; i < (1 << MCX_LOG_RQ_SIZE); i++) { 6826 ms = &rx->rx_slots[i]; 6827 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 1, 6828 sc->sc_hardmtu, 0, 6829 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 6830 &ms->ms_map) != 0) { 6831 printf("%s: failed to allocate rx dma maps\n", 6832 DEVNAME(sc)); 6833 goto destroy_rx_slots; 6834 } 6835 } 6836 6837 tx = &q->q_tx; 6838 tx->tx_slots = mallocarray(sizeof(*ms), (1 << MCX_LOG_SQ_SIZE), 6839 M_DEVBUF, M_WAITOK | M_ZERO); 6840 if (tx->tx_slots == NULL) { 6841 printf("%s: failed to allocate tx slots\n", DEVNAME(sc)); 6842 goto destroy_rx_slots; 6843 } 6844 6845 for (i = 0; i < (1 << MCX_LOG_SQ_SIZE); i++) { 6846 ms = &tx->tx_slots[i]; 6847 if (bus_dmamap_create(sc->sc_dmat, sc->sc_hardmtu, 6848 MCX_SQ_MAX_SEGMENTS, sc->sc_hardmtu, 0, 6849 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 6850 &ms->ms_map) != 0) { 6851 printf("%s: failed to allocate tx dma maps\n", 6852 DEVNAME(sc)); 6853 goto destroy_tx_slots; 6854 } 6855 } 6856 6857 if (mcx_create_cq(sc, &q->q_cq, q->q_uar, q->q_index, 6858 q->q_eq.eq_n) != 0) 6859 return ENOMEM; 6860 6861 if (mcx_create_sq(sc, tx, q->q_uar, q->q_index, q->q_cq.cq_n) 6862 != 0) 6863 return ENOMEM; 6864 6865 if (mcx_create_rq(sc, rx, q->q_index, q->q_cq.cq_n) != 0) 6866 return ENOMEM; 6867 6868 return 0; 6869 destroy_tx_slots: 6870 mcx_free_slots(sc, tx->tx_slots, i, (1 << MCX_LOG_SQ_SIZE)); 6871 tx->tx_slots = NULL; 6872 6873 i = (1 << MCX_LOG_RQ_SIZE); 6874 destroy_rx_slots: 6875 mcx_free_slots(sc, rx->rx_slots, i, (1 << MCX_LOG_RQ_SIZE)); 6876 rx->rx_slots = NULL; 6877 return ENOMEM; 6878 } 6879 6880 static int 6881 mcx_rss_group_entry_count(struct mcx_softc *sc, int group) 6882 { 6883 int i; 6884 int count; 6885 6886 count = 0; 6887 for (i = 0; i < nitems(mcx_rss_config); i++) { 6888 if (mcx_rss_config[i].flow_group == group) 6889 count++; 6890 } 6891 6892 return count; 6893 } 6894 6895 static int 6896 mcx_up(struct mcx_softc *sc) 6897 { 6898 struct ifnet *ifp = &sc->sc_ac.ac_if; 6899 struct mcx_rx *rx; 6900 struct mcx_tx *tx; 6901 int i, start, count, flow_group, flow_index; 6902 struct mcx_flow_match match_crit; 6903 struct mcx_rss_rule *rss; 6904 uint32_t dest; 6905 int rqns[MCX_MAX_QUEUES]; 6906 6907 if (mcx_create_tis(sc, &sc->sc_tis) != 0) 6908 goto down; 6909 6910 for (i = 0; i < sc->sc_nqueues; i++) { 6911 if (mcx_queue_up(sc, &sc->sc_queues[i]) != 0) { 6912 goto down; 6913 } 6914 } 6915 6916 /* RSS flow table and flow groups */ 6917 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 1, 6918 &sc->sc_rss_flow_table_id) != 0) 6919 goto down; 6920 6921 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 6922 sc->sc_rss_flow_table_id; 6923 6924 /* L4 RSS flow group (v4/v6 tcp/udp, no fragments) */ 6925 memset(&match_crit, 0, sizeof(match_crit)); 6926 match_crit.mc_ethertype = 0xffff; 6927 match_crit.mc_ip_proto = 0xff; 6928 match_crit.mc_vlan_flags = MCX_FLOW_MATCH_IP_FRAG; 6929 start = 0; 6930 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L4); 6931 if (count != 0) { 6932 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 6933 MCX_FLOW_GROUP_RSS_L4, start, count, 6934 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 6935 goto down; 6936 start += count; 6937 } 6938 6939 /* L3 RSS flow group (v4/v6, including fragments) */ 6940 memset(&match_crit, 0, sizeof(match_crit)); 6941 match_crit.mc_ethertype = 0xffff; 6942 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_L3); 6943 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 6944 MCX_FLOW_GROUP_RSS_L3, start, count, 6945 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 6946 goto down; 6947 start += count; 6948 6949 /* non-RSS flow group */ 6950 count = mcx_rss_group_entry_count(sc, MCX_FLOW_GROUP_RSS_NONE); 6951 memset(&match_crit, 0, sizeof(match_crit)); 6952 if (mcx_create_flow_group(sc, sc->sc_rss_flow_table_id, 6953 MCX_FLOW_GROUP_RSS_NONE, start, count, 0, &match_crit) != 0) 6954 goto down; 6955 6956 /* Root flow table, matching packets based on mac address */ 6957 if (mcx_create_flow_table(sc, MCX_LOG_FLOW_TABLE_SIZE, 0, 6958 &sc->sc_mac_flow_table_id) != 0) 6959 goto down; 6960 6961 /* promisc flow group */ 6962 start = 0; 6963 memset(&match_crit, 0, sizeof(match_crit)); 6964 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 6965 MCX_FLOW_GROUP_PROMISC, start, 1, 0, &match_crit) != 0) 6966 goto down; 6967 sc->sc_promisc_flow_enabled = 0; 6968 start++; 6969 6970 /* all multicast flow group */ 6971 match_crit.mc_dest_mac[0] = 0x01; 6972 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 6973 MCX_FLOW_GROUP_ALLMULTI, start, 1, 6974 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 6975 goto down; 6976 sc->sc_allmulti_flow_enabled = 0; 6977 start++; 6978 6979 /* mac address matching flow group */ 6980 memset(&match_crit.mc_dest_mac, 0xff, sizeof(match_crit.mc_dest_mac)); 6981 if (mcx_create_flow_group(sc, sc->sc_mac_flow_table_id, 6982 MCX_FLOW_GROUP_MAC, start, (1 << MCX_LOG_FLOW_TABLE_SIZE) - start, 6983 MCX_CREATE_FLOW_GROUP_CRIT_OUTER, &match_crit) != 0) 6984 goto down; 6985 6986 /* flow table entries for unicast and broadcast */ 6987 start = 0; 6988 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 6989 sc->sc_ac.ac_enaddr, dest) != 0) 6990 goto down; 6991 start++; 6992 6993 if (mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, start, 6994 etherbroadcastaddr, dest) != 0) 6995 goto down; 6996 start++; 6997 6998 /* multicast entries go after that */ 6999 sc->sc_mcast_flow_base = start; 7000 7001 /* re-add any existing multicast flows */ 7002 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7003 if (sc->sc_mcast_flows[i][0] != 0) { 7004 mcx_set_flow_table_entry_mac(sc, MCX_FLOW_GROUP_MAC, 7005 sc->sc_mcast_flow_base + i, 7006 sc->sc_mcast_flows[i], dest); 7007 } 7008 } 7009 7010 if (mcx_set_flow_table_root(sc, sc->sc_mac_flow_table_id) != 0) 7011 goto down; 7012 7013 /* 7014 * the RQT can be any size as long as it's a power of two. 7015 * since we also restrict the number of queues to a power of two, 7016 * we can just put each rx queue in once. 7017 */ 7018 for (i = 0; i < sc->sc_nqueues; i++) 7019 rqns[i] = sc->sc_queues[i].q_rx.rx_rqn; 7020 7021 if (mcx_create_rqt(sc, sc->sc_nqueues, rqns, &sc->sc_rqt) != 0) 7022 goto down; 7023 7024 start = 0; 7025 flow_index = 0; 7026 flow_group = -1; 7027 for (i = 0; i < nitems(mcx_rss_config); i++) { 7028 rss = &mcx_rss_config[i]; 7029 if (rss->flow_group != flow_group) { 7030 flow_group = rss->flow_group; 7031 flow_index = 0; 7032 } 7033 7034 if (rss->hash_sel == 0) { 7035 if (mcx_create_tir_direct(sc, &sc->sc_queues[0].q_rx, 7036 &sc->sc_tir[i]) != 0) 7037 goto down; 7038 } else { 7039 if (mcx_create_tir_indirect(sc, sc->sc_rqt, 7040 rss->hash_sel, &sc->sc_tir[i]) != 0) 7041 goto down; 7042 } 7043 7044 if (mcx_set_flow_table_entry_proto(sc, flow_group, 7045 flow_index, rss->ethertype, rss->ip_proto, 7046 MCX_FLOW_CONTEXT_DEST_TYPE_TIR | sc->sc_tir[i]) != 0) 7047 goto down; 7048 flow_index++; 7049 } 7050 7051 for (i = 0; i < sc->sc_nqueues; i++) { 7052 struct mcx_queues *q = &sc->sc_queues[i]; 7053 rx = &q->q_rx; 7054 tx = &q->q_tx; 7055 7056 /* start the queues */ 7057 if (mcx_ready_sq(sc, tx) != 0) 7058 goto down; 7059 7060 if (mcx_ready_rq(sc, rx) != 0) 7061 goto down; 7062 7063 if_rxr_init(&rx->rx_rxr, 1, (1 << MCX_LOG_RQ_SIZE)); 7064 rx->rx_prod = 0; 7065 mcx_rx_fill(sc, rx); 7066 7067 tx->tx_cons = 0; 7068 tx->tx_prod = 0; 7069 ifq_clr_oactive(tx->tx_ifq); 7070 } 7071 7072 mcx_calibrate_first(sc); 7073 7074 SET(ifp->if_flags, IFF_RUNNING); 7075 7076 return ENETRESET; 7077 down: 7078 mcx_down(sc); 7079 return ENOMEM; 7080 } 7081 7082 static void 7083 mcx_down(struct mcx_softc *sc) 7084 { 7085 struct ifnet *ifp = &sc->sc_ac.ac_if; 7086 struct mcx_rss_rule *rss; 7087 int group, i, flow_group, flow_index; 7088 7089 CLR(ifp->if_flags, IFF_RUNNING); 7090 7091 /* 7092 * delete flow table entries first, so no packets can arrive 7093 * after the barriers 7094 */ 7095 if (sc->sc_promisc_flow_enabled) 7096 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_PROMISC, 0); 7097 if (sc->sc_allmulti_flow_enabled) 7098 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_ALLMULTI, 0); 7099 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 0); 7100 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 1); 7101 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7102 if (sc->sc_mcast_flows[i][0] != 0) { 7103 mcx_delete_flow_table_entry(sc, MCX_FLOW_GROUP_MAC, 7104 sc->sc_mcast_flow_base + i); 7105 } 7106 } 7107 7108 flow_group = -1; 7109 flow_index = 0; 7110 for (i = 0; i < nitems(mcx_rss_config); i++) { 7111 rss = &mcx_rss_config[i]; 7112 if (rss->flow_group != flow_group) { 7113 flow_group = rss->flow_group; 7114 flow_index = 0; 7115 } 7116 7117 mcx_delete_flow_table_entry(sc, flow_group, flow_index); 7118 7119 mcx_destroy_tir(sc, sc->sc_tir[i]); 7120 sc->sc_tir[i] = 0; 7121 7122 flow_index++; 7123 } 7124 intr_barrier(sc->sc_ihc); 7125 for (i = 0; i < sc->sc_nqueues; i++) { 7126 struct ifqueue *ifq = sc->sc_queues[i].q_tx.tx_ifq; 7127 ifq_barrier(ifq); 7128 7129 timeout_del_barrier(&sc->sc_queues[i].q_rx.rx_refill); 7130 7131 intr_barrier(sc->sc_queues[i].q_ihc); 7132 } 7133 7134 timeout_del_barrier(&sc->sc_calibrate); 7135 7136 for (group = 0; group < MCX_NUM_FLOW_GROUPS; group++) { 7137 if (sc->sc_flow_group[group].g_id != -1) 7138 mcx_destroy_flow_group(sc, group); 7139 } 7140 7141 if (sc->sc_mac_flow_table_id != -1) { 7142 mcx_destroy_flow_table(sc, sc->sc_mac_flow_table_id); 7143 sc->sc_mac_flow_table_id = -1; 7144 } 7145 if (sc->sc_rss_flow_table_id != -1) { 7146 mcx_destroy_flow_table(sc, sc->sc_rss_flow_table_id); 7147 sc->sc_rss_flow_table_id = -1; 7148 } 7149 if (sc->sc_rqt != -1) { 7150 mcx_destroy_rqt(sc, sc->sc_rqt); 7151 sc->sc_rqt = -1; 7152 } 7153 7154 for (i = 0; i < sc->sc_nqueues; i++) { 7155 struct mcx_queues *q = &sc->sc_queues[i]; 7156 struct mcx_rx *rx = &q->q_rx; 7157 struct mcx_tx *tx = &q->q_tx; 7158 struct mcx_cq *cq = &q->q_cq; 7159 7160 if (rx->rx_rqn != 0) 7161 mcx_destroy_rq(sc, rx); 7162 7163 if (tx->tx_sqn != 0) 7164 mcx_destroy_sq(sc, tx); 7165 7166 if (tx->tx_slots != NULL) { 7167 mcx_free_slots(sc, tx->tx_slots, 7168 (1 << MCX_LOG_SQ_SIZE), (1 << MCX_LOG_SQ_SIZE)); 7169 tx->tx_slots = NULL; 7170 } 7171 if (rx->rx_slots != NULL) { 7172 mcx_free_slots(sc, rx->rx_slots, 7173 (1 << MCX_LOG_RQ_SIZE), (1 << MCX_LOG_RQ_SIZE)); 7174 rx->rx_slots = NULL; 7175 } 7176 7177 if (cq->cq_n != 0) 7178 mcx_destroy_cq(sc, cq); 7179 } 7180 if (sc->sc_tis != 0) { 7181 mcx_destroy_tis(sc, sc->sc_tis); 7182 sc->sc_tis = 0; 7183 } 7184 } 7185 7186 static int 7187 mcx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 7188 { 7189 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7190 struct ifreq *ifr = (struct ifreq *)data; 7191 uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN]; 7192 int s, i, error = 0; 7193 uint32_t dest; 7194 7195 s = splnet(); 7196 switch (cmd) { 7197 case SIOCSIFADDR: 7198 ifp->if_flags |= IFF_UP; 7199 /* FALLTHROUGH */ 7200 7201 case SIOCSIFFLAGS: 7202 if (ISSET(ifp->if_flags, IFF_UP)) { 7203 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7204 error = ENETRESET; 7205 else 7206 error = mcx_up(sc); 7207 } else { 7208 if (ISSET(ifp->if_flags, IFF_RUNNING)) 7209 mcx_down(sc); 7210 } 7211 break; 7212 7213 case SIOCGIFMEDIA: 7214 case SIOCSIFMEDIA: 7215 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 7216 break; 7217 7218 case SIOCGIFSFFPAGE: 7219 error = mcx_get_sffpage(ifp, (struct if_sffpage *)data); 7220 break; 7221 7222 case SIOCGIFRXR: 7223 error = mcx_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 7224 break; 7225 7226 case SIOCADDMULTI: 7227 if (ether_addmulti(ifr, &sc->sc_ac) == ENETRESET) { 7228 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7229 if (error != 0) 7230 return (error); 7231 7232 dest = MCX_FLOW_CONTEXT_DEST_TYPE_TABLE | 7233 sc->sc_rss_flow_table_id; 7234 7235 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7236 if (sc->sc_mcast_flows[i][0] == 0) { 7237 memcpy(sc->sc_mcast_flows[i], addrlo, 7238 ETHER_ADDR_LEN); 7239 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7240 mcx_set_flow_table_entry_mac(sc, 7241 MCX_FLOW_GROUP_MAC, 7242 sc->sc_mcast_flow_base + i, 7243 sc->sc_mcast_flows[i], dest); 7244 } 7245 break; 7246 } 7247 } 7248 7249 if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) { 7250 if (i == MCX_NUM_MCAST_FLOWS) { 7251 SET(ifp->if_flags, IFF_ALLMULTI); 7252 sc->sc_extra_mcast++; 7253 error = ENETRESET; 7254 } 7255 7256 if (sc->sc_ac.ac_multirangecnt > 0) { 7257 SET(ifp->if_flags, IFF_ALLMULTI); 7258 error = ENETRESET; 7259 } 7260 } 7261 } 7262 break; 7263 7264 case SIOCDELMULTI: 7265 if (ether_delmulti(ifr, &sc->sc_ac) == ENETRESET) { 7266 error = ether_multiaddr(&ifr->ifr_addr, addrlo, addrhi); 7267 if (error != 0) 7268 return (error); 7269 7270 for (i = 0; i < MCX_NUM_MCAST_FLOWS; i++) { 7271 if (memcmp(sc->sc_mcast_flows[i], addrlo, 7272 ETHER_ADDR_LEN) == 0) { 7273 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 7274 mcx_delete_flow_table_entry(sc, 7275 MCX_FLOW_GROUP_MAC, 7276 sc->sc_mcast_flow_base + i); 7277 } 7278 sc->sc_mcast_flows[i][0] = 0; 7279 break; 7280 } 7281 } 7282 7283 if (i == MCX_NUM_MCAST_FLOWS) 7284 sc->sc_extra_mcast--; 7285 7286 if (ISSET(ifp->if_flags, IFF_ALLMULTI) && 7287 (sc->sc_extra_mcast == 0) && 7288 (sc->sc_ac.ac_multirangecnt == 0)) { 7289 CLR(ifp->if_flags, IFF_ALLMULTI); 7290 error = ENETRESET; 7291 } 7292 } 7293 break; 7294 7295 default: 7296 error = ether_ioctl(ifp, &sc->sc_ac, cmd, data); 7297 } 7298 7299 if (error == ENETRESET) { 7300 if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == 7301 (IFF_UP | IFF_RUNNING)) 7302 mcx_iff(sc); 7303 error = 0; 7304 } 7305 splx(s); 7306 7307 return (error); 7308 } 7309 7310 static int 7311 mcx_get_sffpage(struct ifnet *ifp, struct if_sffpage *sff) 7312 { 7313 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7314 struct mcx_reg_mcia mcia; 7315 struct mcx_reg_pmlp pmlp; 7316 int offset, error; 7317 7318 /* get module number */ 7319 memset(&pmlp, 0, sizeof(pmlp)); 7320 pmlp.rp_local_port = 1; 7321 error = mcx_access_hca_reg(sc, MCX_REG_PMLP, MCX_REG_OP_READ, &pmlp, 7322 sizeof(pmlp)); 7323 if (error != 0) { 7324 printf("%s: unable to get eeprom module number\n", 7325 DEVNAME(sc)); 7326 return error; 7327 } 7328 7329 for (offset = 0; offset < 256; offset += MCX_MCIA_EEPROM_BYTES) { 7330 memset(&mcia, 0, sizeof(mcia)); 7331 mcia.rm_l = 0; 7332 mcia.rm_module = betoh32(pmlp.rp_lane0_mapping) & 7333 MCX_PMLP_MODULE_NUM_MASK; 7334 mcia.rm_i2c_addr = sff->sff_addr / 2; /* apparently */ 7335 mcia.rm_page_num = sff->sff_page; 7336 mcia.rm_dev_addr = htobe16(offset); 7337 mcia.rm_size = htobe16(MCX_MCIA_EEPROM_BYTES); 7338 7339 error = mcx_access_hca_reg(sc, MCX_REG_MCIA, MCX_REG_OP_READ, 7340 &mcia, sizeof(mcia)); 7341 if (error != 0) { 7342 printf("%s: unable to read eeprom at %x\n", 7343 DEVNAME(sc), offset); 7344 return error; 7345 } 7346 7347 memcpy(sff->sff_data + offset, mcia.rm_data, 7348 MCX_MCIA_EEPROM_BYTES); 7349 } 7350 7351 return 0; 7352 } 7353 7354 static int 7355 mcx_rxrinfo(struct mcx_softc *sc, struct if_rxrinfo *ifri) 7356 { 7357 struct if_rxring_info *ifrs; 7358 unsigned int i; 7359 int error; 7360 7361 ifrs = mallocarray(sc->sc_nqueues, sizeof(*ifrs), M_TEMP, 7362 M_WAITOK|M_ZERO|M_CANFAIL); 7363 if (ifrs == NULL) 7364 return (ENOMEM); 7365 7366 for (i = 0; i < sc->sc_nqueues; i++) { 7367 struct mcx_rx *rx = &sc->sc_queues[i].q_rx; 7368 struct if_rxring_info *ifr = &ifrs[i]; 7369 7370 snprintf(ifr->ifr_name, sizeof(ifr->ifr_name), "%u", i); 7371 ifr->ifr_size = sc->sc_hardmtu; 7372 ifr->ifr_info = rx->rx_rxr; 7373 } 7374 7375 error = if_rxr_info_ioctl(ifri, i, ifrs); 7376 free(ifrs, M_TEMP, i * sizeof(*ifrs)); 7377 7378 return (error); 7379 } 7380 7381 int 7382 mcx_load_mbuf(struct mcx_softc *sc, struct mcx_slot *ms, struct mbuf *m) 7383 { 7384 switch (bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7385 BUS_DMA_STREAMING | BUS_DMA_NOWAIT)) { 7386 case 0: 7387 break; 7388 7389 case EFBIG: 7390 if (m_defrag(m, M_DONTWAIT) == 0 && 7391 bus_dmamap_load_mbuf(sc->sc_dmat, ms->ms_map, m, 7392 BUS_DMA_STREAMING | BUS_DMA_NOWAIT) == 0) 7393 break; 7394 7395 default: 7396 return (1); 7397 } 7398 7399 ms->ms_m = m; 7400 return (0); 7401 } 7402 7403 static void 7404 mcx_start(struct ifqueue *ifq) 7405 { 7406 struct mcx_tx *tx = ifq->ifq_softc; 7407 struct ifnet *ifp = ifq->ifq_if; 7408 struct mcx_softc *sc = ifp->if_softc; 7409 struct mcx_sq_entry *sq, *sqe; 7410 struct mcx_sq_entry_seg *sqs; 7411 struct mcx_slot *ms; 7412 bus_dmamap_t map; 7413 struct mbuf *m; 7414 u_int idx, free, used; 7415 uint64_t *bf; 7416 uint32_t csum; 7417 size_t bf_base; 7418 int i, seg, nseg; 7419 7420 bf_base = (tx->tx_uar * MCX_PAGE_SIZE) + MCX_UAR_BF; 7421 7422 idx = tx->tx_prod % (1 << MCX_LOG_SQ_SIZE); 7423 free = (tx->tx_cons + (1 << MCX_LOG_SQ_SIZE)) - tx->tx_prod; 7424 7425 used = 0; 7426 bf = NULL; 7427 sq = (struct mcx_sq_entry *)MCX_DMA_KVA(&tx->tx_sq_mem); 7428 7429 for (;;) { 7430 if (used + MCX_SQ_ENTRY_MAX_SLOTS >= free) { 7431 ifq_set_oactive(ifq); 7432 break; 7433 } 7434 7435 m = ifq_dequeue(ifq); 7436 if (m == NULL) { 7437 break; 7438 } 7439 7440 sqe = sq + idx; 7441 ms = &tx->tx_slots[idx]; 7442 memset(sqe, 0, sizeof(*sqe)); 7443 7444 /* ctrl segment */ 7445 sqe->sqe_opcode_index = htobe32(MCX_SQE_WQE_OPCODE_SEND | 7446 ((tx->tx_prod & 0xffff) << MCX_SQE_WQE_INDEX_SHIFT)); 7447 /* always generate a completion event */ 7448 sqe->sqe_signature = htobe32(MCX_SQE_CE_CQE_ALWAYS); 7449 7450 /* eth segment */ 7451 csum = 0; 7452 if (m->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT) 7453 csum |= MCX_SQE_L3_CSUM; 7454 if (m->m_pkthdr.csum_flags & (M_TCP_CSUM_OUT | M_UDP_CSUM_OUT)) 7455 csum |= MCX_SQE_L4_CSUM; 7456 sqe->sqe_mss_csum = htobe32(csum); 7457 sqe->sqe_inline_header_size = htobe16(MCX_SQ_INLINE_SIZE); 7458 #if NVLAN > 0 7459 if (m->m_flags & M_VLANTAG) { 7460 struct ether_vlan_header *evh; 7461 evh = (struct ether_vlan_header *) 7462 &sqe->sqe_inline_headers; 7463 7464 /* slightly cheaper vlan_inject() */ 7465 m_copydata(m, 0, ETHER_HDR_LEN, (caddr_t)evh); 7466 evh->evl_proto = evh->evl_encap_proto; 7467 evh->evl_encap_proto = htons(ETHERTYPE_VLAN); 7468 evh->evl_tag = htons(m->m_pkthdr.ether_vtag); 7469 7470 m_adj(m, ETHER_HDR_LEN); 7471 } else 7472 #endif 7473 { 7474 m_copydata(m, 0, MCX_SQ_INLINE_SIZE, 7475 (caddr_t)sqe->sqe_inline_headers); 7476 m_adj(m, MCX_SQ_INLINE_SIZE); 7477 } 7478 7479 if (mcx_load_mbuf(sc, ms, m) != 0) { 7480 m_freem(m); 7481 ifp->if_oerrors++; 7482 continue; 7483 } 7484 bf = (uint64_t *)sqe; 7485 7486 #if NBPFILTER > 0 7487 if (ifp->if_bpf) 7488 bpf_mtap_hdr(ifp->if_bpf, 7489 (caddr_t)sqe->sqe_inline_headers, 7490 MCX_SQ_INLINE_SIZE, m, BPF_DIRECTION_OUT); 7491 #endif 7492 map = ms->ms_map; 7493 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 7494 BUS_DMASYNC_PREWRITE); 7495 7496 sqe->sqe_ds_sq_num = 7497 htobe32((tx->tx_sqn << MCX_SQE_SQ_NUM_SHIFT) | 7498 (map->dm_nsegs + 3)); 7499 7500 /* data segment - first wqe has one segment */ 7501 sqs = sqe->sqe_segs; 7502 seg = 0; 7503 nseg = 1; 7504 for (i = 0; i < map->dm_nsegs; i++) { 7505 if (seg == nseg) { 7506 /* next slot */ 7507 idx++; 7508 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7509 idx = 0; 7510 tx->tx_prod++; 7511 used++; 7512 7513 sqs = (struct mcx_sq_entry_seg *)(sq + idx); 7514 seg = 0; 7515 nseg = MCX_SQ_SEGS_PER_SLOT; 7516 } 7517 sqs[seg].sqs_byte_count = 7518 htobe32(map->dm_segs[i].ds_len); 7519 sqs[seg].sqs_lkey = htobe32(sc->sc_lkey); 7520 sqs[seg].sqs_addr = htobe64(map->dm_segs[i].ds_addr); 7521 seg++; 7522 } 7523 7524 idx++; 7525 if (idx == (1 << MCX_LOG_SQ_SIZE)) 7526 idx = 0; 7527 tx->tx_prod++; 7528 used++; 7529 } 7530 7531 if (used) { 7532 htobem32(tx->tx_doorbell, tx->tx_prod & MCX_WQ_DOORBELL_MASK); 7533 7534 membar_sync(); 7535 7536 /* 7537 * write the first 64 bits of the last sqe we produced 7538 * to the blue flame buffer 7539 */ 7540 bus_space_write_raw_8(sc->sc_memt, sc->sc_memh, 7541 bf_base + tx->tx_bf_offset, *bf); 7542 /* next write goes to the other buffer */ 7543 tx->tx_bf_offset ^= sc->sc_bf_size; 7544 7545 membar_sync(); 7546 } 7547 } 7548 7549 static void 7550 mcx_watchdog(struct ifnet *ifp) 7551 { 7552 } 7553 7554 static void 7555 mcx_media_add_types(struct mcx_softc *sc) 7556 { 7557 struct mcx_reg_ptys ptys; 7558 int i; 7559 uint32_t proto_cap; 7560 7561 memset(&ptys, 0, sizeof(ptys)); 7562 ptys.rp_local_port = 1; 7563 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7564 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7565 sizeof(ptys)) != 0) { 7566 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 7567 return; 7568 } 7569 7570 proto_cap = betoh32(ptys.rp_eth_proto_cap); 7571 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7572 const struct mcx_eth_proto_capability *cap; 7573 if (!ISSET(proto_cap, 1 << i)) 7574 continue; 7575 7576 cap = &mcx_eth_cap_map[i]; 7577 if (cap->cap_media == 0) 7578 continue; 7579 7580 ifmedia_add(&sc->sc_media, IFM_ETHER | cap->cap_media, 0, NULL); 7581 } 7582 } 7583 7584 static void 7585 mcx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 7586 { 7587 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7588 struct mcx_reg_ptys ptys; 7589 int i; 7590 uint32_t proto_oper; 7591 uint64_t media_oper; 7592 7593 memset(&ptys, 0, sizeof(ptys)); 7594 ptys.rp_local_port = 1; 7595 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7596 7597 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7598 sizeof(ptys)) != 0) { 7599 printf("%s: unable to read port type/speed\n", DEVNAME(sc)); 7600 return; 7601 } 7602 7603 proto_oper = betoh32(ptys.rp_eth_proto_oper); 7604 7605 media_oper = 0; 7606 7607 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7608 const struct mcx_eth_proto_capability *cap; 7609 if (!ISSET(proto_oper, 1 << i)) 7610 continue; 7611 7612 cap = &mcx_eth_cap_map[i]; 7613 7614 if (cap->cap_media != 0) 7615 media_oper = cap->cap_media; 7616 } 7617 7618 ifmr->ifm_status = IFM_AVALID; 7619 if (proto_oper != 0) { 7620 ifmr->ifm_status |= IFM_ACTIVE; 7621 ifmr->ifm_active = IFM_ETHER | IFM_AUTO | media_oper; 7622 /* txpause, rxpause, duplex? */ 7623 } 7624 } 7625 7626 static int 7627 mcx_media_change(struct ifnet *ifp) 7628 { 7629 struct mcx_softc *sc = (struct mcx_softc *)ifp->if_softc; 7630 struct mcx_reg_ptys ptys; 7631 struct mcx_reg_paos paos; 7632 uint32_t media; 7633 int i, error; 7634 7635 if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER) 7636 return EINVAL; 7637 7638 error = 0; 7639 7640 if (IFM_SUBTYPE(sc->sc_media.ifm_media) == IFM_AUTO) { 7641 /* read ptys to get supported media */ 7642 memset(&ptys, 0, sizeof(ptys)); 7643 ptys.rp_local_port = 1; 7644 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7645 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, 7646 &ptys, sizeof(ptys)) != 0) { 7647 printf("%s: unable to read port type/speed\n", 7648 DEVNAME(sc)); 7649 return EIO; 7650 } 7651 7652 media = betoh32(ptys.rp_eth_proto_cap); 7653 } else { 7654 /* map media type */ 7655 media = 0; 7656 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7657 const struct mcx_eth_proto_capability *cap; 7658 7659 cap = &mcx_eth_cap_map[i]; 7660 if (cap->cap_media == 7661 IFM_SUBTYPE(sc->sc_media.ifm_media)) { 7662 media = (1 << i); 7663 break; 7664 } 7665 } 7666 } 7667 7668 /* disable the port */ 7669 memset(&paos, 0, sizeof(paos)); 7670 paos.rp_local_port = 1; 7671 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_DOWN; 7672 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 7673 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 7674 sizeof(paos)) != 0) { 7675 printf("%s: unable to set port state to down\n", DEVNAME(sc)); 7676 return EIO; 7677 } 7678 7679 memset(&ptys, 0, sizeof(ptys)); 7680 ptys.rp_local_port = 1; 7681 ptys.rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH; 7682 ptys.rp_eth_proto_admin = htobe32(media); 7683 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_WRITE, &ptys, 7684 sizeof(ptys)) != 0) { 7685 printf("%s: unable to set port media type/speed\n", 7686 DEVNAME(sc)); 7687 error = EIO; 7688 } 7689 7690 /* re-enable the port to start negotiation */ 7691 memset(&paos, 0, sizeof(paos)); 7692 paos.rp_local_port = 1; 7693 paos.rp_admin_status = MCX_REG_PAOS_ADMIN_STATUS_UP; 7694 paos.rp_admin_state_update = MCX_REG_PAOS_ADMIN_STATE_UPDATE_EN; 7695 if (mcx_access_hca_reg(sc, MCX_REG_PAOS, MCX_REG_OP_WRITE, &paos, 7696 sizeof(paos)) != 0) { 7697 printf("%s: unable to set port state to up\n", DEVNAME(sc)); 7698 error = EIO; 7699 } 7700 7701 return error; 7702 } 7703 7704 static void 7705 mcx_port_change(void *xsc) 7706 { 7707 struct mcx_softc *sc = xsc; 7708 struct ifnet *ifp = &sc->sc_ac.ac_if; 7709 struct mcx_reg_ptys ptys = { 7710 .rp_local_port = 1, 7711 .rp_proto_mask = MCX_REG_PTYS_PROTO_MASK_ETH, 7712 }; 7713 int link_state = LINK_STATE_DOWN; 7714 7715 if (mcx_access_hca_reg(sc, MCX_REG_PTYS, MCX_REG_OP_READ, &ptys, 7716 sizeof(ptys)) == 0) { 7717 uint32_t proto_oper = betoh32(ptys.rp_eth_proto_oper); 7718 uint64_t baudrate = 0; 7719 unsigned int i; 7720 7721 if (proto_oper != 0) 7722 link_state = LINK_STATE_FULL_DUPLEX; 7723 7724 for (i = 0; i < nitems(mcx_eth_cap_map); i++) { 7725 const struct mcx_eth_proto_capability *cap; 7726 if (!ISSET(proto_oper, 1 << i)) 7727 continue; 7728 7729 cap = &mcx_eth_cap_map[i]; 7730 if (cap->cap_baudrate == 0) 7731 continue; 7732 7733 baudrate = cap->cap_baudrate; 7734 break; 7735 } 7736 7737 ifp->if_baudrate = baudrate; 7738 } 7739 7740 if (link_state != ifp->if_link_state) { 7741 ifp->if_link_state = link_state; 7742 if_link_state_change(ifp); 7743 } 7744 } 7745 7746 static inline uint32_t 7747 mcx_rd(struct mcx_softc *sc, bus_size_t r) 7748 { 7749 uint32_t word; 7750 7751 word = bus_space_read_raw_4(sc->sc_memt, sc->sc_memh, r); 7752 7753 return (betoh32(word)); 7754 } 7755 7756 static inline void 7757 mcx_wr(struct mcx_softc *sc, bus_size_t r, uint32_t v) 7758 { 7759 bus_space_write_raw_4(sc->sc_memt, sc->sc_memh, r, htobe32(v)); 7760 } 7761 7762 static inline void 7763 mcx_bar(struct mcx_softc *sc, bus_size_t r, bus_size_t l, int f) 7764 { 7765 bus_space_barrier(sc->sc_memt, sc->sc_memh, r, l, f); 7766 } 7767 7768 static uint64_t 7769 mcx_timer(struct mcx_softc *sc) 7770 { 7771 uint32_t hi, lo, ni; 7772 7773 hi = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 7774 for (;;) { 7775 lo = mcx_rd(sc, MCX_INTERNAL_TIMER_L); 7776 mcx_bar(sc, MCX_INTERNAL_TIMER_L, 8, BUS_SPACE_BARRIER_READ); 7777 ni = mcx_rd(sc, MCX_INTERNAL_TIMER_H); 7778 7779 if (ni == hi) 7780 break; 7781 7782 hi = ni; 7783 } 7784 7785 return (((uint64_t)hi << 32) | (uint64_t)lo); 7786 } 7787 7788 static int 7789 mcx_dmamem_alloc(struct mcx_softc *sc, struct mcx_dmamem *mxm, 7790 bus_size_t size, u_int align) 7791 { 7792 mxm->mxm_size = size; 7793 7794 if (bus_dmamap_create(sc->sc_dmat, mxm->mxm_size, 1, 7795 mxm->mxm_size, 0, 7796 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW | BUS_DMA_64BIT, 7797 &mxm->mxm_map) != 0) 7798 return (1); 7799 if (bus_dmamem_alloc(sc->sc_dmat, mxm->mxm_size, 7800 align, 0, &mxm->mxm_seg, 1, &mxm->mxm_nsegs, 7801 BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 7802 goto destroy; 7803 if (bus_dmamem_map(sc->sc_dmat, &mxm->mxm_seg, mxm->mxm_nsegs, 7804 mxm->mxm_size, &mxm->mxm_kva, BUS_DMA_WAITOK) != 0) 7805 goto free; 7806 if (bus_dmamap_load(sc->sc_dmat, mxm->mxm_map, mxm->mxm_kva, 7807 mxm->mxm_size, NULL, BUS_DMA_WAITOK) != 0) 7808 goto unmap; 7809 7810 return (0); 7811 unmap: 7812 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 7813 free: 7814 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 7815 destroy: 7816 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 7817 return (1); 7818 } 7819 7820 static void 7821 mcx_dmamem_zero(struct mcx_dmamem *mxm) 7822 { 7823 memset(MCX_DMA_KVA(mxm), 0, MCX_DMA_LEN(mxm)); 7824 } 7825 7826 static void 7827 mcx_dmamem_free(struct mcx_softc *sc, struct mcx_dmamem *mxm) 7828 { 7829 bus_dmamap_unload(sc->sc_dmat, mxm->mxm_map); 7830 bus_dmamem_unmap(sc->sc_dmat, mxm->mxm_kva, mxm->mxm_size); 7831 bus_dmamem_free(sc->sc_dmat, &mxm->mxm_seg, 1); 7832 bus_dmamap_destroy(sc->sc_dmat, mxm->mxm_map); 7833 } 7834 7835 static int 7836 mcx_hwmem_alloc(struct mcx_softc *sc, struct mcx_hwmem *mhm, unsigned int pages) 7837 { 7838 bus_dma_segment_t *segs; 7839 bus_size_t len = pages * MCX_PAGE_SIZE; 7840 size_t seglen; 7841 7842 segs = mallocarray(sizeof(*segs), pages, M_DEVBUF, M_WAITOK|M_CANFAIL); 7843 if (segs == NULL) 7844 return (-1); 7845 7846 seglen = sizeof(*segs) * pages; 7847 7848 if (bus_dmamem_alloc(sc->sc_dmat, len, MCX_PAGE_SIZE, 0, 7849 segs, pages, &mhm->mhm_seg_count, BUS_DMA_NOWAIT) != 0) 7850 goto free_segs; 7851 7852 if (mhm->mhm_seg_count < pages) { 7853 size_t nseglen; 7854 7855 mhm->mhm_segs = mallocarray(sizeof(*mhm->mhm_segs), 7856 mhm->mhm_seg_count, M_DEVBUF, M_WAITOK|M_CANFAIL); 7857 if (mhm->mhm_segs == NULL) 7858 goto free_dmamem; 7859 7860 nseglen = sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count; 7861 7862 memcpy(mhm->mhm_segs, segs, nseglen); 7863 7864 free(segs, M_DEVBUF, seglen); 7865 7866 segs = mhm->mhm_segs; 7867 seglen = nseglen; 7868 } else 7869 mhm->mhm_segs = segs; 7870 7871 if (bus_dmamap_create(sc->sc_dmat, len, pages, MCX_PAGE_SIZE, 7872 MCX_PAGE_SIZE, BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW /*|BUS_DMA_64BIT*/, 7873 &mhm->mhm_map) != 0) 7874 goto free_dmamem; 7875 7876 if (bus_dmamap_load_raw(sc->sc_dmat, mhm->mhm_map, 7877 mhm->mhm_segs, mhm->mhm_seg_count, len, BUS_DMA_NOWAIT) != 0) 7878 goto destroy; 7879 7880 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 7881 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_PRERW); 7882 7883 mhm->mhm_npages = pages; 7884 7885 return (0); 7886 7887 destroy: 7888 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 7889 free_dmamem: 7890 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 7891 free_segs: 7892 free(segs, M_DEVBUF, seglen); 7893 mhm->mhm_segs = NULL; 7894 7895 return (-1); 7896 } 7897 7898 static void 7899 mcx_hwmem_free(struct mcx_softc *sc, struct mcx_hwmem *mhm) 7900 { 7901 if (mhm->mhm_npages == 0) 7902 return; 7903 7904 bus_dmamap_sync(sc->sc_dmat, mhm->mhm_map, 7905 0, mhm->mhm_map->dm_mapsize, BUS_DMASYNC_POSTRW); 7906 7907 bus_dmamap_unload(sc->sc_dmat, mhm->mhm_map); 7908 bus_dmamap_destroy(sc->sc_dmat, mhm->mhm_map); 7909 bus_dmamem_free(sc->sc_dmat, mhm->mhm_segs, mhm->mhm_seg_count); 7910 free(mhm->mhm_segs, M_DEVBUF, 7911 sizeof(*mhm->mhm_segs) * mhm->mhm_seg_count); 7912 7913 mhm->mhm_npages = 0; 7914 } 7915 7916 #if NKSTAT > 0 7917 struct mcx_ppcnt { 7918 char name[KSTAT_KV_NAMELEN]; 7919 enum kstat_kv_unit unit; 7920 }; 7921 7922 static const struct mcx_ppcnt mcx_ppcnt_ieee8023_tpl[] = { 7923 { "Good Tx", KSTAT_KV_U_PACKETS, }, 7924 { "Good Rx", KSTAT_KV_U_PACKETS, }, 7925 { "FCS errs", KSTAT_KV_U_PACKETS, }, 7926 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 7927 { "Good Tx", KSTAT_KV_U_BYTES, }, 7928 { "Good Rx", KSTAT_KV_U_BYTES, }, 7929 { "Multicast Tx", KSTAT_KV_U_PACKETS, }, 7930 { "Broadcast Tx", KSTAT_KV_U_PACKETS, }, 7931 { "Multicast Rx", KSTAT_KV_U_PACKETS, }, 7932 { "Broadcast Rx", KSTAT_KV_U_PACKETS, }, 7933 { "In Range Len", KSTAT_KV_U_PACKETS, }, 7934 { "Out Of Range Len", KSTAT_KV_U_PACKETS, }, 7935 { "Frame Too Long", KSTAT_KV_U_PACKETS, }, 7936 { "Symbol Errs", KSTAT_KV_U_PACKETS, }, 7937 { "MAC Ctrl Tx", KSTAT_KV_U_PACKETS, }, 7938 { "MAC Ctrl Rx", KSTAT_KV_U_PACKETS, }, 7939 { "MAC Ctrl Unsup", KSTAT_KV_U_PACKETS, }, 7940 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 7941 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 7942 }; 7943 CTASSERT(nitems(mcx_ppcnt_ieee8023_tpl) == mcx_ppcnt_ieee8023_count); 7944 7945 static const struct mcx_ppcnt mcx_ppcnt_rfc2863_tpl[] = { 7946 { "Rx Bytes", KSTAT_KV_U_BYTES, }, 7947 { "Rx Unicast", KSTAT_KV_U_PACKETS, }, 7948 { "Rx Discards", KSTAT_KV_U_PACKETS, }, 7949 { "Rx Errors", KSTAT_KV_U_PACKETS, }, 7950 { "Rx Unknown Proto", KSTAT_KV_U_PACKETS, }, 7951 { "Tx Bytes", KSTAT_KV_U_BYTES, }, 7952 { "Tx Unicast", KSTAT_KV_U_PACKETS, }, 7953 { "Tx Discards", KSTAT_KV_U_PACKETS, }, 7954 { "Tx Errors", KSTAT_KV_U_PACKETS, }, 7955 { "Rx Multicast", KSTAT_KV_U_PACKETS, }, 7956 { "Rx Broadcast", KSTAT_KV_U_PACKETS, }, 7957 { "Tx Multicast", KSTAT_KV_U_PACKETS, }, 7958 { "Tx Broadcast", KSTAT_KV_U_PACKETS, }, 7959 }; 7960 CTASSERT(nitems(mcx_ppcnt_rfc2863_tpl) == mcx_ppcnt_rfc2863_count); 7961 7962 static const struct mcx_ppcnt mcx_ppcnt_rfc2819_tpl[] = { 7963 { "Drop Events", KSTAT_KV_U_PACKETS, }, 7964 { "Octets", KSTAT_KV_U_BYTES, }, 7965 { "Packets", KSTAT_KV_U_PACKETS, }, 7966 { "Broadcasts", KSTAT_KV_U_PACKETS, }, 7967 { "Multicasts", KSTAT_KV_U_PACKETS, }, 7968 { "CRC Align Errs", KSTAT_KV_U_PACKETS, }, 7969 { "Undersize", KSTAT_KV_U_PACKETS, }, 7970 { "Oversize", KSTAT_KV_U_PACKETS, }, 7971 { "Fragments", KSTAT_KV_U_PACKETS, }, 7972 { "Jabbers", KSTAT_KV_U_PACKETS, }, 7973 { "Collisions", KSTAT_KV_U_NONE, }, 7974 { "64B", KSTAT_KV_U_PACKETS, }, 7975 { "65-127B", KSTAT_KV_U_PACKETS, }, 7976 { "128-255B", KSTAT_KV_U_PACKETS, }, 7977 { "256-511B", KSTAT_KV_U_PACKETS, }, 7978 { "512-1023B", KSTAT_KV_U_PACKETS, }, 7979 { "1024-1518B", KSTAT_KV_U_PACKETS, }, 7980 { "1519-2047B", KSTAT_KV_U_PACKETS, }, 7981 { "2048-4095B", KSTAT_KV_U_PACKETS, }, 7982 { "4096-8191B", KSTAT_KV_U_PACKETS, }, 7983 { "8192-10239B", KSTAT_KV_U_PACKETS, }, 7984 }; 7985 CTASSERT(nitems(mcx_ppcnt_rfc2819_tpl) == mcx_ppcnt_rfc2819_count); 7986 7987 static const struct mcx_ppcnt mcx_ppcnt_rfc3635_tpl[] = { 7988 { "Alignment Errs", KSTAT_KV_U_PACKETS, }, 7989 { "FCS Errs", KSTAT_KV_U_PACKETS, }, 7990 { "Single Colls", KSTAT_KV_U_PACKETS, }, 7991 { "Multiple Colls", KSTAT_KV_U_PACKETS, }, 7992 { "SQE Test Errs", KSTAT_KV_U_NONE, }, 7993 { "Deferred Tx", KSTAT_KV_U_PACKETS, }, 7994 { "Late Colls", KSTAT_KV_U_NONE, }, 7995 { "Exess Colls", KSTAT_KV_U_NONE, }, 7996 { "Int MAC Tx Errs", KSTAT_KV_U_PACKETS, }, 7997 { "CSM Sense Errs", KSTAT_KV_U_NONE, }, 7998 { "Too Long", KSTAT_KV_U_PACKETS, }, 7999 { "Int MAC Rx Errs", KSTAT_KV_U_PACKETS, }, 8000 { "Symbol Errs", KSTAT_KV_U_NONE, }, 8001 { "Unknown Control", KSTAT_KV_U_PACKETS, }, 8002 { "Pause Rx", KSTAT_KV_U_PACKETS, }, 8003 { "Pause Tx", KSTAT_KV_U_PACKETS, }, 8004 }; 8005 CTASSERT(nitems(mcx_ppcnt_rfc3635_tpl) == mcx_ppcnt_rfc3635_count); 8006 8007 struct mcx_kstat_ppcnt { 8008 const char *ksp_name; 8009 const struct mcx_ppcnt *ksp_tpl; 8010 unsigned int ksp_n; 8011 uint8_t ksp_grp; 8012 }; 8013 8014 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_ieee8023 = { 8015 .ksp_name = "ieee802.3", 8016 .ksp_tpl = mcx_ppcnt_ieee8023_tpl, 8017 .ksp_n = nitems(mcx_ppcnt_ieee8023_tpl), 8018 .ksp_grp = MCX_REG_PPCNT_GRP_IEEE8023, 8019 }; 8020 8021 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2863 = { 8022 .ksp_name = "rfc2863", 8023 .ksp_tpl = mcx_ppcnt_rfc2863_tpl, 8024 .ksp_n = nitems(mcx_ppcnt_rfc2863_tpl), 8025 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2863, 8026 }; 8027 8028 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc2819 = { 8029 .ksp_name = "rfc2819", 8030 .ksp_tpl = mcx_ppcnt_rfc2819_tpl, 8031 .ksp_n = nitems(mcx_ppcnt_rfc2819_tpl), 8032 .ksp_grp = MCX_REG_PPCNT_GRP_RFC2819, 8033 }; 8034 8035 static const struct mcx_kstat_ppcnt mcx_kstat_ppcnt_rfc3635 = { 8036 .ksp_name = "rfc3635", 8037 .ksp_tpl = mcx_ppcnt_rfc3635_tpl, 8038 .ksp_n = nitems(mcx_ppcnt_rfc3635_tpl), 8039 .ksp_grp = MCX_REG_PPCNT_GRP_RFC3635, 8040 }; 8041 8042 static int mcx_kstat_ppcnt_read(struct kstat *); 8043 8044 static void mcx_kstat_attach_tmps(struct mcx_softc *sc); 8045 static void mcx_kstat_attach_queues(struct mcx_softc *sc); 8046 8047 static struct kstat * 8048 mcx_kstat_attach_ppcnt(struct mcx_softc *sc, 8049 const struct mcx_kstat_ppcnt *ksp) 8050 { 8051 struct kstat *ks; 8052 struct kstat_kv *kvs; 8053 unsigned int i; 8054 8055 ks = kstat_create(DEVNAME(sc), 0, ksp->ksp_name, 0, KSTAT_T_KV, 0); 8056 if (ks == NULL) 8057 return (NULL); 8058 8059 kvs = mallocarray(ksp->ksp_n, sizeof(*kvs), 8060 M_DEVBUF, M_WAITOK); 8061 8062 for (i = 0; i < ksp->ksp_n; i++) { 8063 const struct mcx_ppcnt *tpl = &ksp->ksp_tpl[i]; 8064 8065 kstat_kv_unit_init(&kvs[i], tpl->name, 8066 KSTAT_KV_T_COUNTER64, tpl->unit); 8067 } 8068 8069 ks->ks_softc = sc; 8070 ks->ks_ptr = (void *)ksp; 8071 ks->ks_data = kvs; 8072 ks->ks_datalen = ksp->ksp_n * sizeof(*kvs); 8073 ks->ks_read = mcx_kstat_ppcnt_read; 8074 8075 kstat_install(ks); 8076 8077 return (ks); 8078 } 8079 8080 static void 8081 mcx_kstat_attach(struct mcx_softc *sc) 8082 { 8083 sc->sc_kstat_ieee8023 = mcx_kstat_attach_ppcnt(sc, 8084 &mcx_kstat_ppcnt_ieee8023); 8085 sc->sc_kstat_rfc2863 = mcx_kstat_attach_ppcnt(sc, 8086 &mcx_kstat_ppcnt_rfc2863); 8087 sc->sc_kstat_rfc2819 = mcx_kstat_attach_ppcnt(sc, 8088 &mcx_kstat_ppcnt_rfc2819); 8089 sc->sc_kstat_rfc3635 = mcx_kstat_attach_ppcnt(sc, 8090 &mcx_kstat_ppcnt_rfc3635); 8091 8092 mcx_kstat_attach_tmps(sc); 8093 mcx_kstat_attach_queues(sc); 8094 } 8095 8096 static int 8097 mcx_kstat_ppcnt_read(struct kstat *ks) 8098 { 8099 struct mcx_softc *sc = ks->ks_softc; 8100 struct mcx_kstat_ppcnt *ksp = ks->ks_ptr; 8101 struct mcx_reg_ppcnt ppcnt = { 8102 .ppcnt_grp = ksp->ksp_grp, 8103 .ppcnt_local_port = 1, 8104 }; 8105 struct kstat_kv *kvs = ks->ks_data; 8106 uint64_t *vs = (uint64_t *)&ppcnt.ppcnt_counter_set; 8107 unsigned int i; 8108 int rv; 8109 8110 KERNEL_LOCK(); /* XXX */ 8111 rv = mcx_access_hca_reg(sc, MCX_REG_PPCNT, MCX_REG_OP_READ, 8112 &ppcnt, sizeof(ppcnt)); 8113 KERNEL_UNLOCK(); 8114 if (rv != 0) 8115 return (EIO); 8116 8117 nanouptime(&ks->ks_updated); 8118 8119 for (i = 0; i < ksp->ksp_n; i++) 8120 kstat_kv_u64(&kvs[i]) = bemtoh64(&vs[i]); 8121 8122 return (0); 8123 } 8124 8125 struct mcx_kstat_mtmp { 8126 struct kstat_kv ktmp_name; 8127 struct kstat_kv ktmp_temperature; 8128 struct kstat_kv ktmp_threshold_lo; 8129 struct kstat_kv ktmp_threshold_hi; 8130 }; 8131 8132 static const struct mcx_kstat_mtmp mcx_kstat_mtmp_tpl = { 8133 KSTAT_KV_INITIALIZER("name", KSTAT_KV_T_ISTR), 8134 KSTAT_KV_INITIALIZER("temperature", KSTAT_KV_T_TEMP), 8135 KSTAT_KV_INITIALIZER("lo threshold", KSTAT_KV_T_TEMP), 8136 KSTAT_KV_INITIALIZER("hi threshold", KSTAT_KV_T_TEMP), 8137 }; 8138 8139 static const struct timeval mcx_kstat_mtmp_rate = { 1, 0 }; 8140 8141 static int mcx_kstat_mtmp_read(struct kstat *); 8142 8143 static void 8144 mcx_kstat_attach_tmps(struct mcx_softc *sc) 8145 { 8146 struct kstat *ks; 8147 struct mcx_reg_mtcap mtcap; 8148 struct mcx_kstat_mtmp *ktmp; 8149 uint64_t map; 8150 unsigned int i, n; 8151 8152 memset(&mtcap, 0, sizeof(mtcap)); 8153 8154 if (mcx_access_hca_reg(sc, MCX_REG_MTCAP, MCX_REG_OP_READ, 8155 &mtcap, sizeof(mtcap)) != 0) { 8156 /* unable to find temperature sensors */ 8157 return; 8158 } 8159 8160 sc->sc_kstat_mtmp_count = mtcap.mtcap_sensor_count; 8161 sc->sc_kstat_mtmp = mallocarray(sc->sc_kstat_mtmp_count, 8162 sizeof(*sc->sc_kstat_mtmp), M_DEVBUF, M_WAITOK); 8163 8164 n = 0; 8165 map = bemtoh64(&mtcap.mtcap_sensor_map); 8166 for (i = 0; i < sizeof(map) * NBBY; i++) { 8167 if (!ISSET(map, (1ULL << i))) 8168 continue; 8169 8170 ks = kstat_create(DEVNAME(sc), 0, "temperature", i, 8171 KSTAT_T_KV, 0); 8172 if (ks == NULL) { 8173 /* unable to attach temperature sensor %u, i */ 8174 continue; 8175 } 8176 8177 ktmp = malloc(sizeof(*ktmp), M_DEVBUF, M_WAITOK|M_ZERO); 8178 *ktmp = mcx_kstat_mtmp_tpl; 8179 8180 ks->ks_data = ktmp; 8181 ks->ks_datalen = sizeof(*ktmp); 8182 TIMEVAL_TO_TIMESPEC(&mcx_kstat_mtmp_rate, &ks->ks_interval); 8183 ks->ks_read = mcx_kstat_mtmp_read; 8184 8185 ks->ks_softc = sc; 8186 kstat_install(ks); 8187 8188 sc->sc_kstat_mtmp[n++] = ks; 8189 if (n >= sc->sc_kstat_mtmp_count) 8190 break; 8191 } 8192 } 8193 8194 static uint64_t 8195 mcx_tmp_to_uK(uint16_t *t) 8196 { 8197 int64_t mt = (int16_t)bemtoh16(t); /* 0.125 C units */ 8198 mt *= 1000000 / 8; /* convert to uC */ 8199 mt += 273150000; /* convert to uK */ 8200 8201 return (mt); 8202 } 8203 8204 static int 8205 mcx_kstat_mtmp_read(struct kstat *ks) 8206 { 8207 struct mcx_softc *sc = ks->ks_softc; 8208 struct mcx_kstat_mtmp *ktmp = ks->ks_data; 8209 struct mcx_reg_mtmp mtmp; 8210 int rv; 8211 struct timeval updated; 8212 8213 TIMESPEC_TO_TIMEVAL(&updated, &ks->ks_updated); 8214 8215 if (!ratecheck(&updated, &mcx_kstat_mtmp_rate)) 8216 return (0); 8217 8218 memset(&mtmp, 0, sizeof(mtmp)); 8219 htobem16(&mtmp.mtmp_sensor_index, ks->ks_unit); 8220 8221 KERNEL_LOCK(); /* XXX */ 8222 rv = mcx_access_hca_reg(sc, MCX_REG_MTMP, MCX_REG_OP_READ, 8223 &mtmp, sizeof(mtmp)); 8224 KERNEL_UNLOCK(); 8225 if (rv != 0) 8226 return (EIO); 8227 8228 memset(kstat_kv_istr(&ktmp->ktmp_name), 0, 8229 sizeof(kstat_kv_istr(&ktmp->ktmp_name))); 8230 memcpy(kstat_kv_istr(&ktmp->ktmp_name), 8231 mtmp.mtmp_sensor_name, sizeof(mtmp.mtmp_sensor_name)); 8232 kstat_kv_temp(&ktmp->ktmp_temperature) = 8233 mcx_tmp_to_uK(&mtmp.mtmp_temperature); 8234 kstat_kv_temp(&ktmp->ktmp_threshold_lo) = 8235 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_lo); 8236 kstat_kv_temp(&ktmp->ktmp_threshold_hi) = 8237 mcx_tmp_to_uK(&mtmp.mtmp_temperature_threshold_hi); 8238 8239 TIMEVAL_TO_TIMESPEC(&updated, &ks->ks_updated); 8240 8241 return (0); 8242 } 8243 8244 struct mcx_queuestat { 8245 char name[KSTAT_KV_NAMELEN]; 8246 enum kstat_kv_type type; 8247 }; 8248 8249 static const struct mcx_queuestat mcx_queue_kstat_tpl[] = { 8250 { "RQ SW prod", KSTAT_KV_T_COUNTER64 }, 8251 { "RQ HW prod", KSTAT_KV_T_COUNTER64 }, 8252 { "RQ HW cons", KSTAT_KV_T_COUNTER64 }, 8253 { "RQ HW state", KSTAT_KV_T_ISTR }, 8254 8255 { "SQ SW prod", KSTAT_KV_T_COUNTER64 }, 8256 { "SQ SW cons", KSTAT_KV_T_COUNTER64 }, 8257 { "SQ HW prod", KSTAT_KV_T_COUNTER64 }, 8258 { "SQ HW cons", KSTAT_KV_T_COUNTER64 }, 8259 { "SQ HW state", KSTAT_KV_T_ISTR }, 8260 8261 { "CQ SW cons", KSTAT_KV_T_COUNTER64 }, 8262 { "CQ HW prod", KSTAT_KV_T_COUNTER64 }, 8263 { "CQ HW cons", KSTAT_KV_T_COUNTER64 }, 8264 { "CQ HW notify", KSTAT_KV_T_COUNTER64 }, 8265 { "CQ HW solicit", KSTAT_KV_T_COUNTER64 }, 8266 { "CQ HW status", KSTAT_KV_T_ISTR }, 8267 { "CQ HW state", KSTAT_KV_T_ISTR }, 8268 8269 { "EQ SW cons", KSTAT_KV_T_COUNTER64 }, 8270 { "EQ HW prod", KSTAT_KV_T_COUNTER64 }, 8271 { "EQ HW cons", KSTAT_KV_T_COUNTER64 }, 8272 { "EQ HW status", KSTAT_KV_T_ISTR }, 8273 { "EQ HW state", KSTAT_KV_T_ISTR }, 8274 }; 8275 8276 static int mcx_kstat_queue_read(struct kstat *); 8277 8278 static void 8279 mcx_kstat_attach_queues(struct mcx_softc *sc) 8280 { 8281 struct kstat *ks; 8282 struct kstat_kv *kvs; 8283 int q, i; 8284 8285 for (q = 0; q < sc->sc_nqueues; q++) { 8286 ks = kstat_create(DEVNAME(sc), 0, "mcx-queues", q, 8287 KSTAT_T_KV, 0); 8288 if (ks == NULL) { 8289 /* unable to attach queue stats %u, q */ 8290 continue; 8291 } 8292 8293 kvs = mallocarray(nitems(mcx_queue_kstat_tpl), 8294 sizeof(*kvs), M_DEVBUF, M_WAITOK); 8295 8296 for (i = 0; i < nitems(mcx_queue_kstat_tpl); i++) { 8297 const struct mcx_queuestat *tpl = 8298 &mcx_queue_kstat_tpl[i]; 8299 8300 kstat_kv_init(&kvs[i], tpl->name, tpl->type); 8301 } 8302 8303 ks->ks_softc = &sc->sc_queues[q]; 8304 ks->ks_data = kvs; 8305 ks->ks_datalen = nitems(mcx_queue_kstat_tpl) * sizeof(*kvs); 8306 ks->ks_read = mcx_kstat_queue_read; 8307 8308 sc->sc_queues[q].q_kstat = ks; 8309 kstat_install(ks); 8310 } 8311 } 8312 8313 static int 8314 mcx_kstat_queue_read(struct kstat *ks) 8315 { 8316 struct mcx_queues *q = ks->ks_softc; 8317 struct mcx_softc *sc = q->q_sc; 8318 struct kstat_kv *kvs = ks->ks_data; 8319 union { 8320 struct mcx_rq_ctx rq; 8321 struct mcx_sq_ctx sq; 8322 struct mcx_cq_ctx cq; 8323 struct mcx_eq_ctx eq; 8324 } u; 8325 const char *text; 8326 int error = 0; 8327 8328 KERNEL_LOCK(); 8329 8330 if (mcx_query_rq(sc, &q->q_rx, &u.rq) != 0) { 8331 error = EIO; 8332 goto out; 8333 } 8334 8335 kstat_kv_u64(kvs++) = q->q_rx.rx_prod; 8336 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_sw_counter); 8337 kstat_kv_u64(kvs++) = bemtoh32(&u.rq.rq_wq.wq_hw_counter); 8338 switch ((bemtoh32(&u.rq.rq_flags) & MCX_RQ_CTX_STATE_MASK) >> 8339 MCX_RQ_CTX_STATE_SHIFT) { 8340 case MCX_RQ_CTX_STATE_RST: 8341 text = "RST"; 8342 break; 8343 case MCX_RQ_CTX_STATE_RDY: 8344 text = "RDY"; 8345 break; 8346 case MCX_RQ_CTX_STATE_ERR: 8347 text = "ERR"; 8348 break; 8349 default: 8350 text = "unknown"; 8351 break; 8352 } 8353 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8354 kvs++; 8355 8356 if (mcx_query_sq(sc, &q->q_tx, &u.sq) != 0) { 8357 error = EIO; 8358 goto out; 8359 } 8360 8361 kstat_kv_u64(kvs++) = q->q_tx.tx_prod; 8362 kstat_kv_u64(kvs++) = q->q_tx.tx_cons; 8363 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_sw_counter); 8364 kstat_kv_u64(kvs++) = bemtoh32(&u.sq.sq_wq.wq_hw_counter); 8365 switch ((bemtoh32(&u.sq.sq_flags) & MCX_SQ_CTX_STATE_MASK) >> 8366 MCX_SQ_CTX_STATE_SHIFT) { 8367 case MCX_SQ_CTX_STATE_RST: 8368 text = "RST"; 8369 break; 8370 case MCX_SQ_CTX_STATE_RDY: 8371 text = "RDY"; 8372 break; 8373 case MCX_SQ_CTX_STATE_ERR: 8374 text = "ERR"; 8375 break; 8376 default: 8377 text = "unknown"; 8378 break; 8379 } 8380 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8381 kvs++; 8382 8383 if (mcx_query_cq(sc, &q->q_cq, &u.cq) != 0) { 8384 error = EIO; 8385 goto out; 8386 } 8387 8388 kstat_kv_u64(kvs++) = q->q_cq.cq_cons; 8389 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_producer_counter); 8390 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_consumer_counter); 8391 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_notified); 8392 kstat_kv_u64(kvs++) = bemtoh32(&u.cq.cq_last_solicit); 8393 8394 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATUS_MASK) >> 8395 MCX_CQ_CTX_STATUS_SHIFT) { 8396 case MCX_CQ_CTX_STATUS_OK: 8397 text = "OK"; 8398 break; 8399 case MCX_CQ_CTX_STATUS_OVERFLOW: 8400 text = "overflow"; 8401 break; 8402 case MCX_CQ_CTX_STATUS_WRITE_FAIL: 8403 text = "write fail"; 8404 break; 8405 default: 8406 text = "unknown"; 8407 break; 8408 } 8409 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8410 kvs++; 8411 8412 switch ((bemtoh32(&u.cq.cq_status) & MCX_CQ_CTX_STATE_MASK) >> 8413 MCX_CQ_CTX_STATE_SHIFT) { 8414 case MCX_CQ_CTX_STATE_SOLICITED: 8415 text = "solicited"; 8416 break; 8417 case MCX_CQ_CTX_STATE_ARMED: 8418 text = "armed"; 8419 break; 8420 case MCX_CQ_CTX_STATE_FIRED: 8421 text = "fired"; 8422 break; 8423 default: 8424 text = "unknown"; 8425 break; 8426 } 8427 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8428 kvs++; 8429 8430 if (mcx_query_eq(sc, &q->q_eq, &u.eq) != 0) { 8431 error = EIO; 8432 goto out; 8433 } 8434 8435 kstat_kv_u64(kvs++) = q->q_eq.eq_cons; 8436 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_producer_counter); 8437 kstat_kv_u64(kvs++) = bemtoh32(&u.eq.eq_consumer_counter); 8438 8439 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATUS_MASK) >> 8440 MCX_EQ_CTX_STATUS_SHIFT) { 8441 case MCX_EQ_CTX_STATUS_EQ_WRITE_FAILURE: 8442 text = "write fail"; 8443 break; 8444 case MCX_EQ_CTX_STATUS_OK: 8445 text = "OK"; 8446 break; 8447 default: 8448 text = "unknown"; 8449 break; 8450 } 8451 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8452 kvs++; 8453 8454 switch ((bemtoh32(&u.eq.eq_status) & MCX_EQ_CTX_STATE_MASK) >> 8455 MCX_EQ_CTX_STATE_SHIFT) { 8456 case MCX_EQ_CTX_STATE_ARMED: 8457 text = "armed"; 8458 break; 8459 case MCX_EQ_CTX_STATE_FIRED: 8460 text = "fired"; 8461 break; 8462 default: 8463 text = "unknown"; 8464 break; 8465 } 8466 strlcpy(kstat_kv_istr(kvs), text, sizeof(kstat_kv_istr(kvs))); 8467 kvs++; 8468 8469 nanouptime(&ks->ks_updated); 8470 out: 8471 KERNEL_UNLOCK(); 8472 return (error); 8473 } 8474 8475 #endif /* NKSTAT > 0 */ 8476