1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #ifndef MLX5_DRIVER_H 34 #define MLX5_DRIVER_H 35 36 #include <linux/kernel.h> 37 #include <linux/completion.h> 38 #include <linux/pci.h> 39 #include <linux/irq.h> 40 #include <linux/spinlock_types.h> 41 #include <linux/semaphore.h> 42 #include <linux/slab.h> 43 #include <linux/vmalloc.h> 44 #include <linux/xarray.h> 45 #include <linux/workqueue.h> 46 #include <linux/mempool.h> 47 #include <linux/interrupt.h> 48 #include <linux/idr.h> 49 #include <linux/notifier.h> 50 #include <linux/refcount.h> 51 #include <linux/auxiliary_bus.h> 52 #include <linux/mutex.h> 53 54 #include <linux/mlx5/device.h> 55 #include <linux/mlx5/doorbell.h> 56 #include <linux/mlx5/eq.h> 57 #include <linux/timecounter.h> 58 #include <linux/ptp_clock_kernel.h> 59 #include <net/devlink.h> 60 61 #define MLX5_ADEV_NAME "mlx5_core" 62 63 #define MLX5_IRQ_EQ_CTRL (U8_MAX) 64 65 enum { 66 MLX5_BOARD_ID_LEN = 64, 67 }; 68 69 enum { 70 MLX5_CMD_WQ_MAX_NAME = 32, 71 }; 72 73 enum { 74 CMD_OWNER_SW = 0x0, 75 CMD_OWNER_HW = 0x1, 76 CMD_STATUS_SUCCESS = 0, 77 }; 78 79 enum mlx5_sqp_t { 80 MLX5_SQP_SMI = 0, 81 MLX5_SQP_GSI = 1, 82 MLX5_SQP_IEEE_1588 = 2, 83 MLX5_SQP_SNIFFER = 3, 84 MLX5_SQP_SYNC_UMR = 4, 85 }; 86 87 enum { 88 MLX5_MAX_PORTS = 4, 89 }; 90 91 enum { 92 MLX5_ATOMIC_MODE_OFFSET = 16, 93 MLX5_ATOMIC_MODE_IB_COMP = 1, 94 MLX5_ATOMIC_MODE_CX = 2, 95 MLX5_ATOMIC_MODE_8B = 3, 96 MLX5_ATOMIC_MODE_16B = 4, 97 MLX5_ATOMIC_MODE_32B = 5, 98 MLX5_ATOMIC_MODE_64B = 6, 99 MLX5_ATOMIC_MODE_128B = 7, 100 MLX5_ATOMIC_MODE_256B = 8, 101 }; 102 103 enum { 104 MLX5_REG_SBPR = 0xb001, 105 MLX5_REG_SBCM = 0xb002, 106 MLX5_REG_QPTS = 0x4002, 107 MLX5_REG_QETCR = 0x4005, 108 MLX5_REG_QTCT = 0x400a, 109 MLX5_REG_QPDPM = 0x4013, 110 MLX5_REG_QCAM = 0x4019, 111 MLX5_REG_DCBX_PARAM = 0x4020, 112 MLX5_REG_DCBX_APP = 0x4021, 113 MLX5_REG_FPGA_CAP = 0x4022, 114 MLX5_REG_FPGA_CTRL = 0x4023, 115 MLX5_REG_FPGA_ACCESS_REG = 0x4024, 116 MLX5_REG_CORE_DUMP = 0x402e, 117 MLX5_REG_PCAP = 0x5001, 118 MLX5_REG_PMTU = 0x5003, 119 MLX5_REG_PTYS = 0x5004, 120 MLX5_REG_PAOS = 0x5006, 121 MLX5_REG_PFCC = 0x5007, 122 MLX5_REG_PPCNT = 0x5008, 123 MLX5_REG_PPTB = 0x500b, 124 MLX5_REG_PBMC = 0x500c, 125 MLX5_REG_PMAOS = 0x5012, 126 MLX5_REG_PUDE = 0x5009, 127 MLX5_REG_PMPE = 0x5010, 128 MLX5_REG_PELC = 0x500e, 129 MLX5_REG_PVLC = 0x500f, 130 MLX5_REG_PCMR = 0x5041, 131 MLX5_REG_PDDR = 0x5031, 132 MLX5_REG_PMLP = 0x5002, 133 MLX5_REG_PPLM = 0x5023, 134 MLX5_REG_PCAM = 0x507f, 135 MLX5_REG_NODE_DESC = 0x6001, 136 MLX5_REG_HOST_ENDIANNESS = 0x7004, 137 MLX5_REG_MTCAP = 0x9009, 138 MLX5_REG_MTMP = 0x900A, 139 MLX5_REG_MCIA = 0x9014, 140 MLX5_REG_MFRL = 0x9028, 141 MLX5_REG_MLCR = 0x902b, 142 MLX5_REG_MRTC = 0x902d, 143 MLX5_REG_MTRC_CAP = 0x9040, 144 MLX5_REG_MTRC_CONF = 0x9041, 145 MLX5_REG_MTRC_STDB = 0x9042, 146 MLX5_REG_MTRC_CTRL = 0x9043, 147 MLX5_REG_MPEIN = 0x9050, 148 MLX5_REG_MPCNT = 0x9051, 149 MLX5_REG_MTPPS = 0x9053, 150 MLX5_REG_MTPPSE = 0x9054, 151 MLX5_REG_MTUTC = 0x9055, 152 MLX5_REG_MPEGC = 0x9056, 153 MLX5_REG_MCQS = 0x9060, 154 MLX5_REG_MCQI = 0x9061, 155 MLX5_REG_MCC = 0x9062, 156 MLX5_REG_MCDA = 0x9063, 157 MLX5_REG_MCAM = 0x907f, 158 MLX5_REG_MSECQ = 0x9155, 159 MLX5_REG_MSEES = 0x9156, 160 MLX5_REG_MIRC = 0x9162, 161 MLX5_REG_SBCAM = 0xB01F, 162 MLX5_REG_RESOURCE_DUMP = 0xC000, 163 MLX5_REG_DTOR = 0xC00E, 164 }; 165 166 enum mlx5_qpts_trust_state { 167 MLX5_QPTS_TRUST_PCP = 1, 168 MLX5_QPTS_TRUST_DSCP = 2, 169 }; 170 171 enum mlx5_dcbx_oper_mode { 172 MLX5E_DCBX_PARAM_VER_OPER_HOST = 0x0, 173 MLX5E_DCBX_PARAM_VER_OPER_AUTO = 0x3, 174 }; 175 176 enum { 177 MLX5_ATOMIC_OPS_CMP_SWAP = 1 << 0, 178 MLX5_ATOMIC_OPS_FETCH_ADD = 1 << 1, 179 MLX5_ATOMIC_OPS_EXTENDED_CMP_SWAP = 1 << 2, 180 MLX5_ATOMIC_OPS_EXTENDED_FETCH_ADD = 1 << 3, 181 }; 182 183 enum mlx5_page_fault_resume_flags { 184 MLX5_PAGE_FAULT_RESUME_REQUESTOR = 1 << 0, 185 MLX5_PAGE_FAULT_RESUME_WRITE = 1 << 1, 186 MLX5_PAGE_FAULT_RESUME_RDMA = 1 << 2, 187 MLX5_PAGE_FAULT_RESUME_ERROR = 1 << 7, 188 }; 189 190 enum dbg_rsc_type { 191 MLX5_DBG_RSC_QP, 192 MLX5_DBG_RSC_EQ, 193 MLX5_DBG_RSC_CQ, 194 }; 195 196 enum port_state_policy { 197 MLX5_POLICY_DOWN = 0, 198 MLX5_POLICY_UP = 1, 199 MLX5_POLICY_FOLLOW = 2, 200 MLX5_POLICY_INVALID = 0xffffffff 201 }; 202 203 enum mlx5_coredev_type { 204 MLX5_COREDEV_PF, 205 MLX5_COREDEV_VF, 206 MLX5_COREDEV_SF, 207 }; 208 209 struct mlx5_field_desc { 210 int i; 211 }; 212 213 struct mlx5_rsc_debug { 214 struct mlx5_core_dev *dev; 215 void *object; 216 enum dbg_rsc_type type; 217 struct dentry *root; 218 struct mlx5_field_desc fields[]; 219 }; 220 221 enum mlx5_dev_event { 222 MLX5_DEV_EVENT_SYS_ERROR = 128, /* 0 - 127 are FW events */ 223 MLX5_DEV_EVENT_PORT_AFFINITY = 129, 224 MLX5_DEV_EVENT_MULTIPORT_ESW = 130, 225 }; 226 227 enum mlx5_port_status { 228 MLX5_PORT_UP = 1, 229 MLX5_PORT_DOWN = 2, 230 }; 231 232 enum mlx5_cmdif_state { 233 MLX5_CMDIF_STATE_UNINITIALIZED, 234 MLX5_CMDIF_STATE_UP, 235 MLX5_CMDIF_STATE_DOWN, 236 }; 237 238 struct mlx5_cmd_first { 239 __be32 data[4]; 240 }; 241 242 struct mlx5_cmd_msg { 243 struct list_head list; 244 struct cmd_msg_cache *parent; 245 u32 len; 246 struct mlx5_cmd_first first; 247 struct mlx5_cmd_mailbox *next; 248 }; 249 250 struct mlx5_cmd_debug { 251 struct dentry *dbg_root; 252 void *in_msg; 253 void *out_msg; 254 u8 status; 255 u16 inlen; 256 u16 outlen; 257 }; 258 259 struct cmd_msg_cache { 260 /* protect block chain allocations 261 */ 262 spinlock_t lock; 263 struct list_head head; 264 unsigned int max_inbox_size; 265 unsigned int num_ent; 266 }; 267 268 enum { 269 MLX5_NUM_COMMAND_CACHES = 5, 270 }; 271 272 struct mlx5_cmd_stats { 273 u64 sum; 274 u64 n; 275 /* number of times command failed */ 276 u64 failed; 277 /* number of times command failed on bad status returned by FW */ 278 u64 failed_mbox_status; 279 /* last command failed returned errno */ 280 u32 last_failed_errno; 281 /* last bad status returned by FW */ 282 u8 last_failed_mbox_status; 283 /* last command failed syndrome returned by FW */ 284 u32 last_failed_syndrome; 285 struct dentry *root; 286 /* protect command average calculations */ 287 spinlock_t lock; 288 }; 289 290 struct mlx5_cmd { 291 struct mlx5_nb nb; 292 293 /* members which needs to be queried or reinitialized each reload */ 294 struct { 295 u16 cmdif_rev; 296 u8 log_sz; 297 u8 log_stride; 298 int max_reg_cmds; 299 unsigned long bitmask; 300 struct semaphore sem; 301 struct semaphore pages_sem; 302 struct semaphore throttle_sem; 303 } vars; 304 enum mlx5_cmdif_state state; 305 void *cmd_alloc_buf; 306 dma_addr_t alloc_dma; 307 int alloc_size; 308 void *cmd_buf; 309 dma_addr_t dma; 310 311 /* protect command queue allocations 312 */ 313 spinlock_t alloc_lock; 314 315 /* protect token allocations 316 */ 317 spinlock_t token_lock; 318 u8 token; 319 char wq_name[MLX5_CMD_WQ_MAX_NAME]; 320 struct workqueue_struct *wq; 321 int mode; 322 u16 allowed_opcode; 323 struct mlx5_cmd_work_ent *ent_arr[MLX5_MAX_COMMANDS]; 324 struct dma_pool *pool; 325 struct mlx5_cmd_debug dbg; 326 struct cmd_msg_cache cache[MLX5_NUM_COMMAND_CACHES]; 327 int checksum_disabled; 328 struct xarray stats; 329 }; 330 331 struct mlx5_cmd_mailbox { 332 void *buf; 333 dma_addr_t dma; 334 struct mlx5_cmd_mailbox *next; 335 }; 336 337 struct mlx5_buf_list { 338 void *buf; 339 dma_addr_t map; 340 }; 341 342 struct mlx5_frag_buf { 343 struct mlx5_buf_list *frags; 344 int npages; 345 int size; 346 u8 page_shift; 347 }; 348 349 struct mlx5_frag_buf_ctrl { 350 struct mlx5_buf_list *frags; 351 u32 sz_m1; 352 u16 frag_sz_m1; 353 u16 strides_offset; 354 u8 log_sz; 355 u8 log_stride; 356 u8 log_frag_strides; 357 }; 358 359 struct mlx5_core_psv { 360 u32 psv_idx; 361 struct psv_layout { 362 u32 pd; 363 u16 syndrome; 364 u16 reserved; 365 u16 bg; 366 u16 app_tag; 367 u32 ref_tag; 368 } psv; 369 }; 370 371 struct mlx5_core_sig_ctx { 372 struct mlx5_core_psv psv_memory; 373 struct mlx5_core_psv psv_wire; 374 struct ib_sig_err err_item; 375 bool sig_status_checked; 376 bool sig_err_exists; 377 u32 sigerr_count; 378 }; 379 380 #define MLX5_24BIT_MASK ((1 << 24) - 1) 381 382 enum mlx5_res_type { 383 MLX5_RES_QP = MLX5_EVENT_QUEUE_TYPE_QP, 384 MLX5_RES_RQ = MLX5_EVENT_QUEUE_TYPE_RQ, 385 MLX5_RES_SQ = MLX5_EVENT_QUEUE_TYPE_SQ, 386 MLX5_RES_SRQ = 3, 387 MLX5_RES_XSRQ = 4, 388 MLX5_RES_XRQ = 5, 389 }; 390 391 struct mlx5_core_rsc_common { 392 enum mlx5_res_type res; 393 refcount_t refcount; 394 struct completion free; 395 }; 396 397 struct mlx5_uars_page { 398 void __iomem *map; 399 bool wc; 400 u32 index; 401 struct list_head list; 402 unsigned int bfregs; 403 unsigned long *reg_bitmap; /* for non fast path bf regs */ 404 unsigned long *fp_bitmap; 405 unsigned int reg_avail; 406 unsigned int fp_avail; 407 struct kref ref_count; 408 struct mlx5_core_dev *mdev; 409 }; 410 411 struct mlx5_bfreg_head { 412 /* protect blue flame registers allocations */ 413 struct mutex lock; 414 struct list_head list; 415 }; 416 417 struct mlx5_bfreg_data { 418 struct mlx5_bfreg_head reg_head; 419 struct mlx5_bfreg_head wc_head; 420 }; 421 422 struct mlx5_sq_bfreg { 423 void __iomem *map; 424 struct mlx5_uars_page *up; 425 bool wc; 426 u32 index; 427 unsigned int offset; 428 }; 429 430 struct mlx5_core_health { 431 struct health_buffer __iomem *health; 432 __be32 __iomem *health_counter; 433 struct timer_list timer; 434 u32 prev; 435 int miss_counter; 436 u8 synd; 437 u32 fatal_error; 438 u32 crdump_size; 439 struct workqueue_struct *wq; 440 unsigned long flags; 441 struct work_struct fatal_report_work; 442 struct work_struct report_work; 443 struct devlink_health_reporter *fw_reporter; 444 struct devlink_health_reporter *fw_fatal_reporter; 445 struct devlink_health_reporter *vnic_reporter; 446 struct delayed_work update_fw_log_ts_work; 447 }; 448 449 enum { 450 MLX5_PF_NOTIFY_DISABLE_VF, 451 MLX5_PF_NOTIFY_ENABLE_VF, 452 }; 453 454 struct mlx5_vf_context { 455 int enabled; 456 u64 port_guid; 457 u64 node_guid; 458 /* Valid bits are used to validate administrative guid only. 459 * Enabled after ndo_set_vf_guid 460 */ 461 u8 port_guid_valid:1; 462 u8 node_guid_valid:1; 463 enum port_state_policy policy; 464 struct blocking_notifier_head notifier; 465 }; 466 467 struct mlx5_core_sriov { 468 struct mlx5_vf_context *vfs_ctx; 469 int num_vfs; 470 u16 max_vfs; 471 u16 max_ec_vfs; 472 }; 473 474 struct mlx5_fc_pool { 475 struct mlx5_core_dev *dev; 476 struct mutex pool_lock; /* protects pool lists */ 477 struct list_head fully_used; 478 struct list_head partially_used; 479 struct list_head unused; 480 int available_fcs; 481 int used_fcs; 482 int threshold; 483 }; 484 485 struct mlx5_fc_stats { 486 spinlock_t counters_idr_lock; /* protects counters_idr */ 487 struct idr counters_idr; 488 struct list_head counters; 489 struct llist_head addlist; 490 struct llist_head dellist; 491 492 struct workqueue_struct *wq; 493 struct delayed_work work; 494 unsigned long next_query; 495 unsigned long sampling_interval; /* jiffies */ 496 u32 *bulk_query_out; 497 int bulk_query_len; 498 size_t num_counters; 499 bool bulk_query_alloc_failed; 500 unsigned long next_bulk_query_alloc; 501 struct mlx5_fc_pool fc_pool; 502 }; 503 504 struct mlx5_events; 505 struct mlx5_mpfs; 506 struct mlx5_eswitch; 507 struct mlx5_lag; 508 struct mlx5_devcom_dev; 509 struct mlx5_fw_reset; 510 struct mlx5_eq_table; 511 struct mlx5_irq_table; 512 struct mlx5_vhca_state_notifier; 513 struct mlx5_sf_dev_table; 514 struct mlx5_sf_hw_table; 515 struct mlx5_sf_table; 516 struct mlx5_crypto_dek_priv; 517 518 struct mlx5_rate_limit { 519 u32 rate; 520 u32 max_burst_sz; 521 u16 typical_pkt_sz; 522 }; 523 524 struct mlx5_rl_entry { 525 u8 rl_raw[MLX5_ST_SZ_BYTES(set_pp_rate_limit_context)]; 526 u64 refcount; 527 u16 index; 528 u16 uid; 529 u8 dedicated : 1; 530 }; 531 532 struct mlx5_rl_table { 533 /* protect rate limit table */ 534 struct mutex rl_lock; 535 u16 max_size; 536 u32 max_rate; 537 u32 min_rate; 538 struct mlx5_rl_entry *rl_entry; 539 u64 refcount; 540 }; 541 542 struct mlx5_core_roce { 543 struct mlx5_flow_table *ft; 544 struct mlx5_flow_group *fg; 545 struct mlx5_flow_handle *allow_rule; 546 }; 547 548 enum { 549 MLX5_PRIV_FLAGS_DISABLE_IB_ADEV = 1 << 0, 550 MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV = 1 << 1, 551 /* Set during device detach to block any further devices 552 * creation/deletion on drivers rescan. Unset during device attach. 553 */ 554 MLX5_PRIV_FLAGS_DETACH = 1 << 2, 555 }; 556 557 struct mlx5_adev { 558 struct auxiliary_device adev; 559 struct mlx5_core_dev *mdev; 560 int idx; 561 }; 562 563 struct mlx5_debugfs_entries { 564 struct dentry *dbg_root; 565 struct dentry *qp_debugfs; 566 struct dentry *eq_debugfs; 567 struct dentry *cq_debugfs; 568 struct dentry *cmdif_debugfs; 569 struct dentry *pages_debugfs; 570 struct dentry *lag_debugfs; 571 }; 572 573 enum mlx5_func_type { 574 MLX5_PF, 575 MLX5_VF, 576 MLX5_SF, 577 MLX5_HOST_PF, 578 MLX5_EC_VF, 579 MLX5_FUNC_TYPE_NUM, 580 }; 581 582 struct mlx5_ft_pool; 583 struct mlx5_priv { 584 /* IRQ table valid only for real pci devices PF or VF */ 585 struct mlx5_irq_table *irq_table; 586 struct mlx5_eq_table *eq_table; 587 588 /* pages stuff */ 589 struct mlx5_nb pg_nb; 590 struct workqueue_struct *pg_wq; 591 struct xarray page_root_xa; 592 atomic_t reg_pages; 593 struct list_head free_list; 594 u32 fw_pages; 595 u32 page_counters[MLX5_FUNC_TYPE_NUM]; 596 u32 fw_pages_alloc_failed; 597 u32 give_pages_dropped; 598 u32 reclaim_pages_discard; 599 600 struct mlx5_core_health health; 601 struct list_head traps; 602 603 struct mlx5_debugfs_entries dbg; 604 605 /* start: alloc staff */ 606 /* protect buffer allocation according to numa node */ 607 struct mutex alloc_mutex; 608 int numa_node; 609 610 struct mutex pgdir_mutex; 611 struct list_head pgdir_list; 612 /* end: alloc staff */ 613 614 struct mlx5_adev **adev; 615 int adev_idx; 616 int sw_vhca_id; 617 struct mlx5_events *events; 618 struct mlx5_vhca_events *vhca_events; 619 620 struct mlx5_flow_steering *steering; 621 struct mlx5_mpfs *mpfs; 622 struct mlx5_eswitch *eswitch; 623 struct mlx5_core_sriov sriov; 624 struct mlx5_lag *lag; 625 u32 flags; 626 struct mlx5_devcom_dev *devc; 627 struct mlx5_devcom_comp_dev *hca_devcom_comp; 628 struct mlx5_fw_reset *fw_reset; 629 struct mlx5_core_roce roce; 630 struct mlx5_fc_stats fc_stats; 631 struct mlx5_rl_table rl_table; 632 struct mlx5_ft_pool *ft_pool; 633 634 struct mlx5_bfreg_data bfregs; 635 struct mlx5_uars_page *uar; 636 #ifdef CONFIG_MLX5_SF 637 struct mlx5_vhca_state_notifier *vhca_state_notifier; 638 struct mlx5_sf_dev_table *sf_dev_table; 639 struct mlx5_core_dev *parent_mdev; 640 #endif 641 #ifdef CONFIG_MLX5_SF_MANAGER 642 struct mlx5_sf_hw_table *sf_hw_table; 643 struct mlx5_sf_table *sf_table; 644 #endif 645 }; 646 647 enum mlx5_device_state { 648 MLX5_DEVICE_STATE_UP = 1, 649 MLX5_DEVICE_STATE_INTERNAL_ERROR, 650 }; 651 652 enum mlx5_interface_state { 653 MLX5_INTERFACE_STATE_UP = BIT(0), 654 MLX5_BREAK_FW_WAIT = BIT(1), 655 }; 656 657 enum mlx5_pci_status { 658 MLX5_PCI_STATUS_DISABLED, 659 MLX5_PCI_STATUS_ENABLED, 660 }; 661 662 enum mlx5_pagefault_type_flags { 663 MLX5_PFAULT_REQUESTOR = 1 << 0, 664 MLX5_PFAULT_WRITE = 1 << 1, 665 MLX5_PFAULT_RDMA = 1 << 2, 666 }; 667 668 struct mlx5_td { 669 /* protects tirs list changes while tirs refresh */ 670 struct mutex list_lock; 671 struct list_head tirs_list; 672 u32 tdn; 673 }; 674 675 struct mlx5e_resources { 676 struct mlx5e_hw_objs { 677 u32 pdn; 678 struct mlx5_td td; 679 u32 mkey; 680 struct mlx5_sq_bfreg bfreg; 681 } hw_objs; 682 struct net_device *uplink_netdev; 683 struct mutex uplink_netdev_lock; 684 struct mlx5_crypto_dek_priv *dek_priv; 685 }; 686 687 enum mlx5_sw_icm_type { 688 MLX5_SW_ICM_TYPE_STEERING, 689 MLX5_SW_ICM_TYPE_HEADER_MODIFY, 690 MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN, 691 }; 692 693 #define MLX5_MAX_RESERVED_GIDS 8 694 695 struct mlx5_rsvd_gids { 696 unsigned int start; 697 unsigned int count; 698 struct ida ida; 699 }; 700 701 #define MAX_PIN_NUM 8 702 struct mlx5_pps { 703 u8 pin_caps[MAX_PIN_NUM]; 704 struct work_struct out_work; 705 u64 start[MAX_PIN_NUM]; 706 u8 enabled; 707 u64 min_npps_period; 708 u64 min_out_pulse_duration_ns; 709 }; 710 711 struct mlx5_timer { 712 struct cyclecounter cycles; 713 struct timecounter tc; 714 u32 nominal_c_mult; 715 unsigned long overflow_period; 716 struct delayed_work overflow_work; 717 }; 718 719 struct mlx5_clock { 720 struct mlx5_nb pps_nb; 721 seqlock_t lock; 722 struct hwtstamp_config hwtstamp_config; 723 struct ptp_clock *ptp; 724 struct ptp_clock_info ptp_info; 725 struct mlx5_pps pps_info; 726 struct mlx5_timer timer; 727 }; 728 729 struct mlx5_dm; 730 struct mlx5_fw_tracer; 731 struct mlx5_vxlan; 732 struct mlx5_geneve; 733 struct mlx5_hv_vhca; 734 735 #define MLX5_LOG_SW_ICM_BLOCK_SIZE(dev) (MLX5_CAP_DEV_MEM(dev, log_sw_icm_alloc_granularity)) 736 #define MLX5_SW_ICM_BLOCK_SIZE(dev) (1 << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)) 737 738 enum { 739 MLX5_PROF_MASK_QP_SIZE = (u64)1 << 0, 740 MLX5_PROF_MASK_MR_CACHE = (u64)1 << 1, 741 }; 742 743 enum { 744 MKEY_CACHE_LAST_STD_ENTRY = 20, 745 MLX5_IMR_KSM_CACHE_ENTRY, 746 MAX_MKEY_CACHE_ENTRIES 747 }; 748 749 struct mlx5_profile { 750 u64 mask; 751 u8 log_max_qp; 752 u8 num_cmd_caches; 753 struct { 754 int size; 755 int limit; 756 } mr_cache[MAX_MKEY_CACHE_ENTRIES]; 757 }; 758 759 struct mlx5_hca_cap { 760 u32 cur[MLX5_UN_SZ_DW(hca_cap_union)]; 761 u32 max[MLX5_UN_SZ_DW(hca_cap_union)]; 762 }; 763 764 struct mlx5_core_dev { 765 struct device *device; 766 enum mlx5_coredev_type coredev_type; 767 struct pci_dev *pdev; 768 /* sync pci state */ 769 struct mutex pci_status_mutex; 770 enum mlx5_pci_status pci_status; 771 u8 rev_id; 772 char board_id[MLX5_BOARD_ID_LEN]; 773 struct mlx5_cmd cmd; 774 struct { 775 struct mlx5_hca_cap *hca[MLX5_CAP_NUM]; 776 u32 pcam[MLX5_ST_SZ_DW(pcam_reg)]; 777 u32 mcam[MLX5_MCAM_REGS_NUM][MLX5_ST_SZ_DW(mcam_reg)]; 778 u32 fpga[MLX5_ST_SZ_DW(fpga_cap)]; 779 u32 qcam[MLX5_ST_SZ_DW(qcam_reg)]; 780 u8 embedded_cpu; 781 } caps; 782 struct mlx5_timeouts *timeouts; 783 u64 sys_image_guid; 784 phys_addr_t iseg_base; 785 struct mlx5_init_seg __iomem *iseg; 786 phys_addr_t bar_addr; 787 enum mlx5_device_state state; 788 /* sync interface state */ 789 struct mutex intf_state_mutex; 790 struct lock_class_key lock_key; 791 unsigned long intf_state; 792 struct mlx5_priv priv; 793 struct mlx5_profile profile; 794 u32 issi; 795 struct mlx5e_resources mlx5e_res; 796 struct mlx5_dm *dm; 797 struct mlx5_vxlan *vxlan; 798 struct mlx5_geneve *geneve; 799 struct { 800 struct mlx5_rsvd_gids reserved_gids; 801 u32 roce_en; 802 } roce; 803 #ifdef CONFIG_MLX5_FPGA 804 struct mlx5_fpga_device *fpga; 805 #endif 806 struct mlx5_clock clock; 807 struct mlx5_ib_clock_info *clock_info; 808 struct mlx5_fw_tracer *tracer; 809 struct mlx5_rsc_dump *rsc_dump; 810 u32 vsc_addr; 811 struct mlx5_hv_vhca *hv_vhca; 812 struct mlx5_hwmon *hwmon; 813 u64 num_block_tc; 814 u64 num_block_ipsec; 815 #ifdef CONFIG_MLX5_MACSEC 816 struct mlx5_macsec_fs *macsec_fs; 817 /* MACsec notifier chain to sync MACsec core and IB database */ 818 struct blocking_notifier_head macsec_nh; 819 #endif 820 u64 num_ipsec_offloads; 821 }; 822 823 struct mlx5_db { 824 __be32 *db; 825 union { 826 struct mlx5_db_pgdir *pgdir; 827 struct mlx5_ib_user_db_page *user_page; 828 } u; 829 dma_addr_t dma; 830 int index; 831 }; 832 833 enum { 834 MLX5_COMP_EQ_SIZE = 1024, 835 }; 836 837 enum { 838 MLX5_PTYS_IB = 1 << 0, 839 MLX5_PTYS_EN = 1 << 2, 840 }; 841 842 typedef void (*mlx5_cmd_cbk_t)(int status, void *context); 843 844 enum { 845 MLX5_CMD_ENT_STATE_PENDING_COMP, 846 }; 847 848 struct mlx5_cmd_work_ent { 849 unsigned long state; 850 struct mlx5_cmd_msg *in; 851 struct mlx5_cmd_msg *out; 852 void *uout; 853 int uout_size; 854 mlx5_cmd_cbk_t callback; 855 struct delayed_work cb_timeout_work; 856 void *context; 857 int idx; 858 struct completion handling; 859 struct completion done; 860 struct mlx5_cmd *cmd; 861 struct work_struct work; 862 struct mlx5_cmd_layout *lay; 863 int ret; 864 int page_queue; 865 u8 status; 866 u8 token; 867 u64 ts1; 868 u64 ts2; 869 u16 op; 870 bool polling; 871 /* Track the max comp handlers */ 872 refcount_t refcnt; 873 }; 874 875 enum phy_port_state { 876 MLX5_AAA_111 877 }; 878 879 struct mlx5_hca_vport_context { 880 u32 field_select; 881 bool sm_virt_aware; 882 bool has_smi; 883 bool has_raw; 884 enum port_state_policy policy; 885 enum phy_port_state phys_state; 886 enum ib_port_state vport_state; 887 u8 port_physical_state; 888 u64 sys_image_guid; 889 u64 port_guid; 890 u64 node_guid; 891 u32 cap_mask1; 892 u32 cap_mask1_perm; 893 u16 cap_mask2; 894 u16 cap_mask2_perm; 895 u16 lid; 896 u8 init_type_reply; /* bitmask: see ib spec 14.2.5.6 InitTypeReply */ 897 u8 lmc; 898 u8 subnet_timeout; 899 u16 sm_lid; 900 u8 sm_sl; 901 u16 qkey_violation_counter; 902 u16 pkey_violation_counter; 903 bool grh_required; 904 }; 905 906 #define STRUCT_FIELD(header, field) \ 907 .struct_offset_bytes = offsetof(struct ib_unpacked_ ## header, field), \ 908 .struct_size_bytes = sizeof((struct ib_unpacked_ ## header *)0)->field 909 910 extern struct dentry *mlx5_debugfs_root; 911 912 static inline u16 fw_rev_maj(struct mlx5_core_dev *dev) 913 { 914 return ioread32be(&dev->iseg->fw_rev) & 0xffff; 915 } 916 917 static inline u16 fw_rev_min(struct mlx5_core_dev *dev) 918 { 919 return ioread32be(&dev->iseg->fw_rev) >> 16; 920 } 921 922 static inline u16 fw_rev_sub(struct mlx5_core_dev *dev) 923 { 924 return ioread32be(&dev->iseg->cmdif_rev_fw_sub) & 0xffff; 925 } 926 927 static inline u32 mlx5_base_mkey(const u32 key) 928 { 929 return key & 0xffffff00u; 930 } 931 932 static inline u32 wq_get_byte_sz(u8 log_sz, u8 log_stride) 933 { 934 return ((u32)1 << log_sz) << log_stride; 935 } 936 937 static inline void mlx5_init_fbc_offset(struct mlx5_buf_list *frags, 938 u8 log_stride, u8 log_sz, 939 u16 strides_offset, 940 struct mlx5_frag_buf_ctrl *fbc) 941 { 942 fbc->frags = frags; 943 fbc->log_stride = log_stride; 944 fbc->log_sz = log_sz; 945 fbc->sz_m1 = (1 << fbc->log_sz) - 1; 946 fbc->log_frag_strides = PAGE_SHIFT - fbc->log_stride; 947 fbc->frag_sz_m1 = (1 << fbc->log_frag_strides) - 1; 948 fbc->strides_offset = strides_offset; 949 } 950 951 static inline void mlx5_init_fbc(struct mlx5_buf_list *frags, 952 u8 log_stride, u8 log_sz, 953 struct mlx5_frag_buf_ctrl *fbc) 954 { 955 mlx5_init_fbc_offset(frags, log_stride, log_sz, 0, fbc); 956 } 957 958 static inline void *mlx5_frag_buf_get_wqe(struct mlx5_frag_buf_ctrl *fbc, 959 u32 ix) 960 { 961 unsigned int frag; 962 963 ix += fbc->strides_offset; 964 frag = ix >> fbc->log_frag_strides; 965 966 return fbc->frags[frag].buf + ((fbc->frag_sz_m1 & ix) << fbc->log_stride); 967 } 968 969 static inline u32 970 mlx5_frag_buf_get_idx_last_contig_stride(struct mlx5_frag_buf_ctrl *fbc, u32 ix) 971 { 972 u32 last_frag_stride_idx = (ix + fbc->strides_offset) | fbc->frag_sz_m1; 973 974 return min_t(u32, last_frag_stride_idx - fbc->strides_offset, fbc->sz_m1); 975 } 976 977 enum { 978 CMD_ALLOWED_OPCODE_ALL, 979 }; 980 981 void mlx5_cmd_use_events(struct mlx5_core_dev *dev); 982 void mlx5_cmd_use_polling(struct mlx5_core_dev *dev); 983 void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode); 984 985 struct mlx5_async_ctx { 986 struct mlx5_core_dev *dev; 987 atomic_t num_inflight; 988 struct completion inflight_done; 989 }; 990 991 struct mlx5_async_work; 992 993 typedef void (*mlx5_async_cbk_t)(int status, struct mlx5_async_work *context); 994 995 struct mlx5_async_work { 996 struct mlx5_async_ctx *ctx; 997 mlx5_async_cbk_t user_callback; 998 u16 opcode; /* cmd opcode */ 999 u16 op_mod; /* cmd op_mod */ 1000 void *out; /* pointer to the cmd output buffer */ 1001 }; 1002 1003 void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev, 1004 struct mlx5_async_ctx *ctx); 1005 void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx); 1006 int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size, 1007 void *out, int out_size, mlx5_async_cbk_t callback, 1008 struct mlx5_async_work *work); 1009 void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out); 1010 int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size); 1011 int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out); 1012 int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, 1013 int out_size); 1014 1015 #define mlx5_cmd_exec_inout(dev, ifc_cmd, in, out) \ 1016 ({ \ 1017 mlx5_cmd_exec(dev, in, MLX5_ST_SZ_BYTES(ifc_cmd##_in), out, \ 1018 MLX5_ST_SZ_BYTES(ifc_cmd##_out)); \ 1019 }) 1020 1021 #define mlx5_cmd_exec_in(dev, ifc_cmd, in) \ 1022 ({ \ 1023 u32 _out[MLX5_ST_SZ_DW(ifc_cmd##_out)] = {}; \ 1024 mlx5_cmd_exec_inout(dev, ifc_cmd, in, _out); \ 1025 }) 1026 1027 int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size, 1028 void *out, int out_size); 1029 bool mlx5_cmd_is_down(struct mlx5_core_dev *dev); 1030 1031 void mlx5_core_uplink_netdev_set(struct mlx5_core_dev *mdev, struct net_device *netdev); 1032 void mlx5_core_uplink_netdev_event_replay(struct mlx5_core_dev *mdev); 1033 1034 void mlx5_core_mp_event_replay(struct mlx5_core_dev *dev, u32 event, void *data); 1035 1036 void mlx5_health_cleanup(struct mlx5_core_dev *dev); 1037 int mlx5_health_init(struct mlx5_core_dev *dev); 1038 void mlx5_start_health_poll(struct mlx5_core_dev *dev); 1039 void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health); 1040 void mlx5_start_health_fw_log_up(struct mlx5_core_dev *dev); 1041 void mlx5_drain_health_wq(struct mlx5_core_dev *dev); 1042 void mlx5_trigger_health_work(struct mlx5_core_dev *dev); 1043 int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, 1044 struct mlx5_frag_buf *buf, int node); 1045 void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf); 1046 int mlx5_core_create_mkey(struct mlx5_core_dev *dev, u32 *mkey, u32 *in, 1047 int inlen); 1048 int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, u32 mkey); 1049 int mlx5_core_query_mkey(struct mlx5_core_dev *dev, u32 mkey, u32 *out, 1050 int outlen); 1051 int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn); 1052 int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn); 1053 int mlx5_pagealloc_init(struct mlx5_core_dev *dev); 1054 void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev); 1055 void mlx5_pagealloc_start(struct mlx5_core_dev *dev); 1056 void mlx5_pagealloc_stop(struct mlx5_core_dev *dev); 1057 void mlx5_pages_debugfs_init(struct mlx5_core_dev *dev); 1058 void mlx5_pages_debugfs_cleanup(struct mlx5_core_dev *dev); 1059 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot); 1060 int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev); 1061 void mlx5_register_debugfs(void); 1062 void mlx5_unregister_debugfs(void); 1063 1064 void mlx5_fill_page_frag_array_perm(struct mlx5_frag_buf *buf, __be64 *pas, u8 perm); 1065 void mlx5_fill_page_frag_array(struct mlx5_frag_buf *frag_buf, __be64 *pas); 1066 int mlx5_comp_eqn_get(struct mlx5_core_dev *dev, u16 vecidx, int *eqn); 1067 int mlx5_core_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid, u32 qpn); 1068 int mlx5_core_detach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid, u32 qpn); 1069 1070 struct dentry *mlx5_debugfs_get_dev_root(struct mlx5_core_dev *dev); 1071 void mlx5_qp_debugfs_init(struct mlx5_core_dev *dev); 1072 void mlx5_qp_debugfs_cleanup(struct mlx5_core_dev *dev); 1073 int mlx5_access_reg(struct mlx5_core_dev *dev, void *data_in, int size_in, 1074 void *data_out, int size_out, u16 reg_id, int arg, 1075 int write, bool verbose); 1076 int mlx5_core_access_reg(struct mlx5_core_dev *dev, void *data_in, 1077 int size_in, void *data_out, int size_out, 1078 u16 reg_num, int arg, int write); 1079 1080 int mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db, 1081 int node); 1082 1083 static inline int mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db) 1084 { 1085 return mlx5_db_alloc_node(dev, db, dev->priv.numa_node); 1086 } 1087 1088 void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db); 1089 1090 const char *mlx5_command_str(int command); 1091 void mlx5_cmdif_debugfs_init(struct mlx5_core_dev *dev); 1092 void mlx5_cmdif_debugfs_cleanup(struct mlx5_core_dev *dev); 1093 int mlx5_core_create_psv(struct mlx5_core_dev *dev, u32 pdn, 1094 int npsvs, u32 *sig_index); 1095 int mlx5_core_destroy_psv(struct mlx5_core_dev *dev, int psv_num); 1096 __be32 mlx5_core_get_terminate_scatter_list_mkey(struct mlx5_core_dev *dev); 1097 void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common); 1098 1099 int mlx5_init_rl_table(struct mlx5_core_dev *dev); 1100 void mlx5_cleanup_rl_table(struct mlx5_core_dev *dev); 1101 int mlx5_rl_add_rate(struct mlx5_core_dev *dev, u16 *index, 1102 struct mlx5_rate_limit *rl); 1103 void mlx5_rl_remove_rate(struct mlx5_core_dev *dev, struct mlx5_rate_limit *rl); 1104 bool mlx5_rl_is_in_range(struct mlx5_core_dev *dev, u32 rate); 1105 int mlx5_rl_add_rate_raw(struct mlx5_core_dev *dev, void *rl_in, u16 uid, 1106 bool dedicated_entry, u16 *index); 1107 void mlx5_rl_remove_rate_raw(struct mlx5_core_dev *dev, u16 index); 1108 bool mlx5_rl_are_equal(struct mlx5_rate_limit *rl_0, 1109 struct mlx5_rate_limit *rl_1); 1110 int mlx5_alloc_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg, 1111 bool map_wc, bool fast_path); 1112 void mlx5_free_bfreg(struct mlx5_core_dev *mdev, struct mlx5_sq_bfreg *bfreg); 1113 1114 unsigned int mlx5_comp_vectors_max(struct mlx5_core_dev *dev); 1115 int mlx5_comp_vector_get_cpu(struct mlx5_core_dev *dev, int vector); 1116 unsigned int mlx5_core_reserved_gids_count(struct mlx5_core_dev *dev); 1117 int mlx5_core_roce_gid_set(struct mlx5_core_dev *dev, unsigned int index, 1118 u8 roce_version, u8 roce_l3_type, const u8 *gid, 1119 const u8 *mac, bool vlan, u16 vlan_id, u8 port_num); 1120 1121 static inline u32 mlx5_mkey_to_idx(u32 mkey) 1122 { 1123 return mkey >> 8; 1124 } 1125 1126 static inline u32 mlx5_idx_to_mkey(u32 mkey_idx) 1127 { 1128 return mkey_idx << 8; 1129 } 1130 1131 static inline u8 mlx5_mkey_variant(u32 mkey) 1132 { 1133 return mkey & 0xff; 1134 } 1135 1136 /* Async-atomic event notifier used by mlx5 core to forward FW 1137 * evetns received from event queue to mlx5 consumers. 1138 * Optimise event queue dipatching. 1139 */ 1140 int mlx5_notifier_register(struct mlx5_core_dev *dev, struct notifier_block *nb); 1141 int mlx5_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *nb); 1142 1143 /* Async-atomic event notifier used for forwarding 1144 * evetns from the event queue into the to mlx5 events dispatcher, 1145 * eswitch, clock and others. 1146 */ 1147 int mlx5_eq_notifier_register(struct mlx5_core_dev *dev, struct mlx5_nb *nb); 1148 int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb); 1149 1150 /* Blocking event notifier used to forward SW events, used for slow path */ 1151 int mlx5_blocking_notifier_register(struct mlx5_core_dev *dev, struct notifier_block *nb); 1152 int mlx5_blocking_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *nb); 1153 int mlx5_blocking_notifier_call_chain(struct mlx5_core_dev *dev, unsigned int event, 1154 void *data); 1155 1156 int mlx5_core_query_vendor_id(struct mlx5_core_dev *mdev, u32 *vendor_id); 1157 1158 int mlx5_cmd_create_vport_lag(struct mlx5_core_dev *dev); 1159 int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev); 1160 bool mlx5_lag_is_roce(struct mlx5_core_dev *dev); 1161 bool mlx5_lag_is_sriov(struct mlx5_core_dev *dev); 1162 bool mlx5_lag_is_active(struct mlx5_core_dev *dev); 1163 bool mlx5_lag_mode_is_hash(struct mlx5_core_dev *dev); 1164 bool mlx5_lag_is_master(struct mlx5_core_dev *dev); 1165 bool mlx5_lag_is_shared_fdb(struct mlx5_core_dev *dev); 1166 bool mlx5_lag_is_mpesw(struct mlx5_core_dev *dev); 1167 struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev); 1168 u8 mlx5_lag_get_slave_port(struct mlx5_core_dev *dev, 1169 struct net_device *slave); 1170 int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev, 1171 u64 *values, 1172 int num_counters, 1173 size_t *offsets); 1174 struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int *i); 1175 1176 #define mlx5_lag_for_each_peer_mdev(dev, peer, i) \ 1177 for (i = 0, peer = mlx5_lag_get_next_peer_mdev(dev, &i); \ 1178 peer; \ 1179 peer = mlx5_lag_get_next_peer_mdev(dev, &i)) 1180 1181 u8 mlx5_lag_get_num_ports(struct mlx5_core_dev *dev); 1182 struct mlx5_uars_page *mlx5_get_uars_page(struct mlx5_core_dev *mdev); 1183 void mlx5_put_uars_page(struct mlx5_core_dev *mdev, struct mlx5_uars_page *up); 1184 int mlx5_dm_sw_icm_alloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, 1185 u64 length, u32 log_alignment, u16 uid, 1186 phys_addr_t *addr, u32 *obj_id); 1187 int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, 1188 u64 length, u16 uid, phys_addr_t addr, u32 obj_id); 1189 1190 struct mlx5_core_dev *mlx5_vf_get_core_dev(struct pci_dev *pdev); 1191 void mlx5_vf_put_core_dev(struct mlx5_core_dev *mdev); 1192 1193 int mlx5_sriov_blocking_notifier_register(struct mlx5_core_dev *mdev, 1194 int vf_id, 1195 struct notifier_block *nb); 1196 void mlx5_sriov_blocking_notifier_unregister(struct mlx5_core_dev *mdev, 1197 int vf_id, 1198 struct notifier_block *nb); 1199 int mlx5_rdma_rn_get_params(struct mlx5_core_dev *mdev, 1200 struct ib_device *device, 1201 struct rdma_netdev_alloc_params *params); 1202 1203 enum { 1204 MLX5_PCI_DEV_IS_VF = 1 << 0, 1205 }; 1206 1207 static inline bool mlx5_core_is_pf(const struct mlx5_core_dev *dev) 1208 { 1209 return dev->coredev_type == MLX5_COREDEV_PF; 1210 } 1211 1212 static inline bool mlx5_core_is_vf(const struct mlx5_core_dev *dev) 1213 { 1214 return dev->coredev_type == MLX5_COREDEV_VF; 1215 } 1216 1217 static inline bool mlx5_core_is_ecpf(const struct mlx5_core_dev *dev) 1218 { 1219 return dev->caps.embedded_cpu; 1220 } 1221 1222 static inline bool 1223 mlx5_core_is_ecpf_esw_manager(const struct mlx5_core_dev *dev) 1224 { 1225 return dev->caps.embedded_cpu && MLX5_CAP_GEN(dev, eswitch_manager); 1226 } 1227 1228 static inline bool mlx5_ecpf_vport_exists(const struct mlx5_core_dev *dev) 1229 { 1230 return mlx5_core_is_pf(dev) && MLX5_CAP_ESW(dev, ecpf_vport_exists); 1231 } 1232 1233 static inline u16 mlx5_core_max_vfs(const struct mlx5_core_dev *dev) 1234 { 1235 return dev->priv.sriov.max_vfs; 1236 } 1237 1238 static inline int mlx5_lag_is_lacp_owner(struct mlx5_core_dev *dev) 1239 { 1240 /* LACP owner conditions: 1241 * 1) Function is physical. 1242 * 2) LAG is supported by FW. 1243 * 3) LAG is managed by driver (currently the only option). 1244 */ 1245 return MLX5_CAP_GEN(dev, vport_group_manager) && 1246 (MLX5_CAP_GEN(dev, num_lag_ports) > 1) && 1247 MLX5_CAP_GEN(dev, lag_master); 1248 } 1249 1250 static inline u16 mlx5_core_max_ec_vfs(const struct mlx5_core_dev *dev) 1251 { 1252 return dev->priv.sriov.max_ec_vfs; 1253 } 1254 1255 static inline int mlx5_get_gid_table_len(u16 param) 1256 { 1257 if (param > 4) { 1258 pr_warn("gid table length is zero\n"); 1259 return 0; 1260 } 1261 1262 return 8 * (1 << param); 1263 } 1264 1265 static inline bool mlx5_rl_is_supported(struct mlx5_core_dev *dev) 1266 { 1267 return !!(dev->priv.rl_table.max_size); 1268 } 1269 1270 static inline int mlx5_core_is_mp_slave(struct mlx5_core_dev *dev) 1271 { 1272 return MLX5_CAP_GEN(dev, affiliate_nic_vport_criteria) && 1273 MLX5_CAP_GEN(dev, num_vhca_ports) <= 1; 1274 } 1275 1276 static inline int mlx5_core_is_mp_master(struct mlx5_core_dev *dev) 1277 { 1278 return MLX5_CAP_GEN(dev, num_vhca_ports) > 1; 1279 } 1280 1281 static inline int mlx5_core_mp_enabled(struct mlx5_core_dev *dev) 1282 { 1283 return mlx5_core_is_mp_slave(dev) || 1284 mlx5_core_is_mp_master(dev); 1285 } 1286 1287 static inline int mlx5_core_native_port_num(struct mlx5_core_dev *dev) 1288 { 1289 if (!mlx5_core_mp_enabled(dev)) 1290 return 1; 1291 1292 return MLX5_CAP_GEN(dev, native_port_num); 1293 } 1294 1295 static inline int mlx5_get_dev_index(struct mlx5_core_dev *dev) 1296 { 1297 int idx = MLX5_CAP_GEN(dev, native_port_num); 1298 1299 if (idx >= 1 && idx <= MLX5_MAX_PORTS) 1300 return idx - 1; 1301 else 1302 return PCI_FUNC(dev->pdev->devfn); 1303 } 1304 1305 enum { 1306 MLX5_TRIGGERED_CMD_COMP = (u64)1 << 32, 1307 }; 1308 1309 bool mlx5_is_roce_on(struct mlx5_core_dev *dev); 1310 1311 static inline bool mlx5_get_roce_state(struct mlx5_core_dev *dev) 1312 { 1313 if (MLX5_CAP_GEN(dev, roce_rw_supported)) 1314 return MLX5_CAP_GEN(dev, roce); 1315 1316 /* If RoCE cap is read-only in FW, get RoCE state from devlink 1317 * in order to support RoCE enable/disable feature 1318 */ 1319 return mlx5_is_roce_on(dev); 1320 } 1321 1322 #ifdef CONFIG_MLX5_MACSEC 1323 static inline bool mlx5e_is_macsec_device(const struct mlx5_core_dev *mdev) 1324 { 1325 if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) & 1326 MLX5_GENERAL_OBJ_TYPES_CAP_MACSEC_OFFLOAD)) 1327 return false; 1328 1329 if (!MLX5_CAP_GEN(mdev, log_max_dek)) 1330 return false; 1331 1332 if (!MLX5_CAP_MACSEC(mdev, log_max_macsec_offload)) 1333 return false; 1334 1335 if (!MLX5_CAP_FLOWTABLE_NIC_RX(mdev, macsec_decrypt) || 1336 !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_remove_macsec)) 1337 return false; 1338 1339 if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, macsec_encrypt) || 1340 !MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_macsec)) 1341 return false; 1342 1343 if (!MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_128_encrypt) && 1344 !MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_256_encrypt)) 1345 return false; 1346 1347 if (!MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_128_decrypt) && 1348 !MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_256_decrypt)) 1349 return false; 1350 1351 return true; 1352 } 1353 1354 #define NIC_RDMA_BOTH_DIRS_CAPS (MLX5_FT_NIC_RX_2_NIC_RX_RDMA | MLX5_FT_NIC_TX_RDMA_2_NIC_TX) 1355 1356 static inline bool mlx5_is_macsec_roce_supported(struct mlx5_core_dev *mdev) 1357 { 1358 if (((MLX5_CAP_GEN_2(mdev, flow_table_type_2_type) & 1359 NIC_RDMA_BOTH_DIRS_CAPS) != NIC_RDMA_BOTH_DIRS_CAPS) || 1360 !MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, max_modify_header_actions) || 1361 !mlx5e_is_macsec_device(mdev) || !mdev->macsec_fs) 1362 return false; 1363 1364 return true; 1365 } 1366 #endif 1367 1368 enum { 1369 MLX5_OCTWORD = 16, 1370 }; 1371 1372 struct msi_map mlx5_msix_alloc(struct mlx5_core_dev *dev, 1373 irqreturn_t (*handler)(int, void *), 1374 const struct irq_affinity_desc *affdesc, 1375 const char *name); 1376 void mlx5_msix_free(struct mlx5_core_dev *dev, struct msi_map map); 1377 1378 #endif /* MLX5_DRIVER_H */ 1379