1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #ifndef ADF_ACCEL_DEVICES_H_ 4 #define ADF_ACCEL_DEVICES_H_ 5 #include <linux/interrupt.h> 6 #include <linux/module.h> 7 #include <linux/list.h> 8 #include <linux/io.h> 9 #include <linux/pci.h> 10 #include <linux/ratelimit.h> 11 #include <linux/types.h> 12 #include "adf_cfg_common.h" 13 #include "adf_rl.h" 14 #include "adf_telemetry.h" 15 #include "adf_pfvf_msg.h" 16 #include "icp_qat_hw.h" 17 18 #define ADF_DH895XCC_DEVICE_NAME "dh895xcc" 19 #define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" 20 #define ADF_C62X_DEVICE_NAME "c6xx" 21 #define ADF_C62XVF_DEVICE_NAME "c6xxvf" 22 #define ADF_C3XXX_DEVICE_NAME "c3xxx" 23 #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" 24 #define ADF_4XXX_DEVICE_NAME "4xxx" 25 #define ADF_420XX_DEVICE_NAME "420xx" 26 #define ADF_4XXX_PCI_DEVICE_ID 0x4940 27 #define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 28 #define ADF_401XX_PCI_DEVICE_ID 0x4942 29 #define ADF_401XXIOV_PCI_DEVICE_ID 0x4943 30 #define ADF_402XX_PCI_DEVICE_ID 0x4944 31 #define ADF_402XXIOV_PCI_DEVICE_ID 0x4945 32 #define ADF_420XX_PCI_DEVICE_ID 0x4946 33 #define ADF_420XXIOV_PCI_DEVICE_ID 0x4947 34 #define ADF_DEVICE_FUSECTL_OFFSET 0x40 35 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C 36 #define ADF_DEVICE_FUSECTL_MASK 0x80000000 37 #define ADF_PCI_MAX_BARS 3 38 #define ADF_DEVICE_NAME_LENGTH 32 39 #define ADF_ETR_MAX_RINGS_PER_BANK 16 40 #define ADF_MAX_MSIX_VECTOR_NAME 48 41 #define ADF_DEVICE_NAME_PREFIX "qat_" 42 43 enum adf_accel_capabilities { 44 ADF_ACCEL_CAPABILITIES_NULL = 0, 45 ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = 1, 46 ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = 2, 47 ADF_ACCEL_CAPABILITIES_CIPHER = 4, 48 ADF_ACCEL_CAPABILITIES_AUTHENTICATION = 8, 49 ADF_ACCEL_CAPABILITIES_COMPRESSION = 32, 50 ADF_ACCEL_CAPABILITIES_LZS_COMPRESSION = 64, 51 ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 52 }; 53 54 struct adf_bar { 55 resource_size_t base_addr; 56 void __iomem *virt_addr; 57 resource_size_t size; 58 }; 59 60 struct adf_irq { 61 bool enabled; 62 char name[ADF_MAX_MSIX_VECTOR_NAME]; 63 }; 64 65 struct adf_accel_msix { 66 struct adf_irq *irqs; 67 u32 num_entries; 68 }; 69 70 struct adf_accel_pci { 71 struct pci_dev *pci_dev; 72 struct adf_accel_msix msix_entries; 73 struct adf_bar pci_bars[ADF_PCI_MAX_BARS]; 74 u8 revid; 75 u8 sku; 76 }; 77 78 enum dev_state { 79 DEV_DOWN = 0, 80 DEV_UP 81 }; 82 83 enum dev_sku_info { 84 DEV_SKU_1 = 0, 85 DEV_SKU_2, 86 DEV_SKU_3, 87 DEV_SKU_4, 88 DEV_SKU_VF, 89 DEV_SKU_UNKNOWN, 90 }; 91 92 enum ras_errors { 93 ADF_RAS_CORR, 94 ADF_RAS_UNCORR, 95 ADF_RAS_FATAL, 96 ADF_RAS_ERRORS, 97 }; 98 99 struct adf_error_counters { 100 atomic_t counter[ADF_RAS_ERRORS]; 101 bool sysfs_added; 102 bool enabled; 103 }; 104 105 static inline const char *get_sku_info(enum dev_sku_info info) 106 { 107 switch (info) { 108 case DEV_SKU_1: 109 return "SKU1"; 110 case DEV_SKU_2: 111 return "SKU2"; 112 case DEV_SKU_3: 113 return "SKU3"; 114 case DEV_SKU_4: 115 return "SKU4"; 116 case DEV_SKU_VF: 117 return "SKUVF"; 118 case DEV_SKU_UNKNOWN: 119 default: 120 break; 121 } 122 return "Unknown SKU"; 123 } 124 125 struct adf_hw_device_class { 126 const char *name; 127 const enum adf_device_type type; 128 u32 instances; 129 }; 130 131 struct arb_info { 132 u32 arb_cfg; 133 u32 arb_offset; 134 u32 wt2sam_offset; 135 }; 136 137 struct admin_info { 138 u32 admin_msg_ur; 139 u32 admin_msg_lr; 140 u32 mailbox_offset; 141 }; 142 143 struct adf_hw_csr_ops { 144 u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size); 145 u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, 146 u32 ring); 147 void (*write_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, 148 u32 ring, u32 value); 149 u32 (*read_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, 150 u32 ring); 151 void (*write_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, 152 u32 ring, u32 value); 153 u32 (*read_csr_e_stat)(void __iomem *csr_base_addr, u32 bank); 154 void (*write_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, 155 u32 ring, u32 value); 156 void (*write_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, 157 u32 ring, dma_addr_t addr); 158 void (*write_csr_int_flag)(void __iomem *csr_base_addr, u32 bank, 159 u32 value); 160 void (*write_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); 161 void (*write_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank, 162 u32 value); 163 void (*write_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank, 164 u32 value); 165 void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr, 166 u32 bank, u32 value); 167 void (*write_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank, 168 u32 value); 169 }; 170 171 struct adf_cfg_device_data; 172 struct adf_accel_dev; 173 struct adf_etr_data; 174 struct adf_etr_ring_data; 175 176 struct adf_ras_ops { 177 void (*enable_ras_errors)(struct adf_accel_dev *accel_dev); 178 void (*disable_ras_errors)(struct adf_accel_dev *accel_dev); 179 bool (*handle_interrupt)(struct adf_accel_dev *accel_dev, 180 bool *reset_required); 181 }; 182 183 struct adf_pfvf_ops { 184 int (*enable_comms)(struct adf_accel_dev *accel_dev); 185 u32 (*get_pf2vf_offset)(u32 i); 186 u32 (*get_vf2pf_offset)(u32 i); 187 void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask); 188 void (*disable_all_vf2pf_interrupts)(void __iomem *pmisc_addr); 189 u32 (*disable_pending_vf2pf_interrupts)(void __iomem *pmisc_addr); 190 int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg, 191 u32 pfvf_offset, struct mutex *csr_lock); 192 struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev, 193 u32 pfvf_offset, u8 compat_ver); 194 }; 195 196 struct adf_dc_ops { 197 void (*build_deflate_ctx)(void *ctx); 198 }; 199 200 struct adf_dev_err_mask { 201 u32 cppagentcmdpar_mask; 202 u32 parerr_ath_cph_mask; 203 u32 parerr_cpr_xlt_mask; 204 u32 parerr_dcpr_ucs_mask; 205 u32 parerr_pke_mask; 206 u32 parerr_wat_wcp_mask; 207 u32 ssmfeatren_mask; 208 }; 209 210 struct adf_hw_device_data { 211 struct adf_hw_device_class *dev_class; 212 u32 (*get_accel_mask)(struct adf_hw_device_data *self); 213 u32 (*get_ae_mask)(struct adf_hw_device_data *self); 214 u32 (*get_accel_cap)(struct adf_accel_dev *accel_dev); 215 u32 (*get_sram_bar_id)(struct adf_hw_device_data *self); 216 u32 (*get_misc_bar_id)(struct adf_hw_device_data *self); 217 u32 (*get_etr_bar_id)(struct adf_hw_device_data *self); 218 u32 (*get_num_aes)(struct adf_hw_device_data *self); 219 u32 (*get_num_accels)(struct adf_hw_device_data *self); 220 void (*get_arb_info)(struct arb_info *arb_csrs_info); 221 void (*get_admin_info)(struct admin_info *admin_csrs_info); 222 enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); 223 u16 (*get_ring_to_svc_map)(struct adf_accel_dev *accel_dev); 224 int (*alloc_irq)(struct adf_accel_dev *accel_dev); 225 void (*free_irq)(struct adf_accel_dev *accel_dev); 226 void (*enable_error_correction)(struct adf_accel_dev *accel_dev); 227 int (*init_admin_comms)(struct adf_accel_dev *accel_dev); 228 void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); 229 int (*send_admin_init)(struct adf_accel_dev *accel_dev); 230 int (*start_timer)(struct adf_accel_dev *accel_dev); 231 void (*stop_timer)(struct adf_accel_dev *accel_dev); 232 void (*check_hb_ctrs)(struct adf_accel_dev *accel_dev); 233 uint32_t (*get_hb_clock)(struct adf_hw_device_data *self); 234 int (*measure_clock)(struct adf_accel_dev *accel_dev); 235 int (*init_arb)(struct adf_accel_dev *accel_dev); 236 void (*exit_arb)(struct adf_accel_dev *accel_dev); 237 const u32 *(*get_arb_mapping)(struct adf_accel_dev *accel_dev); 238 int (*init_device)(struct adf_accel_dev *accel_dev); 239 int (*enable_pm)(struct adf_accel_dev *accel_dev); 240 bool (*handle_pm_interrupt)(struct adf_accel_dev *accel_dev); 241 void (*disable_iov)(struct adf_accel_dev *accel_dev); 242 void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, 243 bool enable); 244 void (*enable_ints)(struct adf_accel_dev *accel_dev); 245 void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev); 246 int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr); 247 void (*reset_device)(struct adf_accel_dev *accel_dev); 248 void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); 249 const char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num); 250 u32 (*uof_get_num_objs)(struct adf_accel_dev *accel_dev); 251 u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); 252 int (*get_rp_group)(struct adf_accel_dev *accel_dev, u32 ae_mask); 253 u32 (*get_ena_thd_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); 254 int (*dev_config)(struct adf_accel_dev *accel_dev); 255 struct adf_pfvf_ops pfvf_ops; 256 struct adf_hw_csr_ops csr_ops; 257 struct adf_dc_ops dc_ops; 258 struct adf_ras_ops ras_ops; 259 struct adf_dev_err_mask dev_err_mask; 260 struct adf_rl_hw_data rl_data; 261 struct adf_tl_hw_data tl_data; 262 const char *fw_name; 263 const char *fw_mmp_name; 264 u32 fuses; 265 u32 straps; 266 u32 accel_capabilities_mask; 267 u32 extended_dc_capabilities; 268 u16 fw_capabilities; 269 u32 clock_frequency; 270 u32 instance_id; 271 u16 accel_mask; 272 u32 ae_mask; 273 u32 admin_ae_mask; 274 u16 tx_rings_mask; 275 u16 ring_to_svc_map; 276 u32 thd_to_arb_map[ICP_QAT_HW_AE_DELIMITER]; 277 u8 tx_rx_gap; 278 u8 num_banks; 279 u16 num_banks_per_vf; 280 u8 num_rings_per_bank; 281 u8 num_accel; 282 u8 num_logical_accel; 283 u8 num_engines; 284 u32 num_hb_ctrs; 285 u8 num_rps; 286 }; 287 288 /* CSR write macro */ 289 #define ADF_CSR_WR(csr_base, csr_offset, val) \ 290 __raw_writel(val, csr_base + csr_offset) 291 292 /* CSR read macro */ 293 #define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset) 294 295 #define ADF_CFG_NUM_SERVICES 4 296 #define ADF_SRV_TYPE_BIT_LEN 3 297 #define ADF_SRV_TYPE_MASK 0x7 298 #define ADF_AE_ADMIN_THREAD 7 299 #define ADF_NUM_THREADS_PER_AE 8 300 #define ADF_NUM_PKE_STRAND 2 301 #define ADF_AE_STRAND0_THREAD 8 302 #define ADF_AE_STRAND1_THREAD 9 303 304 #define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev) 305 #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) 306 #define GET_HW_DATA(accel_dev) (accel_dev->hw_device) 307 #define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) 308 #define GET_NUM_RINGS_PER_BANK(accel_dev) \ 309 GET_HW_DATA(accel_dev)->num_rings_per_bank 310 #define GET_SRV_TYPE(accel_dev, idx) \ 311 (((GET_HW_DATA(accel_dev)->ring_to_svc_map) >> (ADF_SRV_TYPE_BIT_LEN * (idx))) \ 312 & ADF_SRV_TYPE_MASK) 313 #define GET_ERR_MASK(accel_dev) (&GET_HW_DATA(accel_dev)->dev_err_mask) 314 #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) 315 #define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) 316 #define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops) 317 #define GET_DC_OPS(accel_dev) (&(accel_dev)->hw_device->dc_ops) 318 #define GET_TL_DATA(accel_dev) GET_HW_DATA(accel_dev)->tl_data 319 #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev 320 321 struct adf_admin_comms; 322 struct icp_qat_fw_loader_handle; 323 struct adf_fw_loader_data { 324 struct icp_qat_fw_loader_handle *fw_loader; 325 const struct firmware *uof_fw; 326 const struct firmware *mmp_fw; 327 }; 328 329 struct adf_accel_vf_info { 330 struct adf_accel_dev *accel_dev; 331 struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */ 332 struct ratelimit_state vf2pf_ratelimit; 333 u32 vf_nr; 334 bool init; 335 u8 vf_compat_ver; 336 }; 337 338 struct adf_dc_data { 339 u8 *ovf_buff; 340 size_t ovf_buff_sz; 341 dma_addr_t ovf_buff_p; 342 }; 343 344 struct adf_pm { 345 struct dentry *debugfs_pm_status; 346 bool present; 347 int idle_irq_counters; 348 int throttle_irq_counters; 349 int fw_irq_counters; 350 int host_ack_counter; 351 int host_nack_counter; 352 ssize_t (*print_pm_status)(struct adf_accel_dev *accel_dev, 353 char __user *buf, size_t count, loff_t *pos); 354 }; 355 356 struct adf_sysfs { 357 int ring_num; 358 struct rw_semaphore lock; /* protects access to the fields in this struct */ 359 }; 360 361 struct adf_accel_dev { 362 struct adf_etr_data *transport; 363 struct adf_hw_device_data *hw_device; 364 struct adf_cfg_device_data *cfg; 365 struct adf_fw_loader_data *fw_loader; 366 struct adf_admin_comms *admin; 367 struct adf_telemetry *telemetry; 368 struct adf_dc_data *dc_data; 369 struct adf_pm power_management; 370 struct list_head crypto_list; 371 struct list_head compression_list; 372 unsigned long status; 373 atomic_t ref_count; 374 struct dentry *debugfs_dir; 375 struct dentry *fw_cntr_dbgfile; 376 struct dentry *cnv_dbgfile; 377 struct list_head list; 378 struct module *owner; 379 struct adf_accel_pci accel_pci_dev; 380 struct adf_timer *timer; 381 struct adf_heartbeat *heartbeat; 382 struct adf_rl *rate_limiting; 383 struct adf_sysfs sysfs; 384 union { 385 struct { 386 /* protects VF2PF interrupts access */ 387 spinlock_t vf2pf_ints_lock; 388 /* vf_info is non-zero when SR-IOV is init'ed */ 389 struct adf_accel_vf_info *vf_info; 390 } pf; 391 struct { 392 bool irq_enabled; 393 char irq_name[ADF_MAX_MSIX_VECTOR_NAME]; 394 struct tasklet_struct pf2vf_bh_tasklet; 395 struct mutex vf2pf_lock; /* protect CSR access */ 396 struct completion msg_received; 397 struct pfvf_message response; /* temp field holding pf2vf response */ 398 u8 pf_compat_ver; 399 } vf; 400 }; 401 struct adf_error_counters ras_errors; 402 struct mutex state_lock; /* protect state of the device */ 403 bool is_vf; 404 u32 accel_id; 405 }; 406 #endif 407