1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 #include <linux/atomic.h> 4 #include <linux/compiler.h> 5 #include <adf_accel_devices.h> 6 #include <adf_common_drv.h> 7 #include <adf_pfvf_msg.h> 8 #include <adf_dev_err.h> 9 #include <adf_cfg.h> 10 #include <adf_fw_counters.h> 11 #include <adf_gen2_hw_data.h> 12 #include <adf_gen2_pfvf.h> 13 #include "adf_c4xxx_hw_data.h" 14 #include "adf_c4xxx_reset.h" 15 #include "adf_c4xxx_inline.h" 16 #include "adf_c4xxx_ras.h" 17 #include "adf_c4xxx_misc_error_stats.h" 18 #include "adf_c4xxx_pke_replay_stats.h" 19 #include "adf_heartbeat.h" 20 #include "icp_qat_fw_init_admin.h" 21 #include "icp_qat_hw.h" 22 23 /* accel unit information */ 24 static struct adf_accel_unit adf_c4xxx_au_32_ae[] = 25 { { 0x1, 0x3, 0x3F, 0x1B, 6, ADF_ACCEL_SERVICE_NULL }, 26 { 0x2, 0xC, 0xFC0, 0x6C0, 6, ADF_ACCEL_SERVICE_NULL }, 27 { 0x4, 0x30, 0xF000, 0xF000, 4, ADF_ACCEL_SERVICE_NULL }, 28 { 0x8, 0xC0, 0x3F0000, 0x1B0000, 6, ADF_ACCEL_SERVICE_NULL }, 29 { 0x10, 0x300, 0xFC00000, 0x6C00000, 6, ADF_ACCEL_SERVICE_NULL }, 30 { 0x20, 0xC00, 0xF0000000, 0xF0000000, 4, ADF_ACCEL_SERVICE_NULL } }; 31 32 static struct adf_accel_unit adf_c4xxx_au_24_ae[] = { 33 { 0x1, 0x3, 0x3F, 0x1B, 6, ADF_ACCEL_SERVICE_NULL }, 34 { 0x2, 0xC, 0xFC0, 0x6C0, 6, ADF_ACCEL_SERVICE_NULL }, 35 { 0x8, 0xC0, 0x3F0000, 0x1B0000, 6, ADF_ACCEL_SERVICE_NULL }, 36 { 0x10, 0x300, 0xFC00000, 0x6C00000, 6, ADF_ACCEL_SERVICE_NULL }, 37 }; 38 39 static struct adf_accel_unit adf_c4xxx_au_12_ae[] = { 40 { 0x1, 0x3, 0x3F, 0x1B, 6, ADF_ACCEL_SERVICE_NULL }, 41 { 0x8, 0xC0, 0x3F0000, 0x1B0000, 6, ADF_ACCEL_SERVICE_NULL }, 42 }; 43 44 static struct adf_accel_unit adf_c4xxx_au_emulation[] = 45 { { 0x1, 0x3, 0x3F, 0x1B, 6, ADF_ACCEL_SERVICE_NULL }, 46 { 0x2, 0xC, 0xC0, 0xC0, 2, ADF_ACCEL_SERVICE_NULL } }; 47 48 /* Accel engine threads for each of the following services 49 * <num_asym_thd> , <num_sym_thd> , <num_dc_thd>, 50 */ 51 52 /* Thread mapping for SKU capable of symmetric cryptography */ 53 static const struct adf_ae_info adf_c4xxx_32_ae_sym[] = 54 { { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, 55 { 1, 7, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, 56 { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, { 2, 6, 3 }, 57 { 2, 6, 3 }, { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, 58 { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, 59 { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, 60 { 2, 6, 3 }, { 2, 6, 3 } }; 61 62 static const struct adf_ae_info adf_c4xxx_24_ae_sym[] = 63 { { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, 64 { 1, 7, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, 65 { 2, 6, 3 }, { 1, 7, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 66 { 0, 0, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, 67 { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, 68 { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 69 { 0, 0, 0 }, { 0, 0, 0 } }; 70 71 static const struct adf_ae_info adf_c4xxx_12_ae_sym[] = 72 { { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, 73 { 1, 7, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 74 { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 75 { 0, 0, 0 }, { 2, 6, 3 }, { 2, 6, 3 }, { 1, 7, 0 }, { 2, 6, 3 }, 76 { 2, 6, 3 }, { 1, 7, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 77 { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 78 { 0, 0, 0 }, { 0, 0, 0 } }; 79 80 /* Thread mapping for SKU capable of asymmetric and symmetric cryptography */ 81 static const struct adf_ae_info adf_c4xxx_32_ae[] = 82 { { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, 83 { 1, 6, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, 84 { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, { 2, 5, 3 }, 85 { 2, 5, 3 }, { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, 86 { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, 87 { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, 88 { 2, 5, 3 }, { 2, 5, 3 } }; 89 90 static const struct adf_ae_info adf_c4xxx_24_ae[] = 91 { { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, 92 { 1, 6, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, 93 { 2, 5, 3 }, { 1, 6, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 94 { 0, 0, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, 95 { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, 96 { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 97 { 0, 0, 0 }, { 0, 0, 0 } }; 98 99 static const struct adf_ae_info adf_c4xxx_12_ae[] = 100 { { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, 101 { 1, 6, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 102 { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 103 { 0, 0, 0 }, { 2, 5, 3 }, { 2, 5, 3 }, { 1, 6, 0 }, { 2, 5, 3 }, 104 { 2, 5, 3 }, { 1, 6, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 105 { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, { 0, 0, 0 }, 106 { 0, 0, 0 }, { 0, 0, 0 } }; 107 108 static struct adf_hw_device_class c4xxx_class = {.name = ADF_C4XXX_DEVICE_NAME, 109 .type = DEV_C4XXX, 110 .instances = 0 }; 111 112 struct icp_qat_fw_init_c4xxx_admin_hb_stats { 113 struct icp_qat_fw_init_admin_hb_cnt stats[ADF_NUM_THREADS_PER_AE]; 114 }; 115 116 struct adf_hb_count { 117 u16 ae_thread[ADF_NUM_THREADS_PER_AE]; 118 }; 119 120 static const int sku_cy_au[] = ADF_C4XXX_NUM_CY_AU; 121 static const int sku_dc_au[] = ADF_C4XXX_NUM_DC_AU; 122 static const int sku_inline_au[] = ADF_C4XXX_NUM_INLINE_AU; 123 124 /* 125 * C4xxx devices introduce new fuses and soft straps and 126 * are different from previous gen device implementations. 127 */ 128 129 static u32 130 get_accel_mask(struct adf_accel_dev *accel_dev) 131 { 132 device_t pdev = accel_dev->accel_pci_dev.pci_dev; 133 u32 fusectl0; 134 u32 softstrappull0; 135 136 fusectl0 = pci_read_config(pdev, ADF_C4XXX_FUSECTL0_OFFSET, 4); 137 softstrappull0 = 138 pci_read_config(pdev, ADF_C4XXX_SOFTSTRAPPULL0_OFFSET, 4); 139 140 return (~(fusectl0 | softstrappull0)) & ADF_C4XXX_ACCELERATORS_MASK; 141 } 142 143 static u32 144 get_ae_mask(struct adf_accel_dev *accel_dev) 145 { 146 device_t pdev = accel_dev->accel_pci_dev.pci_dev; 147 u32 fusectl1; 148 u32 softstrappull1; 149 150 fusectl1 = pci_read_config(pdev, ADF_C4XXX_FUSECTL1_OFFSET, 4); 151 softstrappull1 = 152 pci_read_config(pdev, ADF_C4XXX_SOFTSTRAPPULL1_OFFSET, 4); 153 154 /* Assume that AE and AU disable masks are consistent, so no 155 * checks against the AU mask are performed 156 */ 157 return (~(fusectl1 | softstrappull1)) & ADF_C4XXX_ACCELENGINES_MASK; 158 } 159 160 static u32 161 get_num_accels(struct adf_hw_device_data *self) 162 { 163 return self ? hweight32(self->accel_mask) : 0; 164 } 165 166 static u32 167 get_num_aes(struct adf_hw_device_data *self) 168 { 169 return self ? hweight32(self->ae_mask) : 0; 170 } 171 172 static u32 173 get_misc_bar_id(struct adf_hw_device_data *self) 174 { 175 return ADF_C4XXX_PMISC_BAR; 176 } 177 178 static u32 179 get_etr_bar_id(struct adf_hw_device_data *self) 180 { 181 return ADF_C4XXX_ETR_BAR; 182 } 183 184 static u32 185 get_sram_bar_id(struct adf_hw_device_data *self) 186 { 187 return ADF_C4XXX_SRAM_BAR; 188 } 189 190 static inline void 191 c4xxx_unpack_ssm_wdtimer(u64 value, u32 *upper, u32 *lower) 192 { 193 *lower = lower_32_bits(value); 194 *upper = upper_32_bits(value); 195 } 196 197 /** 198 * c4xxx_set_ssm_wdtimer() - Initialize the slice hang watchdog timer. 199 * 200 * @param accel_dev Structure holding accelerator data. 201 * @return 0 on success, error code otherwise. 202 */ 203 static int 204 c4xxx_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) 205 { 206 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 207 struct adf_bar *misc_bar = 208 &GET_BARS(accel_dev)[hw_device->get_misc_bar_id(hw_device)]; 209 struct resource *csr = misc_bar->virt_addr; 210 unsigned long accel_mask = hw_device->accel_mask; 211 u32 accel = 0; 212 u64 timer_val = ADF_C4XXX_SSM_WDT_64BIT_DEFAULT_VALUE; 213 u64 timer_val_pke = ADF_C4XXX_SSM_WDT_PKE_64BIT_DEFAULT_VALUE; 214 u32 ssm_wdt_low = 0, ssm_wdt_high = 0; 215 u32 ssm_wdt_pke_low = 0, ssm_wdt_pke_high = 0; 216 217 /* Convert 64bit Slice Hang watchdog value into 32bit values for 218 * mmio write to 32bit CSRs. 219 */ 220 c4xxx_unpack_ssm_wdtimer(timer_val, &ssm_wdt_high, &ssm_wdt_low); 221 c4xxx_unpack_ssm_wdtimer(timer_val_pke, 222 &ssm_wdt_pke_high, 223 &ssm_wdt_pke_low); 224 225 /* Configures Slice Hang watchdogs */ 226 for_each_set_bit(accel, &accel_mask, ADF_C4XXX_MAX_ACCELERATORS) 227 { 228 ADF_CSR_WR(csr, ADF_C4XXX_SSMWDTL_OFFSET(accel), ssm_wdt_low); 229 ADF_CSR_WR(csr, ADF_C4XXX_SSMWDTH_OFFSET(accel), ssm_wdt_high); 230 ADF_CSR_WR(csr, 231 ADF_C4XXX_SSMWDTPKEL_OFFSET(accel), 232 ssm_wdt_pke_low); 233 ADF_CSR_WR(csr, 234 ADF_C4XXX_SSMWDTPKEH_OFFSET(accel), 235 ssm_wdt_pke_high); 236 } 237 238 return 0; 239 } 240 241 /** 242 * c4xxx_check_slice_hang() - Check slice hang status 243 * 244 * Return: true if a slice hange interrupt is serviced.. 245 */ 246 static bool 247 c4xxx_check_slice_hang(struct adf_accel_dev *accel_dev) 248 { 249 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 250 struct adf_bar *misc_bar = 251 &GET_BARS(accel_dev)[hw_device->get_misc_bar_id(hw_device)]; 252 struct resource *csr = misc_bar->virt_addr; 253 u32 slice_hang_offset; 254 u32 ia_slice_hang_offset; 255 u32 fw_irq_source; 256 u32 ia_irq_source; 257 u32 accel_num = 0; 258 bool handled = false; 259 u32 errsou10 = ADF_CSR_RD(csr, ADF_C4XXX_ERRSOU10); 260 unsigned long accel_mask; 261 262 accel_mask = hw_device->accel_mask; 263 264 for_each_set_bit(accel_num, &accel_mask, ADF_C4XXX_MAX_ACCELERATORS) 265 { 266 if (!(errsou10 & ADF_C4XXX_IRQ_SRC_MASK(accel_num))) 267 continue; 268 269 fw_irq_source = ADF_CSR_RD(csr, ADF_INTSTATSSM(accel_num)); 270 ia_irq_source = 271 ADF_CSR_RD(csr, ADF_C4XXX_IAINTSTATSSM(accel_num)); 272 ia_slice_hang_offset = 273 ADF_C4XXX_IASLICEHANGSTATUS_OFFSET(accel_num); 274 275 /* FW did not clear SliceHang error, IA logs and clears 276 * the error 277 */ 278 if ((fw_irq_source & ADF_INTSTATSSM_SHANGERR) && 279 (ia_irq_source & ADF_INTSTATSSM_SHANGERR)) { 280 slice_hang_offset = 281 ADF_C4XXX_SLICEHANGSTATUS_OFFSET(accel_num); 282 283 /* Bring hung slice out of reset */ 284 adf_csr_fetch_and_and(csr, slice_hang_offset, ~0); 285 286 /* Log SliceHang error and clear an interrupt */ 287 handled = adf_handle_slice_hang(accel_dev, 288 accel_num, 289 csr, 290 ia_slice_hang_offset); 291 atomic_inc(&accel_dev->ras_counters[ADF_RAS_UNCORR]); 292 } 293 /* FW cleared SliceHang, IA only logs an error */ 294 else if (!(fw_irq_source & ADF_INTSTATSSM_SHANGERR) && 295 (ia_irq_source & ADF_INTSTATSSM_SHANGERR)) { 296 /* Log SliceHang error and clear an interrupt */ 297 handled = adf_handle_slice_hang(accel_dev, 298 accel_num, 299 csr, 300 ia_slice_hang_offset); 301 302 atomic_inc(&accel_dev->ras_counters[ADF_RAS_UNCORR]); 303 } 304 305 /* Clear the associated IA interrupt */ 306 adf_csr_fetch_and_and(csr, 307 ADF_C4XXX_IAINTSTATSSM(accel_num), 308 ~BIT(13)); 309 } 310 311 return handled; 312 } 313 314 static bool 315 get_eth_doorbell_msg(struct adf_accel_dev *accel_dev) 316 { 317 struct resource *csr = 318 (&GET_BARS(accel_dev)[ADF_C4XXX_PMISC_BAR])->virt_addr; 319 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 320 u32 errsou11 = ADF_CSR_RD(csr, ADF_C4XXX_ERRSOU11); 321 u32 doorbell_int = ADF_CSR_RD(csr, ADF_C4XXX_ETH_DOORBELL_INT); 322 u32 eth_doorbell_reg[ADF_C4XXX_NUM_ETH_DOORBELL_REGS]; 323 bool handled = false; 324 u32 data_reg; 325 u8 i; 326 327 /* Reset cannot be acknowledged until the reset */ 328 hw_device->reset_ack = false; 329 330 /* Check if doorbell interrupt occurred. */ 331 if (errsou11 & ADF_C4XXX_DOORBELL_INT_SRC) { 332 /* Decode doorbell messages from ethernet device */ 333 for (i = 0; i < ADF_C4XXX_NUM_ETH_DOORBELL_REGS; i++) { 334 eth_doorbell_reg[i] = 0; 335 if (doorbell_int & BIT(i)) { 336 data_reg = ADF_C4XXX_ETH_DOORBELL(i); 337 eth_doorbell_reg[i] = ADF_CSR_RD(csr, data_reg); 338 device_printf( 339 GET_DEV(accel_dev), 340 "Receives Doorbell message(0x%08x)\n", 341 eth_doorbell_reg[i]); 342 } 343 } 344 /* Only need to check PF0 */ 345 if (eth_doorbell_reg[0] == ADF_C4XXX_IOSFSB_RESET_ACK) { 346 device_printf(GET_DEV(accel_dev), 347 "Receives pending reset ACK\n"); 348 hw_device->reset_ack = true; 349 } 350 /* Clear the interrupt source */ 351 ADF_CSR_WR(csr, 352 ADF_C4XXX_ETH_DOORBELL_INT, 353 ADF_C4XXX_ETH_DOORBELL_MASK); 354 handled = true; 355 } 356 357 return handled; 358 } 359 360 static enum dev_sku_info 361 get_sku(struct adf_hw_device_data *self) 362 { 363 int aes = get_num_aes(self); 364 u32 capabilities = self->accel_capabilities_mask; 365 bool sym_only_sku = false; 366 367 /* Check if SKU is capable only of symmetric cryptography 368 * via device capabilities. 369 */ 370 if ((capabilities & ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC) && 371 !(capabilities & ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC) && 372 !(capabilities & ADF_ACCEL_CAPABILITIES_COMPRESSION)) 373 sym_only_sku = true; 374 375 switch (aes) { 376 case ADF_C4XXX_HIGH_SKU_AES: 377 if (sym_only_sku) 378 return DEV_SKU_1_CY; 379 return DEV_SKU_1; 380 case ADF_C4XXX_MED_SKU_AES: 381 if (sym_only_sku) 382 return DEV_SKU_2_CY; 383 return DEV_SKU_2; 384 case ADF_C4XXX_LOW_SKU_AES: 385 if (sym_only_sku) 386 return DEV_SKU_3_CY; 387 return DEV_SKU_3; 388 }; 389 390 return DEV_SKU_UNKNOWN; 391 } 392 393 static bool 394 c4xxx_check_prod_sku(struct adf_accel_dev *accel_dev) 395 { 396 device_t pdev = accel_dev->accel_pci_dev.pci_dev; 397 u32 fusectl0 = 0; 398 399 fusectl0 = pci_read_config(pdev, ADF_C4XXX_FUSECTL0_OFFSET, 4); 400 401 if (fusectl0 & ADF_C4XXX_FUSE_PROD_SKU_MASK) 402 return true; 403 else 404 return false; 405 } 406 407 static bool 408 adf_check_sym_only_sku_c4xxx(struct adf_accel_dev *accel_dev) 409 { 410 device_t pdev = accel_dev->accel_pci_dev.pci_dev; 411 u32 legfuse = 0; 412 413 legfuse = pci_read_config(pdev, ADF_DEVICE_LEGFUSE_OFFSET, 4); 414 415 if (legfuse & ADF_C4XXX_LEGFUSE_BASE_SKU_MASK) 416 return true; 417 else 418 return false; 419 } 420 421 static void 422 adf_enable_slice_hang_detection(struct adf_accel_dev *accel_dev) 423 { 424 struct resource *csr; 425 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 426 u32 accel = 0; 427 unsigned long accel_mask; 428 429 csr = (&GET_BARS(accel_dev)[ADF_C4XXX_PMISC_BAR])->virt_addr; 430 accel_mask = hw_device->accel_mask; 431 432 for_each_set_bit(accel, &accel_mask, ADF_C4XXX_MAX_ACCELERATORS) 433 { 434 /* Unmasks Slice Hang interrupts so they can be seen by IA. */ 435 ADF_CSR_WR(csr, 436 ADF_C4XXX_SHINTMASKSSM_OFFSET(accel), 437 ADF_C4XXX_SHINTMASKSSM_VAL); 438 } 439 } 440 441 static void 442 adf_enable_ras(struct adf_accel_dev *accel_dev) 443 { 444 struct resource *csr; 445 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 446 u32 accel = 0; 447 unsigned long accel_mask; 448 449 csr = (&GET_BARS(accel_dev)[ADF_C4XXX_PMISC_BAR])->virt_addr; 450 accel_mask = hw_device->accel_mask; 451 452 for_each_set_bit(accel, &accel_mask, ADF_C4XXX_MAX_ACCELERATORS) 453 { 454 ADF_CSR_WR(csr, 455 ADF_C4XXX_GET_SSMFEATREN_OFFSET(accel), 456 ADF_C4XXX_SSMFEATREN_VAL); 457 } 458 } 459 460 static u32 461 get_clock_speed(struct adf_hw_device_data *self) 462 { 463 /* c4xxx CPP clock is equal to high-speed clock */ 464 return self->clock_frequency; 465 } 466 467 static void 468 adf_enable_error_interrupts(struct adf_accel_dev *accel_dev) 469 { 470 struct resource *csr, *aram_csr; 471 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 472 u32 accel = 0; 473 unsigned long accel_mask; 474 475 csr = (&GET_BARS(accel_dev)[ADF_C4XXX_PMISC_BAR])->virt_addr; 476 aram_csr = (&GET_BARS(accel_dev)[ADF_C4XXX_SRAM_BAR])->virt_addr; 477 accel_mask = hw_device->accel_mask; 478 479 for_each_set_bit(accel, &accel_mask, ADF_C4XXX_MAX_ACCELERATORS) 480 { 481 /* Enable shared memory, MMP, CPP, PPERR interrupts 482 * for a given accel 483 */ 484 ADF_CSR_WR(csr, ADF_C4XXX_GET_INTMASKSSM_OFFSET(accel), 0); 485 486 /* Enable SPP parity error interrupts for a given accel */ 487 ADF_CSR_WR(csr, ADF_C4XXX_GET_SPPPARERRMSK_OFFSET(accel), 0); 488 489 /* Enable ssm soft parity errors on given accel */ 490 ADF_CSR_WR(csr, 491 ADF_C4XXX_GET_SSMSOFTERRORPARITY_MASK_OFFSET(accel), 492 ADF_C4XXX_SSMSOFTERRORPARITY_MASK_VAL); 493 } 494 495 /* Enable interrupts for VFtoPF0_127. */ 496 ADF_CSR_WR(csr, ADF_C4XXX_ERRMSK4, ADF_C4XXX_VF2PF0_31); 497 ADF_CSR_WR(csr, ADF_C4XXX_ERRMSK5, ADF_C4XXX_VF2PF32_63); 498 ADF_CSR_WR(csr, ADF_C4XXX_ERRMSK6, ADF_C4XXX_VF2PF64_95); 499 ADF_CSR_WR(csr, ADF_C4XXX_ERRMSK7, ADF_C4XXX_VF2PF96_127); 500 501 /* Enable interrupts signaling ECC correctable errors for all AEs */ 502 ADF_CSR_WR(csr, ADF_C4XXX_ERRMSK8, ADF_C4XXX_ERRMSK8_COERR); 503 ADF_CSR_WR(csr, 504 ADF_C4XXX_HI_ME_COR_ERRLOG_ENABLE, 505 ADF_C4XXX_HI_ME_COR_ERRLOG_ENABLE_MASK); 506 507 /* Enable error interrupts reported by ERRSOU9 */ 508 ADF_CSR_WR(csr, ADF_C4XXX_ERRMSK9, ADF_C4XXX_ERRMSK9_IRQ_MASK); 509 510 /* Enable uncorrectable errors on all the AE */ 511 ADF_CSR_WR(csr, 512 ADF_C4XXX_HI_ME_UNCERR_LOG_ENABLE, 513 ADF_C4XXX_HI_ME_UNCERR_LOG_ENABLE_MASK); 514 515 /* Enable CPP Agent to report command parity errors */ 516 ADF_CSR_WR(csr, 517 ADF_C4XXX_HI_CPP_AGENT_CMD_PAR_ERR_LOG_ENABLE, 518 ADF_C4XXX_HI_CPP_AGENT_CMD_PAR_ERR_LOG_ENABLE_MASK); 519 520 /* Enable reporting of RI memory parity errors */ 521 ADF_CSR_WR(csr, 522 ADF_C4XXX_RI_MEM_PAR_ERR_EN0, 523 ADF_C4XXX_RI_MEM_PAR_ERR_EN0_MASK); 524 525 /* Enable reporting of TI memory parity errors */ 526 ADF_CSR_WR(csr, 527 ADF_C4XXX_TI_MEM_PAR_ERR_EN0, 528 ADF_C4XXX_TI_MEM_PAR_ERR_EN0_MASK); 529 ADF_CSR_WR(csr, 530 ADF_C4XXX_TI_MEM_PAR_ERR_EN1, 531 ADF_C4XXX_TI_MEM_PAR_ERR_EN1_MASK); 532 533 /* Enable SSM errors */ 534 ADF_CSR_WR(csr, ADF_C4XXX_ERRMSK10, ADF_C4XXX_ERRMSK10_SSM_ERR); 535 536 /* Enable miscellaneous errors (ethernet doorbell aram, ici, ice) */ 537 ADF_CSR_WR(csr, ADF_C4XXX_ERRMSK11, ADF_C4XXX_ERRMSK11_ERR); 538 539 /* RI CPP bus interface error detection and reporting. */ 540 ADF_CSR_WR(csr, ADF_C4XXX_RICPPINTCTL, ADF_C4XXX_RICPP_EN); 541 542 /* TI CPP bus interface error detection and reporting. */ 543 ADF_CSR_WR(csr, ADF_C4XXX_TICPPINTCTL, ADF_C4XXX_TICPP_EN); 544 545 /* Enable CFC Error interrupts and logging. */ 546 ADF_CSR_WR(csr, ADF_C4XXX_CPP_CFC_ERR_CTRL, ADF_C4XXX_CPP_CFC_UE); 547 548 /* Enable ARAM correctable error detection. */ 549 ADF_CSR_WR(aram_csr, ADF_C4XXX_ARAMCERR, ADF_C4XXX_ARAM_CERR); 550 551 /* Enable ARAM uncorrectable error detection. */ 552 ADF_CSR_WR(aram_csr, ADF_C4XXX_ARAMUERR, ADF_C4XXX_ARAM_UERR); 553 554 /* Enable Push/Pull Misc Uncorrectable error interrupts and logging */ 555 ADF_CSR_WR(aram_csr, ADF_C4XXX_CPPMEMTGTERR, ADF_C4XXX_TGT_UERR); 556 } 557 558 static void 559 adf_enable_mmp_error_correction(struct resource *csr, 560 struct adf_hw_device_data *hw_data) 561 { 562 unsigned int accel = 0, mmp; 563 unsigned long uerrssmmmp_mask, cerrssmmmp_mask; 564 enum operation op; 565 unsigned long accel_mask; 566 567 /* Prepare values and operation that will be performed on 568 * UERRSSMMMP and CERRSSMMMP registers on each MMP 569 */ 570 if (hw_data->accel_capabilities_mask & 571 ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC) { 572 uerrssmmmp_mask = ADF_C4XXX_UERRSSMMMP_EN; 573 cerrssmmmp_mask = ADF_C4XXX_CERRSSMMMP_EN; 574 op = OR; 575 } else { 576 uerrssmmmp_mask = ~ADF_C4XXX_UERRSSMMMP_EN; 577 cerrssmmmp_mask = ~ADF_C4XXX_CERRSSMMMP_EN; 578 op = AND; 579 } 580 581 accel_mask = hw_data->accel_mask; 582 583 /* Enable MMP Logging */ 584 for_each_set_bit(accel, &accel_mask, ADF_C4XXX_MAX_ACCELERATORS) 585 { 586 /* Set power-up */ 587 adf_csr_fetch_and_and(csr, 588 ADF_C4XXX_SLICEPWRDOWN(accel), 589 ~ADF_C4XXX_MMP_PWR_UP_MSK); 590 591 for (mmp = 0; mmp < ADF_C4XXX_MAX_MMP; ++mmp) { 592 adf_csr_fetch_and_update(op, 593 csr, 594 ADF_C4XXX_UERRSSMMMP(accel, 595 mmp), 596 uerrssmmmp_mask); 597 adf_csr_fetch_and_update(op, 598 csr, 599 ADF_C4XXX_CERRSSMMMP(accel, 600 mmp), 601 cerrssmmmp_mask); 602 } 603 604 /* Restore power-down value */ 605 adf_csr_fetch_and_or(csr, 606 ADF_C4XXX_SLICEPWRDOWN(accel), 607 ADF_C4XXX_MMP_PWR_UP_MSK); 608 } 609 } 610 611 static void 612 get_arb_info(struct arb_info *arb_csrs_info) 613 { 614 arb_csrs_info->arbiter_offset = ADF_C4XXX_ARB_OFFSET; 615 arb_csrs_info->wrk_cfg_offset = ADF_C4XXX_ARB_WQCFG_OFFSET; 616 } 617 618 static void 619 get_admin_info(struct admin_info *admin_csrs_info) 620 { 621 admin_csrs_info->mailbox_offset = ADF_C4XXX_MAILBOX_BASE_OFFSET; 622 admin_csrs_info->admin_msg_ur = ADF_C4XXX_ADMINMSGUR_OFFSET; 623 admin_csrs_info->admin_msg_lr = ADF_C4XXX_ADMINMSGLR_OFFSET; 624 } 625 626 static void 627 get_errsou_offset(u32 *errsou3, u32 *errsou5) 628 { 629 *errsou3 = ADF_C4XXX_ERRSOU3; 630 *errsou5 = ADF_C4XXX_ERRSOU5; 631 } 632 633 static void 634 adf_enable_error_correction(struct adf_accel_dev *accel_dev) 635 { 636 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 637 struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C4XXX_PMISC_BAR]; 638 struct resource *csr = misc_bar->virt_addr; 639 unsigned int val, i = 0; 640 unsigned long ae_mask; 641 unsigned long accel_mask; 642 643 ae_mask = hw_device->ae_mask; 644 645 /* Enable Accel Engine error detection & correction */ 646 for_each_set_bit(i, &ae_mask, ADF_C4XXX_MAX_ACCELENGINES) 647 { 648 val = ADF_CSR_RD(csr, ADF_C4XXX_AE_CTX_ENABLES(i)); 649 val |= ADF_C4XXX_ENABLE_AE_ECC_ERR; 650 ADF_CSR_WR(csr, ADF_C4XXX_AE_CTX_ENABLES(i), val); 651 val = ADF_CSR_RD(csr, ADF_C4XXX_AE_MISC_CONTROL(i)); 652 val |= ADF_C4XXX_ENABLE_AE_ECC_PARITY_CORR; 653 ADF_CSR_WR(csr, ADF_C4XXX_AE_MISC_CONTROL(i), val); 654 } 655 656 accel_mask = hw_device->accel_mask; 657 658 /* Enable shared memory error detection & correction */ 659 for_each_set_bit(i, &accel_mask, ADF_C4XXX_MAX_ACCELERATORS) 660 { 661 val = ADF_CSR_RD(csr, ADF_C4XXX_UERRSSMSH(i)); 662 val |= ADF_C4XXX_ERRSSMSH_EN; 663 ADF_CSR_WR(csr, ADF_C4XXX_UERRSSMSH(i), val); 664 val = ADF_CSR_RD(csr, ADF_C4XXX_CERRSSMSH(i)); 665 val |= ADF_C4XXX_ERRSSMSH_EN; 666 ADF_CSR_WR(csr, ADF_C4XXX_CERRSSMSH(i), val); 667 } 668 669 adf_enable_ras(accel_dev); 670 adf_enable_mmp_error_correction(csr, hw_device); 671 adf_enable_slice_hang_detection(accel_dev); 672 adf_enable_error_interrupts(accel_dev); 673 } 674 675 static void 676 adf_enable_ints(struct adf_accel_dev *accel_dev) 677 { 678 struct resource *addr; 679 680 addr = (&GET_BARS(accel_dev)[ADF_C4XXX_PMISC_BAR])->virt_addr; 681 682 /* Enable bundle interrupts */ 683 ADF_CSR_WR(addr, ADF_C4XXX_SMIAPF0_MASK_OFFSET, ADF_C4XXX_SMIA0_MASK); 684 ADF_CSR_WR(addr, ADF_C4XXX_SMIAPF1_MASK_OFFSET, ADF_C4XXX_SMIA1_MASK); 685 ADF_CSR_WR(addr, ADF_C4XXX_SMIAPF2_MASK_OFFSET, ADF_C4XXX_SMIA2_MASK); 686 ADF_CSR_WR(addr, ADF_C4XXX_SMIAPF3_MASK_OFFSET, ADF_C4XXX_SMIA3_MASK); 687 /*Enable misc interrupts*/ 688 ADF_CSR_WR(addr, ADF_C4XXX_SMIAPF4_MASK_OFFSET, ADF_C4XXX_SMIA4_MASK); 689 } 690 691 static u32 692 get_ae_clock(struct adf_hw_device_data *self) 693 { 694 /* Clock update interval is <16> ticks for c4xxx. */ 695 return self->clock_frequency / 16; 696 } 697 698 static int 699 measure_clock(struct adf_accel_dev *accel_dev) 700 { 701 u32 frequency; 702 int ret = 0; 703 704 ret = adf_dev_measure_clock(accel_dev, 705 &frequency, 706 ADF_C4XXX_MIN_AE_FREQ, 707 ADF_C4XXX_MAX_AE_FREQ); 708 if (ret) 709 return ret; 710 711 accel_dev->hw_device->clock_frequency = frequency; 712 return 0; 713 } 714 715 static int 716 get_storage_enabled(struct adf_accel_dev *accel_dev, uint32_t *storage_enabled) 717 { 718 if (accel_dev->au_info->num_dc_au > 0) { 719 *storage_enabled = 1; 720 GET_HW_DATA(accel_dev)->extended_dc_capabilities = 721 ICP_ACCEL_CAPABILITIES_ADVANCED_COMPRESSION; 722 } 723 return 0; 724 } 725 726 static u32 727 c4xxx_get_hw_cap(struct adf_accel_dev *accel_dev) 728 { 729 device_t pdev = accel_dev->accel_pci_dev.pci_dev; 730 u32 legfuses; 731 u32 softstrappull0, softstrappull2; 732 u32 fusectl0, fusectl2; 733 u32 capabilities; 734 735 /* Read accelerator capabilities mask */ 736 legfuses = pci_read_config(pdev, ADF_DEVICE_LEGFUSE_OFFSET, 4); 737 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | 738 ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | 739 ICP_ACCEL_CAPABILITIES_CIPHER | 740 ICP_ACCEL_CAPABILITIES_AUTHENTICATION | 741 ICP_ACCEL_CAPABILITIES_COMPRESSION | ICP_ACCEL_CAPABILITIES_ZUC | 742 ICP_ACCEL_CAPABILITIES_HKDF | ICP_ACCEL_CAPABILITIES_SHA3_EXT | 743 ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 | 744 ICP_ACCEL_CAPABILITIES_CHACHA_POLY | 745 ICP_ACCEL_CAPABILITIES_AESGCM_SPC | 746 ICP_ACCEL_CAPABILITIES_ECEDMONT; 747 748 if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) { 749 capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; 750 capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER; 751 } 752 if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) 753 capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; 754 if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) 755 capabilities &= ~(ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | 756 ICP_ACCEL_CAPABILITIES_ECEDMONT); 757 if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE) { 758 capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; 759 capabilities &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY; 760 } 761 if (legfuses & ICP_ACCEL_MASK_EIA3_SLICE) 762 capabilities &= ~ICP_ACCEL_CAPABILITIES_ZUC; 763 if (legfuses & ICP_ACCEL_MASK_SM3_SLICE) 764 capabilities &= ~ICP_ACCEL_CAPABILITIES_SM3; 765 if (legfuses & ICP_ACCEL_MASK_SM4_SLICE) 766 capabilities &= ~ICP_ACCEL_CAPABILITIES_SM4; 767 768 /* Read fusectl0 & softstrappull0 registers to ensure inline 769 * acceleration is not disabled 770 */ 771 softstrappull0 = 772 pci_read_config(pdev, ADF_C4XXX_SOFTSTRAPPULL0_OFFSET, 4); 773 fusectl0 = pci_read_config(pdev, ADF_C4XXX_FUSECTL0_OFFSET, 4); 774 if ((fusectl0 | softstrappull0) & ADF_C4XXX_FUSE_DISABLE_INLINE_MASK) 775 capabilities &= ~ICP_ACCEL_CAPABILITIES_INLINE; 776 777 /* Read fusectl2 & softstrappull2 registers to check out if 778 * PKE/DC are enabled/disabled 779 */ 780 softstrappull2 = 781 pci_read_config(pdev, ADF_C4XXX_SOFTSTRAPPULL2_OFFSET, 4); 782 fusectl2 = pci_read_config(pdev, ADF_C4XXX_FUSECTL2_OFFSET, 4); 783 /* Disable PKE/DC cap if there are no PKE/DC-enabled AUs. */ 784 if (!(~fusectl2 & ~softstrappull2 & ADF_C4XXX_FUSE_PKE_MASK)) 785 capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; 786 if (!(~fusectl2 & ~softstrappull2 & ADF_C4XXX_FUSE_COMP_MASK)) 787 capabilities &= ~(ICP_ACCEL_CAPABILITIES_COMPRESSION | 788 ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY); 789 790 return capabilities; 791 } 792 793 static int 794 c4xxx_configure_accel_units(struct adf_accel_dev *accel_dev) 795 { 796 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES] = { 0 }; 797 unsigned long val; 798 char val_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 }; 799 int sku; 800 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 801 802 sku = get_sku(hw_data); 803 804 if (adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC)) 805 goto err; 806 807 snprintf(key, sizeof(key), ADF_SERVICES_ENABLED); 808 809 /* Base station SKU supports symmetric cryptography only. */ 810 if (adf_check_sym_only_sku_c4xxx(accel_dev)) 811 snprintf(val_str, sizeof(val_str), ADF_SERVICE_SYM); 812 else 813 snprintf(val_str, sizeof(val_str), ADF_SERVICE_CY); 814 815 val = sku_dc_au[sku]; 816 if (val) { 817 strncat(val_str, 818 ADF_SERVICES_SEPARATOR ADF_SERVICE_DC, 819 ADF_CFG_MAX_VAL_LEN_IN_BYTES - 820 strnlen(val_str, sizeof(val_str)) - 821 ADF_CFG_NULL_TERM_SIZE); 822 } 823 824 if (adf_cfg_add_key_value_param( 825 accel_dev, ADF_GENERAL_SEC, key, (void *)val_str, ADF_STR)) 826 goto err; 827 828 snprintf(key, sizeof(key), ADF_NUM_CY_ACCEL_UNITS); 829 val = sku_cy_au[sku]; 830 if (adf_cfg_add_key_value_param( 831 accel_dev, ADF_GENERAL_SEC, key, (void *)&val, ADF_DEC)) 832 goto err; 833 834 snprintf(key, sizeof(key), ADF_NUM_DC_ACCEL_UNITS); 835 val = sku_dc_au[sku]; 836 if (adf_cfg_add_key_value_param( 837 accel_dev, ADF_GENERAL_SEC, key, (void *)&val, ADF_DEC)) 838 goto err; 839 840 snprintf(key, sizeof(key), ADF_NUM_INLINE_ACCEL_UNITS); 841 val = sku_inline_au[sku]; 842 if (adf_cfg_add_key_value_param( 843 accel_dev, ADF_GENERAL_SEC, key, (void *)&val, ADF_DEC)) 844 goto err; 845 846 return 0; 847 err: 848 device_printf(GET_DEV(accel_dev), "Failed to configure accel units\n"); 849 return EINVAL; 850 } 851 852 static void 853 update_hw_capability(struct adf_accel_dev *accel_dev) 854 { 855 struct adf_accel_unit_info *au_info = accel_dev->au_info; 856 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 857 u32 disabled_caps = 0; 858 859 if (!au_info->asym_ae_msk) 860 disabled_caps = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | 861 ICP_ACCEL_CAPABILITIES_AUTHENTICATION; 862 863 if (!au_info->sym_ae_msk) 864 disabled_caps |= ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | 865 ICP_ACCEL_CAPABILITIES_CIPHER | ICP_ACCEL_CAPABILITIES_ZUC | 866 ICP_ACCEL_CAPABILITIES_SHA3_EXT | 867 ICP_ACCEL_CAPABILITIES_SM3 | ICP_ACCEL_CAPABILITIES_SM4 | 868 ICP_ACCEL_CAPABILITIES_CHACHA_POLY | 869 ICP_ACCEL_CAPABILITIES_AESGCM_SPC; 870 871 if (!au_info->dc_ae_msk) { 872 disabled_caps |= ICP_ACCEL_CAPABILITIES_COMPRESSION | 873 ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY; 874 hw_device->extended_dc_capabilities = 0; 875 } 876 877 if (!au_info->inline_ingress_msk && !au_info->inline_egress_msk) 878 disabled_caps |= ICP_ACCEL_CAPABILITIES_INLINE; 879 880 hw_device->accel_capabilities_mask = 881 c4xxx_get_hw_cap(accel_dev) & ~disabled_caps; 882 } 883 884 static void 885 c4xxx_set_sadb_size(struct adf_accel_dev *accel_dev) 886 { 887 u32 sadb_reg_value = 0; 888 struct resource *aram_csr_base; 889 890 aram_csr_base = (&GET_BARS(accel_dev)[ADF_C4XXX_SRAM_BAR])->virt_addr; 891 if (accel_dev->au_info->num_inline_au) { 892 /* REG_SA_DB_CTRL register initialisation */ 893 sadb_reg_value = ADF_C4XXX_SADB_REG_VALUE(accel_dev); 894 ADF_CSR_WR(aram_csr_base, 895 ADF_C4XXX_REG_SA_DB_CTRL, 896 sadb_reg_value); 897 } else { 898 /* Zero the SADB size when inline is disabled. */ 899 adf_csr_fetch_and_and(aram_csr_base, 900 ADF_C4XXX_REG_SA_DB_CTRL, 901 ADF_C4XXX_SADB_SIZE_BIT); 902 } 903 /* REG_SA_CTRL_LOCK register initialisation. We set the lock 904 * bit in order to prevent the REG_SA_DB_CTRL to be 905 * overwritten 906 */ 907 ADF_CSR_WR(aram_csr_base, 908 ADF_C4XXX_REG_SA_CTRL_LOCK, 909 ADF_C4XXX_DEFAULT_SA_CTRL_LOCKOUT); 910 } 911 912 static void 913 c4xxx_init_error_notification_configuration(struct adf_accel_dev *accel_dev, 914 u32 offset) 915 { 916 struct resource *aram_csr_base; 917 918 aram_csr_base = (&GET_BARS(accel_dev)[ADF_C4XXX_SRAM_BAR])->virt_addr; 919 920 /* configure error notification configuration registers */ 921 /* Set CD Parity error */ 922 ADF_CSR_WR(aram_csr_base, 923 ADF_C4XXX_IC_CD_RF_PARITY_ERR_0 + offset, 924 ADF_C4XXX_CD_RF_PARITY_ERR_0_VAL); 925 ADF_CSR_WR(aram_csr_base, 926 ADF_C4XXX_IC_CD_RF_PARITY_ERR_1 + offset, 927 ADF_C4XXX_CD_RF_PARITY_ERR_1_VAL); 928 ADF_CSR_WR(aram_csr_base, 929 ADF_C4XXX_IC_CD_RF_PARITY_ERR_2 + offset, 930 ADF_C4XXX_CD_RF_PARITY_ERR_2_VAL); 931 ADF_CSR_WR(aram_csr_base, 932 ADF_C4XXX_IC_CD_RF_PARITY_ERR_3 + offset, 933 ADF_C4XXX_CD_RF_PARITY_ERR_3_VAL); 934 /* Set CD RAM ECC Correctable Error */ 935 ADF_CSR_WR(aram_csr_base, 936 ADF_C4XXX_IC_CD_CERR + offset, 937 ADF_C4XXX_CD_CERR_VAL); 938 /* Set CD RAM ECC UnCorrectable Error */ 939 ADF_CSR_WR(aram_csr_base, 940 ADF_C4XXX_IC_CD_UERR + offset, 941 ADF_C4XXX_CD_UERR_VAL); 942 /* Set Inline (excl cmd_dis) Parity Error */ 943 ADF_CSR_WR(aram_csr_base, 944 ADF_C4XXX_IC_INLN_RF_PARITY_ERR_0 + offset, 945 ADF_C4XXX_INLN_RF_PARITY_ERR_0_VAL); 946 ADF_CSR_WR(aram_csr_base, 947 ADF_C4XXX_IC_INLN_RF_PARITY_ERR_1 + offset, 948 ADF_C4XXX_INLN_RF_PARITY_ERR_1_VAL); 949 ADF_CSR_WR(aram_csr_base, 950 ADF_C4XXX_IC_INLN_RF_PARITY_ERR_2 + offset, 951 ADF_C4XXX_INLN_RF_PARITY_ERR_2_VAL); 952 ADF_CSR_WR(aram_csr_base, 953 ADF_C4XXX_IC_INLN_RF_PARITY_ERR_3 + offset, 954 ADF_C4XXX_INLN_RF_PARITY_ERR_3_VAL); 955 ADF_CSR_WR(aram_csr_base, 956 ADF_C4XXX_IC_INLN_RF_PARITY_ERR_4 + offset, 957 ADF_C4XXX_INLN_RF_PARITY_ERR_4_VAL); 958 ADF_CSR_WR(aram_csr_base, 959 ADF_C4XXX_IC_INLN_RF_PARITY_ERR_5 + offset, 960 ADF_C4XXX_INLN_RF_PARITY_ERR_5_VAL); 961 /* Set Parser RAM ECC Correctable Error */ 962 ADF_CSR_WR(aram_csr_base, 963 ADF_C4XXX_IC_PARSER_CERR + offset, 964 ADF_C4XXX_PARSER_CERR_VAL); 965 /* Set Parser RAM ECC UnCorrectable Error */ 966 ADF_CSR_WR(aram_csr_base, 967 ADF_C4XXX_IC_PARSER_UERR + offset, 968 ADF_C4XXX_PARSER_UERR_VAL); 969 /* Set CTPB RAM ECC Correctable Error */ 970 ADF_CSR_WR(aram_csr_base, 971 ADF_C4XXX_IC_CTPB_CERR + offset, 972 ADF_C4XXX_CTPB_CERR_VAL); 973 /* Set CTPB RAM ECC UnCorrectable Error */ 974 ADF_CSR_WR(aram_csr_base, 975 ADF_C4XXX_IC_CTPB_UERR + offset, 976 ADF_C4XXX_CTPB_UERR_VAL); 977 /* Set CPP Interface Status */ 978 ADF_CSR_WR(aram_csr_base, 979 ADF_C4XXX_IC_CPPM_ERR_STAT + offset, 980 ADF_C4XXX_CPPM_ERR_STAT_VAL); 981 /* Set CGST_MGMT_INT */ 982 ADF_CSR_WR(aram_csr_base, 983 ADF_C4XXX_IC_CONGESTION_MGMT_INT + offset, 984 ADF_C4XXX_CONGESTION_MGMT_INI_VAL); 985 /* CPP Interface Status */ 986 ADF_CSR_WR(aram_csr_base, 987 ADF_C4XXX_IC_CPPT_ERR_STAT + offset, 988 ADF_C4XXX_CPPT_ERR_STAT_VAL); 989 /* MAC Interrupt Mask */ 990 ADF_CSR_WR64(aram_csr_base, 991 ADF_C4XXX_IC_MAC_IM + offset, 992 ADF_C4XXX_MAC_IM_VAL); 993 } 994 995 static void 996 c4xxx_enable_parse_extraction(struct adf_accel_dev *accel_dev) 997 { 998 struct resource *aram_csr_base; 999 1000 aram_csr_base = (&GET_BARS(accel_dev)[ADF_C4XXX_SRAM_BAR])->virt_addr; 1001 1002 /* Enable Inline Parse Extraction CRSs */ 1003 1004 /* Set IC_PARSE_CTRL register */ 1005 ADF_CSR_WR(aram_csr_base, 1006 ADF_C4XXX_IC_PARSE_CTRL_OFFSET, 1007 ADF_C4XXX_IC_PARSE_CTRL_OFFSET_DEFAULT_VALUE); 1008 1009 /* Set IC_PARSE_FIXED_DATA(0) */ 1010 ADF_CSR_WR(aram_csr_base, 1011 ADF_C4XXX_IC_PARSE_FIXED_DATA(0), 1012 ADF_C4XXX_DEFAULT_IC_PARSE_FIXED_DATA_0); 1013 1014 /* Set IC_PARSE_FIXED_LENGTH */ 1015 ADF_CSR_WR(aram_csr_base, 1016 ADF_C4XXX_IC_PARSE_FIXED_LENGTH, 1017 ADF_C4XXX_DEFAULT_IC_PARSE_FIXED_LEN); 1018 1019 /* Configure ESP protocol from an IPv4 header */ 1020 ADF_CSR_WR(aram_csr_base, 1021 ADF_C4XXX_IC_PARSE_IPV4_OFFSET_0, 1022 ADF_C4XXX_DEFAULT_IC_PARSE_IPV4_OFFS_0_VALUE); 1023 ADF_CSR_WR(aram_csr_base, 1024 ADF_C4XXX_IC_PARSE_IPV4_LENGTH_0, 1025 ADF_C4XXX_DEFAULT_IC_PARSE_IPV4_LEN_0_VALUE); 1026 /* Configure protocol extraction field from an IPv4 header */ 1027 ADF_CSR_WR(aram_csr_base, 1028 ADF_C4XXX_IC_PARSE_IPV4_OFFSET_1, 1029 ADF_C4XXX_DEFAULT_IC_PARSE_IPV4_OFFS_1_VALUE); 1030 ADF_CSR_WR(aram_csr_base, 1031 ADF_C4XXX_IC_PARSE_IPV4_LENGTH_1, 1032 ADF_C4XXX_DEFAULT_IC_PARSE_IPV4_LEN_1_VALUE); 1033 /* Configure SPI extraction field from an IPv4 header */ 1034 ADF_CSR_WR(aram_csr_base, 1035 ADF_C4XXX_IC_PARSE_IPV4_OFFSET_2, 1036 ADF_C4XXX_DEFAULT_IC_PARSE_IPV4_OFFS_2_VALUE); 1037 ADF_CSR_WR(aram_csr_base, 1038 ADF_C4XXX_IC_PARSE_IPV4_LENGTH_2, 1039 ADF_C4XXX_DEFAULT_IC_PARSE_IPV4_LEN_2_VALUE); 1040 /* Configure destination field IP address from an IPv4 header */ 1041 ADF_CSR_WR(aram_csr_base, 1042 ADF_C4XXX_IC_PARSE_IPV4_OFFSET_3, 1043 ADF_C4XXX_DEFAULT_IC_PARSE_IPV4_OFFS_3_VALUE); 1044 ADF_CSR_WR(aram_csr_base, 1045 ADF_C4XXX_IC_PARSE_IPV4_LENGTH_3, 1046 ADF_C4XXX_DEFAULT_IC_PARSE_IPV4_LEN_3_VALUE); 1047 1048 /* Configure function number extraction field from an IPv6 header */ 1049 ADF_CSR_WR(aram_csr_base, 1050 ADF_C4XXX_IC_PARSE_IPV6_OFFSET_0, 1051 ADF_C4XXX_DEFAULT_IC_PARSE_IPV6_OFFS_0_VALUE); 1052 ADF_CSR_WR(aram_csr_base, 1053 ADF_C4XXX_IC_PARSE_IPV6_LENGTH_0, 1054 ADF_C4XXX_DEFAULT_IC_PARSE_IPV6_LEN_0_VALUE); 1055 /* Configure protocol extraction field from an IPv6 header */ 1056 ADF_CSR_WR(aram_csr_base, 1057 ADF_C4XXX_IC_PARSE_IPV6_OFFSET_1, 1058 ADF_C4XXX_DEFAULT_IC_PARSE_IPV6_OFFS_1_VALUE); 1059 ADF_CSR_WR(aram_csr_base, 1060 ADF_C4XXX_IC_PARSE_IPV6_LENGTH_1, 1061 ADF_C4XXX_DEFAULT_IC_PARSE_IPV6_LEN_1_VALUE); 1062 /* Configure SPI extraction field from an IPv6 header */ 1063 ADF_CSR_WR(aram_csr_base, 1064 ADF_C4XXX_IC_PARSE_IPV6_OFFSET_2, 1065 ADF_C4XXX_DEFAULT_IC_PARSE_IPV6_OFFS_2_VALUE); 1066 ADF_CSR_WR(aram_csr_base, 1067 ADF_C4XXX_IC_PARSE_IPV6_LENGTH_2, 1068 ADF_C4XXX_DEFAULT_IC_PARSE_IPV6_LEN_2_VALUE); 1069 /* Configure destination field IP address from an IPv6 header */ 1070 ADF_CSR_WR(aram_csr_base, 1071 ADF_C4XXX_IC_PARSE_IPV6_OFFSET_3, 1072 ADF_C4XXX_DEFAULT_IC_PARSE_IPV6_OFFS_3_VALUE); 1073 ADF_CSR_WR(aram_csr_base, 1074 ADF_C4XXX_IC_PARSE_IPV6_LENGTH_3, 1075 ADF_C4XXX_DEFAULT_IC_PARSE_IPV6_LEN_3_VALUE); 1076 } 1077 1078 static int 1079 adf_get_inline_ipsec_algo_group(struct adf_accel_dev *accel_dev, 1080 unsigned long *ipsec_algo_group) 1081 { 1082 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 1083 1084 if (adf_cfg_get_param_value( 1085 accel_dev, ADF_INLINE_SEC, ADF_INLINE_IPSEC_ALGO_GROUP, val)) 1086 return EFAULT; 1087 if (kstrtoul(val, 0, ipsec_algo_group)) 1088 return EFAULT; 1089 1090 /* Verify the ipsec_algo_group */ 1091 if (*ipsec_algo_group >= IPSEC_ALGO_GROUP_DELIMITER) { 1092 device_printf( 1093 GET_DEV(accel_dev), 1094 "Unsupported IPSEC algo group %lu in config file!\n", 1095 *ipsec_algo_group); 1096 return EFAULT; 1097 } 1098 1099 return 0; 1100 } 1101 1102 static int 1103 c4xxx_init_inline_hw(struct adf_accel_dev *accel_dev) 1104 { 1105 u32 sa_entry_reg_value = 0; 1106 u32 sa_fn_lim = 0; 1107 u32 supported_algo = 0; 1108 struct resource *aram_csr_base; 1109 u32 offset; 1110 unsigned long ipsec_algo_group = IPSEC_DEFAUL_ALGO_GROUP; 1111 1112 aram_csr_base = (&GET_BARS(accel_dev)[ADF_C4XXX_SRAM_BAR])->virt_addr; 1113 1114 if (adf_get_inline_ipsec_algo_group(accel_dev, &ipsec_algo_group)) 1115 return EFAULT; 1116 1117 sa_entry_reg_value |= 1118 (ADF_C4XXX_DEFAULT_LU_KEY_LEN << ADF_C4XXX_LU_KEY_LEN_BIT_OFFSET); 1119 if (ipsec_algo_group == IPSEC_DEFAUL_ALGO_GROUP) { 1120 sa_entry_reg_value |= ADF_C4XXX_DEFAULT_SA_SIZE; 1121 sa_fn_lim = 1122 ADF_C4XXX_FUNC_LIMIT(accel_dev, ADF_C4XXX_DEFAULT_SA_SIZE); 1123 supported_algo = ADF_C4XXX_DEFAULT_SUPPORTED_ALGORITHMS; 1124 } else if (ipsec_algo_group == IPSEC_ALGO_GROUP1) { 1125 sa_entry_reg_value |= ADF_C4XXX_ALGO_GROUP1_SA_SIZE; 1126 sa_fn_lim = ADF_C4XXX_FUNC_LIMIT(accel_dev, 1127 ADF_C4XXX_ALGO_GROUP1_SA_SIZE); 1128 supported_algo = ADF_C4XXX_SUPPORTED_ALGORITHMS_GROUP1; 1129 } else { 1130 return EFAULT; 1131 } 1132 1133 /* REG_SA_ENTRY_CTRL register initialisation */ 1134 ADF_CSR_WR(aram_csr_base, 1135 ADF_C4XXX_REG_SA_ENTRY_CTRL, 1136 sa_entry_reg_value); 1137 1138 /* REG_SAL_FUNC_LIMITS register initialisation. Only the first register 1139 * needs to be initialised to enable as it is assigned to a physical 1140 * function. Other registers will be initialised by the LAN PF driver. 1141 * The function limits is initialised to its maximal value. 1142 */ 1143 ADF_CSR_WR(aram_csr_base, ADF_C4XXX_REG_SA_FUNC_LIMITS, sa_fn_lim); 1144 1145 /* Initialize REG_SA_SCRATCH[0] register to 1146 * advertise supported crypto algorithms 1147 */ 1148 ADF_CSR_WR(aram_csr_base, ADF_C4XXX_REG_SA_SCRATCH_0, supported_algo); 1149 1150 /* REG_SA_SCRATCH[2] register initialisation 1151 * to advertise supported crypto offload features. 1152 */ 1153 ADF_CSR_WR(aram_csr_base, 1154 ADF_C4XXX_REG_SA_SCRATCH_2, 1155 ADF_C4XXX_DEFAULT_CY_OFFLOAD_FEATURES); 1156 1157 /* Overwrite default MAC_CFG register in ingress offset */ 1158 ADF_CSR_WR64(aram_csr_base, 1159 ADF_C4XXX_MAC_CFG + ADF_C4XXX_INLINE_INGRESS_OFFSET, 1160 ADF_C4XXX_MAC_CFG_VALUE); 1161 1162 /* Overwrite default MAC_CFG register in egress offset */ 1163 ADF_CSR_WR64(aram_csr_base, 1164 ADF_C4XXX_MAC_CFG + ADF_C4XXX_INLINE_EGRESS_OFFSET, 1165 ADF_C4XXX_MAC_CFG_VALUE); 1166 1167 /* Overwrite default MAC_PIA_CFG 1168 * (Packet Interface Adapter Configuration) registers 1169 * in ingress offset 1170 */ 1171 ADF_CSR_WR64(aram_csr_base, 1172 ADF_C4XXX_MAC_PIA_CFG + ADF_C4XXX_INLINE_INGRESS_OFFSET, 1173 ADF_C4XXX_MAC_PIA_CFG_VALUE); 1174 1175 /* Overwrite default MAC_PIA_CFG in egress offset */ 1176 ADF_CSR_WR64(aram_csr_base, 1177 ADF_C4XXX_MAC_PIA_CFG + ADF_C4XXX_INLINE_EGRESS_OFFSET, 1178 ADF_C4XXX_MAC_PIA_CFG_VALUE); 1179 1180 c4xxx_enable_parse_extraction(accel_dev); 1181 1182 ADF_CSR_WR(aram_csr_base, 1183 ADF_C4XXX_INGRESS_CMD_DIS_MISC, 1184 ADF_C4XXX_REG_CMD_DIS_MISC_DEFAULT_VALUE); 1185 1186 ADF_CSR_WR(aram_csr_base, 1187 ADF_C4XXX_EGRESS_CMD_DIS_MISC, 1188 ADF_C4XXX_REG_CMD_DIS_MISC_DEFAULT_VALUE); 1189 1190 /* Set bits<1:0> in ADF_C4XXX_INLINE_CAPABILITY register to 1191 * advertize that both ingress and egress directions are available 1192 */ 1193 ADF_CSR_WR(aram_csr_base, 1194 ADF_C4XXX_REG_SA_INLINE_CAPABILITY, 1195 ADF_C4XXX_INLINE_CAPABILITIES); 1196 1197 /* Set error notification configuration of ingress */ 1198 offset = ADF_C4XXX_INLINE_INGRESS_OFFSET; 1199 c4xxx_init_error_notification_configuration(accel_dev, offset); 1200 /* Set error notification configuration of egress */ 1201 offset = ADF_C4XXX_INLINE_EGRESS_OFFSET; 1202 c4xxx_init_error_notification_configuration(accel_dev, offset); 1203 1204 return 0; 1205 } 1206 1207 static void 1208 adf_enable_inline_notification(struct adf_accel_dev *accel_dev) 1209 { 1210 struct resource *aram_csr_base; 1211 1212 aram_csr_base = (&GET_BARS(accel_dev)[ADF_C4XXX_SRAM_BAR])->virt_addr; 1213 1214 /* Set bit<0> in ADF_C4XXX_REG_SA_INLINE_ENABLE to advertise 1215 * that inline is enabled. 1216 */ 1217 ADF_CSR_WR(aram_csr_base, 1218 ADF_C4XXX_REG_SA_INLINE_ENABLE, 1219 ADF_C4XXX_INLINE_ENABLED); 1220 } 1221 1222 static int 1223 c4xxx_init_aram_config(struct adf_accel_dev *accel_dev) 1224 { 1225 u32 aram_size = ADF_C4XXX_2MB_ARAM_SIZE; 1226 u32 ibuff_mem_needed = 0; 1227 u32 usable_aram_size = 0; 1228 struct adf_hw_aram_info *aram_info; 1229 u32 sa_db_ctl_value; 1230 struct resource *aram_csr_base; 1231 u8 profile = 0; 1232 u32 sadb_size = 0; 1233 u32 sa_size = 0; 1234 unsigned long ipsec_algo_group = IPSEC_DEFAUL_ALGO_GROUP; 1235 u32 i; 1236 1237 if (accel_dev->au_info->num_inline_au > 0) 1238 if (adf_get_inline_ipsec_algo_group(accel_dev, 1239 &ipsec_algo_group)) 1240 return EFAULT; 1241 1242 /* Allocate memory for adf_hw_aram_info */ 1243 aram_info = kzalloc(sizeof(*accel_dev->aram_info), GFP_KERNEL); 1244 if (!aram_info) 1245 return ENOMEM; 1246 1247 /* Initialise Inline direction */ 1248 aram_info->inline_direction_egress_mask = 0; 1249 if (accel_dev->au_info->num_inline_au) { 1250 /* Set inline direction bitmap in the ARAM to 1251 * inform firmware which ME is egress 1252 */ 1253 aram_info->inline_direction_egress_mask = 1254 accel_dev->au_info->inline_egress_msk; 1255 1256 /* User profile is valid, we can now add it 1257 * in the ARAM partition table 1258 */ 1259 aram_info->inline_congest_mngt_profile = profile; 1260 } 1261 /* Initialise DC ME mask, "1" = ME is used for DC operations */ 1262 aram_info->dc_ae_mask = accel_dev->au_info->dc_ae_msk; 1263 1264 /* Initialise CY ME mask, "1" = ME is used for CY operations 1265 * Since asym service can also be enabled on inline AEs, here 1266 * we use the sym ae mask for configuring the cy_ae_msk 1267 */ 1268 aram_info->cy_ae_mask = accel_dev->au_info->sym_ae_msk; 1269 1270 /* Configure number of long words in the ARAM */ 1271 aram_info->num_aram_lw_entries = ADF_C4XXX_NUM_ARAM_ENTRIES; 1272 1273 /* Reset region offset values to 0xffffffff */ 1274 aram_info->mmp_region_offset = ~aram_info->mmp_region_offset; 1275 aram_info->skm_region_offset = ~aram_info->skm_region_offset; 1276 aram_info->inter_buff_aram_region_offset = 1277 ~aram_info->inter_buff_aram_region_offset; 1278 1279 /* Determine ARAM size */ 1280 aram_csr_base = (&GET_BARS(accel_dev)[ADF_C4XXX_SRAM_BAR])->virt_addr; 1281 sa_db_ctl_value = ADF_CSR_RD(aram_csr_base, ADF_C4XXX_REG_SA_DB_CTRL); 1282 1283 aram_size = (sa_db_ctl_value & ADF_C4XXX_SADB_SIZE_BIT) ? 1284 ADF_C4XXX_2MB_ARAM_SIZE : 1285 ADF_C4XXX_4MB_ARAM_SIZE; 1286 device_printf(GET_DEV(accel_dev), 1287 "Total available accelerator memory: %uMB\n", 1288 aram_size / ADF_C4XXX_1MB_SIZE); 1289 1290 /* Compute MMP region offset */ 1291 aram_info->mmp_region_size = ADF_C4XXX_DEFAULT_MMP_REGION_SIZE; 1292 aram_info->mmp_region_offset = aram_size - aram_info->mmp_region_size; 1293 1294 if (accel_dev->au_info->num_cy_au || 1295 accel_dev->au_info->num_inline_au) { 1296 /* Crypto is available therefore we must 1297 * include space in the ARAM for SKM. 1298 */ 1299 aram_info->skm_region_size = ADF_C4XXX_DEFAULT_SKM_REGION_SIZE; 1300 /* Compute SKM region offset */ 1301 aram_info->skm_region_offset = aram_size - 1302 (aram_info->mmp_region_size + aram_info->skm_region_size); 1303 } 1304 1305 /* SADB always start at offset 0. */ 1306 if (accel_dev->au_info->num_inline_au) { 1307 /* Inline is available therefore we must 1308 * use remaining ARAM for the SADB. 1309 */ 1310 sadb_size = aram_size - 1311 (aram_info->mmp_region_size + aram_info->skm_region_size); 1312 1313 /* 1314 * When the inline service is enabled, the policy is that 1315 * compression gives up it's space in ARAM to allow for a 1316 * larger SADB. Compression must use DRAM instead of ARAM. 1317 */ 1318 aram_info->inter_buff_aram_region_size = 0; 1319 1320 /* the SADB size must be an integral multiple of the SA size */ 1321 if (ipsec_algo_group == IPSEC_DEFAUL_ALGO_GROUP) { 1322 sa_size = ADF_C4XXX_DEFAULT_SA_SIZE; 1323 } else { 1324 /* IPSEC_ALGO_GROUP1 1325 * Total 2 algo groups. 1326 */ 1327 sa_size = ADF_C4XXX_ALGO_GROUP1_SA_SIZE; 1328 } 1329 1330 sadb_size = sadb_size - 1331 (sadb_size % ADF_C4XXX_SA_SIZE_IN_BYTES(sa_size)); 1332 aram_info->sadb_region_size = sadb_size; 1333 } 1334 1335 if (accel_dev->au_info->num_dc_au && 1336 !accel_dev->au_info->num_inline_au) { 1337 /* Compression is available therefore we must see if there is 1338 * space in the ARAM for intermediate buffers. 1339 */ 1340 aram_info->inter_buff_aram_region_size = 0; 1341 usable_aram_size = aram_size - 1342 (aram_info->mmp_region_size + aram_info->skm_region_size); 1343 1344 for (i = 1; i <= accel_dev->au_info->num_dc_au; i++) { 1345 if ((i * ADF_C4XXX_AU_COMPR_INTERM_SIZE) > 1346 usable_aram_size) 1347 break; 1348 1349 ibuff_mem_needed = i * ADF_C4XXX_AU_COMPR_INTERM_SIZE; 1350 } 1351 1352 /* Set remaining ARAM to intermediate buffers. Firmware handles 1353 * fallback to DRAM for cases were number of AU assigned 1354 * to compression exceeds available ARAM memory. 1355 */ 1356 aram_info->inter_buff_aram_region_size = ibuff_mem_needed; 1357 1358 /* If ARAM is used for compression set its initial offset. */ 1359 if (aram_info->inter_buff_aram_region_size) 1360 aram_info->inter_buff_aram_region_offset = 0; 1361 } 1362 1363 accel_dev->aram_info = aram_info; 1364 1365 return 0; 1366 } 1367 1368 static void 1369 c4xxx_exit_aram_config(struct adf_accel_dev *accel_dev) 1370 { 1371 kfree(accel_dev->aram_info); 1372 accel_dev->aram_info = NULL; 1373 } 1374 1375 static u32 1376 get_num_accel_units(struct adf_hw_device_data *self) 1377 { 1378 u32 i = 0, num_accel = 0; 1379 unsigned long accel_mask = 0; 1380 1381 if (!self || !self->accel_mask) 1382 return 0; 1383 1384 accel_mask = self->accel_mask; 1385 1386 for_each_set_bit(i, &accel_mask, ADF_C4XXX_MAX_ACCELERATORS) 1387 { 1388 num_accel++; 1389 } 1390 1391 return num_accel / ADF_C4XXX_NUM_ACCEL_PER_AU; 1392 } 1393 1394 static int 1395 get_accel_unit(struct adf_hw_device_data *self, 1396 struct adf_accel_unit **accel_unit) 1397 { 1398 enum dev_sku_info sku; 1399 1400 sku = get_sku(self); 1401 1402 switch (sku) { 1403 case DEV_SKU_1: 1404 case DEV_SKU_1_CY: 1405 *accel_unit = adf_c4xxx_au_32_ae; 1406 break; 1407 case DEV_SKU_2: 1408 case DEV_SKU_2_CY: 1409 *accel_unit = adf_c4xxx_au_24_ae; 1410 break; 1411 case DEV_SKU_3: 1412 case DEV_SKU_3_CY: 1413 *accel_unit = adf_c4xxx_au_12_ae; 1414 break; 1415 default: 1416 *accel_unit = adf_c4xxx_au_emulation; 1417 break; 1418 } 1419 return 0; 1420 } 1421 1422 static int 1423 get_ae_info(struct adf_hw_device_data *self, const struct adf_ae_info **ae_info) 1424 { 1425 enum dev_sku_info sku; 1426 1427 sku = get_sku(self); 1428 1429 switch (sku) { 1430 case DEV_SKU_1: 1431 *ae_info = adf_c4xxx_32_ae; 1432 break; 1433 case DEV_SKU_1_CY: 1434 *ae_info = adf_c4xxx_32_ae_sym; 1435 break; 1436 case DEV_SKU_2: 1437 *ae_info = adf_c4xxx_24_ae; 1438 break; 1439 case DEV_SKU_2_CY: 1440 *ae_info = adf_c4xxx_24_ae_sym; 1441 break; 1442 case DEV_SKU_3: 1443 *ae_info = adf_c4xxx_12_ae; 1444 break; 1445 case DEV_SKU_3_CY: 1446 *ae_info = adf_c4xxx_12_ae_sym; 1447 break; 1448 default: 1449 *ae_info = adf_c4xxx_12_ae; 1450 break; 1451 } 1452 return 0; 1453 } 1454 1455 static int 1456 adf_add_debugfs_info(struct adf_accel_dev *accel_dev) 1457 { 1458 /* Add Accel Unit configuration table to debug FS interface */ 1459 if (c4xxx_init_ae_config(accel_dev)) { 1460 device_printf(GET_DEV(accel_dev), 1461 "Failed to create entry for AE configuration\n"); 1462 return EFAULT; 1463 } 1464 1465 return 0; 1466 } 1467 1468 static void 1469 adf_remove_debugfs_info(struct adf_accel_dev *accel_dev) 1470 { 1471 /* Remove Accel Unit configuration table from debug FS interface */ 1472 c4xxx_exit_ae_config(accel_dev); 1473 } 1474 1475 static int 1476 check_svc_to_hw_capabilities(struct adf_accel_dev *accel_dev, 1477 const char *svc_name, 1478 enum icp_qat_capabilities_mask cap) 1479 { 1480 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 1481 u32 hw_cap = hw_data->accel_capabilities_mask; 1482 1483 hw_cap &= cap; 1484 if (hw_cap != cap) { 1485 device_printf(GET_DEV(accel_dev), 1486 "Service not supported by accelerator: %s\n", 1487 svc_name); 1488 return EPERM; 1489 } 1490 1491 return 0; 1492 } 1493 1494 static int 1495 check_accel_unit_config(struct adf_accel_dev *accel_dev, 1496 u8 num_cy_au, 1497 u8 num_dc_au, 1498 u8 num_inline_au) 1499 { 1500 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 1501 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 1502 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 1503 u32 num_au = hw_data->get_num_accel_units(hw_data); 1504 u32 service_mask = ADF_ACCEL_SERVICE_NULL; 1505 char *token, *cur_str; 1506 int ret = 0; 1507 1508 /* Get the services enabled by user */ 1509 snprintf(key, sizeof(key), ADF_SERVICES_ENABLED); 1510 if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val)) 1511 return EFAULT; 1512 cur_str = val; 1513 token = strsep(&cur_str, ADF_SERVICES_SEPARATOR); 1514 while (token) { 1515 if (!strncmp(token, ADF_SERVICE_CY, strlen(ADF_SERVICE_CY))) { 1516 service_mask |= ADF_ACCEL_CRYPTO; 1517 ret |= check_svc_to_hw_capabilities( 1518 accel_dev, 1519 token, 1520 ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | 1521 ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC); 1522 } 1523 1524 if (!strncmp(token, ADF_CFG_SYM, strlen(ADF_CFG_SYM))) { 1525 service_mask |= ADF_ACCEL_CRYPTO; 1526 ret |= check_svc_to_hw_capabilities( 1527 accel_dev, 1528 token, 1529 ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC); 1530 } 1531 1532 if (!strncmp(token, ADF_CFG_ASYM, strlen(ADF_CFG_ASYM))) { 1533 /* Handle a special case of services 'asym;inline' 1534 * enabled where ASYM is handled by Inline firmware 1535 * at AE level. This configuration allows to enable 1536 * ASYM service without accel units assigned to 1537 * CRYPTO service, e.g. 1538 * num_inline_au = 6 1539 * num_cy_au = 0 1540 */ 1541 if (num_inline_au < num_au) 1542 service_mask |= ADF_ACCEL_CRYPTO; 1543 1544 ret |= check_svc_to_hw_capabilities( 1545 accel_dev, 1546 token, 1547 ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC); 1548 } 1549 1550 if (!strncmp(token, ADF_SERVICE_DC, strlen(ADF_SERVICE_DC))) { 1551 service_mask |= ADF_ACCEL_COMPRESSION; 1552 ret |= check_svc_to_hw_capabilities( 1553 accel_dev, 1554 token, 1555 ICP_ACCEL_CAPABILITIES_COMPRESSION); 1556 } 1557 1558 if (!strncmp(token, 1559 ADF_SERVICE_INLINE, 1560 strlen(ADF_SERVICE_INLINE))) { 1561 service_mask |= ADF_ACCEL_INLINE_CRYPTO; 1562 ret |= check_svc_to_hw_capabilities( 1563 accel_dev, token, ICP_ACCEL_CAPABILITIES_INLINE); 1564 } 1565 1566 token = strsep(&cur_str, ADF_SERVICES_SEPARATOR); 1567 } 1568 1569 /* Ensure the user doesn't enable services that are not supported by 1570 * accelerator. 1571 */ 1572 if (ret) { 1573 device_printf(GET_DEV(accel_dev), 1574 "Invalid accelerator configuration.\n"); 1575 return EFAULT; 1576 } 1577 1578 if (!(service_mask & ADF_ACCEL_COMPRESSION) && num_dc_au > 0) { 1579 device_printf(GET_DEV(accel_dev), 1580 "Invalid accel unit config.\n"); 1581 device_printf( 1582 GET_DEV(accel_dev), 1583 "DC accel units set when dc service not enabled\n"); 1584 return EFAULT; 1585 } 1586 1587 if (!(service_mask & ADF_ACCEL_CRYPTO) && num_cy_au > 0) { 1588 device_printf(GET_DEV(accel_dev), 1589 "Invalid accel unit config.\n"); 1590 device_printf( 1591 GET_DEV(accel_dev), 1592 "CY accel units set when cy service not enabled\n"); 1593 return EFAULT; 1594 } 1595 1596 if (!(service_mask & ADF_ACCEL_INLINE_CRYPTO) && num_inline_au > 0) { 1597 device_printf(GET_DEV(accel_dev), 1598 "Invalid accel unit config.\n" 1599 "Inline feature not supported.\n"); 1600 return EFAULT; 1601 } 1602 1603 hw_data->service_mask = service_mask; 1604 /* Ensure the user doesn't allocate more than max accel units */ 1605 if (num_au != (num_cy_au + num_dc_au + num_inline_au)) { 1606 device_printf(GET_DEV(accel_dev), 1607 "Invalid accel unit config.\n"); 1608 device_printf(GET_DEV(accel_dev), 1609 "Max accel units is %d\n", 1610 num_au); 1611 return EFAULT; 1612 } 1613 1614 /* Ensure user allocates hardware resources for enabled services */ 1615 if (!num_cy_au && (service_mask & ADF_ACCEL_CRYPTO)) { 1616 device_printf(GET_DEV(accel_dev), 1617 "Failed to enable cy service!\n"); 1618 device_printf(GET_DEV(accel_dev), 1619 "%s should not be 0", 1620 ADF_NUM_CY_ACCEL_UNITS); 1621 return EFAULT; 1622 } 1623 if (!num_dc_au && (service_mask & ADF_ACCEL_COMPRESSION)) { 1624 device_printf(GET_DEV(accel_dev), 1625 "Failed to enable dc service!\n"); 1626 device_printf(GET_DEV(accel_dev), 1627 "%s should not be 0", 1628 ADF_NUM_DC_ACCEL_UNITS); 1629 return EFAULT; 1630 } 1631 if (!num_inline_au && (service_mask & ADF_ACCEL_INLINE_CRYPTO)) { 1632 device_printf(GET_DEV(accel_dev), "Failed to enable"); 1633 device_printf(GET_DEV(accel_dev), " inline service!"); 1634 device_printf(GET_DEV(accel_dev), 1635 " %s should not be 0\n", 1636 ADF_NUM_INLINE_ACCEL_UNITS); 1637 return EFAULT; 1638 } 1639 1640 return 0; 1641 } 1642 1643 static int 1644 get_accel_unit_config(struct adf_accel_dev *accel_dev, 1645 u8 *num_cy_au, 1646 u8 *num_dc_au, 1647 u8 *num_inline_au) 1648 { 1649 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 1650 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 1651 1652 /* Get the number of accel units allocated for each service */ 1653 snprintf(key, sizeof(key), ADF_NUM_CY_ACCEL_UNITS); 1654 if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val)) 1655 return EFAULT; 1656 if (compat_strtou8(val, 10, num_cy_au)) 1657 return EFAULT; 1658 snprintf(key, sizeof(key), ADF_NUM_DC_ACCEL_UNITS); 1659 if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val)) 1660 return EFAULT; 1661 if (compat_strtou8(val, 10, num_dc_au)) 1662 return EFAULT; 1663 1664 snprintf(key, sizeof(key), ADF_NUM_INLINE_ACCEL_UNITS); 1665 if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val)) 1666 return EFAULT; 1667 if (compat_strtou8(val, 10, num_inline_au)) 1668 return EFAULT; 1669 1670 return 0; 1671 } 1672 1673 /* Function reads the inline ingress/egress configuration 1674 * and returns the number of AEs reserved for ingress 1675 * and egress for accel units which are allocated for 1676 * inline service 1677 */ 1678 static int 1679 adf_get_inline_config(struct adf_accel_dev *accel_dev, u32 *num_ingress_aes) 1680 { 1681 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 1682 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 1683 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 1684 char *value; 1685 u32 num_au = hw_data->get_num_accel_units(hw_data); 1686 unsigned long ingress, egress = 0; 1687 struct adf_accel_unit *accel_unit = accel_dev->au_info->au; 1688 u32 num_inline_aes = 0, num_ingress_ae = 0; 1689 u32 i = 0; 1690 1691 snprintf(key, sizeof(key), ADF_INLINE_INGRESS); 1692 if (adf_cfg_get_param_value(accel_dev, ADF_INLINE_SEC, key, val)) { 1693 device_printf(GET_DEV(accel_dev), "Failed to find ingress\n"); 1694 return EFAULT; 1695 } 1696 value = val; 1697 value = strsep(&value, ADF_C4XXX_PERCENTAGE); 1698 if (compat_strtoul(value, 10, &ingress)) 1699 return EFAULT; 1700 1701 snprintf(key, sizeof(key), ADF_INLINE_EGRESS); 1702 if (adf_cfg_get_param_value(accel_dev, ADF_INLINE_SEC, key, val)) { 1703 device_printf(GET_DEV(accel_dev), "Failed to find egress\n"); 1704 return EFAULT; 1705 } 1706 value = val; 1707 value = strsep(&value, ADF_C4XXX_PERCENTAGE); 1708 if (compat_strtoul(value, 10, &egress)) 1709 return EFAULT; 1710 1711 if (ingress + egress != ADF_C4XXX_100) { 1712 device_printf(GET_DEV(accel_dev), 1713 "The sum of ingress and egress should be 100\n"); 1714 return EFAULT; 1715 } 1716 1717 for (i = 0; i < num_au; i++) { 1718 if (accel_unit[i].services == ADF_ACCEL_INLINE_CRYPTO) 1719 num_inline_aes += accel_unit[i].num_ae; 1720 } 1721 1722 num_ingress_ae = num_inline_aes * ingress / ADF_C4XXX_100; 1723 if (((num_inline_aes * ingress) % ADF_C4XXX_100) > 1724 ADF_C4XXX_ROUND_LIMIT) 1725 num_ingress_ae++; 1726 1727 *num_ingress_aes = num_ingress_ae; 1728 return 0; 1729 } 1730 1731 static int 1732 adf_set_inline_ae_mask(struct adf_accel_dev *accel_dev) 1733 { 1734 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 1735 u32 num_au = hw_data->get_num_accel_units(hw_data); 1736 struct adf_accel_unit_info *au_info = accel_dev->au_info; 1737 struct adf_accel_unit *accel_unit = accel_dev->au_info->au; 1738 u32 num_ingress_ae = 0; 1739 u32 ingress_msk = 0; 1740 u32 i, j, ae_mask; 1741 1742 if (adf_get_inline_config(accel_dev, &num_ingress_ae)) 1743 return EFAULT; 1744 1745 for (i = 0; i < num_au; i++) { 1746 j = 0; 1747 if (accel_unit[i].services == ADF_ACCEL_INLINE_CRYPTO) { 1748 /* AEs with inline service enabled are also used 1749 * for asymmetric crypto 1750 */ 1751 au_info->asym_ae_msk |= accel_unit[i].ae_mask; 1752 ae_mask = accel_unit[i].ae_mask; 1753 while (num_ingress_ae && ae_mask) { 1754 if (ae_mask & 1) { 1755 ingress_msk |= BIT(j); 1756 num_ingress_ae--; 1757 } 1758 ae_mask = ae_mask >> 1; 1759 j++; 1760 } 1761 au_info->inline_ingress_msk |= ingress_msk; 1762 1763 au_info->inline_egress_msk |= 1764 ~(au_info->inline_ingress_msk) & 1765 accel_unit[i].ae_mask; 1766 } 1767 } 1768 1769 return 0; 1770 } 1771 1772 static int 1773 adf_set_ae_mask(struct adf_accel_dev *accel_dev) 1774 { 1775 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 1776 u32 num_au = hw_data->get_num_accel_units(hw_data); 1777 struct adf_accel_unit_info *au_info = accel_dev->au_info; 1778 struct adf_accel_unit *accel_unit = accel_dev->au_info->au; 1779 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 1780 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 1781 char *token, *cur_str; 1782 bool asym_en = false, sym_en = false; 1783 u32 i; 1784 1785 /* Get the services enabled by user */ 1786 snprintf(key, sizeof(key), ADF_SERVICES_ENABLED); 1787 if (adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, key, val)) 1788 return EFAULT; 1789 cur_str = val; 1790 token = strsep(&cur_str, ADF_SERVICES_SEPARATOR); 1791 while (token) { 1792 if (!strncmp(token, ADF_CFG_ASYM, strlen(ADF_CFG_ASYM))) 1793 asym_en = true; 1794 if (!strncmp(token, ADF_CFG_SYM, strlen(ADF_CFG_SYM))) 1795 sym_en = true; 1796 if (!strncmp(token, ADF_CFG_CY, strlen(ADF_CFG_CY))) { 1797 sym_en = true; 1798 asym_en = true; 1799 } 1800 token = strsep(&cur_str, ADF_SERVICES_SEPARATOR); 1801 } 1802 1803 for (i = 0; i < num_au; i++) { 1804 if (accel_unit[i].services == ADF_ACCEL_CRYPTO) { 1805 /* AEs that support crypto can perform both 1806 * symmetric and asymmetric crypto, however 1807 * we only enable the threads if the relevant 1808 * service is also enabled 1809 */ 1810 if (asym_en) 1811 au_info->asym_ae_msk |= accel_unit[i].ae_mask; 1812 if (sym_en) 1813 au_info->sym_ae_msk |= accel_unit[i].ae_mask; 1814 } else if (accel_unit[i].services == ADF_ACCEL_COMPRESSION) { 1815 au_info->dc_ae_msk |= accel_unit[i].comp_ae_mask; 1816 } 1817 } 1818 return 0; 1819 } 1820 1821 static int 1822 adf_init_accel_unit_services(struct adf_accel_dev *accel_dev) 1823 { 1824 u8 num_cy_au, num_dc_au, num_inline_au; 1825 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 1826 u32 num_au = hw_data->get_num_accel_units(hw_data); 1827 struct adf_accel_unit *accel_unit; 1828 const struct adf_ae_info *ae_info; 1829 int i; 1830 1831 if (get_accel_unit_config( 1832 accel_dev, &num_cy_au, &num_dc_au, &num_inline_au)) { 1833 device_printf(GET_DEV(accel_dev), "Invalid accel unit cfg\n"); 1834 return EFAULT; 1835 } 1836 1837 if (check_accel_unit_config( 1838 accel_dev, num_cy_au, num_dc_au, num_inline_au)) 1839 return EFAULT; 1840 1841 accel_dev->au_info = kzalloc(sizeof(*accel_dev->au_info), GFP_KERNEL); 1842 if (!accel_dev->au_info) 1843 return ENOMEM; 1844 1845 accel_dev->au_info->num_cy_au = num_cy_au; 1846 accel_dev->au_info->num_dc_au = num_dc_au; 1847 accel_dev->au_info->num_inline_au = num_inline_au; 1848 1849 if (get_ae_info(hw_data, &ae_info)) { 1850 device_printf(GET_DEV(accel_dev), "Failed to get ae info\n"); 1851 goto err_au_info; 1852 } 1853 accel_dev->au_info->ae_info = ae_info; 1854 1855 if (get_accel_unit(hw_data, &accel_unit)) { 1856 device_printf(GET_DEV(accel_dev), "Failed to get accel unit\n"); 1857 goto err_ae_info; 1858 } 1859 1860 /* Enable compression accel units */ 1861 /* Accel units with 4AEs are reserved for compression first */ 1862 for (i = num_au - 1; i >= 0 && num_dc_au > 0; i--) { 1863 if (accel_unit[i].num_ae == ADF_C4XXX_4_AE) { 1864 accel_unit[i].services = ADF_ACCEL_COMPRESSION; 1865 num_dc_au--; 1866 } 1867 } 1868 for (i = num_au - 1; i >= 0 && num_dc_au > 0; i--) { 1869 if (accel_unit[i].services == ADF_ACCEL_SERVICE_NULL) { 1870 accel_unit[i].services = ADF_ACCEL_COMPRESSION; 1871 num_dc_au--; 1872 } 1873 } 1874 1875 /* Enable inline accel units */ 1876 for (i = 0; i < num_au && num_inline_au > 0; i++) { 1877 if (accel_unit[i].services == ADF_ACCEL_SERVICE_NULL) { 1878 accel_unit[i].services = ADF_ACCEL_INLINE_CRYPTO; 1879 num_inline_au--; 1880 } 1881 } 1882 1883 /* Enable crypto accel units */ 1884 for (i = 0; i < num_au && num_cy_au > 0; i++) { 1885 if (accel_unit[i].services == ADF_ACCEL_SERVICE_NULL) { 1886 accel_unit[i].services = ADF_ACCEL_CRYPTO; 1887 num_cy_au--; 1888 } 1889 } 1890 accel_dev->au_info->au = accel_unit; 1891 return 0; 1892 1893 err_ae_info: 1894 accel_dev->au_info->ae_info = NULL; 1895 err_au_info: 1896 kfree(accel_dev->au_info); 1897 accel_dev->au_info = NULL; 1898 return EFAULT; 1899 } 1900 1901 static void 1902 adf_exit_accel_unit_services(struct adf_accel_dev *accel_dev) 1903 { 1904 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 1905 u32 num_au = hw_data->get_num_accel_units(hw_data); 1906 int i; 1907 1908 if (accel_dev->au_info) { 1909 if (accel_dev->au_info->au) { 1910 for (i = 0; i < num_au; i++) { 1911 accel_dev->au_info->au[i].services = 1912 ADF_ACCEL_SERVICE_NULL; 1913 } 1914 } 1915 accel_dev->au_info->au = NULL; 1916 accel_dev->au_info->ae_info = NULL; 1917 kfree(accel_dev->au_info); 1918 accel_dev->au_info = NULL; 1919 } 1920 } 1921 1922 static inline void 1923 adf_c4xxx_reset_hw_units(struct adf_accel_dev *accel_dev) 1924 { 1925 struct resource *pmisc = 1926 (&GET_BARS(accel_dev)[ADF_C4XXX_PMISC_BAR])->virt_addr; 1927 1928 u32 global_clk_enable = ADF_C4XXX_GLOBAL_CLK_ENABLE_GENERIC_ARAM | 1929 ADF_C4XXX_GLOBAL_CLK_ENABLE_GENERIC_ICI_ENABLE | 1930 ADF_C4XXX_GLOBAL_CLK_ENABLE_GENERIC_ICE_ENABLE; 1931 1932 u32 ixp_reset_generic = ADF_C4XXX_IXP_RESET_GENERIC_ARAM | 1933 ADF_C4XXX_IXP_RESET_GENERIC_INLINE_EGRESS | 1934 ADF_C4XXX_IXP_RESET_GENERIC_INLINE_INGRESS; 1935 1936 /* To properly reset each of the units driver must: 1937 * 1)Call out resetactive state using ixp reset generic 1938 * register; 1939 * 2)Disable generic clock; 1940 * 3)Take device out of reset by clearing ixp reset 1941 * generic register; 1942 * 4)Re-enable generic clock; 1943 */ 1944 ADF_CSR_WR(pmisc, ADF_C4XXX_IXP_RESET_GENERIC, ixp_reset_generic); 1945 ADF_CSR_WR(pmisc, 1946 ADF_C4XXX_GLOBAL_CLK_ENABLE_GENERIC, 1947 ADF_C4XXX_GLOBAL_CLK_ENABLE_GENERIC_DISABLE_ALL); 1948 ADF_CSR_WR(pmisc, 1949 ADF_C4XXX_IXP_RESET_GENERIC, 1950 ADF_C4XXX_IXP_RESET_GENERIC_OUT_OF_RESET_TRIGGER); 1951 ADF_CSR_WR(pmisc, 1952 ADF_C4XXX_GLOBAL_CLK_ENABLE_GENERIC, 1953 global_clk_enable); 1954 } 1955 1956 static int 1957 adf_init_accel_units(struct adf_accel_dev *accel_dev) 1958 { 1959 struct resource *csr = 1960 (&GET_BARS(accel_dev)[ADF_C4XXX_PMISC_BAR])->virt_addr; 1961 1962 if (adf_init_accel_unit_services(accel_dev)) 1963 return EFAULT; 1964 1965 /* Set cy and dc enabled AE masks */ 1966 if (accel_dev->au_info->num_cy_au || accel_dev->au_info->num_dc_au) { 1967 if (adf_set_ae_mask(accel_dev)) { 1968 device_printf(GET_DEV(accel_dev), 1969 "Failed to set ae masks\n"); 1970 goto err_au; 1971 } 1972 } 1973 /* Set ingress/egress ae mask if inline is enabled */ 1974 if (accel_dev->au_info->num_inline_au) { 1975 if (adf_set_inline_ae_mask(accel_dev)) { 1976 device_printf(GET_DEV(accel_dev), 1977 "Failed to set inline ae masks\n"); 1978 goto err_au; 1979 } 1980 } 1981 /* Define ARAM regions */ 1982 if (c4xxx_init_aram_config(accel_dev)) { 1983 device_printf(GET_DEV(accel_dev), 1984 "Failed to init aram config\n"); 1985 goto err_au; 1986 } 1987 /* Configure h/w registers for inline operations */ 1988 if (accel_dev->au_info->num_inline_au > 0) 1989 /* Initialise configuration parsing registers */ 1990 if (c4xxx_init_inline_hw(accel_dev)) 1991 goto err_au; 1992 1993 c4xxx_set_sadb_size(accel_dev); 1994 1995 if (accel_dev->au_info->num_inline_au > 0) { 1996 /* ici/ice interrupt shall be enabled after msi-x enabled */ 1997 ADF_CSR_WR(csr, 1998 ADF_C4XXX_ERRMSK11, 1999 ADF_C4XXX_ERRMSK11_ERR_DISABLE_ICI_ICE_INTR); 2000 adf_enable_inline_notification(accel_dev); 2001 } 2002 2003 update_hw_capability(accel_dev); 2004 if (adf_add_debugfs_info(accel_dev)) { 2005 device_printf(GET_DEV(accel_dev), 2006 "Failed to add debug FS information\n"); 2007 goto err_au; 2008 } 2009 return 0; 2010 2011 err_au: 2012 /* Free and clear accel unit data structures */ 2013 adf_exit_accel_unit_services(accel_dev); 2014 return EFAULT; 2015 } 2016 2017 static void 2018 adf_exit_accel_units(struct adf_accel_dev *accel_dev) 2019 { 2020 adf_exit_accel_unit_services(accel_dev); 2021 /* Free aram mapping structure */ 2022 c4xxx_exit_aram_config(accel_dev); 2023 /* Remove entries in debug FS */ 2024 adf_remove_debugfs_info(accel_dev); 2025 } 2026 2027 static const char * 2028 get_obj_name(struct adf_accel_dev *accel_dev, 2029 enum adf_accel_unit_services service) 2030 { 2031 u32 capabilities = GET_HW_DATA(accel_dev)->accel_capabilities_mask; 2032 bool sym_only_sku = false; 2033 2034 /* Check if SKU is capable only of symmetric cryptography 2035 * via device capabilities. 2036 */ 2037 if ((capabilities & ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC) && 2038 !(capabilities & ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC) && 2039 !(capabilities & ADF_ACCEL_CAPABILITIES_COMPRESSION)) 2040 sym_only_sku = true; 2041 2042 switch (service) { 2043 case ADF_ACCEL_INLINE_CRYPTO: 2044 return ADF_C4XXX_INLINE_OBJ; 2045 case ADF_ACCEL_CRYPTO: 2046 if (sym_only_sku) 2047 return ADF_C4XXX_SYM_OBJ; 2048 else 2049 return ADF_C4XXX_CY_OBJ; 2050 break; 2051 case ADF_ACCEL_COMPRESSION: 2052 return ADF_C4XXX_DC_OBJ; 2053 default: 2054 return NULL; 2055 } 2056 } 2057 2058 static uint32_t 2059 get_objs_num(struct adf_accel_dev *accel_dev) 2060 { 2061 u32 srv = 0; 2062 u32 max_srv_id = 0; 2063 unsigned long service_mask = accel_dev->hw_device->service_mask; 2064 2065 /* The objects number corresponds to the number of services */ 2066 for_each_set_bit(srv, &service_mask, ADF_C4XXX_MAX_OBJ) 2067 { 2068 max_srv_id = srv; 2069 } 2070 2071 return (max_srv_id + 1); 2072 } 2073 2074 static uint32_t 2075 get_obj_cfg_ae_mask(struct adf_accel_dev *accel_dev, 2076 enum adf_accel_unit_services service) 2077 { 2078 u32 ae_mask = 0; 2079 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 2080 u32 num_au = hw_data->get_num_accel_units(hw_data); 2081 struct adf_accel_unit *accel_unit = accel_dev->au_info->au; 2082 u32 i = 0; 2083 2084 if (service == ADF_ACCEL_SERVICE_NULL) 2085 return 0; 2086 2087 for (i = 0; i < num_au; i++) { 2088 if (accel_unit[i].services == service) 2089 ae_mask |= accel_unit[i].ae_mask; 2090 } 2091 return ae_mask; 2092 } 2093 2094 static void 2095 configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) 2096 { 2097 struct resource *addr; 2098 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 2099 u32 num_aes = hw_data->get_num_aes(hw_data); 2100 u32 reg = 0x0; 2101 u32 i; 2102 2103 addr = (&GET_BARS(accel_dev)[ADF_C4XXX_PMISC_BAR])->virt_addr; 2104 2105 /* Set/Unset Valid bits in AE Thread to PCIe Function Mapping */ 2106 for (i = 0; i < ADF_C4XXX_AE2FUNC_REG_PER_AE * num_aes; i++) { 2107 reg = ADF_CSR_RD(addr + ADF_C4XXX_AE2FUNC_MAP_OFFSET, 2108 i * ADF_C4XXX_AE2FUNC_MAP_REG_SIZE); 2109 if (enable) 2110 reg |= ADF_C4XXX_AE2FUNC_MAP_VALID; 2111 else 2112 reg &= ~ADF_C4XXX_AE2FUNC_MAP_VALID; 2113 ADF_CSR_WR(addr + ADF_C4XXX_AE2FUNC_MAP_OFFSET, 2114 i * ADF_C4XXX_AE2FUNC_MAP_REG_SIZE, 2115 reg); 2116 } 2117 } 2118 2119 void 2120 adf_init_hw_data_c4xxx(struct adf_hw_device_data *hw_data) 2121 { 2122 hw_data->dev_class = &c4xxx_class; 2123 hw_data->instance_id = c4xxx_class.instances++; 2124 hw_data->num_banks = ADF_C4XXX_ETR_MAX_BANKS; 2125 hw_data->num_rings_per_bank = ADF_C4XXX_NUM_RINGS_PER_BANK; 2126 hw_data->num_accel = ADF_C4XXX_MAX_ACCELERATORS; 2127 hw_data->num_engines = ADF_C4XXX_MAX_ACCELENGINES; 2128 hw_data->num_logical_accel = 1; 2129 hw_data->tx_rx_gap = ADF_C4XXX_RX_RINGS_OFFSET; 2130 hw_data->tx_rings_mask = ADF_C4XXX_TX_RINGS_MASK; 2131 hw_data->alloc_irq = adf_isr_resource_alloc; 2132 hw_data->free_irq = adf_isr_resource_free; 2133 hw_data->enable_error_correction = adf_enable_error_correction; 2134 hw_data->init_ras = adf_init_ras; 2135 hw_data->exit_ras = adf_exit_ras; 2136 hw_data->ras_interrupts = adf_ras_interrupts; 2137 hw_data->get_accel_mask = get_accel_mask; 2138 hw_data->get_ae_mask = get_ae_mask; 2139 hw_data->get_num_accels = get_num_accels; 2140 hw_data->get_num_aes = get_num_aes; 2141 hw_data->get_num_accel_units = get_num_accel_units; 2142 hw_data->get_sram_bar_id = get_sram_bar_id; 2143 hw_data->get_etr_bar_id = get_etr_bar_id; 2144 hw_data->get_misc_bar_id = get_misc_bar_id; 2145 hw_data->get_arb_info = get_arb_info; 2146 hw_data->get_admin_info = get_admin_info; 2147 hw_data->get_errsou_offset = get_errsou_offset; 2148 hw_data->get_clock_speed = get_clock_speed; 2149 hw_data->get_eth_doorbell_msg = get_eth_doorbell_msg; 2150 hw_data->get_sku = get_sku; 2151 hw_data->heartbeat_ctr_num = ADF_NUM_THREADS_PER_AE; 2152 hw_data->check_prod_sku = c4xxx_check_prod_sku; 2153 hw_data->fw_name = ADF_C4XXX_FW; 2154 hw_data->fw_mmp_name = ADF_C4XXX_MMP; 2155 hw_data->get_obj_name = get_obj_name; 2156 hw_data->get_objs_num = get_objs_num; 2157 hw_data->get_obj_cfg_ae_mask = get_obj_cfg_ae_mask; 2158 hw_data->init_admin_comms = adf_init_admin_comms; 2159 hw_data->exit_admin_comms = adf_exit_admin_comms; 2160 hw_data->configure_iov_threads = configure_iov_threads; 2161 hw_data->disable_iov = adf_disable_sriov; 2162 hw_data->send_admin_init = adf_send_admin_init; 2163 hw_data->init_arb = adf_init_arb_c4xxx; 2164 hw_data->exit_arb = adf_exit_arb_c4xxx; 2165 hw_data->disable_arb = adf_disable_arb; 2166 hw_data->enable_ints = adf_enable_ints; 2167 hw_data->set_ssm_wdtimer = c4xxx_set_ssm_wdtimer; 2168 hw_data->check_slice_hang = c4xxx_check_slice_hang; 2169 hw_data->reset_device = adf_reset_flr; 2170 hw_data->restore_device = adf_c4xxx_dev_restore; 2171 hw_data->init_accel_units = adf_init_accel_units; 2172 hw_data->reset_hw_units = adf_c4xxx_reset_hw_units; 2173 hw_data->exit_accel_units = adf_exit_accel_units; 2174 hw_data->ring_to_svc_map = ADF_DEFAULT_RING_TO_SRV_MAP; 2175 hw_data->get_heartbeat_status = adf_get_heartbeat_status; 2176 hw_data->get_ae_clock = get_ae_clock; 2177 hw_data->clock_frequency = ADF_C4XXX_AE_FREQ; 2178 hw_data->measure_clock = measure_clock; 2179 hw_data->add_pke_stats = adf_pke_replay_counters_add_c4xxx; 2180 hw_data->remove_pke_stats = adf_pke_replay_counters_remove_c4xxx; 2181 hw_data->add_misc_error = adf_misc_error_add_c4xxx; 2182 hw_data->remove_misc_error = adf_misc_error_remove_c4xxx; 2183 hw_data->extended_dc_capabilities = 0; 2184 hw_data->get_storage_enabled = get_storage_enabled; 2185 hw_data->query_storage_cap = 0; 2186 hw_data->get_accel_cap = c4xxx_get_hw_cap; 2187 hw_data->configure_accel_units = c4xxx_configure_accel_units; 2188 hw_data->pre_reset = adf_dev_pre_reset; 2189 hw_data->post_reset = adf_dev_post_reset; 2190 hw_data->get_ring_to_svc_map = adf_cfg_get_services_enabled; 2191 hw_data->count_ras_event = adf_fw_count_ras_event; 2192 hw_data->config_device = adf_config_device; 2193 hw_data->set_asym_rings_mask = adf_cfg_set_asym_rings_mask; 2194 2195 adf_gen2_init_hw_csr_info(&hw_data->csr_info); 2196 adf_gen2_init_pf_pfvf_ops(&hw_data->csr_info.pfvf_ops); 2197 hw_data->csr_info.arb_enable_mask = 0xF; 2198 } 2199 2200 void 2201 adf_clean_hw_data_c4xxx(struct adf_hw_device_data *hw_data) 2202 { 2203 hw_data->dev_class->instances--; 2204 } 2205 2206 void 2207 remove_oid(struct adf_accel_dev *accel_dev, struct sysctl_oid *oid) 2208 { 2209 struct sysctl_ctx_list *qat_sysctl_ctx; 2210 int ret; 2211 2212 qat_sysctl_ctx = 2213 device_get_sysctl_ctx(accel_dev->accel_pci_dev.pci_dev); 2214 2215 ret = sysctl_ctx_entry_del(qat_sysctl_ctx, oid); 2216 if (ret) 2217 device_printf(GET_DEV(accel_dev), "Failed to delete entry\n"); 2218 2219 ret = sysctl_remove_oid(oid, 1, 1); 2220 if (ret) 2221 device_printf(GET_DEV(accel_dev), "Failed to delete oid\n"); 2222 } 2223