178ee8d1cSJulian Grajkowski /* SPDX-License-Identifier: BSD-3-Clause */ 278ee8d1cSJulian Grajkowski /* Copyright(c) 2007-2022 Intel Corporation */ 378ee8d1cSJulian Grajkowski /* $FreeBSD$ */ 478ee8d1cSJulian Grajkowski #include "qat_freebsd.h" 578ee8d1cSJulian Grajkowski #include "adf_cfg.h" 678ee8d1cSJulian Grajkowski #include "adf_common_drv.h" 778ee8d1cSJulian Grajkowski #include "adf_accel_devices.h" 878ee8d1cSJulian Grajkowski #include "icp_qat_uclo.h" 978ee8d1cSJulian Grajkowski #include "icp_qat_fw.h" 1078ee8d1cSJulian Grajkowski #include "icp_qat_fw_init_admin.h" 1178ee8d1cSJulian Grajkowski #include "adf_cfg_strings.h" 1278ee8d1cSJulian Grajkowski #include "adf_transport_access_macros.h" 1378ee8d1cSJulian Grajkowski #include "adf_transport_internal.h" 1478ee8d1cSJulian Grajkowski #include <linux/delay.h> 1578ee8d1cSJulian Grajkowski #include "adf_accel_devices.h" 1678ee8d1cSJulian Grajkowski #include "adf_common_drv.h" 1778ee8d1cSJulian Grajkowski #include "icp_qat_hal.h" 1878ee8d1cSJulian Grajkowski #include "icp_qat_uclo.h" 1978ee8d1cSJulian Grajkowski 2078ee8d1cSJulian Grajkowski #define BAD_REGADDR 0xffff 2178ee8d1cSJulian Grajkowski #define MAX_RETRY_TIMES 1000000 2278ee8d1cSJulian Grajkowski #define INIT_CTX_ARB_VALUE 0x0 2378ee8d1cSJulian Grajkowski #define INIT_CTX_ENABLE_VALUE 0x0 2478ee8d1cSJulian Grajkowski #define INIT_PC_VALUE 0x0 2578ee8d1cSJulian Grajkowski #define INIT_WAKEUP_EVENTS_VALUE 0x1 2678ee8d1cSJulian Grajkowski #define INIT_SIG_EVENTS_VALUE 0x1 2778ee8d1cSJulian Grajkowski #define INIT_CCENABLE_VALUE 0x2000 2878ee8d1cSJulian Grajkowski #define RST_CSR_QAT_LSB 20 2978ee8d1cSJulian Grajkowski #define RST_CSR_AE_LSB 0 3078ee8d1cSJulian Grajkowski #define MC_TIMESTAMP_ENABLE (0x1 << 7) 3178ee8d1cSJulian Grajkowski 3278ee8d1cSJulian Grajkowski #define IGNORE_W1C_MASK \ 3378ee8d1cSJulian Grajkowski ((~(1 << CE_BREAKPOINT_BITPOS)) & \ 3478ee8d1cSJulian Grajkowski (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \ 3578ee8d1cSJulian Grajkowski (~(1 << CE_REG_PAR_ERR_BITPOS))) 3678ee8d1cSJulian Grajkowski #define INSERT_IMMED_GPRA_CONST(inst, const_val) \ 3778ee8d1cSJulian Grajkowski (inst = ((inst & 0xFFFF00C03FFull) | \ 3878ee8d1cSJulian Grajkowski ((((const_val) << 12) & 0x0FF00000ull) | \ 3978ee8d1cSJulian Grajkowski (((const_val) << 10) & 0x0003FC00ull)))) 4078ee8d1cSJulian Grajkowski #define INSERT_IMMED_GPRB_CONST(inst, const_val) \ 4178ee8d1cSJulian Grajkowski (inst = ((inst & 0xFFFF00FFF00ull) | \ 4278ee8d1cSJulian Grajkowski ((((const_val) << 12) & 0x0FF00000ull) | \ 4378ee8d1cSJulian Grajkowski (((const_val) << 0) & 0x000000FFull)))) 4478ee8d1cSJulian Grajkowski 4578ee8d1cSJulian Grajkowski #define AE(handle, ae) ((handle)->hal_handle->aes[ae]) 4678ee8d1cSJulian Grajkowski 4778ee8d1cSJulian Grajkowski static const uint64_t inst_4b[] = { 0x0F0400C0000ull, 0x0F4400C0000ull, 4878ee8d1cSJulian Grajkowski 0x0F040000300ull, 0x0F440000300ull, 4978ee8d1cSJulian Grajkowski 0x0FC066C0000ull, 0x0F0000C0300ull, 5078ee8d1cSJulian Grajkowski 0x0F0000C0300ull, 0x0F0000C0300ull, 5178ee8d1cSJulian Grajkowski 0x0A021000000ull }; 5278ee8d1cSJulian Grajkowski 5378ee8d1cSJulian Grajkowski static const uint64_t inst[] = { 5478ee8d1cSJulian Grajkowski 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull, 5578ee8d1cSJulian Grajkowski 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 5678ee8d1cSJulian Grajkowski 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull, 5778ee8d1cSJulian Grajkowski 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 5878ee8d1cSJulian Grajkowski 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull, 5978ee8d1cSJulian Grajkowski 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, 6078ee8d1cSJulian Grajkowski 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, 6178ee8d1cSJulian Grajkowski 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, 6278ee8d1cSJulian Grajkowski 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, 6378ee8d1cSJulian Grajkowski 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, 6478ee8d1cSJulian Grajkowski 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, 6578ee8d1cSJulian Grajkowski 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, 6678ee8d1cSJulian Grajkowski 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, 6778ee8d1cSJulian Grajkowski 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, 6878ee8d1cSJulian Grajkowski 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, 6978ee8d1cSJulian Grajkowski 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, 7078ee8d1cSJulian Grajkowski 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, 7178ee8d1cSJulian Grajkowski 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, 7278ee8d1cSJulian Grajkowski 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, 7378ee8d1cSJulian Grajkowski 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, 7478ee8d1cSJulian Grajkowski 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, 7578ee8d1cSJulian Grajkowski 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull, 7678ee8d1cSJulian Grajkowski }; 7778ee8d1cSJulian Grajkowski 78a977168cSMichal Gulbicki static const uint64_t inst_CPM2X[] = { 79a977168cSMichal Gulbicki 0x0F0000C0000ull, 0x0D802C00011ull, 0x0F0000C0001ull, 0x0FC066C0001ull, 80a977168cSMichal Gulbicki 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F000500300ull, 81a977168cSMichal Gulbicki 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0A0580C0000ull, 82a977168cSMichal Gulbicki 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, 0x0A0584C0000ull, 83a977168cSMichal Gulbicki 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, 0x0A0588C0000ull, 84a977168cSMichal Gulbicki 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, 0x0A058CC0000ull, 85a977168cSMichal Gulbicki 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, 0x0A05C0C0000ull, 86a977168cSMichal Gulbicki 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, 0x0A05C4C0000ull, 87a977168cSMichal Gulbicki 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, 0x0A05C8C0000ull, 88a977168cSMichal Gulbicki 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, 0x0A05CCC0000ull, 89a977168cSMichal Gulbicki 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, 0x0A0400C0000ull, 90a977168cSMichal Gulbicki 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, 0x0A0402C0000ull, 91a977168cSMichal Gulbicki 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, 0x0A0404C0000ull, 92a977168cSMichal Gulbicki 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, 0x0A0406C0000ull, 93a977168cSMichal Gulbicki 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, 0x0A0408C0000ull, 94a977168cSMichal Gulbicki 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, 0x0A040AC0000ull, 95a977168cSMichal Gulbicki 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, 0x0A040CC0000ull, 96a977168cSMichal Gulbicki 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, 0x0A040EC0000ull, 97a977168cSMichal Gulbicki 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, 0x0D81341C010ull, 98a977168cSMichal Gulbicki 0x0E000000001ull, 0x0E000010000ull, 99a977168cSMichal Gulbicki }; 100a977168cSMichal Gulbicki 10178ee8d1cSJulian Grajkowski void 10278ee8d1cSJulian Grajkowski qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, 10378ee8d1cSJulian Grajkowski unsigned char ae, 10478ee8d1cSJulian Grajkowski unsigned int ctx_mask) 10578ee8d1cSJulian Grajkowski { 10678ee8d1cSJulian Grajkowski AE(handle, ae).live_ctx_mask = ctx_mask; 10778ee8d1cSJulian Grajkowski } 10878ee8d1cSJulian Grajkowski 10978ee8d1cSJulian Grajkowski #define CSR_RETRY_TIMES 500 11078ee8d1cSJulian Grajkowski static int 11178ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle, 11278ee8d1cSJulian Grajkowski unsigned char ae, 11378ee8d1cSJulian Grajkowski unsigned int csr, 11478ee8d1cSJulian Grajkowski unsigned int *value) 11578ee8d1cSJulian Grajkowski { 11678ee8d1cSJulian Grajkowski unsigned int iterations = CSR_RETRY_TIMES; 11778ee8d1cSJulian Grajkowski 11878ee8d1cSJulian Grajkowski do { 11978ee8d1cSJulian Grajkowski *value = GET_AE_CSR(handle, ae, csr); 12078ee8d1cSJulian Grajkowski if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) 12178ee8d1cSJulian Grajkowski return 0; 12278ee8d1cSJulian Grajkowski } while (iterations--); 12378ee8d1cSJulian Grajkowski 12478ee8d1cSJulian Grajkowski pr_err("QAT: Read CSR timeout\n"); 12578ee8d1cSJulian Grajkowski return EFAULT; 12678ee8d1cSJulian Grajkowski } 12778ee8d1cSJulian Grajkowski 12878ee8d1cSJulian Grajkowski static int 12978ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle, 13078ee8d1cSJulian Grajkowski unsigned char ae, 13178ee8d1cSJulian Grajkowski unsigned int csr, 13278ee8d1cSJulian Grajkowski unsigned int value) 13378ee8d1cSJulian Grajkowski { 13478ee8d1cSJulian Grajkowski unsigned int iterations = CSR_RETRY_TIMES; 13578ee8d1cSJulian Grajkowski 13678ee8d1cSJulian Grajkowski do { 13778ee8d1cSJulian Grajkowski SET_AE_CSR(handle, ae, csr, value); 13878ee8d1cSJulian Grajkowski if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) 13978ee8d1cSJulian Grajkowski return 0; 14078ee8d1cSJulian Grajkowski } while (iterations--); 14178ee8d1cSJulian Grajkowski 14278ee8d1cSJulian Grajkowski pr_err("QAT: Write CSR Timeout\n"); 14378ee8d1cSJulian Grajkowski return EFAULT; 14478ee8d1cSJulian Grajkowski } 14578ee8d1cSJulian Grajkowski 14678ee8d1cSJulian Grajkowski static void 14778ee8d1cSJulian Grajkowski qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle, 14878ee8d1cSJulian Grajkowski unsigned char ae, 14978ee8d1cSJulian Grajkowski unsigned char ctx, 15078ee8d1cSJulian Grajkowski unsigned int *events) 15178ee8d1cSJulian Grajkowski { 15278ee8d1cSJulian Grajkowski unsigned int cur_ctx; 15378ee8d1cSJulian Grajkowski 15478ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); 15578ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 15678ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events); 15778ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 15878ee8d1cSJulian Grajkowski } 15978ee8d1cSJulian Grajkowski 16078ee8d1cSJulian Grajkowski static int 16178ee8d1cSJulian Grajkowski qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, 16278ee8d1cSJulian Grajkowski unsigned char ae, 16378ee8d1cSJulian Grajkowski unsigned int cycles, 16478ee8d1cSJulian Grajkowski int chk_inactive) 16578ee8d1cSJulian Grajkowski { 16678ee8d1cSJulian Grajkowski unsigned int base_cnt = 0, cur_cnt = 0; 16778ee8d1cSJulian Grajkowski unsigned int csr = (1 << ACS_ABO_BITPOS); 16878ee8d1cSJulian Grajkowski int times = MAX_RETRY_TIMES; 16978ee8d1cSJulian Grajkowski int elapsed_cycles = 0; 17078ee8d1cSJulian Grajkowski 17178ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt); 17278ee8d1cSJulian Grajkowski base_cnt &= 0xffff; 17378ee8d1cSJulian Grajkowski while ((int)cycles > elapsed_cycles && times--) { 17478ee8d1cSJulian Grajkowski if (chk_inactive) 17578ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr); 17678ee8d1cSJulian Grajkowski 17778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt); 17878ee8d1cSJulian Grajkowski cur_cnt &= 0xffff; 17978ee8d1cSJulian Grajkowski elapsed_cycles = cur_cnt - base_cnt; 18078ee8d1cSJulian Grajkowski 18178ee8d1cSJulian Grajkowski if (elapsed_cycles < 0) 18278ee8d1cSJulian Grajkowski elapsed_cycles += 0x10000; 18378ee8d1cSJulian Grajkowski 18478ee8d1cSJulian Grajkowski /* ensure at least 8 time cycles elapsed in wait_cycles */ 18578ee8d1cSJulian Grajkowski if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS))) 18678ee8d1cSJulian Grajkowski return 0; 18778ee8d1cSJulian Grajkowski } 18878ee8d1cSJulian Grajkowski if (times < 0) { 18978ee8d1cSJulian Grajkowski pr_err("QAT: wait_num_cycles time out\n"); 19078ee8d1cSJulian Grajkowski return EFAULT; 19178ee8d1cSJulian Grajkowski } 19278ee8d1cSJulian Grajkowski return 0; 19378ee8d1cSJulian Grajkowski } 19478ee8d1cSJulian Grajkowski 19578ee8d1cSJulian Grajkowski void 19678ee8d1cSJulian Grajkowski qat_hal_get_scs_neigh_ae(unsigned char ae, unsigned char *ae_neigh) 19778ee8d1cSJulian Grajkowski { 19878ee8d1cSJulian Grajkowski *ae_neigh = (ae & 0x1) ? (ae - 1) : (ae + 1); 19978ee8d1cSJulian Grajkowski } 20078ee8d1cSJulian Grajkowski 20178ee8d1cSJulian Grajkowski #define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit))) 20278ee8d1cSJulian Grajkowski #define SET_BIT(wrd, bit) ((wrd) | 1 << (bit)) 20378ee8d1cSJulian Grajkowski 20478ee8d1cSJulian Grajkowski int 20578ee8d1cSJulian Grajkowski qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, 20678ee8d1cSJulian Grajkowski unsigned char ae, 20778ee8d1cSJulian Grajkowski unsigned char mode) 20878ee8d1cSJulian Grajkowski { 20978ee8d1cSJulian Grajkowski unsigned int csr, new_csr; 21078ee8d1cSJulian Grajkowski 21178ee8d1cSJulian Grajkowski if (mode != 4 && mode != 8) { 21278ee8d1cSJulian Grajkowski pr_err("QAT: bad ctx mode=%d\n", mode); 21378ee8d1cSJulian Grajkowski return EINVAL; 21478ee8d1cSJulian Grajkowski } 21578ee8d1cSJulian Grajkowski 21678ee8d1cSJulian Grajkowski /* Sets the accelaration engine context mode to either four or eight */ 21778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); 21878ee8d1cSJulian Grajkowski csr = IGNORE_W1C_MASK & csr; 21978ee8d1cSJulian Grajkowski new_csr = (mode == 4) ? SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) : 22078ee8d1cSJulian Grajkowski CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS); 22178ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 22278ee8d1cSJulian Grajkowski return 0; 22378ee8d1cSJulian Grajkowski } 22478ee8d1cSJulian Grajkowski 22578ee8d1cSJulian Grajkowski int 22678ee8d1cSJulian Grajkowski qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, 22778ee8d1cSJulian Grajkowski unsigned char ae, 22878ee8d1cSJulian Grajkowski unsigned char mode) 22978ee8d1cSJulian Grajkowski { 23078ee8d1cSJulian Grajkowski unsigned int csr, new_csr; 23178ee8d1cSJulian Grajkowski 232a977168cSMichal Gulbicki if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 233a977168cSMichal Gulbicki pr_err("QAT: No next neigh for CPM2X\n"); 234a977168cSMichal Gulbicki return EINVAL; 235a977168cSMichal Gulbicki } 236a977168cSMichal Gulbicki 23778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); 23878ee8d1cSJulian Grajkowski csr &= IGNORE_W1C_MASK; 23978ee8d1cSJulian Grajkowski 24078ee8d1cSJulian Grajkowski new_csr = (mode) ? SET_BIT(csr, CE_NN_MODE_BITPOS) : 24178ee8d1cSJulian Grajkowski CLR_BIT(csr, CE_NN_MODE_BITPOS); 24278ee8d1cSJulian Grajkowski 24378ee8d1cSJulian Grajkowski if (new_csr != csr) 24478ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 24578ee8d1cSJulian Grajkowski 24678ee8d1cSJulian Grajkowski return 0; 24778ee8d1cSJulian Grajkowski } 24878ee8d1cSJulian Grajkowski 24978ee8d1cSJulian Grajkowski int 25078ee8d1cSJulian Grajkowski qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, 25178ee8d1cSJulian Grajkowski unsigned char ae, 25278ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype lm_type, 25378ee8d1cSJulian Grajkowski unsigned char mode) 25478ee8d1cSJulian Grajkowski { 25578ee8d1cSJulian Grajkowski unsigned int csr, new_csr; 25678ee8d1cSJulian Grajkowski 25778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); 25878ee8d1cSJulian Grajkowski csr &= IGNORE_W1C_MASK; 25978ee8d1cSJulian Grajkowski switch (lm_type) { 26078ee8d1cSJulian Grajkowski case ICP_LMEM0: 26178ee8d1cSJulian Grajkowski new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) : 26278ee8d1cSJulian Grajkowski CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS); 26378ee8d1cSJulian Grajkowski break; 26478ee8d1cSJulian Grajkowski case ICP_LMEM1: 26578ee8d1cSJulian Grajkowski new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) : 26678ee8d1cSJulian Grajkowski CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS); 26778ee8d1cSJulian Grajkowski break; 26878ee8d1cSJulian Grajkowski case ICP_LMEM2: 26978ee8d1cSJulian Grajkowski new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) : 27078ee8d1cSJulian Grajkowski CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS); 27178ee8d1cSJulian Grajkowski break; 27278ee8d1cSJulian Grajkowski case ICP_LMEM3: 27378ee8d1cSJulian Grajkowski new_csr = (mode) ? SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) : 27478ee8d1cSJulian Grajkowski CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS); 27578ee8d1cSJulian Grajkowski break; 27678ee8d1cSJulian Grajkowski default: 27778ee8d1cSJulian Grajkowski pr_err("QAT: lmType = 0x%x\n", lm_type); 27878ee8d1cSJulian Grajkowski return EINVAL; 27978ee8d1cSJulian Grajkowski } 28078ee8d1cSJulian Grajkowski 28178ee8d1cSJulian Grajkowski if (new_csr != csr) 28278ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 28378ee8d1cSJulian Grajkowski return 0; 28478ee8d1cSJulian Grajkowski } 28578ee8d1cSJulian Grajkowski 28678ee8d1cSJulian Grajkowski void 28778ee8d1cSJulian Grajkowski qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, 28878ee8d1cSJulian Grajkowski unsigned char ae, 28978ee8d1cSJulian Grajkowski unsigned char mode) 29078ee8d1cSJulian Grajkowski { 29178ee8d1cSJulian Grajkowski unsigned int csr, new_csr; 29278ee8d1cSJulian Grajkowski 29378ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); 29478ee8d1cSJulian Grajkowski csr &= IGNORE_W1C_MASK; 29578ee8d1cSJulian Grajkowski new_csr = (mode) ? SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) : 29678ee8d1cSJulian Grajkowski CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS); 29778ee8d1cSJulian Grajkowski if (new_csr != csr) 29878ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); 29978ee8d1cSJulian Grajkowski } 30078ee8d1cSJulian Grajkowski 30178ee8d1cSJulian Grajkowski void 30278ee8d1cSJulian Grajkowski qat_hal_set_ae_scs_mode(struct icp_qat_fw_loader_handle *handle, 30378ee8d1cSJulian Grajkowski unsigned char ae, 30478ee8d1cSJulian Grajkowski unsigned char mode) 30578ee8d1cSJulian Grajkowski { 30678ee8d1cSJulian Grajkowski unsigned int csr, new_csr; 30778ee8d1cSJulian Grajkowski 30878ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr); 30978ee8d1cSJulian Grajkowski new_csr = (mode) ? SET_BIT(csr, MMC_SHARE_CS_BITPOS) : 31078ee8d1cSJulian Grajkowski CLR_BIT(csr, MMC_SHARE_CS_BITPOS); 31178ee8d1cSJulian Grajkowski if (new_csr != csr) 31278ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, new_csr); 31378ee8d1cSJulian Grajkowski } 31478ee8d1cSJulian Grajkowski 31578ee8d1cSJulian Grajkowski static unsigned short 31678ee8d1cSJulian Grajkowski qat_hal_get_reg_addr(unsigned int type, unsigned short reg_num) 31778ee8d1cSJulian Grajkowski { 31878ee8d1cSJulian Grajkowski unsigned short reg_addr; 31978ee8d1cSJulian Grajkowski 32078ee8d1cSJulian Grajkowski switch (type) { 32178ee8d1cSJulian Grajkowski case ICP_GPA_ABS: 32278ee8d1cSJulian Grajkowski case ICP_GPB_ABS: 32378ee8d1cSJulian Grajkowski reg_addr = 0x80 | (reg_num & 0x7f); 32478ee8d1cSJulian Grajkowski break; 32578ee8d1cSJulian Grajkowski case ICP_GPA_REL: 32678ee8d1cSJulian Grajkowski case ICP_GPB_REL: 32778ee8d1cSJulian Grajkowski reg_addr = reg_num & 0x1f; 32878ee8d1cSJulian Grajkowski break; 32978ee8d1cSJulian Grajkowski case ICP_SR_RD_REL: 33078ee8d1cSJulian Grajkowski case ICP_SR_WR_REL: 33178ee8d1cSJulian Grajkowski case ICP_SR_REL: 33278ee8d1cSJulian Grajkowski reg_addr = 0x180 | (reg_num & 0x1f); 33378ee8d1cSJulian Grajkowski break; 33478ee8d1cSJulian Grajkowski case ICP_SR_ABS: 33578ee8d1cSJulian Grajkowski reg_addr = 0x140 | ((reg_num & 0x3) << 1); 33678ee8d1cSJulian Grajkowski break; 33778ee8d1cSJulian Grajkowski case ICP_DR_RD_REL: 33878ee8d1cSJulian Grajkowski case ICP_DR_WR_REL: 33978ee8d1cSJulian Grajkowski case ICP_DR_REL: 34078ee8d1cSJulian Grajkowski reg_addr = 0x1c0 | (reg_num & 0x1f); 34178ee8d1cSJulian Grajkowski break; 34278ee8d1cSJulian Grajkowski case ICP_DR_ABS: 34378ee8d1cSJulian Grajkowski reg_addr = 0x100 | ((reg_num & 0x3) << 1); 34478ee8d1cSJulian Grajkowski break; 34578ee8d1cSJulian Grajkowski case ICP_NEIGH_REL: 34678ee8d1cSJulian Grajkowski reg_addr = 0x280 | (reg_num & 0x1f); 34778ee8d1cSJulian Grajkowski break; 34878ee8d1cSJulian Grajkowski case ICP_LMEM0: 34978ee8d1cSJulian Grajkowski reg_addr = 0x200; 35078ee8d1cSJulian Grajkowski break; 35178ee8d1cSJulian Grajkowski case ICP_LMEM1: 35278ee8d1cSJulian Grajkowski reg_addr = 0x220; 35378ee8d1cSJulian Grajkowski break; 35478ee8d1cSJulian Grajkowski case ICP_LMEM2: 35578ee8d1cSJulian Grajkowski reg_addr = 0x2c0; 35678ee8d1cSJulian Grajkowski break; 35778ee8d1cSJulian Grajkowski case ICP_LMEM3: 35878ee8d1cSJulian Grajkowski reg_addr = 0x2e0; 35978ee8d1cSJulian Grajkowski break; 36078ee8d1cSJulian Grajkowski case ICP_NO_DEST: 36178ee8d1cSJulian Grajkowski reg_addr = 0x300 | (reg_num & 0xff); 36278ee8d1cSJulian Grajkowski break; 36378ee8d1cSJulian Grajkowski default: 36478ee8d1cSJulian Grajkowski reg_addr = BAD_REGADDR; 36578ee8d1cSJulian Grajkowski break; 36678ee8d1cSJulian Grajkowski } 36778ee8d1cSJulian Grajkowski return reg_addr; 36878ee8d1cSJulian Grajkowski } 36978ee8d1cSJulian Grajkowski 370a977168cSMichal Gulbicki static u32 371a977168cSMichal Gulbicki qat_hal_get_ae_mask_gen4(struct icp_qat_fw_loader_handle *handle) 372a977168cSMichal Gulbicki { 373a977168cSMichal Gulbicki u32 tg = 0, ae; 374a977168cSMichal Gulbicki u32 valid_ae_mask = 0; 375a977168cSMichal Gulbicki 376a977168cSMichal Gulbicki for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { 377a977168cSMichal Gulbicki if (handle->hal_handle->ae_mask & (1 << ae)) { 378a977168cSMichal Gulbicki tg = ae / 4; 379a977168cSMichal Gulbicki valid_ae_mask |= (1 << (tg * 2)); 380a977168cSMichal Gulbicki } 381a977168cSMichal Gulbicki } 382a977168cSMichal Gulbicki return valid_ae_mask; 383a977168cSMichal Gulbicki } 384a977168cSMichal Gulbicki 38578ee8d1cSJulian Grajkowski void 38678ee8d1cSJulian Grajkowski qat_hal_reset(struct icp_qat_fw_loader_handle *handle) 38778ee8d1cSJulian Grajkowski { 38878ee8d1cSJulian Grajkowski unsigned int ae_reset_csr[MAX_CPP_NUM]; 38978ee8d1cSJulian Grajkowski unsigned int ae_reset_val[MAX_CPP_NUM]; 39078ee8d1cSJulian Grajkowski unsigned int valid_ae_mask, valid_slice_mask; 39178ee8d1cSJulian Grajkowski unsigned int cpp_num = 1; 39278ee8d1cSJulian Grajkowski unsigned int i; 39378ee8d1cSJulian Grajkowski 39478ee8d1cSJulian Grajkowski if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { 39578ee8d1cSJulian Grajkowski ae_reset_csr[0] = ICP_RESET_CPP0; 39678ee8d1cSJulian Grajkowski ae_reset_csr[1] = ICP_RESET_CPP1; 39778ee8d1cSJulian Grajkowski if (handle->hal_handle->ae_mask > 0xffff) 39878ee8d1cSJulian Grajkowski ++cpp_num; 399a977168cSMichal Gulbicki } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 400a977168cSMichal Gulbicki ae_reset_csr[0] = ICP_RESET_CPP0; 40178ee8d1cSJulian Grajkowski } else { 40278ee8d1cSJulian Grajkowski ae_reset_csr[0] = ICP_RESET; 40378ee8d1cSJulian Grajkowski } 40478ee8d1cSJulian Grajkowski 40578ee8d1cSJulian Grajkowski for (i = 0; i < cpp_num; i++) { 40678ee8d1cSJulian Grajkowski if (i == 0) { 407a977168cSMichal Gulbicki if (IS_QAT_GEN4( 408a977168cSMichal Gulbicki pci_get_device(GET_DEV(handle->accel_dev)))) { 409a977168cSMichal Gulbicki valid_ae_mask = 410a977168cSMichal Gulbicki qat_hal_get_ae_mask_gen4(handle); 411a977168cSMichal Gulbicki valid_slice_mask = 412a977168cSMichal Gulbicki handle->hal_handle->slice_mask; 413a977168cSMichal Gulbicki } else { 414a977168cSMichal Gulbicki valid_ae_mask = 415a977168cSMichal Gulbicki handle->hal_handle->ae_mask & 0xFFFF; 41678ee8d1cSJulian Grajkowski valid_slice_mask = 41778ee8d1cSJulian Grajkowski handle->hal_handle->slice_mask & 0x3F; 418a977168cSMichal Gulbicki } 41978ee8d1cSJulian Grajkowski } else { 42078ee8d1cSJulian Grajkowski valid_ae_mask = 42178ee8d1cSJulian Grajkowski (handle->hal_handle->ae_mask >> AES_PER_CPP) & 42278ee8d1cSJulian Grajkowski 0xFFFF; 42378ee8d1cSJulian Grajkowski valid_slice_mask = 42478ee8d1cSJulian Grajkowski (handle->hal_handle->slice_mask >> SLICES_PER_CPP) & 42578ee8d1cSJulian Grajkowski 0x3F; 42678ee8d1cSJulian Grajkowski } 42778ee8d1cSJulian Grajkowski 42878ee8d1cSJulian Grajkowski ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]); 42978ee8d1cSJulian Grajkowski ae_reset_val[i] |= valid_ae_mask << RST_CSR_AE_LSB; 43078ee8d1cSJulian Grajkowski ae_reset_val[i] |= valid_slice_mask << RST_CSR_QAT_LSB; 43178ee8d1cSJulian Grajkowski SET_GLB_CSR(handle, ae_reset_csr[i], ae_reset_val[i]); 43278ee8d1cSJulian Grajkowski } 43378ee8d1cSJulian Grajkowski } 43478ee8d1cSJulian Grajkowski 43578ee8d1cSJulian Grajkowski static void 43678ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, 43778ee8d1cSJulian Grajkowski unsigned char ae, 43878ee8d1cSJulian Grajkowski unsigned int ctx_mask, 43978ee8d1cSJulian Grajkowski unsigned int ae_csr, 44078ee8d1cSJulian Grajkowski unsigned int csr_val) 44178ee8d1cSJulian Grajkowski { 44278ee8d1cSJulian Grajkowski unsigned int ctx, cur_ctx; 44378ee8d1cSJulian Grajkowski 44478ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); 44578ee8d1cSJulian Grajkowski 44678ee8d1cSJulian Grajkowski for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { 44778ee8d1cSJulian Grajkowski if (!(ctx_mask & (1 << ctx))) 44878ee8d1cSJulian Grajkowski continue; 44978ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 45078ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val); 45178ee8d1cSJulian Grajkowski } 45278ee8d1cSJulian Grajkowski 45378ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 45478ee8d1cSJulian Grajkowski } 45578ee8d1cSJulian Grajkowski 45678ee8d1cSJulian Grajkowski static void 45778ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle, 45878ee8d1cSJulian Grajkowski unsigned char ae, 45978ee8d1cSJulian Grajkowski unsigned char ctx, 46078ee8d1cSJulian Grajkowski unsigned int ae_csr, 46178ee8d1cSJulian Grajkowski unsigned int *csr_val) 46278ee8d1cSJulian Grajkowski { 46378ee8d1cSJulian Grajkowski unsigned int cur_ctx; 46478ee8d1cSJulian Grajkowski 46578ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); 46678ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 46778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val); 46878ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 46978ee8d1cSJulian Grajkowski } 47078ee8d1cSJulian Grajkowski 47178ee8d1cSJulian Grajkowski static void 47278ee8d1cSJulian Grajkowski qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle, 47378ee8d1cSJulian Grajkowski unsigned char ae, 47478ee8d1cSJulian Grajkowski unsigned int ctx_mask, 47578ee8d1cSJulian Grajkowski unsigned int events) 47678ee8d1cSJulian Grajkowski { 47778ee8d1cSJulian Grajkowski unsigned int ctx, cur_ctx; 47878ee8d1cSJulian Grajkowski 47978ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); 48078ee8d1cSJulian Grajkowski for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { 48178ee8d1cSJulian Grajkowski if (!(ctx_mask & (1 << ctx))) 48278ee8d1cSJulian Grajkowski continue; 48378ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 48478ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events); 48578ee8d1cSJulian Grajkowski } 48678ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 48778ee8d1cSJulian Grajkowski } 48878ee8d1cSJulian Grajkowski 48978ee8d1cSJulian Grajkowski static void 49078ee8d1cSJulian Grajkowski qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, 49178ee8d1cSJulian Grajkowski unsigned char ae, 49278ee8d1cSJulian Grajkowski unsigned int ctx_mask, 49378ee8d1cSJulian Grajkowski unsigned int events) 49478ee8d1cSJulian Grajkowski { 49578ee8d1cSJulian Grajkowski unsigned int ctx, cur_ctx; 49678ee8d1cSJulian Grajkowski 49778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); 49878ee8d1cSJulian Grajkowski for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { 49978ee8d1cSJulian Grajkowski if (!(ctx_mask & (1 << ctx))) 50078ee8d1cSJulian Grajkowski continue; 50178ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); 50278ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, 50378ee8d1cSJulian Grajkowski ae, 50478ee8d1cSJulian Grajkowski CTX_WAKEUP_EVENTS_INDIRECT, 50578ee8d1cSJulian Grajkowski events); 50678ee8d1cSJulian Grajkowski } 50778ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); 50878ee8d1cSJulian Grajkowski } 50978ee8d1cSJulian Grajkowski 51078ee8d1cSJulian Grajkowski static int 51178ee8d1cSJulian Grajkowski qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) 51278ee8d1cSJulian Grajkowski { 51378ee8d1cSJulian Grajkowski unsigned int base_cnt, cur_cnt; 51478ee8d1cSJulian Grajkowski unsigned char ae; 51578ee8d1cSJulian Grajkowski unsigned long ae_mask = handle->hal_handle->ae_mask; 51678ee8d1cSJulian Grajkowski int times = MAX_RETRY_TIMES; 51778ee8d1cSJulian Grajkowski 51878ee8d1cSJulian Grajkowski for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 51978ee8d1cSJulian Grajkowski { 52078ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, 52178ee8d1cSJulian Grajkowski ae, 52278ee8d1cSJulian Grajkowski PROFILE_COUNT, 52378ee8d1cSJulian Grajkowski (unsigned int *)&base_cnt); 52478ee8d1cSJulian Grajkowski base_cnt &= 0xffff; 52578ee8d1cSJulian Grajkowski 52678ee8d1cSJulian Grajkowski do { 52778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, 52878ee8d1cSJulian Grajkowski ae, 52978ee8d1cSJulian Grajkowski PROFILE_COUNT, 53078ee8d1cSJulian Grajkowski (unsigned int *)&cur_cnt); 53178ee8d1cSJulian Grajkowski cur_cnt &= 0xffff; 53278ee8d1cSJulian Grajkowski } while (times-- && (cur_cnt == base_cnt)); 53378ee8d1cSJulian Grajkowski 53478ee8d1cSJulian Grajkowski if (times < 0) { 53578ee8d1cSJulian Grajkowski pr_err("QAT: AE%d is inactive!!\n", ae); 53678ee8d1cSJulian Grajkowski return EFAULT; 53778ee8d1cSJulian Grajkowski } 53878ee8d1cSJulian Grajkowski } 53978ee8d1cSJulian Grajkowski 54078ee8d1cSJulian Grajkowski return 0; 54178ee8d1cSJulian Grajkowski } 54278ee8d1cSJulian Grajkowski 54378ee8d1cSJulian Grajkowski int 54478ee8d1cSJulian Grajkowski qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, 54578ee8d1cSJulian Grajkowski unsigned int ae) 54678ee8d1cSJulian Grajkowski { 54778ee8d1cSJulian Grajkowski unsigned int enable = 0, active = 0; 54878ee8d1cSJulian Grajkowski 54978ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &enable); 55078ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &active); 55178ee8d1cSJulian Grajkowski if ((enable & (0xff << CE_ENABLE_BITPOS)) || 55278ee8d1cSJulian Grajkowski (active & (1 << ACS_ABO_BITPOS))) 55378ee8d1cSJulian Grajkowski return 1; 55478ee8d1cSJulian Grajkowski else 55578ee8d1cSJulian Grajkowski return 0; 55678ee8d1cSJulian Grajkowski } 55778ee8d1cSJulian Grajkowski 55878ee8d1cSJulian Grajkowski static void 55978ee8d1cSJulian Grajkowski qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) 56078ee8d1cSJulian Grajkowski { 56178ee8d1cSJulian Grajkowski unsigned int misc_ctl_csr, misc_ctl; 56278ee8d1cSJulian Grajkowski unsigned char ae; 56378ee8d1cSJulian Grajkowski unsigned long ae_mask = handle->hal_handle->ae_mask; 56478ee8d1cSJulian Grajkowski 56578ee8d1cSJulian Grajkowski misc_ctl_csr = 566a977168cSMichal Gulbicki (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) ? 56778ee8d1cSJulian Grajkowski MISC_CONTROL_C4XXX : 56878ee8d1cSJulian Grajkowski MISC_CONTROL; 56978ee8d1cSJulian Grajkowski /* stop the timestamp timers */ 57078ee8d1cSJulian Grajkowski misc_ctl = GET_GLB_CSR(handle, misc_ctl_csr); 57178ee8d1cSJulian Grajkowski if (misc_ctl & MC_TIMESTAMP_ENABLE) 57278ee8d1cSJulian Grajkowski SET_GLB_CSR(handle, 57378ee8d1cSJulian Grajkowski misc_ctl_csr, 57478ee8d1cSJulian Grajkowski misc_ctl & (~MC_TIMESTAMP_ENABLE)); 57578ee8d1cSJulian Grajkowski 57678ee8d1cSJulian Grajkowski for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 57778ee8d1cSJulian Grajkowski { 57878ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); 57978ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); 58078ee8d1cSJulian Grajkowski } 58178ee8d1cSJulian Grajkowski /* start timestamp timers */ 58278ee8d1cSJulian Grajkowski SET_GLB_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE); 58378ee8d1cSJulian Grajkowski } 58478ee8d1cSJulian Grajkowski 58578ee8d1cSJulian Grajkowski #define ESRAM_AUTO_TINIT BIT(2) 58678ee8d1cSJulian Grajkowski #define ESRAM_AUTO_TINIT_DONE BIT(3) 58778ee8d1cSJulian Grajkowski #define ESRAM_AUTO_INIT_USED_CYCLES (1640) 58878ee8d1cSJulian Grajkowski #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C 58978ee8d1cSJulian Grajkowski 59078ee8d1cSJulian Grajkowski static int 59178ee8d1cSJulian Grajkowski qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) 59278ee8d1cSJulian Grajkowski { 59378ee8d1cSJulian Grajkowski uintptr_t csr_addr = 59478ee8d1cSJulian Grajkowski ((uintptr_t)handle->hal_ep_csr_addr_v + ESRAM_AUTO_INIT_CSR_OFFSET); 59578ee8d1cSJulian Grajkowski unsigned int csr_val; 59678ee8d1cSJulian Grajkowski int times = 30; 59778ee8d1cSJulian Grajkowski 59878ee8d1cSJulian Grajkowski if (pci_get_device(GET_DEV(handle->accel_dev)) != 59978ee8d1cSJulian Grajkowski ADF_DH895XCC_PCI_DEVICE_ID) 60078ee8d1cSJulian Grajkowski return 0; 60178ee8d1cSJulian Grajkowski 60278ee8d1cSJulian Grajkowski csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr); 60378ee8d1cSJulian Grajkowski if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE)) 60478ee8d1cSJulian Grajkowski return 0; 60578ee8d1cSJulian Grajkowski csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr); 60678ee8d1cSJulian Grajkowski csr_val |= ESRAM_AUTO_TINIT; 60778ee8d1cSJulian Grajkowski 60878ee8d1cSJulian Grajkowski ADF_CSR_WR(handle->hal_misc_addr_v, csr_addr, csr_val); 60978ee8d1cSJulian Grajkowski do { 61078ee8d1cSJulian Grajkowski qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); 61178ee8d1cSJulian Grajkowski csr_val = ADF_CSR_RD(handle->hal_misc_addr_v, csr_addr); 61278ee8d1cSJulian Grajkowski 61378ee8d1cSJulian Grajkowski } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--); 61478ee8d1cSJulian Grajkowski if (times < 0) { 61578ee8d1cSJulian Grajkowski pr_err("QAT: Fail to init eSram!\n"); 61678ee8d1cSJulian Grajkowski return EFAULT; 61778ee8d1cSJulian Grajkowski } 61878ee8d1cSJulian Grajkowski return 0; 61978ee8d1cSJulian Grajkowski } 62078ee8d1cSJulian Grajkowski 62178ee8d1cSJulian Grajkowski #define SHRAM_INIT_CYCLES 2060 62278ee8d1cSJulian Grajkowski int 62378ee8d1cSJulian Grajkowski qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) 62478ee8d1cSJulian Grajkowski { 62578ee8d1cSJulian Grajkowski unsigned int ae_reset_csr[MAX_CPP_NUM]; 62678ee8d1cSJulian Grajkowski unsigned int ae_reset_val[MAX_CPP_NUM]; 62778ee8d1cSJulian Grajkowski unsigned int cpp_num = 1; 62878ee8d1cSJulian Grajkowski unsigned int valid_ae_mask, valid_slice_mask; 62978ee8d1cSJulian Grajkowski unsigned char ae; 63078ee8d1cSJulian Grajkowski unsigned int i; 63178ee8d1cSJulian Grajkowski unsigned int clk_csr[MAX_CPP_NUM]; 63278ee8d1cSJulian Grajkowski unsigned int clk_val[MAX_CPP_NUM]; 63378ee8d1cSJulian Grajkowski unsigned int times = 100; 63478ee8d1cSJulian Grajkowski unsigned long ae_mask = handle->hal_handle->ae_mask; 63578ee8d1cSJulian Grajkowski 63678ee8d1cSJulian Grajkowski if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { 63778ee8d1cSJulian Grajkowski ae_reset_csr[0] = ICP_RESET_CPP0; 63878ee8d1cSJulian Grajkowski ae_reset_csr[1] = ICP_RESET_CPP1; 63978ee8d1cSJulian Grajkowski clk_csr[0] = ICP_GLOBAL_CLK_ENABLE_CPP0; 64078ee8d1cSJulian Grajkowski clk_csr[1] = ICP_GLOBAL_CLK_ENABLE_CPP1; 64178ee8d1cSJulian Grajkowski if (handle->hal_handle->ae_mask > 0xffff) 64278ee8d1cSJulian Grajkowski ++cpp_num; 643a977168cSMichal Gulbicki } else if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 644a977168cSMichal Gulbicki ae_reset_csr[0] = ICP_RESET_CPP0; 645a977168cSMichal Gulbicki clk_csr[0] = ICP_GLOBAL_CLK_ENABLE_CPP0; 64678ee8d1cSJulian Grajkowski } else { 64778ee8d1cSJulian Grajkowski ae_reset_csr[0] = ICP_RESET; 64878ee8d1cSJulian Grajkowski clk_csr[0] = ICP_GLOBAL_CLK_ENABLE; 64978ee8d1cSJulian Grajkowski } 65078ee8d1cSJulian Grajkowski 65178ee8d1cSJulian Grajkowski for (i = 0; i < cpp_num; i++) { 65278ee8d1cSJulian Grajkowski if (i == 0) { 653a977168cSMichal Gulbicki if (IS_QAT_GEN4( 654a977168cSMichal Gulbicki pci_get_device(GET_DEV(handle->accel_dev)))) { 655a977168cSMichal Gulbicki valid_ae_mask = 656a977168cSMichal Gulbicki qat_hal_get_ae_mask_gen4(handle); 657a977168cSMichal Gulbicki valid_slice_mask = 658a977168cSMichal Gulbicki handle->hal_handle->slice_mask; 659a977168cSMichal Gulbicki } else { 660a977168cSMichal Gulbicki valid_ae_mask = 661a977168cSMichal Gulbicki handle->hal_handle->ae_mask & 0xFFFF; 66278ee8d1cSJulian Grajkowski valid_slice_mask = 66378ee8d1cSJulian Grajkowski handle->hal_handle->slice_mask & 0x3F; 664a977168cSMichal Gulbicki } 66578ee8d1cSJulian Grajkowski } else { 66678ee8d1cSJulian Grajkowski valid_ae_mask = 66778ee8d1cSJulian Grajkowski (handle->hal_handle->ae_mask >> AES_PER_CPP) & 66878ee8d1cSJulian Grajkowski 0xFFFF; 66978ee8d1cSJulian Grajkowski valid_slice_mask = 67078ee8d1cSJulian Grajkowski (handle->hal_handle->slice_mask >> SLICES_PER_CPP) & 67178ee8d1cSJulian Grajkowski 0x3F; 67278ee8d1cSJulian Grajkowski } 67378ee8d1cSJulian Grajkowski /* write to the reset csr */ 67478ee8d1cSJulian Grajkowski ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]); 67578ee8d1cSJulian Grajkowski ae_reset_val[i] &= ~(valid_ae_mask << RST_CSR_AE_LSB); 67678ee8d1cSJulian Grajkowski ae_reset_val[i] &= ~(valid_slice_mask << RST_CSR_QAT_LSB); 67778ee8d1cSJulian Grajkowski do { 67878ee8d1cSJulian Grajkowski SET_GLB_CSR(handle, ae_reset_csr[i], ae_reset_val[i]); 67978ee8d1cSJulian Grajkowski if (!(times--)) 68078ee8d1cSJulian Grajkowski goto out_err; 68178ee8d1cSJulian Grajkowski ae_reset_val[i] = GET_GLB_CSR(handle, ae_reset_csr[i]); 68278ee8d1cSJulian Grajkowski } while ( 68378ee8d1cSJulian Grajkowski (valid_ae_mask | (valid_slice_mask << RST_CSR_QAT_LSB)) & 68478ee8d1cSJulian Grajkowski ae_reset_val[i]); 68578ee8d1cSJulian Grajkowski /* enable clock */ 68678ee8d1cSJulian Grajkowski clk_val[i] = GET_GLB_CSR(handle, clk_csr[i]); 68778ee8d1cSJulian Grajkowski clk_val[i] |= valid_ae_mask << 0; 68878ee8d1cSJulian Grajkowski clk_val[i] |= valid_slice_mask << 20; 68978ee8d1cSJulian Grajkowski SET_GLB_CSR(handle, clk_csr[i], clk_val[i]); 69078ee8d1cSJulian Grajkowski } 69178ee8d1cSJulian Grajkowski if (qat_hal_check_ae_alive(handle)) 69278ee8d1cSJulian Grajkowski goto out_err; 69378ee8d1cSJulian Grajkowski 69478ee8d1cSJulian Grajkowski /* Set undefined power-up/reset states to reasonable default values */ 69578ee8d1cSJulian Grajkowski for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 69678ee8d1cSJulian Grajkowski { 69778ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, 69878ee8d1cSJulian Grajkowski ae, 69978ee8d1cSJulian Grajkowski CTX_ENABLES, 70078ee8d1cSJulian Grajkowski INIT_CTX_ENABLE_VALUE); 70178ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, 70278ee8d1cSJulian Grajkowski ae, 70378ee8d1cSJulian Grajkowski ICP_QAT_UCLO_AE_ALL_CTX, 70478ee8d1cSJulian Grajkowski CTX_STS_INDIRECT, 70578ee8d1cSJulian Grajkowski handle->hal_handle->upc_mask & 70678ee8d1cSJulian Grajkowski INIT_PC_VALUE); 70778ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); 70878ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); 70978ee8d1cSJulian Grajkowski qat_hal_put_wakeup_event(handle, 71078ee8d1cSJulian Grajkowski ae, 71178ee8d1cSJulian Grajkowski ICP_QAT_UCLO_AE_ALL_CTX, 71278ee8d1cSJulian Grajkowski INIT_WAKEUP_EVENTS_VALUE); 71378ee8d1cSJulian Grajkowski qat_hal_put_sig_event(handle, 71478ee8d1cSJulian Grajkowski ae, 71578ee8d1cSJulian Grajkowski ICP_QAT_UCLO_AE_ALL_CTX, 71678ee8d1cSJulian Grajkowski INIT_SIG_EVENTS_VALUE); 71778ee8d1cSJulian Grajkowski } 71878ee8d1cSJulian Grajkowski if (qat_hal_init_esram(handle)) 71978ee8d1cSJulian Grajkowski goto out_err; 72078ee8d1cSJulian Grajkowski if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0)) 72178ee8d1cSJulian Grajkowski goto out_err; 72278ee8d1cSJulian Grajkowski qat_hal_reset_timestamp(handle); 72378ee8d1cSJulian Grajkowski 72478ee8d1cSJulian Grajkowski return 0; 72578ee8d1cSJulian Grajkowski out_err: 72678ee8d1cSJulian Grajkowski pr_err("QAT: failed to get device out of reset\n"); 72778ee8d1cSJulian Grajkowski return EFAULT; 72878ee8d1cSJulian Grajkowski } 72978ee8d1cSJulian Grajkowski 73078ee8d1cSJulian Grajkowski static void 73178ee8d1cSJulian Grajkowski qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle, 73278ee8d1cSJulian Grajkowski unsigned char ae, 73378ee8d1cSJulian Grajkowski unsigned int ctx_mask) 73478ee8d1cSJulian Grajkowski { 73578ee8d1cSJulian Grajkowski unsigned int ctx; 73678ee8d1cSJulian Grajkowski 73778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx); 73878ee8d1cSJulian Grajkowski ctx &= IGNORE_W1C_MASK & 73978ee8d1cSJulian Grajkowski (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS)); 74078ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); 74178ee8d1cSJulian Grajkowski } 74278ee8d1cSJulian Grajkowski 74378ee8d1cSJulian Grajkowski static uint64_t 74478ee8d1cSJulian Grajkowski qat_hal_parity_64bit(uint64_t word) 74578ee8d1cSJulian Grajkowski { 74678ee8d1cSJulian Grajkowski word ^= word >> 1; 74778ee8d1cSJulian Grajkowski word ^= word >> 2; 74878ee8d1cSJulian Grajkowski word ^= word >> 4; 74978ee8d1cSJulian Grajkowski word ^= word >> 8; 75078ee8d1cSJulian Grajkowski word ^= word >> 16; 75178ee8d1cSJulian Grajkowski word ^= word >> 32; 75278ee8d1cSJulian Grajkowski return word & 1; 75378ee8d1cSJulian Grajkowski } 75478ee8d1cSJulian Grajkowski 75578ee8d1cSJulian Grajkowski static uint64_t 75678ee8d1cSJulian Grajkowski qat_hal_set_uword_ecc(uint64_t uword) 75778ee8d1cSJulian Grajkowski { 75878ee8d1cSJulian Grajkowski uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL, 75978ee8d1cSJulian Grajkowski bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL, 76078ee8d1cSJulian Grajkowski bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL, 76178ee8d1cSJulian Grajkowski bit6_mask = 0xdaf69a46910ULL; 76278ee8d1cSJulian Grajkowski 76378ee8d1cSJulian Grajkowski /* clear the ecc bits */ 76478ee8d1cSJulian Grajkowski uword &= ~(0x7fULL << 0x2C); 76578ee8d1cSJulian Grajkowski uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C; 76678ee8d1cSJulian Grajkowski uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D; 76778ee8d1cSJulian Grajkowski uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E; 76878ee8d1cSJulian Grajkowski uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F; 76978ee8d1cSJulian Grajkowski uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30; 77078ee8d1cSJulian Grajkowski uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31; 77178ee8d1cSJulian Grajkowski uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32; 77278ee8d1cSJulian Grajkowski return uword; 77378ee8d1cSJulian Grajkowski } 77478ee8d1cSJulian Grajkowski 77578ee8d1cSJulian Grajkowski void 77678ee8d1cSJulian Grajkowski qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, 77778ee8d1cSJulian Grajkowski unsigned char ae, 77878ee8d1cSJulian Grajkowski unsigned int uaddr, 77978ee8d1cSJulian Grajkowski unsigned int words_num, 78078ee8d1cSJulian Grajkowski const uint64_t *uword) 78178ee8d1cSJulian Grajkowski { 78278ee8d1cSJulian Grajkowski unsigned int ustore_addr; 783a977168cSMichal Gulbicki unsigned int i, ae_in_group; 784a977168cSMichal Gulbicki 785a977168cSMichal Gulbicki if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 786a977168cSMichal Gulbicki ae_in_group = ae / 4 * 4; 787a977168cSMichal Gulbicki 788a977168cSMichal Gulbicki for (i = 0; i < AE_TG_NUM_CPM2X; i++) { 789a977168cSMichal Gulbicki if (ae_in_group + i == ae) 790a977168cSMichal Gulbicki continue; 791a977168cSMichal Gulbicki if (ae_in_group + i >= handle->hal_handle->ae_max_num) 792a977168cSMichal Gulbicki break; 793a977168cSMichal Gulbicki if (qat_hal_check_ae_active(handle, ae_in_group + i)) { 794a977168cSMichal Gulbicki pr_err( 795a977168cSMichal Gulbicki "ae%d in T_group is active, cannot write to ustore!\n", 796a977168cSMichal Gulbicki ae_in_group + i); 797a977168cSMichal Gulbicki return; 798a977168cSMichal Gulbicki } 799a977168cSMichal Gulbicki } 800a977168cSMichal Gulbicki } 80178ee8d1cSJulian Grajkowski 80278ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); 80378ee8d1cSJulian Grajkowski uaddr |= UA_ECS; 80478ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 80578ee8d1cSJulian Grajkowski for (i = 0; i < words_num; i++) { 80678ee8d1cSJulian Grajkowski unsigned int uwrd_lo, uwrd_hi; 80778ee8d1cSJulian Grajkowski uint64_t tmp; 80878ee8d1cSJulian Grajkowski 80978ee8d1cSJulian Grajkowski tmp = qat_hal_set_uword_ecc(uword[i]); 81078ee8d1cSJulian Grajkowski uwrd_lo = (unsigned int)(tmp & 0xffffffff); 81178ee8d1cSJulian Grajkowski uwrd_hi = (unsigned int)(tmp >> 0x20); 81278ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); 81378ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); 81478ee8d1cSJulian Grajkowski } 81578ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 81678ee8d1cSJulian Grajkowski } 81778ee8d1cSJulian Grajkowski 81878ee8d1cSJulian Grajkowski void 81978ee8d1cSJulian Grajkowski qat_hal_wr_coalesce_uwords(struct icp_qat_fw_loader_handle *handle, 82078ee8d1cSJulian Grajkowski unsigned char ae, 82178ee8d1cSJulian Grajkowski unsigned int uaddr, 82278ee8d1cSJulian Grajkowski unsigned int words_num, 82378ee8d1cSJulian Grajkowski u64 *uword) 82478ee8d1cSJulian Grajkowski { 82578ee8d1cSJulian Grajkowski u64 *even_uwrods, *odd_uwords; 82678ee8d1cSJulian Grajkowski unsigned char neigh_ae, odd_ae, even_ae; 82778ee8d1cSJulian Grajkowski int i, even_cpy_cnt = 0, odd_cpy_cnt = 0; 82878ee8d1cSJulian Grajkowski 82978ee8d1cSJulian Grajkowski even_uwrods = 83078ee8d1cSJulian Grajkowski malloc(16 * 1024 * sizeof(*uword), M_QAT, M_WAITOK | M_ZERO); 83178ee8d1cSJulian Grajkowski odd_uwords = 83278ee8d1cSJulian Grajkowski malloc(16 * 1024 * sizeof(*uword), M_QAT, M_WAITOK | M_ZERO); 83378ee8d1cSJulian Grajkowski qat_hal_get_scs_neigh_ae(ae, &neigh_ae); 83478ee8d1cSJulian Grajkowski if (ae & 1) { 83578ee8d1cSJulian Grajkowski odd_ae = ae; 83678ee8d1cSJulian Grajkowski even_ae = neigh_ae; 83778ee8d1cSJulian Grajkowski } else { 83878ee8d1cSJulian Grajkowski odd_ae = neigh_ae; 83978ee8d1cSJulian Grajkowski even_ae = ae; 84078ee8d1cSJulian Grajkowski } 84178ee8d1cSJulian Grajkowski for (i = 0; i < words_num; i++) { 84278ee8d1cSJulian Grajkowski if ((uaddr + i) & 1) 84378ee8d1cSJulian Grajkowski odd_uwords[odd_cpy_cnt++] = uword[i]; 84478ee8d1cSJulian Grajkowski else 84578ee8d1cSJulian Grajkowski even_uwrods[even_cpy_cnt++] = uword[i]; 84678ee8d1cSJulian Grajkowski } 84778ee8d1cSJulian Grajkowski if (even_cpy_cnt) 84878ee8d1cSJulian Grajkowski qat_hal_wr_uwords(handle, 84978ee8d1cSJulian Grajkowski even_ae, 85078ee8d1cSJulian Grajkowski (uaddr + 1) / 2, 85178ee8d1cSJulian Grajkowski even_cpy_cnt, 85278ee8d1cSJulian Grajkowski even_uwrods); 85378ee8d1cSJulian Grajkowski if (odd_cpy_cnt) 85478ee8d1cSJulian Grajkowski qat_hal_wr_uwords( 85578ee8d1cSJulian Grajkowski handle, odd_ae, uaddr / 2, odd_cpy_cnt, odd_uwords); 85678ee8d1cSJulian Grajkowski free(even_uwrods, M_QAT); 85778ee8d1cSJulian Grajkowski free(odd_uwords, M_QAT); 85878ee8d1cSJulian Grajkowski } 85978ee8d1cSJulian Grajkowski 86078ee8d1cSJulian Grajkowski static void 86178ee8d1cSJulian Grajkowski qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, 86278ee8d1cSJulian Grajkowski unsigned char ae, 86378ee8d1cSJulian Grajkowski unsigned int ctx_mask) 86478ee8d1cSJulian Grajkowski { 86578ee8d1cSJulian Grajkowski unsigned int ctx; 86678ee8d1cSJulian Grajkowski 86778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx); 86878ee8d1cSJulian Grajkowski ctx &= IGNORE_W1C_MASK; 86978ee8d1cSJulian Grajkowski ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF; 87078ee8d1cSJulian Grajkowski ctx |= (ctx_mask << CE_ENABLE_BITPOS); 87178ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); 87278ee8d1cSJulian Grajkowski } 87378ee8d1cSJulian Grajkowski 87478ee8d1cSJulian Grajkowski static void 87578ee8d1cSJulian Grajkowski qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) 87678ee8d1cSJulian Grajkowski { 87778ee8d1cSJulian Grajkowski unsigned char ae; 87878ee8d1cSJulian Grajkowski unsigned short reg; 87978ee8d1cSJulian Grajkowski unsigned long ae_mask = handle->hal_handle->ae_mask; 88078ee8d1cSJulian Grajkowski 88178ee8d1cSJulian Grajkowski for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 88278ee8d1cSJulian Grajkowski { 88378ee8d1cSJulian Grajkowski for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { 88478ee8d1cSJulian Grajkowski qat_hal_init_rd_xfer( 88578ee8d1cSJulian Grajkowski handle, ae, 0, ICP_SR_RD_ABS, reg, 0); 88678ee8d1cSJulian Grajkowski qat_hal_init_rd_xfer( 88778ee8d1cSJulian Grajkowski handle, ae, 0, ICP_DR_RD_ABS, reg, 0); 88878ee8d1cSJulian Grajkowski } 88978ee8d1cSJulian Grajkowski } 89078ee8d1cSJulian Grajkowski } 89178ee8d1cSJulian Grajkowski 89278ee8d1cSJulian Grajkowski static int 89378ee8d1cSJulian Grajkowski qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) 89478ee8d1cSJulian Grajkowski { 89578ee8d1cSJulian Grajkowski unsigned char ae; 89678ee8d1cSJulian Grajkowski unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; 89778ee8d1cSJulian Grajkowski int times = MAX_RETRY_TIMES; 89878ee8d1cSJulian Grajkowski unsigned int csr_val = 0; 89978ee8d1cSJulian Grajkowski unsigned int savctx = 0; 90078ee8d1cSJulian Grajkowski unsigned int scs_flag = 0; 90178ee8d1cSJulian Grajkowski unsigned long ae_mask = handle->hal_handle->ae_mask; 90278ee8d1cSJulian Grajkowski int ret = 0; 90378ee8d1cSJulian Grajkowski 90478ee8d1cSJulian Grajkowski for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 90578ee8d1cSJulian Grajkowski { 90678ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 90778ee8d1cSJulian Grajkowski scs_flag = csr_val & (1 << MMC_SHARE_CS_BITPOS); 90878ee8d1cSJulian Grajkowski csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); 90978ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); 91078ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val); 91178ee8d1cSJulian Grajkowski csr_val &= IGNORE_W1C_MASK; 912a977168cSMichal Gulbicki if (!IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 91378ee8d1cSJulian Grajkowski csr_val |= CE_NN_MODE; 914a977168cSMichal Gulbicki } 91578ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); 916a977168cSMichal Gulbicki 917a977168cSMichal Gulbicki if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 918a977168cSMichal Gulbicki if (ae % 4 == 0) 919a977168cSMichal Gulbicki qat_hal_wr_uwords(handle, 920a977168cSMichal Gulbicki ae, 921a977168cSMichal Gulbicki 0, 922a977168cSMichal Gulbicki ARRAY_SIZE(inst_CPM2X), 923a977168cSMichal Gulbicki (const uint64_t *)inst_CPM2X); 924a977168cSMichal Gulbicki } else { 925a977168cSMichal Gulbicki qat_hal_wr_uwords(handle, 926a977168cSMichal Gulbicki ae, 927a977168cSMichal Gulbicki 0, 928a977168cSMichal Gulbicki ARRAY_SIZE(inst), 929a977168cSMichal Gulbicki (const uint64_t *)inst); 930a977168cSMichal Gulbicki } 93178ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, 93278ee8d1cSJulian Grajkowski ae, 93378ee8d1cSJulian Grajkowski ctx_mask, 93478ee8d1cSJulian Grajkowski CTX_STS_INDIRECT, 93578ee8d1cSJulian Grajkowski handle->hal_handle->upc_mask & 93678ee8d1cSJulian Grajkowski INIT_PC_VALUE); 93778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); 93878ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0); 93978ee8d1cSJulian Grajkowski qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY); 94078ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr( 94178ee8d1cSJulian Grajkowski handle, ae, ctx_mask, CTX_SIG_EVENTS_INDIRECT, 0); 94278ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); 94378ee8d1cSJulian Grajkowski qat_hal_enable_ctx(handle, ae, ctx_mask); 94478ee8d1cSJulian Grajkowski } 94578ee8d1cSJulian Grajkowski 94678ee8d1cSJulian Grajkowski for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 94778ee8d1cSJulian Grajkowski { 94878ee8d1cSJulian Grajkowski /* wait for AE to finish */ 94978ee8d1cSJulian Grajkowski do { 95078ee8d1cSJulian Grajkowski ret = qat_hal_wait_cycles(handle, ae, 20, 1); 95178ee8d1cSJulian Grajkowski } while (ret && times--); 95278ee8d1cSJulian Grajkowski 95378ee8d1cSJulian Grajkowski if (times < 0) { 95478ee8d1cSJulian Grajkowski pr_err("QAT: clear GPR of AE %d failed", ae); 95578ee8d1cSJulian Grajkowski return EINVAL; 95678ee8d1cSJulian Grajkowski } 95778ee8d1cSJulian Grajkowski qat_hal_disable_ctx(handle, ae, ctx_mask); 95878ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 95978ee8d1cSJulian Grajkowski if (scs_flag) 96078ee8d1cSJulian Grajkowski csr_val |= (1 << MMC_SHARE_CS_BITPOS); 96178ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); 96278ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, 96378ee8d1cSJulian Grajkowski ae, 96478ee8d1cSJulian Grajkowski ACTIVE_CTX_STATUS, 96578ee8d1cSJulian Grajkowski savctx & ACS_ACNO); 96678ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, 96778ee8d1cSJulian Grajkowski ae, 96878ee8d1cSJulian Grajkowski CTX_ENABLES, 96978ee8d1cSJulian Grajkowski INIT_CTX_ENABLE_VALUE); 97078ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, 97178ee8d1cSJulian Grajkowski ae, 97278ee8d1cSJulian Grajkowski ctx_mask, 97378ee8d1cSJulian Grajkowski CTX_STS_INDIRECT, 97478ee8d1cSJulian Grajkowski handle->hal_handle->upc_mask & 97578ee8d1cSJulian Grajkowski INIT_PC_VALUE); 97678ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); 97778ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); 97878ee8d1cSJulian Grajkowski qat_hal_put_wakeup_event(handle, 97978ee8d1cSJulian Grajkowski ae, 98078ee8d1cSJulian Grajkowski ctx_mask, 98178ee8d1cSJulian Grajkowski INIT_WAKEUP_EVENTS_VALUE); 98278ee8d1cSJulian Grajkowski qat_hal_put_sig_event(handle, 98378ee8d1cSJulian Grajkowski ae, 98478ee8d1cSJulian Grajkowski ctx_mask, 98578ee8d1cSJulian Grajkowski INIT_SIG_EVENTS_VALUE); 98678ee8d1cSJulian Grajkowski } 98778ee8d1cSJulian Grajkowski return 0; 98878ee8d1cSJulian Grajkowski } 98978ee8d1cSJulian Grajkowski 99078ee8d1cSJulian Grajkowski static int 99178ee8d1cSJulian Grajkowski qat_hal_check_imr(struct icp_qat_fw_loader_handle *handle) 99278ee8d1cSJulian Grajkowski { 99378ee8d1cSJulian Grajkowski device_t dev = accel_to_pci_dev(handle->accel_dev); 99478ee8d1cSJulian Grajkowski u8 reg_val = 0; 99578ee8d1cSJulian Grajkowski 99678ee8d1cSJulian Grajkowski if (pci_get_device(GET_DEV(handle->accel_dev)) != 99778ee8d1cSJulian Grajkowski ADF_C3XXX_PCI_DEVICE_ID && 99878ee8d1cSJulian Grajkowski pci_get_device(GET_DEV(handle->accel_dev)) != 99978ee8d1cSJulian Grajkowski ADF_200XX_PCI_DEVICE_ID) 100078ee8d1cSJulian Grajkowski return 0; 100178ee8d1cSJulian Grajkowski 100278ee8d1cSJulian Grajkowski reg_val = pci_read_config(dev, 0x04, 1); 100378ee8d1cSJulian Grajkowski /* 100478ee8d1cSJulian Grajkowski * PCI command register memory bit and rambaseaddr_lo address 100578ee8d1cSJulian Grajkowski * are checked to confirm IMR2 is enabled in BIOS settings 100678ee8d1cSJulian Grajkowski */ 100778ee8d1cSJulian Grajkowski if ((reg_val & 0x2) && GET_FCU_CSR(handle, FCU_RAMBASE_ADDR_LO)) 100878ee8d1cSJulian Grajkowski return 0; 100978ee8d1cSJulian Grajkowski 101078ee8d1cSJulian Grajkowski return EINVAL; 101178ee8d1cSJulian Grajkowski } 101278ee8d1cSJulian Grajkowski 101378ee8d1cSJulian Grajkowski int 101478ee8d1cSJulian Grajkowski qat_hal_init(struct adf_accel_dev *accel_dev) 101578ee8d1cSJulian Grajkowski { 101678ee8d1cSJulian Grajkowski unsigned char ae; 101778ee8d1cSJulian Grajkowski unsigned int cap_offset, ae_offset, ep_offset; 101878ee8d1cSJulian Grajkowski unsigned int sram_offset = 0; 101978ee8d1cSJulian Grajkowski unsigned int max_en_ae_id = 0; 102078ee8d1cSJulian Grajkowski int ret = 0; 102178ee8d1cSJulian Grajkowski unsigned long ae_mask; 102278ee8d1cSJulian Grajkowski struct icp_qat_fw_loader_handle *handle; 102378ee8d1cSJulian Grajkowski if (!accel_dev) { 102478ee8d1cSJulian Grajkowski return EFAULT; 102578ee8d1cSJulian Grajkowski } 102678ee8d1cSJulian Grajkowski struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; 102778ee8d1cSJulian Grajkowski struct adf_hw_device_data *hw_data = accel_dev->hw_device; 102878ee8d1cSJulian Grajkowski struct adf_bar *misc_bar = 102978ee8d1cSJulian Grajkowski &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; 103078ee8d1cSJulian Grajkowski struct adf_bar *sram_bar; 103178ee8d1cSJulian Grajkowski 103278ee8d1cSJulian Grajkowski handle = malloc(sizeof(*handle), M_QAT, M_WAITOK | M_ZERO); 103378ee8d1cSJulian Grajkowski 103478ee8d1cSJulian Grajkowski handle->hal_misc_addr_v = misc_bar->virt_addr; 103578ee8d1cSJulian Grajkowski handle->accel_dev = accel_dev; 103678ee8d1cSJulian Grajkowski if (pci_get_device(GET_DEV(handle->accel_dev)) == 103778ee8d1cSJulian Grajkowski ADF_DH895XCC_PCI_DEVICE_ID || 103878ee8d1cSJulian Grajkowski IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { 103978ee8d1cSJulian Grajkowski sram_bar = 104078ee8d1cSJulian Grajkowski &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; 104178ee8d1cSJulian Grajkowski if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) 104278ee8d1cSJulian Grajkowski sram_offset = 104378ee8d1cSJulian Grajkowski 0x400000 + accel_dev->aram_info->mmp_region_offset; 104478ee8d1cSJulian Grajkowski handle->hal_sram_addr_v = sram_bar->virt_addr; 104578ee8d1cSJulian Grajkowski handle->hal_sram_offset = sram_offset; 104678ee8d1cSJulian Grajkowski handle->hal_sram_size = sram_bar->size; 104778ee8d1cSJulian Grajkowski } 104878ee8d1cSJulian Grajkowski GET_CSR_OFFSET(pci_get_device(GET_DEV(handle->accel_dev)), 104978ee8d1cSJulian Grajkowski cap_offset, 105078ee8d1cSJulian Grajkowski ae_offset, 105178ee8d1cSJulian Grajkowski ep_offset); 105278ee8d1cSJulian Grajkowski handle->hal_cap_g_ctl_csr_addr_v = cap_offset; 105378ee8d1cSJulian Grajkowski handle->hal_cap_ae_xfer_csr_addr_v = ae_offset; 105478ee8d1cSJulian Grajkowski handle->hal_ep_csr_addr_v = ep_offset; 105578ee8d1cSJulian Grajkowski handle->hal_cap_ae_local_csr_addr_v = 105678ee8d1cSJulian Grajkowski ((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + 105778ee8d1cSJulian Grajkowski LOCAL_TO_XFER_REG_OFFSET); 105878ee8d1cSJulian Grajkowski handle->fw_auth = (pci_get_device(GET_DEV(handle->accel_dev)) == 105978ee8d1cSJulian Grajkowski ADF_DH895XCC_PCI_DEVICE_ID) ? 106078ee8d1cSJulian Grajkowski false : 106178ee8d1cSJulian Grajkowski true; 106278ee8d1cSJulian Grajkowski if (handle->fw_auth && qat_hal_check_imr(handle)) { 106378ee8d1cSJulian Grajkowski device_printf(GET_DEV(accel_dev), "IMR2 not enabled in BIOS\n"); 106478ee8d1cSJulian Grajkowski ret = EINVAL; 106578ee8d1cSJulian Grajkowski goto out_hal_handle; 106678ee8d1cSJulian Grajkowski } 106778ee8d1cSJulian Grajkowski 106878ee8d1cSJulian Grajkowski handle->hal_handle = 106978ee8d1cSJulian Grajkowski malloc(sizeof(*handle->hal_handle), M_QAT, M_WAITOK | M_ZERO); 107078ee8d1cSJulian Grajkowski handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; 107178ee8d1cSJulian Grajkowski handle->hal_handle->ae_mask = hw_data->ae_mask; 1072a977168cSMichal Gulbicki handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; 107378ee8d1cSJulian Grajkowski handle->hal_handle->slice_mask = hw_data->accel_mask; 107478ee8d1cSJulian Grajkowski handle->cfg_ae_mask = 0xFFFFFFFF; 107578ee8d1cSJulian Grajkowski /* create AE objects */ 107678ee8d1cSJulian Grajkowski if (IS_QAT_GEN3(pci_get_device(GET_DEV(handle->accel_dev)))) { 107778ee8d1cSJulian Grajkowski handle->hal_handle->upc_mask = 0xffff; 107878ee8d1cSJulian Grajkowski handle->hal_handle->max_ustore = 0x2000; 107978ee8d1cSJulian Grajkowski } else { 108078ee8d1cSJulian Grajkowski handle->hal_handle->upc_mask = 0x1ffff; 108178ee8d1cSJulian Grajkowski handle->hal_handle->max_ustore = 0x4000; 108278ee8d1cSJulian Grajkowski } 108378ee8d1cSJulian Grajkowski 108478ee8d1cSJulian Grajkowski ae_mask = hw_data->ae_mask; 108578ee8d1cSJulian Grajkowski 108678ee8d1cSJulian Grajkowski for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) 108778ee8d1cSJulian Grajkowski { 108878ee8d1cSJulian Grajkowski handle->hal_handle->aes[ae].free_addr = 0; 108978ee8d1cSJulian Grajkowski handle->hal_handle->aes[ae].free_size = 109078ee8d1cSJulian Grajkowski handle->hal_handle->max_ustore; 109178ee8d1cSJulian Grajkowski handle->hal_handle->aes[ae].ustore_size = 109278ee8d1cSJulian Grajkowski handle->hal_handle->max_ustore; 109378ee8d1cSJulian Grajkowski handle->hal_handle->aes[ae].live_ctx_mask = 109478ee8d1cSJulian Grajkowski ICP_QAT_UCLO_AE_ALL_CTX; 109578ee8d1cSJulian Grajkowski max_en_ae_id = ae; 109678ee8d1cSJulian Grajkowski } 109778ee8d1cSJulian Grajkowski handle->hal_handle->ae_max_num = max_en_ae_id + 1; 109878ee8d1cSJulian Grajkowski /* take all AEs out of reset */ 109978ee8d1cSJulian Grajkowski if (qat_hal_clr_reset(handle)) { 110078ee8d1cSJulian Grajkowski device_printf(GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); 110178ee8d1cSJulian Grajkowski ret = EIO; 110278ee8d1cSJulian Grajkowski goto out_err; 110378ee8d1cSJulian Grajkowski } 110478ee8d1cSJulian Grajkowski qat_hal_clear_xfer(handle); 110578ee8d1cSJulian Grajkowski if (!handle->fw_auth) { 110678ee8d1cSJulian Grajkowski if (qat_hal_clear_gpr(handle)) { 110778ee8d1cSJulian Grajkowski ret = EIO; 110878ee8d1cSJulian Grajkowski goto out_err; 110978ee8d1cSJulian Grajkowski } 111078ee8d1cSJulian Grajkowski } 111178ee8d1cSJulian Grajkowski 111278ee8d1cSJulian Grajkowski /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ 111378ee8d1cSJulian Grajkowski for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 111478ee8d1cSJulian Grajkowski { 111578ee8d1cSJulian Grajkowski unsigned int csr_val = 0; 111678ee8d1cSJulian Grajkowski 111778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val); 111878ee8d1cSJulian Grajkowski csr_val |= 0x1; 111978ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); 112078ee8d1cSJulian Grajkowski } 112178ee8d1cSJulian Grajkowski accel_dev->fw_loader->fw_loader = handle; 112278ee8d1cSJulian Grajkowski return 0; 112378ee8d1cSJulian Grajkowski 112478ee8d1cSJulian Grajkowski out_err: 112578ee8d1cSJulian Grajkowski free(handle->hal_handle, M_QAT); 112678ee8d1cSJulian Grajkowski out_hal_handle: 112778ee8d1cSJulian Grajkowski free(handle, M_QAT); 112878ee8d1cSJulian Grajkowski return ret; 112978ee8d1cSJulian Grajkowski } 113078ee8d1cSJulian Grajkowski 113178ee8d1cSJulian Grajkowski void 113278ee8d1cSJulian Grajkowski qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) 113378ee8d1cSJulian Grajkowski { 113478ee8d1cSJulian Grajkowski if (!handle) 113578ee8d1cSJulian Grajkowski return; 113678ee8d1cSJulian Grajkowski free(handle->hal_handle, M_QAT); 113778ee8d1cSJulian Grajkowski free(handle, M_QAT); 113878ee8d1cSJulian Grajkowski } 113978ee8d1cSJulian Grajkowski 1140a977168cSMichal Gulbicki int 1141a977168cSMichal Gulbicki qat_hal_start(struct icp_qat_fw_loader_handle *handle) 114278ee8d1cSJulian Grajkowski { 1143a977168cSMichal Gulbicki unsigned char ae = 0; 114478ee8d1cSJulian Grajkowski int retry = 0; 114578ee8d1cSJulian Grajkowski unsigned int fcu_sts = 0; 114678ee8d1cSJulian Grajkowski unsigned int fcu_ctl_csr, fcu_sts_csr; 1147a977168cSMichal Gulbicki unsigned long ae_mask = handle->hal_handle->ae_mask; 1148a977168cSMichal Gulbicki u32 ae_ctr = 0; 114978ee8d1cSJulian Grajkowski 115078ee8d1cSJulian Grajkowski if (handle->fw_auth) { 1151a977168cSMichal Gulbicki for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 1152a977168cSMichal Gulbicki { 1153a977168cSMichal Gulbicki ae_ctr++; 1154a977168cSMichal Gulbicki } 1155a977168cSMichal Gulbicki if (IS_QAT_GEN3_OR_GEN4( 1156a977168cSMichal Gulbicki pci_get_device(GET_DEV(handle->accel_dev)))) { 115778ee8d1cSJulian Grajkowski fcu_ctl_csr = FCU_CONTROL_C4XXX; 115878ee8d1cSJulian Grajkowski fcu_sts_csr = FCU_STATUS_C4XXX; 115978ee8d1cSJulian Grajkowski 116078ee8d1cSJulian Grajkowski } else { 116178ee8d1cSJulian Grajkowski fcu_ctl_csr = FCU_CONTROL; 116278ee8d1cSJulian Grajkowski fcu_sts_csr = FCU_STATUS; 116378ee8d1cSJulian Grajkowski } 116478ee8d1cSJulian Grajkowski SET_FCU_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START); 116578ee8d1cSJulian Grajkowski do { 116678ee8d1cSJulian Grajkowski pause_ms("adfstop", FW_AUTH_WAIT_PERIOD); 116778ee8d1cSJulian Grajkowski fcu_sts = GET_FCU_CSR(handle, fcu_sts_csr); 116878ee8d1cSJulian Grajkowski if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) 1169a977168cSMichal Gulbicki return ae_ctr; 117078ee8d1cSJulian Grajkowski } while (retry++ < FW_AUTH_MAX_RETRY); 117178ee8d1cSJulian Grajkowski pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", 117278ee8d1cSJulian Grajkowski ae, 117378ee8d1cSJulian Grajkowski fcu_sts); 1174a977168cSMichal Gulbicki return 0; 117578ee8d1cSJulian Grajkowski } else { 1176a977168cSMichal Gulbicki for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) 1177a977168cSMichal Gulbicki { 117878ee8d1cSJulian Grajkowski qat_hal_put_wakeup_event(handle, 117978ee8d1cSJulian Grajkowski ae, 1180a977168cSMichal Gulbicki 0, 1181a977168cSMichal Gulbicki IS_QAT_GEN4( 1182a977168cSMichal Gulbicki pci_get_device(GET_DEV( 1183a977168cSMichal Gulbicki handle->accel_dev))) ? 1184a977168cSMichal Gulbicki 0x80000000 : 118578ee8d1cSJulian Grajkowski 0x10000); 1186a977168cSMichal Gulbicki qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); 1187a977168cSMichal Gulbicki ae_ctr++; 1188a977168cSMichal Gulbicki } 1189a977168cSMichal Gulbicki return ae_ctr; 119078ee8d1cSJulian Grajkowski } 119178ee8d1cSJulian Grajkowski } 119278ee8d1cSJulian Grajkowski 119378ee8d1cSJulian Grajkowski void 119478ee8d1cSJulian Grajkowski qat_hal_stop(struct icp_qat_fw_loader_handle *handle, 119578ee8d1cSJulian Grajkowski unsigned char ae, 119678ee8d1cSJulian Grajkowski unsigned int ctx_mask) 119778ee8d1cSJulian Grajkowski { 119878ee8d1cSJulian Grajkowski if (!handle->fw_auth) 119978ee8d1cSJulian Grajkowski qat_hal_disable_ctx(handle, ae, ctx_mask); 120078ee8d1cSJulian Grajkowski } 120178ee8d1cSJulian Grajkowski 120278ee8d1cSJulian Grajkowski void 120378ee8d1cSJulian Grajkowski qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, 120478ee8d1cSJulian Grajkowski unsigned char ae, 120578ee8d1cSJulian Grajkowski unsigned int ctx_mask, 120678ee8d1cSJulian Grajkowski unsigned int upc) 120778ee8d1cSJulian Grajkowski { 120878ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, 120978ee8d1cSJulian Grajkowski ae, 121078ee8d1cSJulian Grajkowski ctx_mask, 121178ee8d1cSJulian Grajkowski CTX_STS_INDIRECT, 121278ee8d1cSJulian Grajkowski handle->hal_handle->upc_mask & upc); 121378ee8d1cSJulian Grajkowski } 121478ee8d1cSJulian Grajkowski 121578ee8d1cSJulian Grajkowski static void 121678ee8d1cSJulian Grajkowski qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle, 121778ee8d1cSJulian Grajkowski unsigned char ae, 121878ee8d1cSJulian Grajkowski unsigned int uaddr, 121978ee8d1cSJulian Grajkowski unsigned int words_num, 122078ee8d1cSJulian Grajkowski uint64_t *uword) 122178ee8d1cSJulian Grajkowski { 122278ee8d1cSJulian Grajkowski unsigned int i, uwrd_lo, uwrd_hi; 122378ee8d1cSJulian Grajkowski unsigned int ustore_addr, misc_control; 122478ee8d1cSJulian Grajkowski unsigned int scs_flag = 0; 122578ee8d1cSJulian Grajkowski 122678ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control); 122778ee8d1cSJulian Grajkowski scs_flag = misc_control & (0x1 << MMC_SHARE_CS_BITPOS); 122878ee8d1cSJulian Grajkowski /*disable scs*/ 122978ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, 123078ee8d1cSJulian Grajkowski ae, 123178ee8d1cSJulian Grajkowski AE_MISC_CONTROL, 123278ee8d1cSJulian Grajkowski misc_control & 0xfffffffb); 123378ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); 123478ee8d1cSJulian Grajkowski uaddr |= UA_ECS; 123578ee8d1cSJulian Grajkowski for (i = 0; i < words_num; i++) { 123678ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 123778ee8d1cSJulian Grajkowski uaddr++; 123878ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo); 123978ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi); 124078ee8d1cSJulian Grajkowski uword[i] = uwrd_hi; 124178ee8d1cSJulian Grajkowski uword[i] = (uword[i] << 0x20) | uwrd_lo; 124278ee8d1cSJulian Grajkowski } 124378ee8d1cSJulian Grajkowski if (scs_flag) 124478ee8d1cSJulian Grajkowski misc_control |= (0x1 << MMC_SHARE_CS_BITPOS); 124578ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control); 124678ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 124778ee8d1cSJulian Grajkowski } 124878ee8d1cSJulian Grajkowski 124978ee8d1cSJulian Grajkowski void 125078ee8d1cSJulian Grajkowski qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, 125178ee8d1cSJulian Grajkowski unsigned char ae, 125278ee8d1cSJulian Grajkowski unsigned int uaddr, 125378ee8d1cSJulian Grajkowski unsigned int words_num, 125478ee8d1cSJulian Grajkowski unsigned int *data) 125578ee8d1cSJulian Grajkowski { 125678ee8d1cSJulian Grajkowski unsigned int i, ustore_addr; 125778ee8d1cSJulian Grajkowski 125878ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); 125978ee8d1cSJulian Grajkowski uaddr |= UA_ECS; 126078ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 126178ee8d1cSJulian Grajkowski for (i = 0; i < words_num; i++) { 126278ee8d1cSJulian Grajkowski unsigned int uwrd_lo, uwrd_hi, tmp; 126378ee8d1cSJulian Grajkowski 126478ee8d1cSJulian Grajkowski uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) | 126578ee8d1cSJulian Grajkowski ((data[i] & 0xff00) << 2) | (0x3 << 8) | (data[i] & 0xff); 126678ee8d1cSJulian Grajkowski uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28); 126778ee8d1cSJulian Grajkowski uwrd_hi |= (bitcount32(data[i] & 0xffff) & 0x1) << 8; 126878ee8d1cSJulian Grajkowski tmp = ((data[i] >> 0x10) & 0xffff); 126978ee8d1cSJulian Grajkowski uwrd_hi |= (bitcount32(tmp) & 0x1) << 9; 127078ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); 127178ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); 127278ee8d1cSJulian Grajkowski } 127378ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 127478ee8d1cSJulian Grajkowski } 127578ee8d1cSJulian Grajkowski 127678ee8d1cSJulian Grajkowski #define MAX_EXEC_INST 100 127778ee8d1cSJulian Grajkowski static int 127878ee8d1cSJulian Grajkowski qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, 127978ee8d1cSJulian Grajkowski unsigned char ae, 128078ee8d1cSJulian Grajkowski unsigned char ctx, 128178ee8d1cSJulian Grajkowski uint64_t *micro_inst, 128278ee8d1cSJulian Grajkowski unsigned int inst_num, 128378ee8d1cSJulian Grajkowski int code_off, 128478ee8d1cSJulian Grajkowski unsigned int max_cycle, 128578ee8d1cSJulian Grajkowski unsigned int *endpc) 128678ee8d1cSJulian Grajkowski { 128778ee8d1cSJulian Grajkowski uint64_t savuwords[MAX_EXEC_INST]; 128878ee8d1cSJulian Grajkowski unsigned int ind_lm_addr0, ind_lm_addr1; 128978ee8d1cSJulian Grajkowski unsigned int ind_lm_addr2, ind_lm_addr3; 129078ee8d1cSJulian Grajkowski unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1; 129178ee8d1cSJulian Grajkowski unsigned int ind_lm_addr_byte2, ind_lm_addr_byte3; 129278ee8d1cSJulian Grajkowski unsigned int ind_t_index, ind_t_index_byte; 129378ee8d1cSJulian Grajkowski unsigned int ind_cnt_sig; 129478ee8d1cSJulian Grajkowski unsigned int ind_sig, act_sig; 129578ee8d1cSJulian Grajkowski unsigned int csr_val = 0, newcsr_val; 129678ee8d1cSJulian Grajkowski unsigned int savctx, scs_flag; 129778ee8d1cSJulian Grajkowski unsigned int savcc, wakeup_events, savpc; 129878ee8d1cSJulian Grajkowski unsigned int ctxarb_ctl, ctx_enables; 129978ee8d1cSJulian Grajkowski 130078ee8d1cSJulian Grajkowski if (inst_num > handle->hal_handle->max_ustore || !micro_inst) { 130178ee8d1cSJulian Grajkowski pr_err("QAT: invalid instruction num %d\n", inst_num); 130278ee8d1cSJulian Grajkowski return EINVAL; 130378ee8d1cSJulian Grajkowski } 130478ee8d1cSJulian Grajkowski /* save current context */ 130578ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0); 130678ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1); 130778ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr( 130878ee8d1cSJulian Grajkowski handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, &ind_lm_addr_byte0); 130978ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr( 131078ee8d1cSJulian Grajkowski handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, &ind_lm_addr_byte1); 1311a977168cSMichal Gulbicki if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 131278ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr( 131378ee8d1cSJulian Grajkowski handle, ae, ctx, LM_ADDR_2_INDIRECT, &ind_lm_addr2); 131478ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr( 131578ee8d1cSJulian Grajkowski handle, ae, ctx, LM_ADDR_3_INDIRECT, &ind_lm_addr3); 131678ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr(handle, 131778ee8d1cSJulian Grajkowski ae, 131878ee8d1cSJulian Grajkowski ctx, 131978ee8d1cSJulian Grajkowski INDIRECT_LM_ADDR_2_BYTE_INDEX, 132078ee8d1cSJulian Grajkowski &ind_lm_addr_byte2); 132178ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr(handle, 132278ee8d1cSJulian Grajkowski ae, 132378ee8d1cSJulian Grajkowski ctx, 132478ee8d1cSJulian Grajkowski INDIRECT_LM_ADDR_3_BYTE_INDEX, 132578ee8d1cSJulian Grajkowski &ind_lm_addr_byte3); 132678ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr( 132778ee8d1cSJulian Grajkowski handle, ae, ctx, INDIRECT_T_INDEX, &ind_t_index); 132878ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr(handle, 132978ee8d1cSJulian Grajkowski ae, 133078ee8d1cSJulian Grajkowski ctx, 133178ee8d1cSJulian Grajkowski INDIRECT_T_INDEX_BYTE_INDEX, 133278ee8d1cSJulian Grajkowski &ind_t_index_byte); 133378ee8d1cSJulian Grajkowski } 133478ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 133578ee8d1cSJulian Grajkowski scs_flag = csr_val & (1 << MMC_SHARE_CS_BITPOS); 133678ee8d1cSJulian Grajkowski newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); 133778ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); 133878ee8d1cSJulian Grajkowski if (inst_num <= MAX_EXEC_INST) 133978ee8d1cSJulian Grajkowski qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); 134078ee8d1cSJulian Grajkowski qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); 134178ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc); 134278ee8d1cSJulian Grajkowski savpc = (savpc & handle->hal_handle->upc_mask) >> 0; 134378ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 134478ee8d1cSJulian Grajkowski ctx_enables &= IGNORE_W1C_MASK; 134578ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc); 134678ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); 134778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl); 134878ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr( 134978ee8d1cSJulian Grajkowski handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, &ind_cnt_sig); 135078ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig); 135178ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig); 135278ee8d1cSJulian Grajkowski /* execute micro codes */ 135378ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 135478ee8d1cSJulian Grajkowski qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst); 135578ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0); 135678ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO); 135778ee8d1cSJulian Grajkowski if (code_off) 135878ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff); 135978ee8d1cSJulian Grajkowski qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY); 136078ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0); 136178ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); 136278ee8d1cSJulian Grajkowski qat_hal_enable_ctx(handle, ae, (1 << ctx)); 136378ee8d1cSJulian Grajkowski /* wait for micro codes to finish */ 136478ee8d1cSJulian Grajkowski if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0) 136578ee8d1cSJulian Grajkowski return EFAULT; 136678ee8d1cSJulian Grajkowski if (endpc) { 136778ee8d1cSJulian Grajkowski unsigned int ctx_status; 136878ee8d1cSJulian Grajkowski 136978ee8d1cSJulian Grajkowski qat_hal_rd_indr_csr( 137078ee8d1cSJulian Grajkowski handle, ae, ctx, CTX_STS_INDIRECT, &ctx_status); 137178ee8d1cSJulian Grajkowski *endpc = ctx_status & handle->hal_handle->upc_mask; 137278ee8d1cSJulian Grajkowski } 137378ee8d1cSJulian Grajkowski /* retore to saved context */ 137478ee8d1cSJulian Grajkowski qat_hal_disable_ctx(handle, ae, (1 << ctx)); 137578ee8d1cSJulian Grajkowski if (inst_num <= MAX_EXEC_INST) 137678ee8d1cSJulian Grajkowski qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords); 137778ee8d1cSJulian Grajkowski qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events); 137878ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, 137978ee8d1cSJulian Grajkowski ae, 138078ee8d1cSJulian Grajkowski (1 << ctx), 138178ee8d1cSJulian Grajkowski CTX_STS_INDIRECT, 138278ee8d1cSJulian Grajkowski handle->hal_handle->upc_mask & savpc); 138378ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 138478ee8d1cSJulian Grajkowski newcsr_val = scs_flag ? SET_BIT(csr_val, MMC_SHARE_CS_BITPOS) : 138578ee8d1cSJulian Grajkowski CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); 138678ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); 138778ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc); 138878ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO); 138978ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl); 139078ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr( 139178ee8d1cSJulian Grajkowski handle, ae, (1 << ctx), LM_ADDR_0_INDIRECT, ind_lm_addr0); 139278ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr( 139378ee8d1cSJulian Grajkowski handle, ae, (1 << ctx), LM_ADDR_1_INDIRECT, ind_lm_addr1); 139478ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, 139578ee8d1cSJulian Grajkowski ae, 139678ee8d1cSJulian Grajkowski (1 << ctx), 139778ee8d1cSJulian Grajkowski INDIRECT_LM_ADDR_0_BYTE_INDEX, 139878ee8d1cSJulian Grajkowski ind_lm_addr_byte0); 139978ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, 140078ee8d1cSJulian Grajkowski ae, 140178ee8d1cSJulian Grajkowski (1 << ctx), 140278ee8d1cSJulian Grajkowski INDIRECT_LM_ADDR_1_BYTE_INDEX, 140378ee8d1cSJulian Grajkowski ind_lm_addr_byte1); 1404a977168cSMichal Gulbicki if (IS_QAT_GEN3_OR_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 140578ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr( 140678ee8d1cSJulian Grajkowski handle, ae, (1 << ctx), LM_ADDR_2_INDIRECT, ind_lm_addr2); 140778ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr( 140878ee8d1cSJulian Grajkowski handle, ae, (1 << ctx), LM_ADDR_3_INDIRECT, ind_lm_addr3); 140978ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, 141078ee8d1cSJulian Grajkowski ae, 141178ee8d1cSJulian Grajkowski (1 << ctx), 141278ee8d1cSJulian Grajkowski INDIRECT_LM_ADDR_2_BYTE_INDEX, 141378ee8d1cSJulian Grajkowski ind_lm_addr_byte2); 141478ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, 141578ee8d1cSJulian Grajkowski ae, 141678ee8d1cSJulian Grajkowski (1 << ctx), 141778ee8d1cSJulian Grajkowski INDIRECT_LM_ADDR_3_BYTE_INDEX, 141878ee8d1cSJulian Grajkowski ind_lm_addr_byte3); 141978ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr( 142078ee8d1cSJulian Grajkowski handle, ae, (1 << ctx), INDIRECT_T_INDEX, ind_t_index); 142178ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr(handle, 142278ee8d1cSJulian Grajkowski ae, 142378ee8d1cSJulian Grajkowski (1 << ctx), 142478ee8d1cSJulian Grajkowski INDIRECT_T_INDEX_BYTE_INDEX, 142578ee8d1cSJulian Grajkowski ind_t_index_byte); 142678ee8d1cSJulian Grajkowski } 142778ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr( 142878ee8d1cSJulian Grajkowski handle, ae, (1 << ctx), FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); 142978ee8d1cSJulian Grajkowski qat_hal_wr_indr_csr( 143078ee8d1cSJulian Grajkowski handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, ind_sig); 143178ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig); 143278ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 143378ee8d1cSJulian Grajkowski 143478ee8d1cSJulian Grajkowski return 0; 143578ee8d1cSJulian Grajkowski } 143678ee8d1cSJulian Grajkowski 143778ee8d1cSJulian Grajkowski static int 143878ee8d1cSJulian Grajkowski qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle, 143978ee8d1cSJulian Grajkowski unsigned char ae, 144078ee8d1cSJulian Grajkowski unsigned char ctx, 144178ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype reg_type, 144278ee8d1cSJulian Grajkowski unsigned short reg_num, 144378ee8d1cSJulian Grajkowski unsigned int *data) 144478ee8d1cSJulian Grajkowski { 144578ee8d1cSJulian Grajkowski unsigned int savctx, uaddr, uwrd_lo, uwrd_hi; 144678ee8d1cSJulian Grajkowski unsigned int ctxarb_cntl, ustore_addr, ctx_enables; 144778ee8d1cSJulian Grajkowski unsigned short reg_addr; 144878ee8d1cSJulian Grajkowski int status = 0; 144978ee8d1cSJulian Grajkowski unsigned int scs_flag = 0; 145078ee8d1cSJulian Grajkowski unsigned int csr_val = 0, newcsr_val = 0; 145178ee8d1cSJulian Grajkowski u64 insts, savuword; 145278ee8d1cSJulian Grajkowski 145378ee8d1cSJulian Grajkowski reg_addr = qat_hal_get_reg_addr(reg_type, reg_num); 145478ee8d1cSJulian Grajkowski if (reg_addr == BAD_REGADDR) { 145578ee8d1cSJulian Grajkowski pr_err("QAT: bad regaddr=0x%x\n", reg_addr); 145678ee8d1cSJulian Grajkowski return EINVAL; 145778ee8d1cSJulian Grajkowski } 145878ee8d1cSJulian Grajkowski switch (reg_type) { 145978ee8d1cSJulian Grajkowski case ICP_GPA_REL: 146078ee8d1cSJulian Grajkowski insts = 0xA070000000ull | (reg_addr & 0x3ff); 146178ee8d1cSJulian Grajkowski break; 146278ee8d1cSJulian Grajkowski default: 146378ee8d1cSJulian Grajkowski insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10); 146478ee8d1cSJulian Grajkowski break; 146578ee8d1cSJulian Grajkowski } 146678ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 146778ee8d1cSJulian Grajkowski scs_flag = csr_val & (1 << MMC_SHARE_CS_BITPOS); 146878ee8d1cSJulian Grajkowski newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); 146978ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); 147078ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); 147178ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl); 147278ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 147378ee8d1cSJulian Grajkowski ctx_enables &= IGNORE_W1C_MASK; 147478ee8d1cSJulian Grajkowski if (ctx != (savctx & ACS_ACNO)) 147578ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, 147678ee8d1cSJulian Grajkowski ae, 147778ee8d1cSJulian Grajkowski ACTIVE_CTX_STATUS, 147878ee8d1cSJulian Grajkowski ctx & ACS_ACNO); 147978ee8d1cSJulian Grajkowski qat_hal_get_uwords(handle, ae, 0, 1, &savuword); 148078ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 148178ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); 148278ee8d1cSJulian Grajkowski uaddr = UA_ECS; 148378ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 148478ee8d1cSJulian Grajkowski insts = qat_hal_set_uword_ecc(insts); 148578ee8d1cSJulian Grajkowski uwrd_lo = (unsigned int)(insts & 0xffffffff); 148678ee8d1cSJulian Grajkowski uwrd_hi = (unsigned int)(insts >> 0x20); 148778ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); 148878ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); 148978ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); 149078ee8d1cSJulian Grajkowski /* delay for at least 8 cycles */ 149178ee8d1cSJulian Grajkowski qat_hal_wait_cycles(handle, ae, 0x8, 0); 149278ee8d1cSJulian Grajkowski /* 149378ee8d1cSJulian Grajkowski * read ALU output 149478ee8d1cSJulian Grajkowski * the instruction should have been executed 149578ee8d1cSJulian Grajkowski * prior to clearing the ECS in putUwords 149678ee8d1cSJulian Grajkowski */ 149778ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data); 149878ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); 149978ee8d1cSJulian Grajkowski qat_hal_wr_uwords(handle, ae, 0, 1, &savuword); 150078ee8d1cSJulian Grajkowski if (ctx != (savctx & ACS_ACNO)) 150178ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, 150278ee8d1cSJulian Grajkowski ae, 150378ee8d1cSJulian Grajkowski ACTIVE_CTX_STATUS, 150478ee8d1cSJulian Grajkowski savctx & ACS_ACNO); 150578ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl); 150678ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); 150778ee8d1cSJulian Grajkowski newcsr_val = scs_flag ? SET_BIT(csr_val, MMC_SHARE_CS_BITPOS) : 150878ee8d1cSJulian Grajkowski CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); 150978ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); 151078ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 151178ee8d1cSJulian Grajkowski 151278ee8d1cSJulian Grajkowski return status; 151378ee8d1cSJulian Grajkowski } 151478ee8d1cSJulian Grajkowski 151578ee8d1cSJulian Grajkowski static int 151678ee8d1cSJulian Grajkowski qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle, 151778ee8d1cSJulian Grajkowski unsigned char ae, 151878ee8d1cSJulian Grajkowski unsigned char ctx, 151978ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype reg_type, 152078ee8d1cSJulian Grajkowski unsigned short reg_num, 152178ee8d1cSJulian Grajkowski unsigned int data) 152278ee8d1cSJulian Grajkowski { 152378ee8d1cSJulian Grajkowski unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo; 152478ee8d1cSJulian Grajkowski uint64_t insts[] = { 0x0F440000000ull, 152578ee8d1cSJulian Grajkowski 0x0F040000000ull, 152678ee8d1cSJulian Grajkowski 0x0F0000C0300ull, 152778ee8d1cSJulian Grajkowski 0x0E000010000ull }; 152878ee8d1cSJulian Grajkowski const int num_inst = ARRAY_SIZE(insts), code_off = 1; 152978ee8d1cSJulian Grajkowski const int imm_w1 = 0, imm_w0 = 1; 153078ee8d1cSJulian Grajkowski 153178ee8d1cSJulian Grajkowski dest_addr = qat_hal_get_reg_addr(reg_type, reg_num); 153278ee8d1cSJulian Grajkowski if (dest_addr == BAD_REGADDR) { 153378ee8d1cSJulian Grajkowski pr_err("QAT: bad destAddr=0x%x\n", dest_addr); 153478ee8d1cSJulian Grajkowski return EINVAL; 153578ee8d1cSJulian Grajkowski } 153678ee8d1cSJulian Grajkowski 153778ee8d1cSJulian Grajkowski data16lo = 0xffff & data; 153878ee8d1cSJulian Grajkowski data16hi = 0xffff & (data >> 0x10); 153978ee8d1cSJulian Grajkowski src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, 154078ee8d1cSJulian Grajkowski (unsigned short)(0xff & data16hi)); 154178ee8d1cSJulian Grajkowski src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, 154278ee8d1cSJulian Grajkowski (unsigned short)(0xff & data16lo)); 154378ee8d1cSJulian Grajkowski switch (reg_type) { 154478ee8d1cSJulian Grajkowski case ICP_GPA_REL: 154578ee8d1cSJulian Grajkowski insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | 154678ee8d1cSJulian Grajkowski ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); 154778ee8d1cSJulian Grajkowski insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | 154878ee8d1cSJulian Grajkowski ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); 154978ee8d1cSJulian Grajkowski break; 155078ee8d1cSJulian Grajkowski default: 155178ee8d1cSJulian Grajkowski insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | 155278ee8d1cSJulian Grajkowski ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); 155378ee8d1cSJulian Grajkowski 155478ee8d1cSJulian Grajkowski insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | 155578ee8d1cSJulian Grajkowski ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); 155678ee8d1cSJulian Grajkowski break; 155778ee8d1cSJulian Grajkowski } 155878ee8d1cSJulian Grajkowski 155978ee8d1cSJulian Grajkowski return qat_hal_exec_micro_inst( 156078ee8d1cSJulian Grajkowski handle, ae, ctx, insts, num_inst, code_off, num_inst * 0x5, NULL); 156178ee8d1cSJulian Grajkowski } 156278ee8d1cSJulian Grajkowski 156378ee8d1cSJulian Grajkowski int 156478ee8d1cSJulian Grajkowski qat_hal_get_ins_num(void) 156578ee8d1cSJulian Grajkowski { 156678ee8d1cSJulian Grajkowski return ARRAY_SIZE(inst_4b); 156778ee8d1cSJulian Grajkowski } 156878ee8d1cSJulian Grajkowski 156978ee8d1cSJulian Grajkowski static int 157078ee8d1cSJulian Grajkowski qat_hal_concat_micro_code(uint64_t *micro_inst, 157178ee8d1cSJulian Grajkowski unsigned int inst_num, 157278ee8d1cSJulian Grajkowski unsigned int size, 157378ee8d1cSJulian Grajkowski unsigned int addr, 157478ee8d1cSJulian Grajkowski unsigned int *value) 157578ee8d1cSJulian Grajkowski { 157678ee8d1cSJulian Grajkowski int i; 157778ee8d1cSJulian Grajkowski unsigned int cur_value; 157878ee8d1cSJulian Grajkowski const uint64_t *inst_arr; 157978ee8d1cSJulian Grajkowski unsigned int fixup_offset; 158078ee8d1cSJulian Grajkowski int usize = 0; 158178ee8d1cSJulian Grajkowski unsigned int orig_num; 158278ee8d1cSJulian Grajkowski unsigned int delta; 158378ee8d1cSJulian Grajkowski 158478ee8d1cSJulian Grajkowski orig_num = inst_num; 158578ee8d1cSJulian Grajkowski fixup_offset = inst_num; 158678ee8d1cSJulian Grajkowski cur_value = value[0]; 158778ee8d1cSJulian Grajkowski inst_arr = inst_4b; 158878ee8d1cSJulian Grajkowski usize = ARRAY_SIZE(inst_4b); 158978ee8d1cSJulian Grajkowski for (i = 0; i < usize; i++) 159078ee8d1cSJulian Grajkowski micro_inst[inst_num++] = inst_arr[i]; 159178ee8d1cSJulian Grajkowski INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr)); 159278ee8d1cSJulian Grajkowski fixup_offset++; 159378ee8d1cSJulian Grajkowski INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0); 159478ee8d1cSJulian Grajkowski fixup_offset++; 159578ee8d1cSJulian Grajkowski INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0)); 159678ee8d1cSJulian Grajkowski fixup_offset++; 159778ee8d1cSJulian Grajkowski INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10)); 159878ee8d1cSJulian Grajkowski 159978ee8d1cSJulian Grajkowski delta = inst_num - orig_num; 160078ee8d1cSJulian Grajkowski 160178ee8d1cSJulian Grajkowski return (int)delta; 160278ee8d1cSJulian Grajkowski } 160378ee8d1cSJulian Grajkowski 160478ee8d1cSJulian Grajkowski static int 160578ee8d1cSJulian Grajkowski qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle, 160678ee8d1cSJulian Grajkowski unsigned char ae, 160778ee8d1cSJulian Grajkowski unsigned char ctx, 160878ee8d1cSJulian Grajkowski int *pfirst_exec, 160978ee8d1cSJulian Grajkowski uint64_t *micro_inst, 161078ee8d1cSJulian Grajkowski unsigned int inst_num) 161178ee8d1cSJulian Grajkowski { 161278ee8d1cSJulian Grajkowski int stat = 0; 161378ee8d1cSJulian Grajkowski unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0; 161478ee8d1cSJulian Grajkowski unsigned int gprb0 = 0, gprb1 = 0; 161578ee8d1cSJulian Grajkowski 161678ee8d1cSJulian Grajkowski if (*pfirst_exec) { 161778ee8d1cSJulian Grajkowski qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0); 161878ee8d1cSJulian Grajkowski qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1); 161978ee8d1cSJulian Grajkowski qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2); 162078ee8d1cSJulian Grajkowski qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0); 162178ee8d1cSJulian Grajkowski qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1); 162278ee8d1cSJulian Grajkowski *pfirst_exec = 0; 162378ee8d1cSJulian Grajkowski } 162478ee8d1cSJulian Grajkowski stat = qat_hal_exec_micro_inst( 162578ee8d1cSJulian Grajkowski handle, ae, ctx, micro_inst, inst_num, 1, inst_num * 0x5, NULL); 162678ee8d1cSJulian Grajkowski if (stat != 0) 162778ee8d1cSJulian Grajkowski return EFAULT; 162878ee8d1cSJulian Grajkowski qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0); 162978ee8d1cSJulian Grajkowski qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1); 163078ee8d1cSJulian Grajkowski qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2); 163178ee8d1cSJulian Grajkowski qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0); 163278ee8d1cSJulian Grajkowski qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1); 163378ee8d1cSJulian Grajkowski 163478ee8d1cSJulian Grajkowski return 0; 163578ee8d1cSJulian Grajkowski } 163678ee8d1cSJulian Grajkowski 163778ee8d1cSJulian Grajkowski int 163878ee8d1cSJulian Grajkowski qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, 163978ee8d1cSJulian Grajkowski unsigned char ae, 164078ee8d1cSJulian Grajkowski struct icp_qat_uof_batch_init *lm_init_header) 164178ee8d1cSJulian Grajkowski { 164278ee8d1cSJulian Grajkowski struct icp_qat_uof_batch_init *plm_init; 164378ee8d1cSJulian Grajkowski uint64_t *micro_inst_arry; 164478ee8d1cSJulian Grajkowski int micro_inst_num; 164578ee8d1cSJulian Grajkowski int alloc_inst_size; 164678ee8d1cSJulian Grajkowski int first_exec = 1; 164778ee8d1cSJulian Grajkowski int stat = 0; 164878ee8d1cSJulian Grajkowski 164978ee8d1cSJulian Grajkowski if (!lm_init_header) 165078ee8d1cSJulian Grajkowski return 0; 165178ee8d1cSJulian Grajkowski plm_init = lm_init_header->next; 165278ee8d1cSJulian Grajkowski alloc_inst_size = lm_init_header->size; 165378ee8d1cSJulian Grajkowski if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore) 165478ee8d1cSJulian Grajkowski alloc_inst_size = handle->hal_handle->max_ustore; 165578ee8d1cSJulian Grajkowski micro_inst_arry = malloc(alloc_inst_size * sizeof(uint64_t), 165678ee8d1cSJulian Grajkowski M_QAT, 165778ee8d1cSJulian Grajkowski M_WAITOK | M_ZERO); 165878ee8d1cSJulian Grajkowski micro_inst_num = 0; 165978ee8d1cSJulian Grajkowski while (plm_init) { 166078ee8d1cSJulian Grajkowski unsigned int addr, *value, size; 166178ee8d1cSJulian Grajkowski 166278ee8d1cSJulian Grajkowski ae = plm_init->ae; 166378ee8d1cSJulian Grajkowski addr = plm_init->addr; 166478ee8d1cSJulian Grajkowski value = plm_init->value; 166578ee8d1cSJulian Grajkowski size = plm_init->size; 166678ee8d1cSJulian Grajkowski micro_inst_num += qat_hal_concat_micro_code( 166778ee8d1cSJulian Grajkowski micro_inst_arry, micro_inst_num, size, addr, value); 166878ee8d1cSJulian Grajkowski plm_init = plm_init->next; 166978ee8d1cSJulian Grajkowski } 167078ee8d1cSJulian Grajkowski /* exec micro codes */ 167178ee8d1cSJulian Grajkowski if (micro_inst_arry && micro_inst_num > 0) { 167278ee8d1cSJulian Grajkowski micro_inst_arry[micro_inst_num++] = 0x0E000010000ull; 167378ee8d1cSJulian Grajkowski stat = qat_hal_exec_micro_init_lm(handle, 167478ee8d1cSJulian Grajkowski ae, 167578ee8d1cSJulian Grajkowski 0, 167678ee8d1cSJulian Grajkowski &first_exec, 167778ee8d1cSJulian Grajkowski micro_inst_arry, 167878ee8d1cSJulian Grajkowski micro_inst_num); 167978ee8d1cSJulian Grajkowski } 168078ee8d1cSJulian Grajkowski free(micro_inst_arry, M_QAT); 168178ee8d1cSJulian Grajkowski return stat; 168278ee8d1cSJulian Grajkowski } 168378ee8d1cSJulian Grajkowski 168478ee8d1cSJulian Grajkowski static int 168578ee8d1cSJulian Grajkowski qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, 168678ee8d1cSJulian Grajkowski unsigned char ae, 168778ee8d1cSJulian Grajkowski unsigned char ctx, 168878ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype reg_type, 168978ee8d1cSJulian Grajkowski unsigned short reg_num, 169078ee8d1cSJulian Grajkowski unsigned int val) 169178ee8d1cSJulian Grajkowski { 169278ee8d1cSJulian Grajkowski int status = 0; 169378ee8d1cSJulian Grajkowski unsigned int reg_addr; 169478ee8d1cSJulian Grajkowski unsigned int ctx_enables; 169578ee8d1cSJulian Grajkowski unsigned short mask; 169678ee8d1cSJulian Grajkowski unsigned short dr_offset = 0x10; 169778ee8d1cSJulian Grajkowski 169878ee8d1cSJulian Grajkowski status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 169978ee8d1cSJulian Grajkowski if (CE_INUSE_CONTEXTS & ctx_enables) { 170078ee8d1cSJulian Grajkowski if (ctx & 0x1) { 170178ee8d1cSJulian Grajkowski pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); 170278ee8d1cSJulian Grajkowski return EINVAL; 170378ee8d1cSJulian Grajkowski } 170478ee8d1cSJulian Grajkowski mask = 0x1f; 170578ee8d1cSJulian Grajkowski dr_offset = 0x20; 170678ee8d1cSJulian Grajkowski } else { 170778ee8d1cSJulian Grajkowski mask = 0x0f; 170878ee8d1cSJulian Grajkowski } 170978ee8d1cSJulian Grajkowski if (reg_num & ~mask) 171078ee8d1cSJulian Grajkowski return EINVAL; 171178ee8d1cSJulian Grajkowski reg_addr = reg_num + (ctx << 0x5); 171278ee8d1cSJulian Grajkowski switch (reg_type) { 171378ee8d1cSJulian Grajkowski case ICP_SR_RD_REL: 171478ee8d1cSJulian Grajkowski case ICP_SR_REL: 171578ee8d1cSJulian Grajkowski SET_AE_XFER(handle, ae, reg_addr, val); 171678ee8d1cSJulian Grajkowski break; 171778ee8d1cSJulian Grajkowski case ICP_DR_RD_REL: 171878ee8d1cSJulian Grajkowski case ICP_DR_REL: 171978ee8d1cSJulian Grajkowski SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val); 172078ee8d1cSJulian Grajkowski break; 172178ee8d1cSJulian Grajkowski default: 172278ee8d1cSJulian Grajkowski status = EINVAL; 172378ee8d1cSJulian Grajkowski break; 172478ee8d1cSJulian Grajkowski } 172578ee8d1cSJulian Grajkowski return status; 172678ee8d1cSJulian Grajkowski } 172778ee8d1cSJulian Grajkowski 172878ee8d1cSJulian Grajkowski static int 172978ee8d1cSJulian Grajkowski qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle, 173078ee8d1cSJulian Grajkowski unsigned char ae, 173178ee8d1cSJulian Grajkowski unsigned char ctx, 173278ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype reg_type, 173378ee8d1cSJulian Grajkowski unsigned short reg_num, 173478ee8d1cSJulian Grajkowski unsigned int data) 173578ee8d1cSJulian Grajkowski { 173678ee8d1cSJulian Grajkowski unsigned int gprval, ctx_enables; 173778ee8d1cSJulian Grajkowski unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi, 173878ee8d1cSJulian Grajkowski data16low; 173978ee8d1cSJulian Grajkowski unsigned short reg_mask; 174078ee8d1cSJulian Grajkowski int status = 0; 174178ee8d1cSJulian Grajkowski uint64_t micro_inst[] = { 0x0F440000000ull, 174278ee8d1cSJulian Grajkowski 0x0F040000000ull, 174378ee8d1cSJulian Grajkowski 0x0A000000000ull, 174478ee8d1cSJulian Grajkowski 0x0F0000C0300ull, 174578ee8d1cSJulian Grajkowski 0x0E000010000ull }; 174678ee8d1cSJulian Grajkowski const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1; 174778ee8d1cSJulian Grajkowski const unsigned short gprnum = 0, dly = num_inst * 0x5; 174878ee8d1cSJulian Grajkowski 174978ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 175078ee8d1cSJulian Grajkowski if (CE_INUSE_CONTEXTS & ctx_enables) { 175178ee8d1cSJulian Grajkowski if (ctx & 0x1) { 175278ee8d1cSJulian Grajkowski pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx); 175378ee8d1cSJulian Grajkowski return EINVAL; 175478ee8d1cSJulian Grajkowski } 175578ee8d1cSJulian Grajkowski reg_mask = (unsigned short)~0x1f; 175678ee8d1cSJulian Grajkowski } else { 175778ee8d1cSJulian Grajkowski reg_mask = (unsigned short)~0xf; 175878ee8d1cSJulian Grajkowski } 175978ee8d1cSJulian Grajkowski if (reg_num & reg_mask) 176078ee8d1cSJulian Grajkowski return EINVAL; 176178ee8d1cSJulian Grajkowski xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num); 176278ee8d1cSJulian Grajkowski if (xfr_addr == BAD_REGADDR) { 176378ee8d1cSJulian Grajkowski pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr); 176478ee8d1cSJulian Grajkowski return EINVAL; 176578ee8d1cSJulian Grajkowski } 176678ee8d1cSJulian Grajkowski qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval); 176778ee8d1cSJulian Grajkowski gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum); 176878ee8d1cSJulian Grajkowski data16low = 0xffff & data; 176978ee8d1cSJulian Grajkowski data16hi = 0xffff & (data >> 0x10); 177078ee8d1cSJulian Grajkowski src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, 177178ee8d1cSJulian Grajkowski (unsigned short)(0xff & data16hi)); 177278ee8d1cSJulian Grajkowski src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, 177378ee8d1cSJulian Grajkowski (unsigned short)(0xff & data16low)); 177478ee8d1cSJulian Grajkowski micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) | 177578ee8d1cSJulian Grajkowski ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); 177678ee8d1cSJulian Grajkowski micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) | 177778ee8d1cSJulian Grajkowski ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); 177878ee8d1cSJulian Grajkowski micro_inst[0x2] = micro_inst[0x2] | ((xfr_addr & 0x3ff) << 20) | 177978ee8d1cSJulian Grajkowski ((gpr_addr & 0x3ff) << 10); 178078ee8d1cSJulian Grajkowski status = qat_hal_exec_micro_inst( 178178ee8d1cSJulian Grajkowski handle, ae, ctx, micro_inst, num_inst, code_off, dly, NULL); 178278ee8d1cSJulian Grajkowski qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval); 178378ee8d1cSJulian Grajkowski return status; 178478ee8d1cSJulian Grajkowski } 178578ee8d1cSJulian Grajkowski 178678ee8d1cSJulian Grajkowski static int 178778ee8d1cSJulian Grajkowski qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle, 178878ee8d1cSJulian Grajkowski unsigned char ae, 178978ee8d1cSJulian Grajkowski unsigned char ctx, 179078ee8d1cSJulian Grajkowski unsigned short nn, 179178ee8d1cSJulian Grajkowski unsigned int val) 179278ee8d1cSJulian Grajkowski { 179378ee8d1cSJulian Grajkowski unsigned int ctx_enables; 179478ee8d1cSJulian Grajkowski int stat = 0; 179578ee8d1cSJulian Grajkowski 179678ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 179778ee8d1cSJulian Grajkowski ctx_enables &= IGNORE_W1C_MASK; 179878ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE); 179978ee8d1cSJulian Grajkowski 180078ee8d1cSJulian Grajkowski stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val); 180178ee8d1cSJulian Grajkowski qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); 180278ee8d1cSJulian Grajkowski return stat; 180378ee8d1cSJulian Grajkowski } 180478ee8d1cSJulian Grajkowski 180578ee8d1cSJulian Grajkowski static int 180678ee8d1cSJulian Grajkowski qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle *handle, 180778ee8d1cSJulian Grajkowski unsigned char ae, 180878ee8d1cSJulian Grajkowski unsigned short absreg_num, 180978ee8d1cSJulian Grajkowski unsigned short *relreg, 181078ee8d1cSJulian Grajkowski unsigned char *ctx) 181178ee8d1cSJulian Grajkowski { 181278ee8d1cSJulian Grajkowski unsigned int ctx_enables; 181378ee8d1cSJulian Grajkowski 181478ee8d1cSJulian Grajkowski qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); 181578ee8d1cSJulian Grajkowski if (ctx_enables & CE_INUSE_CONTEXTS) { 181678ee8d1cSJulian Grajkowski /* 4-ctx mode */ 181778ee8d1cSJulian Grajkowski *relreg = absreg_num & 0x1F; 181878ee8d1cSJulian Grajkowski *ctx = (absreg_num >> 0x4) & 0x6; 181978ee8d1cSJulian Grajkowski } else { 182078ee8d1cSJulian Grajkowski /* 8-ctx mode */ 182178ee8d1cSJulian Grajkowski *relreg = absreg_num & 0x0F; 182278ee8d1cSJulian Grajkowski *ctx = (absreg_num >> 0x4) & 0x7; 182378ee8d1cSJulian Grajkowski } 182478ee8d1cSJulian Grajkowski return 0; 182578ee8d1cSJulian Grajkowski } 182678ee8d1cSJulian Grajkowski 182778ee8d1cSJulian Grajkowski int 182878ee8d1cSJulian Grajkowski qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, 182978ee8d1cSJulian Grajkowski unsigned char ae, 183078ee8d1cSJulian Grajkowski unsigned long ctx_mask, 183178ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype reg_type, 183278ee8d1cSJulian Grajkowski unsigned short reg_num, 183378ee8d1cSJulian Grajkowski unsigned int regdata) 183478ee8d1cSJulian Grajkowski { 183578ee8d1cSJulian Grajkowski int stat = 0; 183678ee8d1cSJulian Grajkowski unsigned short reg; 183778ee8d1cSJulian Grajkowski unsigned char ctx = 0; 183878ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype type; 183978ee8d1cSJulian Grajkowski 184078ee8d1cSJulian Grajkowski if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG) 184178ee8d1cSJulian Grajkowski return EINVAL; 184278ee8d1cSJulian Grajkowski 184378ee8d1cSJulian Grajkowski do { 184478ee8d1cSJulian Grajkowski if (ctx_mask == 0) { 184578ee8d1cSJulian Grajkowski qat_hal_convert_abs_to_rel( 184678ee8d1cSJulian Grajkowski handle, ae, reg_num, ®, &ctx); 184778ee8d1cSJulian Grajkowski type = reg_type - 1; 184878ee8d1cSJulian Grajkowski } else { 184978ee8d1cSJulian Grajkowski reg = reg_num; 185078ee8d1cSJulian Grajkowski type = reg_type; 185178ee8d1cSJulian Grajkowski if (!test_bit(ctx, &ctx_mask)) 185278ee8d1cSJulian Grajkowski continue; 185378ee8d1cSJulian Grajkowski } 185478ee8d1cSJulian Grajkowski stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); 185578ee8d1cSJulian Grajkowski if (stat) { 185678ee8d1cSJulian Grajkowski pr_err("QAT: write gpr fail\n"); 185778ee8d1cSJulian Grajkowski return EINVAL; 185878ee8d1cSJulian Grajkowski } 185978ee8d1cSJulian Grajkowski } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); 186078ee8d1cSJulian Grajkowski 186178ee8d1cSJulian Grajkowski return 0; 186278ee8d1cSJulian Grajkowski } 186378ee8d1cSJulian Grajkowski 186478ee8d1cSJulian Grajkowski int 186578ee8d1cSJulian Grajkowski qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, 186678ee8d1cSJulian Grajkowski unsigned char ae, 186778ee8d1cSJulian Grajkowski unsigned long ctx_mask, 186878ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype reg_type, 186978ee8d1cSJulian Grajkowski unsigned short reg_num, 187078ee8d1cSJulian Grajkowski unsigned int regdata) 187178ee8d1cSJulian Grajkowski { 187278ee8d1cSJulian Grajkowski int stat = 0; 187378ee8d1cSJulian Grajkowski unsigned short reg; 187478ee8d1cSJulian Grajkowski unsigned char ctx = 0; 187578ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype type; 187678ee8d1cSJulian Grajkowski 187778ee8d1cSJulian Grajkowski if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) 187878ee8d1cSJulian Grajkowski return EINVAL; 187978ee8d1cSJulian Grajkowski 188078ee8d1cSJulian Grajkowski do { 188178ee8d1cSJulian Grajkowski if (ctx_mask == 0) { 188278ee8d1cSJulian Grajkowski qat_hal_convert_abs_to_rel( 188378ee8d1cSJulian Grajkowski handle, ae, reg_num, ®, &ctx); 188478ee8d1cSJulian Grajkowski type = reg_type - 3; 188578ee8d1cSJulian Grajkowski } else { 188678ee8d1cSJulian Grajkowski reg = reg_num; 188778ee8d1cSJulian Grajkowski type = reg_type; 188878ee8d1cSJulian Grajkowski if (!test_bit(ctx, &ctx_mask)) 188978ee8d1cSJulian Grajkowski continue; 189078ee8d1cSJulian Grajkowski } 189178ee8d1cSJulian Grajkowski stat = qat_hal_put_rel_wr_xfer( 189278ee8d1cSJulian Grajkowski handle, ae, ctx, type, reg, regdata); 189378ee8d1cSJulian Grajkowski if (stat) { 189478ee8d1cSJulian Grajkowski pr_err("QAT: write wr xfer fail\n"); 189578ee8d1cSJulian Grajkowski return EINVAL; 189678ee8d1cSJulian Grajkowski } 189778ee8d1cSJulian Grajkowski } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); 189878ee8d1cSJulian Grajkowski 189978ee8d1cSJulian Grajkowski return 0; 190078ee8d1cSJulian Grajkowski } 190178ee8d1cSJulian Grajkowski 190278ee8d1cSJulian Grajkowski int 190378ee8d1cSJulian Grajkowski qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, 190478ee8d1cSJulian Grajkowski unsigned char ae, 190578ee8d1cSJulian Grajkowski unsigned long ctx_mask, 190678ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype reg_type, 190778ee8d1cSJulian Grajkowski unsigned short reg_num, 190878ee8d1cSJulian Grajkowski unsigned int regdata) 190978ee8d1cSJulian Grajkowski { 191078ee8d1cSJulian Grajkowski int stat = 0; 191178ee8d1cSJulian Grajkowski unsigned short reg; 191278ee8d1cSJulian Grajkowski unsigned char ctx = 0; 191378ee8d1cSJulian Grajkowski enum icp_qat_uof_regtype type; 191478ee8d1cSJulian Grajkowski 191578ee8d1cSJulian Grajkowski if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) 191678ee8d1cSJulian Grajkowski return EINVAL; 191778ee8d1cSJulian Grajkowski 191878ee8d1cSJulian Grajkowski do { 191978ee8d1cSJulian Grajkowski if (ctx_mask == 0) { 192078ee8d1cSJulian Grajkowski qat_hal_convert_abs_to_rel( 192178ee8d1cSJulian Grajkowski handle, ae, reg_num, ®, &ctx); 192278ee8d1cSJulian Grajkowski type = reg_type - 3; 192378ee8d1cSJulian Grajkowski } else { 192478ee8d1cSJulian Grajkowski reg = reg_num; 192578ee8d1cSJulian Grajkowski type = reg_type; 192678ee8d1cSJulian Grajkowski if (!test_bit(ctx, &ctx_mask)) 192778ee8d1cSJulian Grajkowski continue; 192878ee8d1cSJulian Grajkowski } 192978ee8d1cSJulian Grajkowski stat = qat_hal_put_rel_rd_xfer( 193078ee8d1cSJulian Grajkowski handle, ae, ctx, type, reg, regdata); 193178ee8d1cSJulian Grajkowski if (stat) { 193278ee8d1cSJulian Grajkowski pr_err("QAT: write rd xfer fail\n"); 193378ee8d1cSJulian Grajkowski return EINVAL; 193478ee8d1cSJulian Grajkowski } 193578ee8d1cSJulian Grajkowski } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); 193678ee8d1cSJulian Grajkowski 193778ee8d1cSJulian Grajkowski return 0; 193878ee8d1cSJulian Grajkowski } 193978ee8d1cSJulian Grajkowski 194078ee8d1cSJulian Grajkowski int 194178ee8d1cSJulian Grajkowski qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, 194278ee8d1cSJulian Grajkowski unsigned char ae, 194378ee8d1cSJulian Grajkowski unsigned long ctx_mask, 194478ee8d1cSJulian Grajkowski unsigned short reg_num, 194578ee8d1cSJulian Grajkowski unsigned int regdata) 194678ee8d1cSJulian Grajkowski { 194778ee8d1cSJulian Grajkowski int stat = 0; 194878ee8d1cSJulian Grajkowski unsigned char ctx; 194978ee8d1cSJulian Grajkowski 1950a977168cSMichal Gulbicki if (IS_QAT_GEN4(pci_get_device(GET_DEV(handle->accel_dev)))) { 1951a977168cSMichal Gulbicki pr_err("QAT: No next neigh for CPM2X\n"); 1952a977168cSMichal Gulbicki return EINVAL; 1953a977168cSMichal Gulbicki } 1954a977168cSMichal Gulbicki 195578ee8d1cSJulian Grajkowski if (ctx_mask == 0) 195678ee8d1cSJulian Grajkowski return EINVAL; 195778ee8d1cSJulian Grajkowski 195878ee8d1cSJulian Grajkowski for_each_set_bit(ctx, &ctx_mask, ICP_QAT_UCLO_MAX_CTX) 195978ee8d1cSJulian Grajkowski { 196078ee8d1cSJulian Grajkowski stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); 196178ee8d1cSJulian Grajkowski if (stat) { 196278ee8d1cSJulian Grajkowski pr_err("QAT: write neigh error\n"); 196378ee8d1cSJulian Grajkowski return EINVAL; 196478ee8d1cSJulian Grajkowski } 196578ee8d1cSJulian Grajkowski } 196678ee8d1cSJulian Grajkowski 196778ee8d1cSJulian Grajkowski return 0; 196878ee8d1cSJulian Grajkowski } 1969