1*fcf60f4bSJie Wang // SPDX-License-Identifier: GPL-2.0-only
2*fcf60f4bSJie Wang /* Copyright(c) 2023 Intel Corporation */
3*fcf60f4bSJie Wang #include <linux/device.h>
4*fcf60f4bSJie Wang #include <linux/module.h>
5*fcf60f4bSJie Wang #include <linux/pci.h>
6*fcf60f4bSJie Wang
7*fcf60f4bSJie Wang #include <adf_accel_devices.h>
8*fcf60f4bSJie Wang #include <adf_gen4_hw_data.h>
9*fcf60f4bSJie Wang #include <adf_gen4_config.h>
10*fcf60f4bSJie Wang #include <adf_cfg.h>
11*fcf60f4bSJie Wang #include <adf_common_drv.h>
12*fcf60f4bSJie Wang #include <adf_dbgfs.h>
13*fcf60f4bSJie Wang
14*fcf60f4bSJie Wang #include "adf_420xx_hw_data.h"
15*fcf60f4bSJie Wang
16*fcf60f4bSJie Wang static const struct pci_device_id adf_pci_tbl[] = {
17*fcf60f4bSJie Wang { PCI_VDEVICE(INTEL, ADF_420XX_PCI_DEVICE_ID), },
18*fcf60f4bSJie Wang { }
19*fcf60f4bSJie Wang };
20*fcf60f4bSJie Wang MODULE_DEVICE_TABLE(pci, adf_pci_tbl);
21*fcf60f4bSJie Wang
adf_cleanup_accel(struct adf_accel_dev * accel_dev)22*fcf60f4bSJie Wang static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
23*fcf60f4bSJie Wang {
24*fcf60f4bSJie Wang if (accel_dev->hw_device) {
25*fcf60f4bSJie Wang adf_clean_hw_data_420xx(accel_dev->hw_device);
26*fcf60f4bSJie Wang accel_dev->hw_device = NULL;
27*fcf60f4bSJie Wang }
28*fcf60f4bSJie Wang adf_dbgfs_exit(accel_dev);
29*fcf60f4bSJie Wang adf_cfg_dev_remove(accel_dev);
30*fcf60f4bSJie Wang adf_devmgr_rm_dev(accel_dev, NULL);
31*fcf60f4bSJie Wang }
32*fcf60f4bSJie Wang
adf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)33*fcf60f4bSJie Wang static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
34*fcf60f4bSJie Wang {
35*fcf60f4bSJie Wang struct adf_accel_dev *accel_dev;
36*fcf60f4bSJie Wang struct adf_accel_pci *accel_pci_dev;
37*fcf60f4bSJie Wang struct adf_hw_device_data *hw_data;
38*fcf60f4bSJie Wang unsigned int i, bar_nr;
39*fcf60f4bSJie Wang unsigned long bar_mask;
40*fcf60f4bSJie Wang struct adf_bar *bar;
41*fcf60f4bSJie Wang int ret;
42*fcf60f4bSJie Wang
43*fcf60f4bSJie Wang if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) {
44*fcf60f4bSJie Wang /*
45*fcf60f4bSJie Wang * If the accelerator is connected to a node with no memory
46*fcf60f4bSJie Wang * there is no point in using the accelerator since the remote
47*fcf60f4bSJie Wang * memory transaction will be very slow.
48*fcf60f4bSJie Wang */
49*fcf60f4bSJie Wang dev_err(&pdev->dev, "Invalid NUMA configuration.\n");
50*fcf60f4bSJie Wang return -EINVAL;
51*fcf60f4bSJie Wang }
52*fcf60f4bSJie Wang
53*fcf60f4bSJie Wang accel_dev = devm_kzalloc(&pdev->dev, sizeof(*accel_dev), GFP_KERNEL);
54*fcf60f4bSJie Wang if (!accel_dev)
55*fcf60f4bSJie Wang return -ENOMEM;
56*fcf60f4bSJie Wang
57*fcf60f4bSJie Wang INIT_LIST_HEAD(&accel_dev->crypto_list);
58*fcf60f4bSJie Wang accel_pci_dev = &accel_dev->accel_pci_dev;
59*fcf60f4bSJie Wang accel_pci_dev->pci_dev = pdev;
60*fcf60f4bSJie Wang
61*fcf60f4bSJie Wang /*
62*fcf60f4bSJie Wang * Add accel device to accel table
63*fcf60f4bSJie Wang * This should be called before adf_cleanup_accel is called
64*fcf60f4bSJie Wang */
65*fcf60f4bSJie Wang if (adf_devmgr_add_dev(accel_dev, NULL)) {
66*fcf60f4bSJie Wang dev_err(&pdev->dev, "Failed to add new accelerator device.\n");
67*fcf60f4bSJie Wang return -EFAULT;
68*fcf60f4bSJie Wang }
69*fcf60f4bSJie Wang
70*fcf60f4bSJie Wang accel_dev->owner = THIS_MODULE;
71*fcf60f4bSJie Wang /* Allocate and initialise device hardware meta-data structure */
72*fcf60f4bSJie Wang hw_data = devm_kzalloc(&pdev->dev, sizeof(*hw_data), GFP_KERNEL);
73*fcf60f4bSJie Wang if (!hw_data) {
74*fcf60f4bSJie Wang ret = -ENOMEM;
75*fcf60f4bSJie Wang goto out_err;
76*fcf60f4bSJie Wang }
77*fcf60f4bSJie Wang
78*fcf60f4bSJie Wang accel_dev->hw_device = hw_data;
79*fcf60f4bSJie Wang adf_init_hw_data_420xx(accel_dev->hw_device, ent->device);
80*fcf60f4bSJie Wang
81*fcf60f4bSJie Wang pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid);
82*fcf60f4bSJie Wang pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses);
83*fcf60f4bSJie Wang
84*fcf60f4bSJie Wang /* Get Accelerators and Accelerators Engines masks */
85*fcf60f4bSJie Wang hw_data->accel_mask = hw_data->get_accel_mask(hw_data);
86*fcf60f4bSJie Wang hw_data->ae_mask = hw_data->get_ae_mask(hw_data);
87*fcf60f4bSJie Wang accel_pci_dev->sku = hw_data->get_sku(hw_data);
88*fcf60f4bSJie Wang /* If the device has no acceleration engines then ignore it */
89*fcf60f4bSJie Wang if (!hw_data->accel_mask || !hw_data->ae_mask ||
90*fcf60f4bSJie Wang (~hw_data->ae_mask & 0x01)) {
91*fcf60f4bSJie Wang dev_err(&pdev->dev, "No acceleration units found.\n");
92*fcf60f4bSJie Wang ret = -EFAULT;
93*fcf60f4bSJie Wang goto out_err;
94*fcf60f4bSJie Wang }
95*fcf60f4bSJie Wang
96*fcf60f4bSJie Wang /* Create device configuration table */
97*fcf60f4bSJie Wang ret = adf_cfg_dev_add(accel_dev);
98*fcf60f4bSJie Wang if (ret)
99*fcf60f4bSJie Wang goto out_err;
100*fcf60f4bSJie Wang
101*fcf60f4bSJie Wang /* Enable PCI device */
102*fcf60f4bSJie Wang ret = pcim_enable_device(pdev);
103*fcf60f4bSJie Wang if (ret) {
104*fcf60f4bSJie Wang dev_err(&pdev->dev, "Can't enable PCI device.\n");
105*fcf60f4bSJie Wang goto out_err;
106*fcf60f4bSJie Wang }
107*fcf60f4bSJie Wang
108*fcf60f4bSJie Wang /* Set DMA identifier */
109*fcf60f4bSJie Wang ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
110*fcf60f4bSJie Wang if (ret) {
111*fcf60f4bSJie Wang dev_err(&pdev->dev, "No usable DMA configuration.\n");
112*fcf60f4bSJie Wang goto out_err;
113*fcf60f4bSJie Wang }
114*fcf60f4bSJie Wang
115*fcf60f4bSJie Wang ret = adf_gen4_cfg_dev_init(accel_dev);
116*fcf60f4bSJie Wang if (ret) {
117*fcf60f4bSJie Wang dev_err(&pdev->dev, "Failed to initialize configuration.\n");
118*fcf60f4bSJie Wang goto out_err;
119*fcf60f4bSJie Wang }
120*fcf60f4bSJie Wang
121*fcf60f4bSJie Wang /* Get accelerator capabilities mask */
122*fcf60f4bSJie Wang hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev);
123*fcf60f4bSJie Wang if (!hw_data->accel_capabilities_mask) {
124*fcf60f4bSJie Wang dev_err(&pdev->dev, "Failed to get capabilities mask.\n");
125*fcf60f4bSJie Wang ret = -EINVAL;
126*fcf60f4bSJie Wang goto out_err;
127*fcf60f4bSJie Wang }
128*fcf60f4bSJie Wang
129*fcf60f4bSJie Wang /* Find and map all the device's BARS */
130*fcf60f4bSJie Wang bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_GEN4_BAR_MASK;
131*fcf60f4bSJie Wang
132*fcf60f4bSJie Wang ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev));
133*fcf60f4bSJie Wang if (ret) {
134*fcf60f4bSJie Wang dev_err(&pdev->dev, "Failed to map pci regions.\n");
135*fcf60f4bSJie Wang goto out_err;
136*fcf60f4bSJie Wang }
137*fcf60f4bSJie Wang
138*fcf60f4bSJie Wang i = 0;
139*fcf60f4bSJie Wang for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) {
140*fcf60f4bSJie Wang bar = &accel_pci_dev->pci_bars[i++];
141*fcf60f4bSJie Wang bar->virt_addr = pcim_iomap_table(pdev)[bar_nr];
142*fcf60f4bSJie Wang }
143*fcf60f4bSJie Wang
144*fcf60f4bSJie Wang pci_set_master(pdev);
145*fcf60f4bSJie Wang
146*fcf60f4bSJie Wang if (pci_save_state(pdev)) {
147*fcf60f4bSJie Wang dev_err(&pdev->dev, "Failed to save pci state.\n");
148*fcf60f4bSJie Wang ret = -ENOMEM;
149*fcf60f4bSJie Wang goto out_err;
150*fcf60f4bSJie Wang }
151*fcf60f4bSJie Wang
152*fcf60f4bSJie Wang accel_dev->ras_errors.enabled = true;
153*fcf60f4bSJie Wang adf_dbgfs_init(accel_dev);
154*fcf60f4bSJie Wang
155*fcf60f4bSJie Wang ret = adf_dev_up(accel_dev, true);
156*fcf60f4bSJie Wang if (ret)
157*fcf60f4bSJie Wang goto out_err_dev_stop;
158*fcf60f4bSJie Wang
159*fcf60f4bSJie Wang ret = adf_sysfs_init(accel_dev);
160*fcf60f4bSJie Wang if (ret)
161*fcf60f4bSJie Wang goto out_err_dev_stop;
162*fcf60f4bSJie Wang
163*fcf60f4bSJie Wang return ret;
164*fcf60f4bSJie Wang
165*fcf60f4bSJie Wang out_err_dev_stop:
166*fcf60f4bSJie Wang adf_dev_down(accel_dev, false);
167*fcf60f4bSJie Wang out_err:
168*fcf60f4bSJie Wang adf_cleanup_accel(accel_dev);
169*fcf60f4bSJie Wang return ret;
170*fcf60f4bSJie Wang }
171*fcf60f4bSJie Wang
adf_remove(struct pci_dev * pdev)172*fcf60f4bSJie Wang static void adf_remove(struct pci_dev *pdev)
173*fcf60f4bSJie Wang {
174*fcf60f4bSJie Wang struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev);
175*fcf60f4bSJie Wang
176*fcf60f4bSJie Wang if (!accel_dev) {
177*fcf60f4bSJie Wang pr_err("QAT: Driver removal failed\n");
178*fcf60f4bSJie Wang return;
179*fcf60f4bSJie Wang }
180*fcf60f4bSJie Wang adf_dev_down(accel_dev, false);
181*fcf60f4bSJie Wang adf_cleanup_accel(accel_dev);
182*fcf60f4bSJie Wang }
183*fcf60f4bSJie Wang
184*fcf60f4bSJie Wang static struct pci_driver adf_driver = {
185*fcf60f4bSJie Wang .id_table = adf_pci_tbl,
186*fcf60f4bSJie Wang .name = ADF_420XX_DEVICE_NAME,
187*fcf60f4bSJie Wang .probe = adf_probe,
188*fcf60f4bSJie Wang .remove = adf_remove,
189*fcf60f4bSJie Wang .sriov_configure = adf_sriov_configure,
190*fcf60f4bSJie Wang .err_handler = &adf_err_handler,
191*fcf60f4bSJie Wang };
192*fcf60f4bSJie Wang
193*fcf60f4bSJie Wang module_pci_driver(adf_driver);
194*fcf60f4bSJie Wang
195*fcf60f4bSJie Wang MODULE_LICENSE("GPL");
196*fcf60f4bSJie Wang MODULE_AUTHOR("Intel");
197*fcf60f4bSJie Wang MODULE_FIRMWARE(ADF_420XX_FW);
198*fcf60f4bSJie Wang MODULE_FIRMWARE(ADF_420XX_MMP);
199*fcf60f4bSJie Wang MODULE_DESCRIPTION("Intel(R) QuickAssist Technology");
200*fcf60f4bSJie Wang MODULE_VERSION(ADF_DRV_VERSION);
201*fcf60f4bSJie Wang MODULE_SOFTDEP("pre: crypto-intel_qat");
202*fcf60f4bSJie Wang MODULE_IMPORT_NS(CRYPTO_QAT);
203