1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <linux/dma-mapping.h> 4 #include <linux/module.h> 5 #include "./sgl.h" 6 7 #define HISI_ACC_SGL_SGE_NR_MIN 1 8 #define HISI_ACC_SGL_SGE_NR_MAX 255 9 #define HISI_ACC_SGL_SGE_NR_DEF 10 10 #define HISI_ACC_SGL_NR_MAX 256 11 #define HISI_ACC_SGL_ALIGN_SIZE 64 12 13 static int acc_sgl_sge_set(const char *val, const struct kernel_param *kp) 14 { 15 int ret; 16 u32 n; 17 18 if (!val) 19 return -EINVAL; 20 21 ret = kstrtou32(val, 10, &n); 22 if (ret != 0 || n > HISI_ACC_SGL_SGE_NR_MAX || n == 0) 23 return -EINVAL; 24 25 return param_set_int(val, kp); 26 } 27 28 static const struct kernel_param_ops acc_sgl_sge_ops = { 29 .set = acc_sgl_sge_set, 30 .get = param_get_int, 31 }; 32 33 static u32 acc_sgl_sge_nr = HISI_ACC_SGL_SGE_NR_DEF; 34 module_param_cb(acc_sgl_sge_nr, &acc_sgl_sge_ops, &acc_sgl_sge_nr, 0444); 35 MODULE_PARM_DESC(acc_sgl_sge_nr, "Number of sge in sgl(1-255)"); 36 37 struct acc_hw_sge { 38 dma_addr_t buf; 39 void *page_ctrl; 40 __le32 len; 41 __le32 pad; 42 __le32 pad0; 43 __le32 pad1; 44 }; 45 46 /* use default sgl head size 64B */ 47 struct hisi_acc_hw_sgl { 48 dma_addr_t next_dma; 49 __le16 entry_sum_in_chain; 50 __le16 entry_sum_in_sgl; 51 __le16 entry_length_in_sgl; 52 __le16 pad0; 53 __le64 pad1[5]; 54 struct hisi_acc_hw_sgl *next; 55 struct acc_hw_sge sge_entries[]; 56 } __aligned(1); 57 58 /** 59 * hisi_acc_create_sgl_pool() - Create a hw sgl pool. 60 * @dev: The device which hw sgl pool belongs to. 61 * @pool: Pointer of pool. 62 * @count: Count of hisi_acc_hw_sgl in pool. 63 * 64 * This function creates a hw sgl pool, after this user can get hw sgl memory 65 * from it. 66 */ 67 int hisi_acc_create_sgl_pool(struct device *dev, 68 struct hisi_acc_sgl_pool *pool, u32 count) 69 { 70 u32 sgl_size; 71 u32 size; 72 73 if (!dev || !pool || !count) 74 return -EINVAL; 75 76 sgl_size = sizeof(struct acc_hw_sge) * acc_sgl_sge_nr + 77 sizeof(struct hisi_acc_hw_sgl); 78 size = sgl_size * count; 79 80 pool->sgl = dma_alloc_coherent(dev, size, &pool->sgl_dma, GFP_KERNEL); 81 if (!pool->sgl) 82 return -ENOMEM; 83 84 pool->size = size; 85 pool->count = count; 86 pool->sgl_size = sgl_size; 87 88 return 0; 89 } 90 EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool); 91 92 /** 93 * hisi_acc_free_sgl_pool() - Free a hw sgl pool. 94 * @dev: The device which hw sgl pool belongs to. 95 * @pool: Pointer of pool. 96 * 97 * This function frees memory of a hw sgl pool. 98 */ 99 void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool) 100 { 101 dma_free_coherent(dev, pool->size, pool->sgl, pool->sgl_dma); 102 memset(pool, 0, sizeof(struct hisi_acc_sgl_pool)); 103 } 104 EXPORT_SYMBOL_GPL(hisi_acc_free_sgl_pool); 105 106 struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool, u32 index, 107 dma_addr_t *hw_sgl_dma) 108 { 109 if (!pool || !hw_sgl_dma || index >= pool->count || !pool->sgl) 110 return ERR_PTR(-EINVAL); 111 112 *hw_sgl_dma = pool->sgl_dma + pool->sgl_size * index; 113 return (void *)pool->sgl + pool->sgl_size * index; 114 } 115 116 void acc_put_sgl(struct hisi_acc_sgl_pool *pool, u32 index) {} 117 118 static void sg_map_to_hw_sg(struct scatterlist *sgl, 119 struct acc_hw_sge *hw_sge) 120 { 121 hw_sge->buf = sgl->dma_address; 122 hw_sge->len = sgl->dma_length; 123 } 124 125 static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl) 126 { 127 hw_sgl->entry_sum_in_sgl++; 128 } 129 130 static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum) 131 { 132 hw_sgl->entry_sum_in_chain = sum; 133 } 134 135 /** 136 * hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl. 137 * @dev: The device which hw sgl belongs to. 138 * @sgl: Scatterlist which will be mapped to hw sgl. 139 * @pool: Pool which hw sgl memory will be allocated in. 140 * @index: Index of hisi_acc_hw_sgl in pool. 141 * @hw_sgl_dma: The dma address of allocated hw sgl. 142 * 143 * This function builds hw sgl according input sgl, user can use hw_sgl_dma 144 * as src/dst in its BD. Only support single hw sgl currently. 145 */ 146 struct hisi_acc_hw_sgl * 147 hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, 148 struct scatterlist *sgl, 149 struct hisi_acc_sgl_pool *pool, 150 u32 index, dma_addr_t *hw_sgl_dma) 151 { 152 struct hisi_acc_hw_sgl *curr_hw_sgl; 153 dma_addr_t curr_sgl_dma = 0; 154 struct acc_hw_sge *curr_hw_sge; 155 struct scatterlist *sg; 156 int sg_n = sg_nents(sgl); 157 int i, ret; 158 159 if (!dev || !sgl || !pool || !hw_sgl_dma || sg_n > acc_sgl_sge_nr) 160 return ERR_PTR(-EINVAL); 161 162 ret = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); 163 if (!ret) 164 return ERR_PTR(-EINVAL); 165 166 curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma); 167 if (!curr_hw_sgl) { 168 ret = -ENOMEM; 169 goto err_unmap_sg; 170 } 171 curr_hw_sgl->entry_length_in_sgl = acc_sgl_sge_nr; 172 curr_hw_sge = curr_hw_sgl->sge_entries; 173 174 for_each_sg(sgl, sg, sg_n, i) { 175 sg_map_to_hw_sg(sg, curr_hw_sge); 176 inc_hw_sgl_sge(curr_hw_sgl); 177 curr_hw_sge++; 178 } 179 180 update_hw_sgl_sum_sge(curr_hw_sgl, acc_sgl_sge_nr); 181 *hw_sgl_dma = curr_sgl_dma; 182 183 return curr_hw_sgl; 184 185 err_unmap_sg: 186 dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); 187 return ERR_PTR(ret); 188 } 189 EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl); 190 191 /** 192 * hisi_acc_sg_buf_unmap() - Unmap allocated hw sgl. 193 * @dev: The device which hw sgl belongs to. 194 * @sgl: Related scatterlist. 195 * @hw_sgl: Virtual address of hw sgl. 196 * @hw_sgl_dma: DMA address of hw sgl. 197 * @pool: Pool which hw sgl is allocated in. 198 * 199 * This function unmaps allocated hw sgl. 200 */ 201 void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl, 202 struct hisi_acc_hw_sgl *hw_sgl) 203 { 204 dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL); 205 206 hw_sgl->entry_sum_in_chain = 0; 207 hw_sgl->entry_sum_in_sgl = 0; 208 hw_sgl->entry_length_in_sgl = 0; 209 } 210 EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_unmap); 211 212 MODULE_LICENSE("GPL v2"); 213 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>"); 214 MODULE_DESCRIPTION("HiSilicon Accelerator SGL support"); 215