1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3 #include <linux/align.h>
4 #include <linux/dma-mapping.h>
5 #include <linux/hisi_acc_qm.h>
6 #include <linux/module.h>
7 #include <linux/slab.h>
8
9 #define HISI_ACC_SGL_SGE_NR_MIN 1
10 #define HISI_ACC_SGL_NR_MAX 256
11 #define HISI_ACC_SGL_ALIGN_SIZE 64
12 #define HISI_ACC_MEM_BLOCK_NR 5
13
14 struct acc_hw_sge {
15 dma_addr_t buf;
16 void *page_ctrl;
17 __le32 len;
18 __le32 pad;
19 __le32 pad0;
20 __le32 pad1;
21 };
22
23 /* use default sgl head size 64B */
24 struct hisi_acc_hw_sgl {
25 dma_addr_t next_dma;
26 __le16 entry_sum_in_chain;
27 __le16 entry_sum_in_sgl;
28 __le16 entry_length_in_sgl;
29 __le16 pad0;
30 __le64 pad1[5];
31 struct hisi_acc_hw_sgl *next;
32 struct acc_hw_sge sge_entries[];
33 } __aligned(1);
34
35 struct hisi_acc_sgl_pool {
36 struct mem_block {
37 struct hisi_acc_hw_sgl *sgl;
38 dma_addr_t sgl_dma;
39 size_t size;
40 } mem_block[HISI_ACC_MEM_BLOCK_NR];
41 u32 sgl_num_per_block;
42 u32 block_num;
43 u32 count;
44 u32 sge_nr;
45 size_t sgl_size;
46 };
47
48 /**
49 * hisi_acc_create_sgl_pool() - Create a hw sgl pool.
50 * @dev: The device which hw sgl pool belongs to.
51 * @count: Count of hisi_acc_hw_sgl in pool.
52 * @sge_nr: The count of sge in hw_sgl
53 *
54 * This function creates a hw sgl pool, after this user can get hw sgl memory
55 * from it.
56 */
hisi_acc_create_sgl_pool(struct device * dev,u32 count,u32 sge_nr)57 struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,
58 u32 count, u32 sge_nr)
59 {
60 u32 sgl_size, block_size, sgl_num_per_block, block_num, remain_sgl;
61 struct hisi_acc_sgl_pool *pool;
62 struct mem_block *block;
63 u32 i, j;
64
65 if (!dev || !count || !sge_nr || sge_nr > HISI_ACC_SGL_SGE_NR_MAX)
66 return ERR_PTR(-EINVAL);
67
68 sgl_size = ALIGN(sizeof(struct acc_hw_sge) * sge_nr +
69 sizeof(struct hisi_acc_hw_sgl),
70 HISI_ACC_SGL_ALIGN_SIZE);
71
72 /*
73 * the pool may allocate a block of memory of size PAGE_SIZE * 2^MAX_PAGE_ORDER,
74 * block size may exceed 2^31 on ia64, so the max of block size is 2^31
75 */
76 block_size = 1 << (PAGE_SHIFT + MAX_PAGE_ORDER < 32 ?
77 PAGE_SHIFT + MAX_PAGE_ORDER : 31);
78 sgl_num_per_block = block_size / sgl_size;
79 block_num = count / sgl_num_per_block;
80 remain_sgl = count % sgl_num_per_block;
81
82 if ((!remain_sgl && block_num > HISI_ACC_MEM_BLOCK_NR) ||
83 (remain_sgl > 0 && block_num > HISI_ACC_MEM_BLOCK_NR - 1))
84 return ERR_PTR(-EINVAL);
85
86 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
87 if (!pool)
88 return ERR_PTR(-ENOMEM);
89 block = pool->mem_block;
90
91 for (i = 0; i < block_num; i++) {
92 block[i].sgl = dma_alloc_coherent(dev, block_size,
93 &block[i].sgl_dma,
94 GFP_KERNEL);
95 if (!block[i].sgl) {
96 dev_err(dev, "Fail to allocate hw SG buffer!\n");
97 goto err_free_mem;
98 }
99
100 block[i].size = block_size;
101 }
102
103 if (remain_sgl > 0) {
104 block[i].sgl = dma_alloc_coherent(dev, remain_sgl * sgl_size,
105 &block[i].sgl_dma,
106 GFP_KERNEL);
107 if (!block[i].sgl) {
108 dev_err(dev, "Fail to allocate remained hw SG buffer!\n");
109 goto err_free_mem;
110 }
111
112 block[i].size = remain_sgl * sgl_size;
113 }
114
115 pool->sgl_num_per_block = sgl_num_per_block;
116 pool->block_num = remain_sgl ? block_num + 1 : block_num;
117 pool->count = count;
118 pool->sgl_size = sgl_size;
119 pool->sge_nr = sge_nr;
120
121 return pool;
122
123 err_free_mem:
124 for (j = 0; j < i; j++)
125 dma_free_coherent(dev, block_size, block[j].sgl,
126 block[j].sgl_dma);
127
128 kfree_sensitive(pool);
129 return ERR_PTR(-ENOMEM);
130 }
131 EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool);
132
133 /**
134 * hisi_acc_free_sgl_pool() - Free a hw sgl pool.
135 * @dev: The device which hw sgl pool belongs to.
136 * @pool: Pointer of pool.
137 *
138 * This function frees memory of a hw sgl pool.
139 */
hisi_acc_free_sgl_pool(struct device * dev,struct hisi_acc_sgl_pool * pool)140 void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool)
141 {
142 struct mem_block *block;
143 u32 i;
144
145 if (!dev || !pool)
146 return;
147
148 block = pool->mem_block;
149
150 for (i = 0; i < pool->block_num; i++)
151 dma_free_coherent(dev, block[i].size, block[i].sgl,
152 block[i].sgl_dma);
153
154 kfree(pool);
155 }
156 EXPORT_SYMBOL_GPL(hisi_acc_free_sgl_pool);
157
acc_get_sgl(struct hisi_acc_sgl_pool * pool,u32 index,dma_addr_t * hw_sgl_dma)158 static struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool,
159 u32 index, dma_addr_t *hw_sgl_dma)
160 {
161 struct mem_block *block;
162 u32 block_index, offset;
163
164 block = pool->mem_block;
165 block_index = index / pool->sgl_num_per_block;
166 offset = index % pool->sgl_num_per_block;
167
168 *hw_sgl_dma = block[block_index].sgl_dma + pool->sgl_size * offset;
169 return (void *)block[block_index].sgl + pool->sgl_size * offset;
170 }
171
sg_map_to_hw_sg(struct scatterlist * sgl,struct acc_hw_sge * hw_sge)172 static void sg_map_to_hw_sg(struct scatterlist *sgl,
173 struct acc_hw_sge *hw_sge)
174 {
175 hw_sge->buf = sg_dma_address(sgl);
176 hw_sge->len = cpu_to_le32(sg_dma_len(sgl));
177 hw_sge->page_ctrl = sg_virt(sgl);
178 }
179
inc_hw_sgl_sge(struct hisi_acc_hw_sgl * hw_sgl)180 static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)
181 {
182 u16 var = le16_to_cpu(hw_sgl->entry_sum_in_sgl);
183
184 var++;
185 hw_sgl->entry_sum_in_sgl = cpu_to_le16(var);
186 }
187
update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl * hw_sgl,u16 sum)188 static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum)
189 {
190 hw_sgl->entry_sum_in_chain = cpu_to_le16(sum);
191 }
192
clear_hw_sgl_sge(struct hisi_acc_hw_sgl * hw_sgl)193 static void clear_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)
194 {
195 struct acc_hw_sge *hw_sge = hw_sgl->sge_entries;
196 u16 entry_sum = le16_to_cpu(hw_sgl->entry_sum_in_sgl);
197 int i;
198
199 for (i = 0; i < entry_sum; i++) {
200 hw_sge[i].page_ctrl = NULL;
201 hw_sge[i].buf = 0;
202 hw_sge[i].len = 0;
203 }
204 }
205
206 /**
207 * hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl.
208 * @dev: The device which hw sgl belongs to.
209 * @sgl: Scatterlist which will be mapped to hw sgl.
210 * @pool: Pool which hw sgl memory will be allocated in.
211 * @index: Index of hisi_acc_hw_sgl in pool.
212 * @hw_sgl_dma: The dma address of allocated hw sgl.
213 *
214 * This function builds hw sgl according input sgl, user can use hw_sgl_dma
215 * as src/dst in its BD. Only support single hw sgl currently.
216 */
217 struct hisi_acc_hw_sgl *
hisi_acc_sg_buf_map_to_hw_sgl(struct device * dev,struct scatterlist * sgl,struct hisi_acc_sgl_pool * pool,u32 index,dma_addr_t * hw_sgl_dma)218 hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
219 struct scatterlist *sgl,
220 struct hisi_acc_sgl_pool *pool,
221 u32 index, dma_addr_t *hw_sgl_dma)
222 {
223 struct hisi_acc_hw_sgl *curr_hw_sgl;
224 unsigned int i, sg_n_mapped;
225 dma_addr_t curr_sgl_dma = 0;
226 struct acc_hw_sge *curr_hw_sge;
227 struct scatterlist *sg;
228 int sg_n;
229
230 if (!dev || !sgl || !pool || !hw_sgl_dma || index >= pool->count)
231 return ERR_PTR(-EINVAL);
232
233 sg_n = sg_nents(sgl);
234
235 sg_n_mapped = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
236 if (!sg_n_mapped) {
237 dev_err(dev, "DMA mapping for SG error!\n");
238 return ERR_PTR(-EINVAL);
239 }
240
241 if (sg_n_mapped > pool->sge_nr) {
242 dev_err(dev, "the number of entries in input scatterlist is bigger than SGL pool setting.\n");
243 return ERR_PTR(-EINVAL);
244 }
245
246 curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma);
247 if (IS_ERR(curr_hw_sgl)) {
248 dev_err(dev, "Get SGL error!\n");
249 dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
250 return ERR_PTR(-ENOMEM);
251 }
252 curr_hw_sgl->entry_length_in_sgl = cpu_to_le16(pool->sge_nr);
253 curr_hw_sge = curr_hw_sgl->sge_entries;
254
255 for_each_sg(sgl, sg, sg_n_mapped, i) {
256 sg_map_to_hw_sg(sg, curr_hw_sge);
257 inc_hw_sgl_sge(curr_hw_sgl);
258 curr_hw_sge++;
259 }
260
261 update_hw_sgl_sum_sge(curr_hw_sgl, pool->sge_nr);
262 *hw_sgl_dma = curr_sgl_dma;
263
264 return curr_hw_sgl;
265 }
266 EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl);
267
268 /**
269 * hisi_acc_sg_buf_unmap() - Unmap allocated hw sgl.
270 * @dev: The device which hw sgl belongs to.
271 * @sgl: Related scatterlist.
272 * @hw_sgl: Virtual address of hw sgl.
273 *
274 * This function unmaps allocated hw sgl.
275 */
hisi_acc_sg_buf_unmap(struct device * dev,struct scatterlist * sgl,struct hisi_acc_hw_sgl * hw_sgl)276 void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
277 struct hisi_acc_hw_sgl *hw_sgl)
278 {
279 if (!dev || !sgl || !hw_sgl)
280 return;
281
282 dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL);
283 clear_hw_sgl_sge(hw_sgl);
284 hw_sgl->entry_sum_in_chain = 0;
285 hw_sgl->entry_sum_in_sgl = 0;
286 hw_sgl->entry_length_in_sgl = 0;
287 }
288 EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_unmap);
289