1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2018-2020 Broadcom.
4 */
5 #include <linux/dma-mapping.h>
6 #include <linux/mm.h>
7 #include <linux/pagemap.h>
8 #include <linux/pgtable.h>
9 #include <linux/vmalloc.h>
10
11 #include <asm/page.h>
12 #include <asm/unaligned.h>
13
14 #include <uapi/linux/misc/bcm_vk.h>
15
16 #include "bcm_vk.h"
17 #include "bcm_vk_msg.h"
18 #include "bcm_vk_sg.h"
19
20 /*
21 * Valkyrie has a hardware limitation of 16M transfer size.
22 * So limit the SGL chunks to 16M.
23 */
24 #define BCM_VK_MAX_SGL_CHUNK SZ_16M
25
26 static int bcm_vk_dma_alloc(struct device *dev,
27 struct bcm_vk_dma *dma,
28 int dir,
29 struct _vk_data *vkdata);
30 static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma);
31
32 /* Uncomment to dump SGLIST */
33 /* #define BCM_VK_DUMP_SGLIST */
34
bcm_vk_dma_alloc(struct device * dev,struct bcm_vk_dma * dma,int direction,struct _vk_data * vkdata)35 static int bcm_vk_dma_alloc(struct device *dev,
36 struct bcm_vk_dma *dma,
37 int direction,
38 struct _vk_data *vkdata)
39 {
40 dma_addr_t addr, sg_addr;
41 int err;
42 int i;
43 int offset;
44 u32 size;
45 u32 remaining_size;
46 u32 transfer_size;
47 u64 data;
48 unsigned long first, last;
49 struct _vk_data *sgdata;
50
51 /* Get 64-bit user address */
52 data = get_unaligned(&vkdata->address);
53
54 /* offset into first page */
55 offset = offset_in_page(data);
56
57 /* Calculate number of pages */
58 first = (data & PAGE_MASK) >> PAGE_SHIFT;
59 last = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT;
60 dma->nr_pages = last - first + 1;
61
62 /* Allocate DMA pages */
63 dma->pages = kmalloc_array(dma->nr_pages,
64 sizeof(struct page *),
65 GFP_KERNEL);
66 if (!dma->pages)
67 return -ENOMEM;
68
69 dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n",
70 data, vkdata->size, dma->nr_pages);
71
72 dma->direction = direction;
73
74 /* Get user pages into memory */
75 err = get_user_pages_fast(data & PAGE_MASK,
76 dma->nr_pages,
77 direction == DMA_FROM_DEVICE,
78 dma->pages);
79 if (err != dma->nr_pages) {
80 dma->nr_pages = (err >= 0) ? err : 0;
81 dev_err(dev, "get_user_pages_fast, err=%d [%d]\n",
82 err, dma->nr_pages);
83 return err < 0 ? err : -EINVAL;
84 }
85
86 /* Max size of sg list is 1 per mapped page + fields at start */
87 dma->sglen = (dma->nr_pages * sizeof(*sgdata)) +
88 (sizeof(u32) * SGLIST_VKDATA_START);
89
90 /* Allocate sglist */
91 dma->sglist = dma_alloc_coherent(dev,
92 dma->sglen,
93 &dma->handle,
94 GFP_KERNEL);
95 if (!dma->sglist)
96 return -ENOMEM;
97
98 dma->sglist[SGLIST_NUM_SG] = 0;
99 dma->sglist[SGLIST_TOTALSIZE] = vkdata->size;
100 remaining_size = vkdata->size;
101 sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
102
103 /* Map all pages into DMA */
104 size = min_t(size_t, PAGE_SIZE - offset, remaining_size);
105 remaining_size -= size;
106 sg_addr = dma_map_page(dev,
107 dma->pages[0],
108 offset,
109 size,
110 dma->direction);
111 transfer_size = size;
112 if (unlikely(dma_mapping_error(dev, sg_addr))) {
113 __free_page(dma->pages[0]);
114 return -EIO;
115 }
116
117 for (i = 1; i < dma->nr_pages; i++) {
118 size = min_t(size_t, PAGE_SIZE, remaining_size);
119 remaining_size -= size;
120 addr = dma_map_page(dev,
121 dma->pages[i],
122 0,
123 size,
124 dma->direction);
125 if (unlikely(dma_mapping_error(dev, addr))) {
126 __free_page(dma->pages[i]);
127 return -EIO;
128 }
129
130 /*
131 * Compress SG list entry when pages are contiguous
132 * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK
133 */
134 if ((addr == (sg_addr + transfer_size)) &&
135 ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) {
136 /* pages are contiguous, add to same sg entry */
137 transfer_size += size;
138 } else {
139 /* pages are not contiguous, write sg entry */
140 sgdata->size = transfer_size;
141 put_unaligned(sg_addr, (u64 *)&sgdata->address);
142 dma->sglist[SGLIST_NUM_SG]++;
143
144 /* start new sg entry */
145 sgdata++;
146 sg_addr = addr;
147 transfer_size = size;
148 }
149 }
150 /* Write last sg list entry */
151 sgdata->size = transfer_size;
152 put_unaligned(sg_addr, (u64 *)&sgdata->address);
153 dma->sglist[SGLIST_NUM_SG]++;
154
155 /* Update pointers and size field to point to sglist */
156 put_unaligned((u64)dma->handle, &vkdata->address);
157 vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) +
158 (sizeof(u32) * SGLIST_VKDATA_START);
159
160 #ifdef BCM_VK_DUMP_SGLIST
161 dev_dbg(dev,
162 "sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n",
163 (u64)dma->sglist,
164 dma->handle,
165 dma->sglen,
166 vkdata->size);
167 for (i = 0; i < vkdata->size / sizeof(u32); i++)
168 dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]);
169 #endif
170
171 return 0;
172 }
173
bcm_vk_sg_alloc(struct device * dev,struct bcm_vk_dma * dma,int dir,struct _vk_data * vkdata,int num)174 int bcm_vk_sg_alloc(struct device *dev,
175 struct bcm_vk_dma *dma,
176 int dir,
177 struct _vk_data *vkdata,
178 int num)
179 {
180 int i;
181 int rc = -EINVAL;
182
183 /* Convert user addresses to DMA SG List */
184 for (i = 0; i < num; i++) {
185 if (vkdata[i].size && vkdata[i].address) {
186 /*
187 * If both size and address are non-zero
188 * then DMA alloc.
189 */
190 rc = bcm_vk_dma_alloc(dev,
191 &dma[i],
192 dir,
193 &vkdata[i]);
194 } else if (vkdata[i].size ||
195 vkdata[i].address) {
196 /*
197 * If one of size and address are zero
198 * there is a problem.
199 */
200 dev_err(dev,
201 "Invalid vkdata %x 0x%x 0x%llx\n",
202 i, vkdata[i].size, vkdata[i].address);
203 rc = -EINVAL;
204 } else {
205 /*
206 * If size and address are both zero
207 * don't convert, but return success.
208 */
209 rc = 0;
210 }
211
212 if (rc)
213 goto fail_alloc;
214 }
215 return rc;
216
217 fail_alloc:
218 while (i > 0) {
219 i--;
220 if (dma[i].sglist)
221 bcm_vk_dma_free(dev, &dma[i]);
222 }
223 return rc;
224 }
225
bcm_vk_dma_free(struct device * dev,struct bcm_vk_dma * dma)226 static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma)
227 {
228 dma_addr_t addr;
229 int i;
230 int num_sg;
231 u32 size;
232 struct _vk_data *vkdata;
233
234 dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen);
235
236 /* Unmap all pages in the sglist */
237 num_sg = dma->sglist[SGLIST_NUM_SG];
238 vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
239 for (i = 0; i < num_sg; i++) {
240 size = vkdata[i].size;
241 addr = get_unaligned(&vkdata[i].address);
242
243 dma_unmap_page(dev, addr, size, dma->direction);
244 }
245
246 /* Free allocated sglist */
247 dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle);
248
249 /* Release lock on all pages */
250 for (i = 0; i < dma->nr_pages; i++)
251 put_page(dma->pages[i]);
252
253 /* Free allocated dma pages */
254 kfree(dma->pages);
255 dma->sglist = NULL;
256
257 return 0;
258 }
259
bcm_vk_sg_free(struct device * dev,struct bcm_vk_dma * dma,int num,int * proc_cnt)260 int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num,
261 int *proc_cnt)
262 {
263 int i;
264
265 *proc_cnt = 0;
266 /* Unmap and free all pages and sglists */
267 for (i = 0; i < num; i++) {
268 if (dma[i].sglist) {
269 bcm_vk_dma_free(dev, &dma[i]);
270 *proc_cnt += 1;
271 }
272 }
273
274 return 0;
275 }
276