1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2013--2024 Intel Corporation
4 */
5 #include <asm/barrier.h>
6
7 #include <linux/align.h>
8 #include <linux/atomic.h>
9 #include <linux/bitops.h>
10 #include <linux/bits.h>
11 #include <linux/bug.h>
12 #include <linux/cacheflush.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/err.h>
15 #include <linux/gfp.h>
16 #include <linux/io.h>
17 #include <linux/iova.h>
18 #include <linux/math.h>
19 #include <linux/minmax.h>
20 #include <linux/mm.h>
21 #include <linux/pfn.h>
22 #include <linux/slab.h>
23 #include <linux/spinlock.h>
24 #include <linux/types.h>
25 #include <linux/vmalloc.h>
26
27 #include "ipu6.h"
28 #include "ipu6-dma.h"
29 #include "ipu6-mmu.h"
30 #include "ipu6-platform-regs.h"
31
32 #define ISP_PAGE_SHIFT 12
33 #define ISP_PAGE_SIZE BIT(ISP_PAGE_SHIFT)
34 #define ISP_PAGE_MASK (~(ISP_PAGE_SIZE - 1))
35
36 #define ISP_L1PT_SHIFT 22
37 #define ISP_L1PT_MASK (~((1U << ISP_L1PT_SHIFT) - 1))
38
39 #define ISP_L2PT_SHIFT 12
40 #define ISP_L2PT_MASK (~(ISP_L1PT_MASK | (~(ISP_PAGE_MASK))))
41
42 #define ISP_L1PT_PTES 1024
43 #define ISP_L2PT_PTES 1024
44
45 #define ISP_PADDR_SHIFT 12
46
47 #define REG_TLB_INVALIDATE 0x0000
48
49 #define REG_L1_PHYS 0x0004 /* 27-bit pfn */
50 #define REG_INFO 0x0008
51
52 #define TBL_PHYS_ADDR(a) ((phys_addr_t)(a) << ISP_PADDR_SHIFT)
53
tlb_invalidate(struct ipu6_mmu * mmu)54 static void tlb_invalidate(struct ipu6_mmu *mmu)
55 {
56 unsigned long flags;
57 unsigned int i;
58
59 spin_lock_irqsave(&mmu->ready_lock, flags);
60 if (!mmu->ready) {
61 spin_unlock_irqrestore(&mmu->ready_lock, flags);
62 return;
63 }
64
65 for (i = 0; i < mmu->nr_mmus; i++) {
66 /*
67 * To avoid the HW bug induced dead lock in some of the IPU6
68 * MMUs on successive invalidate calls, we need to first do a
69 * read to the page table base before writing the invalidate
70 * register. MMUs which need to implement this WA, will have
71 * the insert_read_before_invalidate flags set as true.
72 * Disregard the return value of the read.
73 */
74 if (mmu->mmu_hw[i].insert_read_before_invalidate)
75 readl(mmu->mmu_hw[i].base + REG_L1_PHYS);
76
77 writel(0xffffffff, mmu->mmu_hw[i].base +
78 REG_TLB_INVALIDATE);
79 /*
80 * The TLB invalidation is a "single cycle" (IOMMU clock cycles)
81 * When the actual MMIO write reaches the IPU6 TLB Invalidate
82 * register, wmb() will force the TLB invalidate out if the CPU
83 * attempts to update the IOMMU page table (or sooner).
84 */
85 wmb();
86 }
87 spin_unlock_irqrestore(&mmu->ready_lock, flags);
88 }
89
90 #ifdef DEBUG
page_table_dump(struct ipu6_mmu_info * mmu_info)91 static void page_table_dump(struct ipu6_mmu_info *mmu_info)
92 {
93 u32 l1_idx;
94
95 dev_dbg(mmu_info->dev, "begin IOMMU page table dump\n");
96
97 for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {
98 u32 l2_idx;
99 u32 iova = (phys_addr_t)l1_idx << ISP_L1PT_SHIFT;
100
101 if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval)
102 continue;
103 dev_dbg(mmu_info->dev,
104 "l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %pa\n",
105 l1_idx, iova, iova + ISP_PAGE_SIZE,
106 TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]));
107
108 for (l2_idx = 0; l2_idx < ISP_L2PT_PTES; l2_idx++) {
109 u32 *l2_pt = mmu_info->l2_pts[l1_idx];
110 u32 iova2 = iova + (l2_idx << ISP_L2PT_SHIFT);
111
112 if (l2_pt[l2_idx] == mmu_info->dummy_page_pteval)
113 continue;
114
115 dev_dbg(mmu_info->dev,
116 "\tl2 entry %u; iova 0x%8.8x, phys %pa\n",
117 l2_idx, iova2,
118 TBL_PHYS_ADDR(l2_pt[l2_idx]));
119 }
120 }
121
122 dev_dbg(mmu_info->dev, "end IOMMU page table dump\n");
123 }
124 #endif /* DEBUG */
125
map_single(struct ipu6_mmu_info * mmu_info,void * ptr)126 static dma_addr_t map_single(struct ipu6_mmu_info *mmu_info, void *ptr)
127 {
128 dma_addr_t dma;
129
130 dma = dma_map_single(mmu_info->dev, ptr, PAGE_SIZE, DMA_BIDIRECTIONAL);
131 if (dma_mapping_error(mmu_info->dev, dma))
132 return 0;
133
134 return dma;
135 }
136
get_dummy_page(struct ipu6_mmu_info * mmu_info)137 static int get_dummy_page(struct ipu6_mmu_info *mmu_info)
138 {
139 void *pt = (void *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
140 dma_addr_t dma;
141
142 if (!pt)
143 return -ENOMEM;
144
145 dev_dbg(mmu_info->dev, "dummy_page: get_zeroed_page() == %p\n", pt);
146
147 dma = map_single(mmu_info, pt);
148 if (!dma) {
149 dev_err(mmu_info->dev, "Failed to map dummy page\n");
150 goto err_free_page;
151 }
152
153 mmu_info->dummy_page = pt;
154 mmu_info->dummy_page_pteval = dma >> ISP_PAGE_SHIFT;
155
156 return 0;
157
158 err_free_page:
159 free_page((unsigned long)pt);
160 return -ENOMEM;
161 }
162
free_dummy_page(struct ipu6_mmu_info * mmu_info)163 static void free_dummy_page(struct ipu6_mmu_info *mmu_info)
164 {
165 dma_unmap_single(mmu_info->dev,
166 TBL_PHYS_ADDR(mmu_info->dummy_page_pteval),
167 PAGE_SIZE, DMA_BIDIRECTIONAL);
168 free_page((unsigned long)mmu_info->dummy_page);
169 }
170
alloc_dummy_l2_pt(struct ipu6_mmu_info * mmu_info)171 static int alloc_dummy_l2_pt(struct ipu6_mmu_info *mmu_info)
172 {
173 u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
174 dma_addr_t dma;
175 unsigned int i;
176
177 if (!pt)
178 return -ENOMEM;
179
180 dev_dbg(mmu_info->dev, "dummy_l2: get_zeroed_page() = %p\n", pt);
181
182 dma = map_single(mmu_info, pt);
183 if (!dma) {
184 dev_err(mmu_info->dev, "Failed to map l2pt page\n");
185 goto err_free_page;
186 }
187
188 for (i = 0; i < ISP_L2PT_PTES; i++)
189 pt[i] = mmu_info->dummy_page_pteval;
190
191 mmu_info->dummy_l2_pt = pt;
192 mmu_info->dummy_l2_pteval = dma >> ISP_PAGE_SHIFT;
193
194 return 0;
195
196 err_free_page:
197 free_page((unsigned long)pt);
198 return -ENOMEM;
199 }
200
free_dummy_l2_pt(struct ipu6_mmu_info * mmu_info)201 static void free_dummy_l2_pt(struct ipu6_mmu_info *mmu_info)
202 {
203 dma_unmap_single(mmu_info->dev,
204 TBL_PHYS_ADDR(mmu_info->dummy_l2_pteval),
205 PAGE_SIZE, DMA_BIDIRECTIONAL);
206 free_page((unsigned long)mmu_info->dummy_l2_pt);
207 }
208
alloc_l1_pt(struct ipu6_mmu_info * mmu_info)209 static u32 *alloc_l1_pt(struct ipu6_mmu_info *mmu_info)
210 {
211 u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
212 dma_addr_t dma;
213 unsigned int i;
214
215 if (!pt)
216 return NULL;
217
218 dev_dbg(mmu_info->dev, "alloc_l1: get_zeroed_page() = %p\n", pt);
219
220 for (i = 0; i < ISP_L1PT_PTES; i++)
221 pt[i] = mmu_info->dummy_l2_pteval;
222
223 dma = map_single(mmu_info, pt);
224 if (!dma) {
225 dev_err(mmu_info->dev, "Failed to map l1pt page\n");
226 goto err_free_page;
227 }
228
229 mmu_info->l1_pt_dma = dma >> ISP_PADDR_SHIFT;
230 dev_dbg(mmu_info->dev, "l1 pt %p mapped at %llx\n", pt, dma);
231
232 return pt;
233
234 err_free_page:
235 free_page((unsigned long)pt);
236 return NULL;
237 }
238
alloc_l2_pt(struct ipu6_mmu_info * mmu_info)239 static u32 *alloc_l2_pt(struct ipu6_mmu_info *mmu_info)
240 {
241 u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
242 unsigned int i;
243
244 if (!pt)
245 return NULL;
246
247 dev_dbg(mmu_info->dev, "alloc_l2: get_zeroed_page() = %p\n", pt);
248
249 for (i = 0; i < ISP_L1PT_PTES; i++)
250 pt[i] = mmu_info->dummy_page_pteval;
251
252 return pt;
253 }
254
l2_map(struct ipu6_mmu_info * mmu_info,unsigned long iova,phys_addr_t paddr,size_t size)255 static int l2_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,
256 phys_addr_t paddr, size_t size)
257 {
258 u32 l1_idx = iova >> ISP_L1PT_SHIFT;
259 u32 iova_start = iova;
260 u32 *l2_pt, *l2_virt;
261 unsigned int l2_idx;
262 unsigned long flags;
263 dma_addr_t dma;
264 u32 l1_entry;
265
266 dev_dbg(mmu_info->dev,
267 "mapping l2 page table for l1 index %u (iova %8.8x)\n",
268 l1_idx, (u32)iova);
269
270 spin_lock_irqsave(&mmu_info->lock, flags);
271 l1_entry = mmu_info->l1_pt[l1_idx];
272 if (l1_entry == mmu_info->dummy_l2_pteval) {
273 l2_virt = mmu_info->l2_pts[l1_idx];
274 if (likely(!l2_virt)) {
275 l2_virt = alloc_l2_pt(mmu_info);
276 if (!l2_virt) {
277 spin_unlock_irqrestore(&mmu_info->lock, flags);
278 return -ENOMEM;
279 }
280 }
281
282 dma = map_single(mmu_info, l2_virt);
283 if (!dma) {
284 dev_err(mmu_info->dev, "Failed to map l2pt page\n");
285 free_page((unsigned long)l2_virt);
286 spin_unlock_irqrestore(&mmu_info->lock, flags);
287 return -EINVAL;
288 }
289
290 l1_entry = dma >> ISP_PADDR_SHIFT;
291
292 dev_dbg(mmu_info->dev, "page for l1_idx %u %p allocated\n",
293 l1_idx, l2_virt);
294 mmu_info->l1_pt[l1_idx] = l1_entry;
295 mmu_info->l2_pts[l1_idx] = l2_virt;
296 clflush_cache_range((void *)&mmu_info->l1_pt[l1_idx],
297 sizeof(mmu_info->l1_pt[l1_idx]));
298 }
299
300 l2_pt = mmu_info->l2_pts[l1_idx];
301
302 dev_dbg(mmu_info->dev, "l2_pt at %p with dma 0x%x\n", l2_pt, l1_entry);
303
304 paddr = ALIGN(paddr, ISP_PAGE_SIZE);
305
306 l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT;
307
308 dev_dbg(mmu_info->dev, "l2_idx %u, phys 0x%8.8x\n", l2_idx,
309 l2_pt[l2_idx]);
310 if (l2_pt[l2_idx] != mmu_info->dummy_page_pteval) {
311 spin_unlock_irqrestore(&mmu_info->lock, flags);
312 return -EINVAL;
313 }
314
315 l2_pt[l2_idx] = paddr >> ISP_PADDR_SHIFT;
316
317 clflush_cache_range((void *)&l2_pt[l2_idx], sizeof(l2_pt[l2_idx]));
318 spin_unlock_irqrestore(&mmu_info->lock, flags);
319
320 dev_dbg(mmu_info->dev, "l2 index %u mapped as 0x%8.8x\n", l2_idx,
321 l2_pt[l2_idx]);
322
323 return 0;
324 }
325
__ipu6_mmu_map(struct ipu6_mmu_info * mmu_info,unsigned long iova,phys_addr_t paddr,size_t size)326 static int __ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,
327 phys_addr_t paddr, size_t size)
328 {
329 u32 iova_start = round_down(iova, ISP_PAGE_SIZE);
330 u32 iova_end = ALIGN(iova + size, ISP_PAGE_SIZE);
331
332 dev_dbg(mmu_info->dev,
333 "mapping iova 0x%8.8x--0x%8.8x, size %zu at paddr 0x%10.10llx\n",
334 iova_start, iova_end, size, paddr);
335
336 return l2_map(mmu_info, iova_start, paddr, size);
337 }
338
l2_unmap(struct ipu6_mmu_info * mmu_info,unsigned long iova,phys_addr_t dummy,size_t size)339 static size_t l2_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova,
340 phys_addr_t dummy, size_t size)
341 {
342 u32 l1_idx = iova >> ISP_L1PT_SHIFT;
343 u32 iova_start = iova;
344 unsigned int l2_idx;
345 size_t unmapped = 0;
346 unsigned long flags;
347 u32 *l2_pt;
348
349 dev_dbg(mmu_info->dev, "unmapping l2 page table for l1 index %u (iova 0x%8.8lx)\n",
350 l1_idx, iova);
351
352 spin_lock_irqsave(&mmu_info->lock, flags);
353 if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) {
354 spin_unlock_irqrestore(&mmu_info->lock, flags);
355 dev_err(mmu_info->dev,
356 "unmap iova 0x%8.8lx l1 idx %u which was not mapped\n",
357 iova, l1_idx);
358 return 0;
359 }
360
361 for (l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT;
362 (iova_start & ISP_L1PT_MASK) + (l2_idx << ISP_PAGE_SHIFT)
363 < iova_start + size && l2_idx < ISP_L2PT_PTES; l2_idx++) {
364 l2_pt = mmu_info->l2_pts[l1_idx];
365 dev_dbg(mmu_info->dev,
366 "unmap l2 index %u with pteval 0x%10.10llx\n",
367 l2_idx, TBL_PHYS_ADDR(l2_pt[l2_idx]));
368 l2_pt[l2_idx] = mmu_info->dummy_page_pteval;
369
370 clflush_cache_range((void *)&l2_pt[l2_idx],
371 sizeof(l2_pt[l2_idx]));
372 unmapped++;
373 }
374 spin_unlock_irqrestore(&mmu_info->lock, flags);
375
376 return unmapped << ISP_PAGE_SHIFT;
377 }
378
__ipu6_mmu_unmap(struct ipu6_mmu_info * mmu_info,unsigned long iova,size_t size)379 static size_t __ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info,
380 unsigned long iova, size_t size)
381 {
382 return l2_unmap(mmu_info, iova, 0, size);
383 }
384
allocate_trash_buffer(struct ipu6_mmu * mmu)385 static int allocate_trash_buffer(struct ipu6_mmu *mmu)
386 {
387 unsigned int n_pages = PHYS_PFN(PAGE_ALIGN(IPU6_MMUV2_TRASH_RANGE));
388 struct iova *iova;
389 unsigned int i;
390 dma_addr_t dma;
391 unsigned long iova_addr;
392 int ret;
393
394 /* Allocate 8MB in iova range */
395 iova = alloc_iova(&mmu->dmap->iovad, n_pages,
396 PHYS_PFN(mmu->dmap->mmu_info->aperture_end), 0);
397 if (!iova) {
398 dev_err(mmu->dev, "cannot allocate iova range for trash\n");
399 return -ENOMEM;
400 }
401
402 dma = dma_map_page(mmu->dmap->mmu_info->dev, mmu->trash_page, 0,
403 PAGE_SIZE, DMA_BIDIRECTIONAL);
404 if (dma_mapping_error(mmu->dmap->mmu_info->dev, dma)) {
405 dev_err(mmu->dmap->mmu_info->dev, "Failed to map trash page\n");
406 ret = -ENOMEM;
407 goto out_free_iova;
408 }
409
410 mmu->pci_trash_page = dma;
411
412 /*
413 * Map the 8MB iova address range to the same physical trash page
414 * mmu->trash_page which is already reserved at the probe
415 */
416 iova_addr = iova->pfn_lo;
417 for (i = 0; i < n_pages; i++) {
418 ret = ipu6_mmu_map(mmu->dmap->mmu_info, PFN_PHYS(iova_addr),
419 mmu->pci_trash_page, PAGE_SIZE);
420 if (ret) {
421 dev_err(mmu->dev,
422 "mapping trash buffer range failed\n");
423 goto out_unmap;
424 }
425
426 iova_addr++;
427 }
428
429 mmu->iova_trash_page = PFN_PHYS(iova->pfn_lo);
430 dev_dbg(mmu->dev, "iova trash buffer for MMUID: %d is %u\n",
431 mmu->mmid, (unsigned int)mmu->iova_trash_page);
432 return 0;
433
434 out_unmap:
435 ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),
436 PFN_PHYS(iova_size(iova)));
437 dma_unmap_page(mmu->dmap->mmu_info->dev, mmu->pci_trash_page,
438 PAGE_SIZE, DMA_BIDIRECTIONAL);
439 out_free_iova:
440 __free_iova(&mmu->dmap->iovad, iova);
441 return ret;
442 }
443
ipu6_mmu_hw_init(struct ipu6_mmu * mmu)444 int ipu6_mmu_hw_init(struct ipu6_mmu *mmu)
445 {
446 struct ipu6_mmu_info *mmu_info;
447 unsigned long flags;
448 unsigned int i;
449
450 mmu_info = mmu->dmap->mmu_info;
451
452 /* Initialise the each MMU HW block */
453 for (i = 0; i < mmu->nr_mmus; i++) {
454 struct ipu6_mmu_hw *mmu_hw = &mmu->mmu_hw[i];
455 unsigned int j;
456 u16 block_addr;
457
458 /* Write page table address per MMU */
459 writel((phys_addr_t)mmu_info->l1_pt_dma,
460 mmu->mmu_hw[i].base + REG_L1_PHYS);
461
462 /* Set info bits per MMU */
463 writel(mmu->mmu_hw[i].info_bits,
464 mmu->mmu_hw[i].base + REG_INFO);
465
466 /* Configure MMU TLB stream configuration for L1 */
467 for (j = 0, block_addr = 0; j < mmu_hw->nr_l1streams;
468 block_addr += mmu->mmu_hw[i].l1_block_sz[j], j++) {
469 if (block_addr > IPU6_MAX_LI_BLOCK_ADDR) {
470 dev_err(mmu->dev, "invalid L1 configuration\n");
471 return -EINVAL;
472 }
473
474 /* Write block start address for each streams */
475 writel(block_addr, mmu_hw->base +
476 mmu_hw->l1_stream_id_reg_offset + 4 * j);
477 }
478
479 /* Configure MMU TLB stream configuration for L2 */
480 for (j = 0, block_addr = 0; j < mmu_hw->nr_l2streams;
481 block_addr += mmu->mmu_hw[i].l2_block_sz[j], j++) {
482 if (block_addr > IPU6_MAX_L2_BLOCK_ADDR) {
483 dev_err(mmu->dev, "invalid L2 configuration\n");
484 return -EINVAL;
485 }
486
487 writel(block_addr, mmu_hw->base +
488 mmu_hw->l2_stream_id_reg_offset + 4 * j);
489 }
490 }
491
492 if (!mmu->trash_page) {
493 int ret;
494
495 mmu->trash_page = alloc_page(GFP_KERNEL);
496 if (!mmu->trash_page) {
497 dev_err(mmu->dev, "insufficient memory for trash buffer\n");
498 return -ENOMEM;
499 }
500
501 ret = allocate_trash_buffer(mmu);
502 if (ret) {
503 __free_page(mmu->trash_page);
504 mmu->trash_page = NULL;
505 dev_err(mmu->dev, "trash buffer allocation failed\n");
506 return ret;
507 }
508 }
509
510 spin_lock_irqsave(&mmu->ready_lock, flags);
511 mmu->ready = true;
512 spin_unlock_irqrestore(&mmu->ready_lock, flags);
513
514 return 0;
515 }
516 EXPORT_SYMBOL_NS_GPL(ipu6_mmu_hw_init, INTEL_IPU6);
517
ipu6_mmu_alloc(struct ipu6_device * isp)518 static struct ipu6_mmu_info *ipu6_mmu_alloc(struct ipu6_device *isp)
519 {
520 struct ipu6_mmu_info *mmu_info;
521 int ret;
522
523 mmu_info = kzalloc(sizeof(*mmu_info), GFP_KERNEL);
524 if (!mmu_info)
525 return NULL;
526
527 mmu_info->aperture_start = 0;
528 mmu_info->aperture_end = DMA_BIT_MASK(isp->secure_mode ?
529 IPU6_MMU_ADDR_BITS :
530 IPU6_MMU_ADDR_BITS_NON_SECURE);
531 mmu_info->pgsize_bitmap = SZ_4K;
532 mmu_info->dev = &isp->pdev->dev;
533
534 ret = get_dummy_page(mmu_info);
535 if (ret)
536 goto err_free_info;
537
538 ret = alloc_dummy_l2_pt(mmu_info);
539 if (ret)
540 goto err_free_dummy_page;
541
542 mmu_info->l2_pts = vzalloc(ISP_L2PT_PTES * sizeof(*mmu_info->l2_pts));
543 if (!mmu_info->l2_pts)
544 goto err_free_dummy_l2_pt;
545
546 /*
547 * We always map the L1 page table (a single page as well as
548 * the L2 page tables).
549 */
550 mmu_info->l1_pt = alloc_l1_pt(mmu_info);
551 if (!mmu_info->l1_pt)
552 goto err_free_l2_pts;
553
554 spin_lock_init(&mmu_info->lock);
555
556 dev_dbg(mmu_info->dev, "domain initialised\n");
557
558 return mmu_info;
559
560 err_free_l2_pts:
561 vfree(mmu_info->l2_pts);
562 err_free_dummy_l2_pt:
563 free_dummy_l2_pt(mmu_info);
564 err_free_dummy_page:
565 free_dummy_page(mmu_info);
566 err_free_info:
567 kfree(mmu_info);
568
569 return NULL;
570 }
571
ipu6_mmu_hw_cleanup(struct ipu6_mmu * mmu)572 void ipu6_mmu_hw_cleanup(struct ipu6_mmu *mmu)
573 {
574 unsigned long flags;
575
576 spin_lock_irqsave(&mmu->ready_lock, flags);
577 mmu->ready = false;
578 spin_unlock_irqrestore(&mmu->ready_lock, flags);
579 }
580 EXPORT_SYMBOL_NS_GPL(ipu6_mmu_hw_cleanup, INTEL_IPU6);
581
alloc_dma_mapping(struct ipu6_device * isp)582 static struct ipu6_dma_mapping *alloc_dma_mapping(struct ipu6_device *isp)
583 {
584 struct ipu6_dma_mapping *dmap;
585
586 dmap = kzalloc(sizeof(*dmap), GFP_KERNEL);
587 if (!dmap)
588 return NULL;
589
590 dmap->mmu_info = ipu6_mmu_alloc(isp);
591 if (!dmap->mmu_info) {
592 kfree(dmap);
593 return NULL;
594 }
595
596 init_iova_domain(&dmap->iovad, SZ_4K, 1);
597 dmap->mmu_info->dmap = dmap;
598
599 dev_dbg(&isp->pdev->dev, "alloc mapping\n");
600
601 iova_cache_get();
602
603 return dmap;
604 }
605
ipu6_mmu_iova_to_phys(struct ipu6_mmu_info * mmu_info,dma_addr_t iova)606 phys_addr_t ipu6_mmu_iova_to_phys(struct ipu6_mmu_info *mmu_info,
607 dma_addr_t iova)
608 {
609 phys_addr_t phy_addr;
610 unsigned long flags;
611 u32 *l2_pt;
612
613 spin_lock_irqsave(&mmu_info->lock, flags);
614 l2_pt = mmu_info->l2_pts[iova >> ISP_L1PT_SHIFT];
615 phy_addr = (phys_addr_t)l2_pt[(iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT];
616 phy_addr <<= ISP_PAGE_SHIFT;
617 spin_unlock_irqrestore(&mmu_info->lock, flags);
618
619 return phy_addr;
620 }
621
ipu6_mmu_pgsize(unsigned long pgsize_bitmap,unsigned long addr_merge,size_t size)622 static size_t ipu6_mmu_pgsize(unsigned long pgsize_bitmap,
623 unsigned long addr_merge, size_t size)
624 {
625 unsigned int pgsize_idx;
626 size_t pgsize;
627
628 /* Max page size that still fits into 'size' */
629 pgsize_idx = __fls(size);
630
631 if (likely(addr_merge)) {
632 /* Max page size allowed by address */
633 unsigned int align_pgsize_idx = __ffs(addr_merge);
634
635 pgsize_idx = min(pgsize_idx, align_pgsize_idx);
636 }
637
638 pgsize = (1UL << (pgsize_idx + 1)) - 1;
639 pgsize &= pgsize_bitmap;
640
641 WARN_ON(!pgsize);
642
643 /* pick the biggest page */
644 pgsize_idx = __fls(pgsize);
645 pgsize = 1UL << pgsize_idx;
646
647 return pgsize;
648 }
649
ipu6_mmu_unmap(struct ipu6_mmu_info * mmu_info,unsigned long iova,size_t size)650 size_t ipu6_mmu_unmap(struct ipu6_mmu_info *mmu_info, unsigned long iova,
651 size_t size)
652 {
653 size_t unmapped_page, unmapped = 0;
654 unsigned int min_pagesz;
655
656 /* find out the minimum page size supported */
657 min_pagesz = 1 << __ffs(mmu_info->pgsize_bitmap);
658
659 /*
660 * The virtual address and the size of the mapping must be
661 * aligned (at least) to the size of the smallest page supported
662 * by the hardware
663 */
664 if (!IS_ALIGNED(iova | size, min_pagesz)) {
665 dev_err(NULL, "unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
666 iova, size, min_pagesz);
667 return -EINVAL;
668 }
669
670 /*
671 * Keep iterating until we either unmap 'size' bytes (or more)
672 * or we hit an area that isn't mapped.
673 */
674 while (unmapped < size) {
675 size_t pgsize = ipu6_mmu_pgsize(mmu_info->pgsize_bitmap,
676 iova, size - unmapped);
677
678 unmapped_page = __ipu6_mmu_unmap(mmu_info, iova, pgsize);
679 if (!unmapped_page)
680 break;
681
682 dev_dbg(mmu_info->dev, "unmapped: iova 0x%lx size 0x%zx\n",
683 iova, unmapped_page);
684
685 iova += unmapped_page;
686 unmapped += unmapped_page;
687 }
688
689 return unmapped;
690 }
691
ipu6_mmu_map(struct ipu6_mmu_info * mmu_info,unsigned long iova,phys_addr_t paddr,size_t size)692 int ipu6_mmu_map(struct ipu6_mmu_info *mmu_info, unsigned long iova,
693 phys_addr_t paddr, size_t size)
694 {
695 unsigned long orig_iova = iova;
696 unsigned int min_pagesz;
697 size_t orig_size = size;
698 int ret = 0;
699
700 if (mmu_info->pgsize_bitmap == 0UL)
701 return -ENODEV;
702
703 /* find out the minimum page size supported */
704 min_pagesz = 1 << __ffs(mmu_info->pgsize_bitmap);
705
706 /*
707 * both the virtual address and the physical one, as well as
708 * the size of the mapping, must be aligned (at least) to the
709 * size of the smallest page supported by the hardware
710 */
711 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
712 dev_err(mmu_info->dev,
713 "unaligned: iova %lx pa %pa size %zx min_pagesz %x\n",
714 iova, &paddr, size, min_pagesz);
715 return -EINVAL;
716 }
717
718 dev_dbg(mmu_info->dev, "map: iova 0x%lx pa %pa size 0x%zx\n",
719 iova, &paddr, size);
720
721 while (size) {
722 size_t pgsize = ipu6_mmu_pgsize(mmu_info->pgsize_bitmap,
723 iova | paddr, size);
724
725 dev_dbg(mmu_info->dev,
726 "mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
727 iova, &paddr, pgsize);
728
729 ret = __ipu6_mmu_map(mmu_info, iova, paddr, pgsize);
730 if (ret)
731 break;
732
733 iova += pgsize;
734 paddr += pgsize;
735 size -= pgsize;
736 }
737
738 /* unroll mapping in case something went wrong */
739 if (ret)
740 ipu6_mmu_unmap(mmu_info, orig_iova, orig_size - size);
741
742 return ret;
743 }
744
ipu6_mmu_destroy(struct ipu6_mmu * mmu)745 static void ipu6_mmu_destroy(struct ipu6_mmu *mmu)
746 {
747 struct ipu6_dma_mapping *dmap = mmu->dmap;
748 struct ipu6_mmu_info *mmu_info = dmap->mmu_info;
749 struct iova *iova;
750 u32 l1_idx;
751
752 if (mmu->iova_trash_page) {
753 iova = find_iova(&dmap->iovad, PHYS_PFN(mmu->iova_trash_page));
754 if (iova) {
755 /* unmap and free the trash buffer iova */
756 ipu6_mmu_unmap(mmu_info, PFN_PHYS(iova->pfn_lo),
757 PFN_PHYS(iova_size(iova)));
758 __free_iova(&dmap->iovad, iova);
759 } else {
760 dev_err(mmu->dev, "trash buffer iova not found.\n");
761 }
762
763 mmu->iova_trash_page = 0;
764 dma_unmap_page(mmu_info->dev, mmu->pci_trash_page,
765 PAGE_SIZE, DMA_BIDIRECTIONAL);
766 mmu->pci_trash_page = 0;
767 __free_page(mmu->trash_page);
768 }
769
770 for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {
771 if (mmu_info->l1_pt[l1_idx] != mmu_info->dummy_l2_pteval) {
772 dma_unmap_single(mmu_info->dev,
773 TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]),
774 PAGE_SIZE, DMA_BIDIRECTIONAL);
775 free_page((unsigned long)mmu_info->l2_pts[l1_idx]);
776 }
777 }
778
779 vfree(mmu_info->l2_pts);
780 free_dummy_page(mmu_info);
781 dma_unmap_single(mmu_info->dev, TBL_PHYS_ADDR(mmu_info->l1_pt_dma),
782 PAGE_SIZE, DMA_BIDIRECTIONAL);
783 free_page((unsigned long)mmu_info->dummy_l2_pt);
784 free_page((unsigned long)mmu_info->l1_pt);
785 kfree(mmu_info);
786 }
787
ipu6_mmu_init(struct device * dev,void __iomem * base,int mmid,const struct ipu6_hw_variants * hw)788 struct ipu6_mmu *ipu6_mmu_init(struct device *dev,
789 void __iomem *base, int mmid,
790 const struct ipu6_hw_variants *hw)
791 {
792 struct ipu6_device *isp = pci_get_drvdata(to_pci_dev(dev));
793 struct ipu6_mmu_pdata *pdata;
794 struct ipu6_mmu *mmu;
795 unsigned int i;
796
797 if (hw->nr_mmus > IPU6_MMU_MAX_DEVICES)
798 return ERR_PTR(-EINVAL);
799
800 pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
801 if (!pdata)
802 return ERR_PTR(-ENOMEM);
803
804 for (i = 0; i < hw->nr_mmus; i++) {
805 struct ipu6_mmu_hw *pdata_mmu = &pdata->mmu_hw[i];
806 const struct ipu6_mmu_hw *src_mmu = &hw->mmu_hw[i];
807
808 if (src_mmu->nr_l1streams > IPU6_MMU_MAX_TLB_L1_STREAMS ||
809 src_mmu->nr_l2streams > IPU6_MMU_MAX_TLB_L2_STREAMS)
810 return ERR_PTR(-EINVAL);
811
812 *pdata_mmu = *src_mmu;
813 pdata_mmu->base = base + src_mmu->offset;
814 }
815
816 mmu = devm_kzalloc(dev, sizeof(*mmu), GFP_KERNEL);
817 if (!mmu)
818 return ERR_PTR(-ENOMEM);
819
820 mmu->mmid = mmid;
821 mmu->mmu_hw = pdata->mmu_hw;
822 mmu->nr_mmus = hw->nr_mmus;
823 mmu->tlb_invalidate = tlb_invalidate;
824 mmu->ready = false;
825 INIT_LIST_HEAD(&mmu->vma_list);
826 spin_lock_init(&mmu->ready_lock);
827
828 mmu->dmap = alloc_dma_mapping(isp);
829 if (!mmu->dmap) {
830 dev_err(dev, "can't alloc dma mapping\n");
831 return ERR_PTR(-ENOMEM);
832 }
833
834 return mmu;
835 }
836
ipu6_mmu_cleanup(struct ipu6_mmu * mmu)837 void ipu6_mmu_cleanup(struct ipu6_mmu *mmu)
838 {
839 struct ipu6_dma_mapping *dmap = mmu->dmap;
840
841 ipu6_mmu_destroy(mmu);
842 mmu->dmap = NULL;
843 iova_cache_put();
844 put_iova_domain(&dmap->iovad);
845 kfree(dmap);
846 }
847