1459121c9SGlauber Costa #include <linux/dma-mapping.h> 2cb5867a5SGlauber Costa #include <linux/dmar.h> 3116890d5SGlauber Costa #include <linux/bootmem.h> 4bca5c096SGlauber Costa #include <linux/pci.h> 5cb5867a5SGlauber Costa 6116890d5SGlauber Costa #include <asm/proto.h> 7116890d5SGlauber Costa #include <asm/dma.h> 846a7fa27SFUJITA Tomonori #include <asm/iommu.h> 91d9b16d1SJoerg Roedel #include <asm/gart.h> 10cb5867a5SGlauber Costa #include <asm/calgary.h> 11a69ca340SJoerg Roedel #include <asm/amd_iommu.h> 12459121c9SGlauber Costa 133b15e581SFenghua Yu static int forbid_dac __read_mostly; 143b15e581SFenghua Yu 15*160c1d8eSFUJITA Tomonori struct dma_map_ops *dma_ops; 1685c246eeSGlauber Costa EXPORT_SYMBOL(dma_ops); 1785c246eeSGlauber Costa 18b4cdc430SDmitri Vorobiev static int iommu_sac_force __read_mostly; 198e0c3797SGlauber Costa 20f9c258deSGlauber Costa #ifdef CONFIG_IOMMU_DEBUG 21f9c258deSGlauber Costa int panic_on_overflow __read_mostly = 1; 22f9c258deSGlauber Costa int force_iommu __read_mostly = 1; 23f9c258deSGlauber Costa #else 24f9c258deSGlauber Costa int panic_on_overflow __read_mostly = 0; 25f9c258deSGlauber Costa int force_iommu __read_mostly = 0; 26f9c258deSGlauber Costa #endif 27f9c258deSGlauber Costa 28fae9a0d8SGlauber Costa int iommu_merge __read_mostly = 0; 29fae9a0d8SGlauber Costa 30fae9a0d8SGlauber Costa int no_iommu __read_mostly; 31fae9a0d8SGlauber Costa /* Set this to 1 if there is a HW IOMMU in the system */ 32fae9a0d8SGlauber Costa int iommu_detected __read_mostly = 0; 33fae9a0d8SGlauber Costa 34cac67877SGlauber Costa dma_addr_t bad_dma_address __read_mostly = 0; 35cac67877SGlauber Costa EXPORT_SYMBOL(bad_dma_address); 36fae9a0d8SGlauber Costa 37098cb7f2SGlauber Costa /* Dummy device used for NULL arguments (normally ISA). Better would 38098cb7f2SGlauber Costa be probably a smaller DMA mask, but this is bug-to-bug compatible 39098cb7f2SGlauber Costa to older i386. */ 406c505ce3SJoerg Roedel struct device x86_dma_fallback_dev = { 41098cb7f2SGlauber Costa .bus_id = "fallback device", 42098cb7f2SGlauber Costa .coherent_dma_mask = DMA_32BIT_MASK, 436c505ce3SJoerg Roedel .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask, 44098cb7f2SGlauber Costa }; 456c505ce3SJoerg Roedel EXPORT_SYMBOL(x86_dma_fallback_dev); 46098cb7f2SGlauber Costa 47459121c9SGlauber Costa int dma_set_mask(struct device *dev, u64 mask) 48459121c9SGlauber Costa { 49459121c9SGlauber Costa if (!dev->dma_mask || !dma_supported(dev, mask)) 50459121c9SGlauber Costa return -EIO; 51459121c9SGlauber Costa 52459121c9SGlauber Costa *dev->dma_mask = mask; 53459121c9SGlauber Costa 54459121c9SGlauber Costa return 0; 55459121c9SGlauber Costa } 56459121c9SGlauber Costa EXPORT_SYMBOL(dma_set_mask); 57459121c9SGlauber Costa 58116890d5SGlauber Costa #ifdef CONFIG_X86_64 59116890d5SGlauber Costa static __initdata void *dma32_bootmem_ptr; 60116890d5SGlauber Costa static unsigned long dma32_bootmem_size __initdata = (128ULL<<20); 61116890d5SGlauber Costa 62116890d5SGlauber Costa static int __init parse_dma32_size_opt(char *p) 63116890d5SGlauber Costa { 64116890d5SGlauber Costa if (!p) 65116890d5SGlauber Costa return -EINVAL; 66116890d5SGlauber Costa dma32_bootmem_size = memparse(p, &p); 67116890d5SGlauber Costa return 0; 68116890d5SGlauber Costa } 69116890d5SGlauber Costa early_param("dma32_size", parse_dma32_size_opt); 70116890d5SGlauber Costa 71116890d5SGlauber Costa void __init dma32_reserve_bootmem(void) 72116890d5SGlauber Costa { 73116890d5SGlauber Costa unsigned long size, align; 74c987d12fSYinghai Lu if (max_pfn <= MAX_DMA32_PFN) 75116890d5SGlauber Costa return; 76116890d5SGlauber Costa 777677b2efSYinghai Lu /* 787677b2efSYinghai Lu * check aperture_64.c allocate_aperture() for reason about 797677b2efSYinghai Lu * using 512M as goal 807677b2efSYinghai Lu */ 81116890d5SGlauber Costa align = 64ULL<<20; 821ddb5518SJoerg Roedel size = roundup(dma32_bootmem_size, align); 83116890d5SGlauber Costa dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align, 847677b2efSYinghai Lu 512ULL<<20); 85116890d5SGlauber Costa if (dma32_bootmem_ptr) 86116890d5SGlauber Costa dma32_bootmem_size = size; 87116890d5SGlauber Costa else 88116890d5SGlauber Costa dma32_bootmem_size = 0; 89116890d5SGlauber Costa } 90116890d5SGlauber Costa static void __init dma32_free_bootmem(void) 91116890d5SGlauber Costa { 92116890d5SGlauber Costa 93c987d12fSYinghai Lu if (max_pfn <= MAX_DMA32_PFN) 94116890d5SGlauber Costa return; 95116890d5SGlauber Costa 96116890d5SGlauber Costa if (!dma32_bootmem_ptr) 97116890d5SGlauber Costa return; 98116890d5SGlauber Costa 99330fce23SYinghai Lu free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size); 100116890d5SGlauber Costa 101116890d5SGlauber Costa dma32_bootmem_ptr = NULL; 102116890d5SGlauber Costa dma32_bootmem_size = 0; 103116890d5SGlauber Costa } 104cfb80c9eSJeremy Fitzhardinge #endif 105116890d5SGlauber Costa 106116890d5SGlauber Costa void __init pci_iommu_alloc(void) 107116890d5SGlauber Costa { 108cfb80c9eSJeremy Fitzhardinge #ifdef CONFIG_X86_64 109116890d5SGlauber Costa /* free the range so iommu could get some range less than 4G */ 110116890d5SGlauber Costa dma32_free_bootmem(); 111cfb80c9eSJeremy Fitzhardinge #endif 112cfb80c9eSJeremy Fitzhardinge 113116890d5SGlauber Costa /* 114116890d5SGlauber Costa * The order of these functions is important for 115116890d5SGlauber Costa * fall-back/fail-over reasons 116116890d5SGlauber Costa */ 117116890d5SGlauber Costa gart_iommu_hole_init(); 118116890d5SGlauber Costa 119116890d5SGlauber Costa detect_calgary(); 120116890d5SGlauber Costa 121116890d5SGlauber Costa detect_intel_iommu(); 122116890d5SGlauber Costa 123a69ca340SJoerg Roedel amd_iommu_detect(); 124a69ca340SJoerg Roedel 125116890d5SGlauber Costa pci_swiotlb_init(); 126116890d5SGlauber Costa } 1278978b742SFUJITA Tomonori 1289f6ac577SFUJITA Tomonori void *dma_generic_alloc_coherent(struct device *dev, size_t size, 1299f6ac577SFUJITA Tomonori dma_addr_t *dma_addr, gfp_t flag) 1309f6ac577SFUJITA Tomonori { 1319f6ac577SFUJITA Tomonori unsigned long dma_mask; 1329f6ac577SFUJITA Tomonori struct page *page; 1339f6ac577SFUJITA Tomonori dma_addr_t addr; 1349f6ac577SFUJITA Tomonori 1359f6ac577SFUJITA Tomonori dma_mask = dma_alloc_coherent_mask(dev, flag); 1369f6ac577SFUJITA Tomonori 1379f6ac577SFUJITA Tomonori flag |= __GFP_ZERO; 1389f6ac577SFUJITA Tomonori again: 1399f6ac577SFUJITA Tomonori page = alloc_pages_node(dev_to_node(dev), flag, get_order(size)); 1409f6ac577SFUJITA Tomonori if (!page) 1419f6ac577SFUJITA Tomonori return NULL; 1429f6ac577SFUJITA Tomonori 1439f6ac577SFUJITA Tomonori addr = page_to_phys(page); 1449f6ac577SFUJITA Tomonori if (!is_buffer_dma_capable(dma_mask, addr, size)) { 1459f6ac577SFUJITA Tomonori __free_pages(page, get_order(size)); 1469f6ac577SFUJITA Tomonori 1479f6ac577SFUJITA Tomonori if (dma_mask < DMA_32BIT_MASK && !(flag & GFP_DMA)) { 1489f6ac577SFUJITA Tomonori flag = (flag & ~GFP_DMA32) | GFP_DMA; 1499f6ac577SFUJITA Tomonori goto again; 1509f6ac577SFUJITA Tomonori } 1519f6ac577SFUJITA Tomonori 1529f6ac577SFUJITA Tomonori return NULL; 1539f6ac577SFUJITA Tomonori } 1549f6ac577SFUJITA Tomonori 1559f6ac577SFUJITA Tomonori *dma_addr = addr; 1569f6ac577SFUJITA Tomonori return page_address(page); 1579f6ac577SFUJITA Tomonori } 1589f6ac577SFUJITA Tomonori 159fae9a0d8SGlauber Costa /* 160fae9a0d8SGlauber Costa * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter 161fae9a0d8SGlauber Costa * documentation. 162fae9a0d8SGlauber Costa */ 163fae9a0d8SGlauber Costa static __init int iommu_setup(char *p) 164fae9a0d8SGlauber Costa { 165fae9a0d8SGlauber Costa iommu_merge = 1; 166fae9a0d8SGlauber Costa 167fae9a0d8SGlauber Costa if (!p) 168fae9a0d8SGlauber Costa return -EINVAL; 169fae9a0d8SGlauber Costa 170fae9a0d8SGlauber Costa while (*p) { 171fae9a0d8SGlauber Costa if (!strncmp(p, "off", 3)) 172fae9a0d8SGlauber Costa no_iommu = 1; 173fae9a0d8SGlauber Costa /* gart_parse_options has more force support */ 174fae9a0d8SGlauber Costa if (!strncmp(p, "force", 5)) 175fae9a0d8SGlauber Costa force_iommu = 1; 176fae9a0d8SGlauber Costa if (!strncmp(p, "noforce", 7)) { 177fae9a0d8SGlauber Costa iommu_merge = 0; 178fae9a0d8SGlauber Costa force_iommu = 0; 179fae9a0d8SGlauber Costa } 180fae9a0d8SGlauber Costa 181fae9a0d8SGlauber Costa if (!strncmp(p, "biomerge", 8)) { 182fae9a0d8SGlauber Costa iommu_merge = 1; 183fae9a0d8SGlauber Costa force_iommu = 1; 184fae9a0d8SGlauber Costa } 185fae9a0d8SGlauber Costa if (!strncmp(p, "panic", 5)) 186fae9a0d8SGlauber Costa panic_on_overflow = 1; 187fae9a0d8SGlauber Costa if (!strncmp(p, "nopanic", 7)) 188fae9a0d8SGlauber Costa panic_on_overflow = 0; 189fae9a0d8SGlauber Costa if (!strncmp(p, "merge", 5)) { 190fae9a0d8SGlauber Costa iommu_merge = 1; 191fae9a0d8SGlauber Costa force_iommu = 1; 192fae9a0d8SGlauber Costa } 193fae9a0d8SGlauber Costa if (!strncmp(p, "nomerge", 7)) 194fae9a0d8SGlauber Costa iommu_merge = 0; 195fae9a0d8SGlauber Costa if (!strncmp(p, "forcesac", 8)) 196fae9a0d8SGlauber Costa iommu_sac_force = 1; 197fae9a0d8SGlauber Costa if (!strncmp(p, "allowdac", 8)) 198fae9a0d8SGlauber Costa forbid_dac = 0; 199fae9a0d8SGlauber Costa if (!strncmp(p, "nodac", 5)) 200fae9a0d8SGlauber Costa forbid_dac = -1; 201fae9a0d8SGlauber Costa if (!strncmp(p, "usedac", 6)) { 202fae9a0d8SGlauber Costa forbid_dac = -1; 203fae9a0d8SGlauber Costa return 1; 204fae9a0d8SGlauber Costa } 205fae9a0d8SGlauber Costa #ifdef CONFIG_SWIOTLB 206fae9a0d8SGlauber Costa if (!strncmp(p, "soft", 4)) 207fae9a0d8SGlauber Costa swiotlb = 1; 208fae9a0d8SGlauber Costa #endif 209fae9a0d8SGlauber Costa 210fae9a0d8SGlauber Costa gart_parse_options(p); 211fae9a0d8SGlauber Costa 212fae9a0d8SGlauber Costa #ifdef CONFIG_CALGARY_IOMMU 213fae9a0d8SGlauber Costa if (!strncmp(p, "calgary", 7)) 214fae9a0d8SGlauber Costa use_calgary = 1; 215fae9a0d8SGlauber Costa #endif /* CONFIG_CALGARY_IOMMU */ 216fae9a0d8SGlauber Costa 217fae9a0d8SGlauber Costa p += strcspn(p, ","); 218fae9a0d8SGlauber Costa if (*p == ',') 219fae9a0d8SGlauber Costa ++p; 220fae9a0d8SGlauber Costa } 221fae9a0d8SGlauber Costa return 0; 222fae9a0d8SGlauber Costa } 223fae9a0d8SGlauber Costa early_param("iommu", iommu_setup); 224fae9a0d8SGlauber Costa 2258e0c3797SGlauber Costa int dma_supported(struct device *dev, u64 mask) 2268e0c3797SGlauber Costa { 227*160c1d8eSFUJITA Tomonori struct dma_map_ops *ops = get_dma_ops(dev); 2288d8bb39bSFUJITA Tomonori 2298e0c3797SGlauber Costa #ifdef CONFIG_PCI 2308e0c3797SGlauber Costa if (mask > 0xffffffff && forbid_dac > 0) { 231fc3a8828SGreg Kroah-Hartman dev_info(dev, "PCI: Disallowing DAC for device\n"); 2328e0c3797SGlauber Costa return 0; 2338e0c3797SGlauber Costa } 2348e0c3797SGlauber Costa #endif 2358e0c3797SGlauber Costa 2368d8bb39bSFUJITA Tomonori if (ops->dma_supported) 2378d8bb39bSFUJITA Tomonori return ops->dma_supported(dev, mask); 2388e0c3797SGlauber Costa 2398e0c3797SGlauber Costa /* Copied from i386. Doesn't make much sense, because it will 2408e0c3797SGlauber Costa only work for pci_alloc_coherent. 2418e0c3797SGlauber Costa The caller just has to use GFP_DMA in this case. */ 2428e0c3797SGlauber Costa if (mask < DMA_24BIT_MASK) 2438e0c3797SGlauber Costa return 0; 2448e0c3797SGlauber Costa 2458e0c3797SGlauber Costa /* Tell the device to use SAC when IOMMU force is on. This 2468e0c3797SGlauber Costa allows the driver to use cheaper accesses in some cases. 2478e0c3797SGlauber Costa 2488e0c3797SGlauber Costa Problem with this is that if we overflow the IOMMU area and 2498e0c3797SGlauber Costa return DAC as fallback address the device may not handle it 2508e0c3797SGlauber Costa correctly. 2518e0c3797SGlauber Costa 2528e0c3797SGlauber Costa As a special case some controllers have a 39bit address 2538e0c3797SGlauber Costa mode that is as efficient as 32bit (aic79xx). Don't force 2548e0c3797SGlauber Costa SAC for these. Assume all masks <= 40 bits are of this 2558e0c3797SGlauber Costa type. Normally this doesn't make any difference, but gives 2568e0c3797SGlauber Costa more gentle handling of IOMMU overflow. */ 2578e0c3797SGlauber Costa if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) { 258fc3a8828SGreg Kroah-Hartman dev_info(dev, "Force SAC with mask %Lx\n", mask); 2598e0c3797SGlauber Costa return 0; 2608e0c3797SGlauber Costa } 2618e0c3797SGlauber Costa 2628e0c3797SGlauber Costa return 1; 2638e0c3797SGlauber Costa } 2648e0c3797SGlauber Costa EXPORT_SYMBOL(dma_supported); 2658e0c3797SGlauber Costa 266cb5867a5SGlauber Costa static int __init pci_iommu_init(void) 267cb5867a5SGlauber Costa { 268cb5867a5SGlauber Costa calgary_iommu_init(); 269459121c9SGlauber Costa 270cb5867a5SGlauber Costa intel_iommu_init(); 271cb5867a5SGlauber Costa 272a69ca340SJoerg Roedel amd_iommu_init(); 273a69ca340SJoerg Roedel 274cb5867a5SGlauber Costa gart_iommu_init(); 275cb5867a5SGlauber Costa 276cb5867a5SGlauber Costa no_iommu_init(); 277cb5867a5SGlauber Costa return 0; 278cb5867a5SGlauber Costa } 279cb5867a5SGlauber Costa 280cb5867a5SGlauber Costa void pci_iommu_shutdown(void) 281cb5867a5SGlauber Costa { 282cb5867a5SGlauber Costa gart_iommu_shutdown(); 283cb5867a5SGlauber Costa } 284cb5867a5SGlauber Costa /* Must execute after PCI subsystem */ 285cb5867a5SGlauber Costa fs_initcall(pci_iommu_init); 2863b15e581SFenghua Yu 2873b15e581SFenghua Yu #ifdef CONFIG_PCI 2883b15e581SFenghua Yu /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ 2893b15e581SFenghua Yu 2903b15e581SFenghua Yu static __devinit void via_no_dac(struct pci_dev *dev) 2913b15e581SFenghua Yu { 2923b15e581SFenghua Yu if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { 293a0286c94SMichael Tokarev printk(KERN_INFO 294a0286c94SMichael Tokarev "PCI: VIA PCI bridge detected. Disabling DAC.\n"); 2953b15e581SFenghua Yu forbid_dac = 1; 2963b15e581SFenghua Yu } 2973b15e581SFenghua Yu } 2983b15e581SFenghua Yu DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac); 2993b15e581SFenghua Yu #endif 300