1459121c9SGlauber Costa #include <linux/dma-mapping.h> 22118d0c5SJoerg Roedel #include <linux/dma-debug.h> 3cb5867a5SGlauber Costa #include <linux/dmar.h> 4116890d5SGlauber Costa #include <linux/bootmem.h> 5bca5c096SGlauber Costa #include <linux/pci.h> 6cb5867a5SGlauber Costa 7116890d5SGlauber Costa #include <asm/proto.h> 8116890d5SGlauber Costa #include <asm/dma.h> 946a7fa27SFUJITA Tomonori #include <asm/iommu.h> 101d9b16d1SJoerg Roedel #include <asm/gart.h> 11cb5867a5SGlauber Costa #include <asm/calgary.h> 12a69ca340SJoerg Roedel #include <asm/amd_iommu.h> 13459121c9SGlauber Costa 143b15e581SFenghua Yu static int forbid_dac __read_mostly; 153b15e581SFenghua Yu 16160c1d8eSFUJITA Tomonori struct dma_map_ops *dma_ops; 1785c246eeSGlauber Costa EXPORT_SYMBOL(dma_ops); 1885c246eeSGlauber Costa 19b4cdc430SDmitri Vorobiev static int iommu_sac_force __read_mostly; 208e0c3797SGlauber Costa 21f9c258deSGlauber Costa #ifdef CONFIG_IOMMU_DEBUG 22f9c258deSGlauber Costa int panic_on_overflow __read_mostly = 1; 23f9c258deSGlauber Costa int force_iommu __read_mostly = 1; 24f9c258deSGlauber Costa #else 25f9c258deSGlauber Costa int panic_on_overflow __read_mostly = 0; 26f9c258deSGlauber Costa int force_iommu __read_mostly = 0; 27f9c258deSGlauber Costa #endif 28f9c258deSGlauber Costa 29fae9a0d8SGlauber Costa int iommu_merge __read_mostly = 0; 30fae9a0d8SGlauber Costa 31fae9a0d8SGlauber Costa int no_iommu __read_mostly; 32fae9a0d8SGlauber Costa /* Set this to 1 if there is a HW IOMMU in the system */ 33fae9a0d8SGlauber Costa int iommu_detected __read_mostly = 0; 34fae9a0d8SGlauber Costa 35cac67877SGlauber Costa dma_addr_t bad_dma_address __read_mostly = 0; 36cac67877SGlauber Costa EXPORT_SYMBOL(bad_dma_address); 37fae9a0d8SGlauber Costa 38098cb7f2SGlauber Costa /* Dummy device used for NULL arguments (normally ISA). Better would 39098cb7f2SGlauber Costa be probably a smaller DMA mask, but this is bug-to-bug compatible 40098cb7f2SGlauber Costa to older i386. */ 416c505ce3SJoerg Roedel struct device x86_dma_fallback_dev = { 421a927133SKay Sievers .init_name = "fallback device", 43284901a9SYang Hongyang .coherent_dma_mask = DMA_BIT_MASK(32), 446c505ce3SJoerg Roedel .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask, 45098cb7f2SGlauber Costa }; 466c505ce3SJoerg Roedel EXPORT_SYMBOL(x86_dma_fallback_dev); 47098cb7f2SGlauber Costa 482118d0c5SJoerg Roedel /* Number of entries preallocated for DMA-API debugging */ 492118d0c5SJoerg Roedel #define PREALLOC_DMA_DEBUG_ENTRIES 32768 502118d0c5SJoerg Roedel 51459121c9SGlauber Costa int dma_set_mask(struct device *dev, u64 mask) 52459121c9SGlauber Costa { 53459121c9SGlauber Costa if (!dev->dma_mask || !dma_supported(dev, mask)) 54459121c9SGlauber Costa return -EIO; 55459121c9SGlauber Costa 56459121c9SGlauber Costa *dev->dma_mask = mask; 57459121c9SGlauber Costa 58459121c9SGlauber Costa return 0; 59459121c9SGlauber Costa } 60459121c9SGlauber Costa EXPORT_SYMBOL(dma_set_mask); 61459121c9SGlauber Costa 62116890d5SGlauber Costa #ifdef CONFIG_X86_64 63116890d5SGlauber Costa static __initdata void *dma32_bootmem_ptr; 64116890d5SGlauber Costa static unsigned long dma32_bootmem_size __initdata = (128ULL<<20); 65116890d5SGlauber Costa 66116890d5SGlauber Costa static int __init parse_dma32_size_opt(char *p) 67116890d5SGlauber Costa { 68116890d5SGlauber Costa if (!p) 69116890d5SGlauber Costa return -EINVAL; 70116890d5SGlauber Costa dma32_bootmem_size = memparse(p, &p); 71116890d5SGlauber Costa return 0; 72116890d5SGlauber Costa } 73116890d5SGlauber Costa early_param("dma32_size", parse_dma32_size_opt); 74116890d5SGlauber Costa 75116890d5SGlauber Costa void __init dma32_reserve_bootmem(void) 76116890d5SGlauber Costa { 77116890d5SGlauber Costa unsigned long size, align; 78c987d12fSYinghai Lu if (max_pfn <= MAX_DMA32_PFN) 79116890d5SGlauber Costa return; 80116890d5SGlauber Costa 817677b2efSYinghai Lu /* 827677b2efSYinghai Lu * check aperture_64.c allocate_aperture() for reason about 837677b2efSYinghai Lu * using 512M as goal 847677b2efSYinghai Lu */ 85116890d5SGlauber Costa align = 64ULL<<20; 861ddb5518SJoerg Roedel size = roundup(dma32_bootmem_size, align); 87116890d5SGlauber Costa dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align, 887677b2efSYinghai Lu 512ULL<<20); 89116890d5SGlauber Costa if (dma32_bootmem_ptr) 90116890d5SGlauber Costa dma32_bootmem_size = size; 91116890d5SGlauber Costa else 92116890d5SGlauber Costa dma32_bootmem_size = 0; 93116890d5SGlauber Costa } 94116890d5SGlauber Costa static void __init dma32_free_bootmem(void) 95116890d5SGlauber Costa { 96116890d5SGlauber Costa 97c987d12fSYinghai Lu if (max_pfn <= MAX_DMA32_PFN) 98116890d5SGlauber Costa return; 99116890d5SGlauber Costa 100116890d5SGlauber Costa if (!dma32_bootmem_ptr) 101116890d5SGlauber Costa return; 102116890d5SGlauber Costa 103330fce23SYinghai Lu free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size); 104116890d5SGlauber Costa 105116890d5SGlauber Costa dma32_bootmem_ptr = NULL; 106116890d5SGlauber Costa dma32_bootmem_size = 0; 107116890d5SGlauber Costa } 108cfb80c9eSJeremy Fitzhardinge #endif 109116890d5SGlauber Costa 110116890d5SGlauber Costa void __init pci_iommu_alloc(void) 111116890d5SGlauber Costa { 112cfb80c9eSJeremy Fitzhardinge #ifdef CONFIG_X86_64 113116890d5SGlauber Costa /* free the range so iommu could get some range less than 4G */ 114116890d5SGlauber Costa dma32_free_bootmem(); 115cfb80c9eSJeremy Fitzhardinge #endif 116cfb80c9eSJeremy Fitzhardinge 117116890d5SGlauber Costa /* 118116890d5SGlauber Costa * The order of these functions is important for 119116890d5SGlauber Costa * fall-back/fail-over reasons 120116890d5SGlauber Costa */ 121116890d5SGlauber Costa gart_iommu_hole_init(); 122116890d5SGlauber Costa 123116890d5SGlauber Costa detect_calgary(); 124116890d5SGlauber Costa 125116890d5SGlauber Costa detect_intel_iommu(); 126116890d5SGlauber Costa 127a69ca340SJoerg Roedel amd_iommu_detect(); 128a69ca340SJoerg Roedel 129116890d5SGlauber Costa pci_swiotlb_init(); 130116890d5SGlauber Costa } 1318978b742SFUJITA Tomonori 1329f6ac577SFUJITA Tomonori void *dma_generic_alloc_coherent(struct device *dev, size_t size, 1339f6ac577SFUJITA Tomonori dma_addr_t *dma_addr, gfp_t flag) 1349f6ac577SFUJITA Tomonori { 1359f6ac577SFUJITA Tomonori unsigned long dma_mask; 1369f6ac577SFUJITA Tomonori struct page *page; 1379f6ac577SFUJITA Tomonori dma_addr_t addr; 1389f6ac577SFUJITA Tomonori 1399f6ac577SFUJITA Tomonori dma_mask = dma_alloc_coherent_mask(dev, flag); 1409f6ac577SFUJITA Tomonori 1419f6ac577SFUJITA Tomonori flag |= __GFP_ZERO; 1429f6ac577SFUJITA Tomonori again: 1439f6ac577SFUJITA Tomonori page = alloc_pages_node(dev_to_node(dev), flag, get_order(size)); 1449f6ac577SFUJITA Tomonori if (!page) 1459f6ac577SFUJITA Tomonori return NULL; 1469f6ac577SFUJITA Tomonori 1479f6ac577SFUJITA Tomonori addr = page_to_phys(page); 1489f6ac577SFUJITA Tomonori if (!is_buffer_dma_capable(dma_mask, addr, size)) { 1499f6ac577SFUJITA Tomonori __free_pages(page, get_order(size)); 1509f6ac577SFUJITA Tomonori 151284901a9SYang Hongyang if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) { 1529f6ac577SFUJITA Tomonori flag = (flag & ~GFP_DMA32) | GFP_DMA; 1539f6ac577SFUJITA Tomonori goto again; 1549f6ac577SFUJITA Tomonori } 1559f6ac577SFUJITA Tomonori 1569f6ac577SFUJITA Tomonori return NULL; 1579f6ac577SFUJITA Tomonori } 1589f6ac577SFUJITA Tomonori 1599f6ac577SFUJITA Tomonori *dma_addr = addr; 1609f6ac577SFUJITA Tomonori return page_address(page); 1619f6ac577SFUJITA Tomonori } 1629f6ac577SFUJITA Tomonori 163fae9a0d8SGlauber Costa /* 164fae9a0d8SGlauber Costa * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter 165fae9a0d8SGlauber Costa * documentation. 166fae9a0d8SGlauber Costa */ 167fae9a0d8SGlauber Costa static __init int iommu_setup(char *p) 168fae9a0d8SGlauber Costa { 169fae9a0d8SGlauber Costa iommu_merge = 1; 170fae9a0d8SGlauber Costa 171fae9a0d8SGlauber Costa if (!p) 172fae9a0d8SGlauber Costa return -EINVAL; 173fae9a0d8SGlauber Costa 174fae9a0d8SGlauber Costa while (*p) { 175fae9a0d8SGlauber Costa if (!strncmp(p, "off", 3)) 176fae9a0d8SGlauber Costa no_iommu = 1; 177fae9a0d8SGlauber Costa /* gart_parse_options has more force support */ 178fae9a0d8SGlauber Costa if (!strncmp(p, "force", 5)) 179fae9a0d8SGlauber Costa force_iommu = 1; 180fae9a0d8SGlauber Costa if (!strncmp(p, "noforce", 7)) { 181fae9a0d8SGlauber Costa iommu_merge = 0; 182fae9a0d8SGlauber Costa force_iommu = 0; 183fae9a0d8SGlauber Costa } 184fae9a0d8SGlauber Costa 185fae9a0d8SGlauber Costa if (!strncmp(p, "biomerge", 8)) { 186fae9a0d8SGlauber Costa iommu_merge = 1; 187fae9a0d8SGlauber Costa force_iommu = 1; 188fae9a0d8SGlauber Costa } 189fae9a0d8SGlauber Costa if (!strncmp(p, "panic", 5)) 190fae9a0d8SGlauber Costa panic_on_overflow = 1; 191fae9a0d8SGlauber Costa if (!strncmp(p, "nopanic", 7)) 192fae9a0d8SGlauber Costa panic_on_overflow = 0; 193fae9a0d8SGlauber Costa if (!strncmp(p, "merge", 5)) { 194fae9a0d8SGlauber Costa iommu_merge = 1; 195fae9a0d8SGlauber Costa force_iommu = 1; 196fae9a0d8SGlauber Costa } 197fae9a0d8SGlauber Costa if (!strncmp(p, "nomerge", 7)) 198fae9a0d8SGlauber Costa iommu_merge = 0; 199fae9a0d8SGlauber Costa if (!strncmp(p, "forcesac", 8)) 200fae9a0d8SGlauber Costa iommu_sac_force = 1; 201fae9a0d8SGlauber Costa if (!strncmp(p, "allowdac", 8)) 202fae9a0d8SGlauber Costa forbid_dac = 0; 203fae9a0d8SGlauber Costa if (!strncmp(p, "nodac", 5)) 204fae9a0d8SGlauber Costa forbid_dac = -1; 205fae9a0d8SGlauber Costa if (!strncmp(p, "usedac", 6)) { 206fae9a0d8SGlauber Costa forbid_dac = -1; 207fae9a0d8SGlauber Costa return 1; 208fae9a0d8SGlauber Costa } 209fae9a0d8SGlauber Costa #ifdef CONFIG_SWIOTLB 210fae9a0d8SGlauber Costa if (!strncmp(p, "soft", 4)) 211fae9a0d8SGlauber Costa swiotlb = 1; 212fae9a0d8SGlauber Costa #endif 213fae9a0d8SGlauber Costa 214fae9a0d8SGlauber Costa gart_parse_options(p); 215fae9a0d8SGlauber Costa 216fae9a0d8SGlauber Costa #ifdef CONFIG_CALGARY_IOMMU 217fae9a0d8SGlauber Costa if (!strncmp(p, "calgary", 7)) 218fae9a0d8SGlauber Costa use_calgary = 1; 219fae9a0d8SGlauber Costa #endif /* CONFIG_CALGARY_IOMMU */ 220fae9a0d8SGlauber Costa 221fae9a0d8SGlauber Costa p += strcspn(p, ","); 222fae9a0d8SGlauber Costa if (*p == ',') 223fae9a0d8SGlauber Costa ++p; 224fae9a0d8SGlauber Costa } 225fae9a0d8SGlauber Costa return 0; 226fae9a0d8SGlauber Costa } 227fae9a0d8SGlauber Costa early_param("iommu", iommu_setup); 228fae9a0d8SGlauber Costa 2298e0c3797SGlauber Costa int dma_supported(struct device *dev, u64 mask) 2308e0c3797SGlauber Costa { 231160c1d8eSFUJITA Tomonori struct dma_map_ops *ops = get_dma_ops(dev); 2328d8bb39bSFUJITA Tomonori 2338e0c3797SGlauber Costa #ifdef CONFIG_PCI 2348e0c3797SGlauber Costa if (mask > 0xffffffff && forbid_dac > 0) { 235fc3a8828SGreg Kroah-Hartman dev_info(dev, "PCI: Disallowing DAC for device\n"); 2368e0c3797SGlauber Costa return 0; 2378e0c3797SGlauber Costa } 2388e0c3797SGlauber Costa #endif 2398e0c3797SGlauber Costa 2408d8bb39bSFUJITA Tomonori if (ops->dma_supported) 2418d8bb39bSFUJITA Tomonori return ops->dma_supported(dev, mask); 2428e0c3797SGlauber Costa 2438e0c3797SGlauber Costa /* Copied from i386. Doesn't make much sense, because it will 2448e0c3797SGlauber Costa only work for pci_alloc_coherent. 2458e0c3797SGlauber Costa The caller just has to use GFP_DMA in this case. */ 2462f4f27d4SYang Hongyang if (mask < DMA_BIT_MASK(24)) 2478e0c3797SGlauber Costa return 0; 2488e0c3797SGlauber Costa 2498e0c3797SGlauber Costa /* Tell the device to use SAC when IOMMU force is on. This 2508e0c3797SGlauber Costa allows the driver to use cheaper accesses in some cases. 2518e0c3797SGlauber Costa 2528e0c3797SGlauber Costa Problem with this is that if we overflow the IOMMU area and 2538e0c3797SGlauber Costa return DAC as fallback address the device may not handle it 2548e0c3797SGlauber Costa correctly. 2558e0c3797SGlauber Costa 2568e0c3797SGlauber Costa As a special case some controllers have a 39bit address 2578e0c3797SGlauber Costa mode that is as efficient as 32bit (aic79xx). Don't force 2588e0c3797SGlauber Costa SAC for these. Assume all masks <= 40 bits are of this 2598e0c3797SGlauber Costa type. Normally this doesn't make any difference, but gives 2608e0c3797SGlauber Costa more gentle handling of IOMMU overflow. */ 26150cf156aSYang Hongyang if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) { 262fc3a8828SGreg Kroah-Hartman dev_info(dev, "Force SAC with mask %Lx\n", mask); 2638e0c3797SGlauber Costa return 0; 2648e0c3797SGlauber Costa } 2658e0c3797SGlauber Costa 2668e0c3797SGlauber Costa return 1; 2678e0c3797SGlauber Costa } 2688e0c3797SGlauber Costa EXPORT_SYMBOL(dma_supported); 2698e0c3797SGlauber Costa 270cb5867a5SGlauber Costa static int __init pci_iommu_init(void) 271cb5867a5SGlauber Costa { 2722118d0c5SJoerg Roedel dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 2732118d0c5SJoerg Roedel 27486f31952SJoerg Roedel #ifdef CONFIG_PCI 27586f31952SJoerg Roedel dma_debug_add_bus(&pci_bus_type); 27686f31952SJoerg Roedel #endif 27786f31952SJoerg Roedel 278cb5867a5SGlauber Costa calgary_iommu_init(); 279459121c9SGlauber Costa 280cb5867a5SGlauber Costa intel_iommu_init(); 281cb5867a5SGlauber Costa 282a69ca340SJoerg Roedel amd_iommu_init(); 283a69ca340SJoerg Roedel 284cb5867a5SGlauber Costa gart_iommu_init(); 285cb5867a5SGlauber Costa 286cb5867a5SGlauber Costa no_iommu_init(); 287cb5867a5SGlauber Costa return 0; 288cb5867a5SGlauber Costa } 289cb5867a5SGlauber Costa 290cb5867a5SGlauber Costa void pci_iommu_shutdown(void) 291cb5867a5SGlauber Costa { 292cb5867a5SGlauber Costa gart_iommu_shutdown(); 293*09759042SJoerg Roedel 294*09759042SJoerg Roedel amd_iommu_shutdown(); 295cb5867a5SGlauber Costa } 296cb5867a5SGlauber Costa /* Must execute after PCI subsystem */ 297cb5867a5SGlauber Costa fs_initcall(pci_iommu_init); 2983b15e581SFenghua Yu 2993b15e581SFenghua Yu #ifdef CONFIG_PCI 3003b15e581SFenghua Yu /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ 3013b15e581SFenghua Yu 3023b15e581SFenghua Yu static __devinit void via_no_dac(struct pci_dev *dev) 3033b15e581SFenghua Yu { 3043b15e581SFenghua Yu if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { 30513bf7576SBjorn Helgaas dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n"); 3063b15e581SFenghua Yu forbid_dac = 1; 3073b15e581SFenghua Yu } 3083b15e581SFenghua Yu } 3093b15e581SFenghua Yu DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac); 3103b15e581SFenghua Yu #endif 311