1459121c9SGlauber Costa #include <linux/dma-mapping.h> 22118d0c5SJoerg Roedel #include <linux/dma-debug.h> 3cb5867a5SGlauber Costa #include <linux/dmar.h> 4116890d5SGlauber Costa #include <linux/bootmem.h> 5bca5c096SGlauber Costa #include <linux/pci.h> 6cb5867a5SGlauber Costa 7116890d5SGlauber Costa #include <asm/proto.h> 8116890d5SGlauber Costa #include <asm/dma.h> 946a7fa27SFUJITA Tomonori #include <asm/iommu.h> 101d9b16d1SJoerg Roedel #include <asm/gart.h> 11cb5867a5SGlauber Costa #include <asm/calgary.h> 12a69ca340SJoerg Roedel #include <asm/amd_iommu.h> 13459121c9SGlauber Costa 143b15e581SFenghua Yu static int forbid_dac __read_mostly; 153b15e581SFenghua Yu 16160c1d8eSFUJITA Tomonori struct dma_map_ops *dma_ops; 1785c246eeSGlauber Costa EXPORT_SYMBOL(dma_ops); 1885c246eeSGlauber Costa 19b4cdc430SDmitri Vorobiev static int iommu_sac_force __read_mostly; 208e0c3797SGlauber Costa 21f9c258deSGlauber Costa #ifdef CONFIG_IOMMU_DEBUG 22f9c258deSGlauber Costa int panic_on_overflow __read_mostly = 1; 23f9c258deSGlauber Costa int force_iommu __read_mostly = 1; 24f9c258deSGlauber Costa #else 25f9c258deSGlauber Costa int panic_on_overflow __read_mostly = 0; 26f9c258deSGlauber Costa int force_iommu __read_mostly = 0; 27f9c258deSGlauber Costa #endif 28f9c258deSGlauber Costa 29fae9a0d8SGlauber Costa int iommu_merge __read_mostly = 0; 30fae9a0d8SGlauber Costa 31fae9a0d8SGlauber Costa int no_iommu __read_mostly; 32fae9a0d8SGlauber Costa /* Set this to 1 if there is a HW IOMMU in the system */ 33fae9a0d8SGlauber Costa int iommu_detected __read_mostly = 0; 34fae9a0d8SGlauber Costa 35aed5d5f4SFenghua Yu int iommu_pass_through; 36aed5d5f4SFenghua Yu 37cac67877SGlauber Costa dma_addr_t bad_dma_address __read_mostly = 0; 38cac67877SGlauber Costa EXPORT_SYMBOL(bad_dma_address); 39fae9a0d8SGlauber Costa 40098cb7f2SGlauber Costa /* Dummy device used for NULL arguments (normally ISA). Better would 41098cb7f2SGlauber Costa be probably a smaller DMA mask, but this is bug-to-bug compatible 42098cb7f2SGlauber Costa to older i386. */ 436c505ce3SJoerg Roedel struct device x86_dma_fallback_dev = { 441a927133SKay Sievers .init_name = "fallback device", 45284901a9SYang Hongyang .coherent_dma_mask = DMA_BIT_MASK(32), 466c505ce3SJoerg Roedel .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask, 47098cb7f2SGlauber Costa }; 486c505ce3SJoerg Roedel EXPORT_SYMBOL(x86_dma_fallback_dev); 49098cb7f2SGlauber Costa 502118d0c5SJoerg Roedel /* Number of entries preallocated for DMA-API debugging */ 512118d0c5SJoerg Roedel #define PREALLOC_DMA_DEBUG_ENTRIES 32768 522118d0c5SJoerg Roedel 53459121c9SGlauber Costa int dma_set_mask(struct device *dev, u64 mask) 54459121c9SGlauber Costa { 55459121c9SGlauber Costa if (!dev->dma_mask || !dma_supported(dev, mask)) 56459121c9SGlauber Costa return -EIO; 57459121c9SGlauber Costa 58459121c9SGlauber Costa *dev->dma_mask = mask; 59459121c9SGlauber Costa 60459121c9SGlauber Costa return 0; 61459121c9SGlauber Costa } 62459121c9SGlauber Costa EXPORT_SYMBOL(dma_set_mask); 63459121c9SGlauber Costa 64116890d5SGlauber Costa #ifdef CONFIG_X86_64 65116890d5SGlauber Costa static __initdata void *dma32_bootmem_ptr; 66116890d5SGlauber Costa static unsigned long dma32_bootmem_size __initdata = (128ULL<<20); 67116890d5SGlauber Costa 68116890d5SGlauber Costa static int __init parse_dma32_size_opt(char *p) 69116890d5SGlauber Costa { 70116890d5SGlauber Costa if (!p) 71116890d5SGlauber Costa return -EINVAL; 72116890d5SGlauber Costa dma32_bootmem_size = memparse(p, &p); 73116890d5SGlauber Costa return 0; 74116890d5SGlauber Costa } 75116890d5SGlauber Costa early_param("dma32_size", parse_dma32_size_opt); 76116890d5SGlauber Costa 77116890d5SGlauber Costa void __init dma32_reserve_bootmem(void) 78116890d5SGlauber Costa { 79116890d5SGlauber Costa unsigned long size, align; 80c987d12fSYinghai Lu if (max_pfn <= MAX_DMA32_PFN) 81116890d5SGlauber Costa return; 82116890d5SGlauber Costa 837677b2efSYinghai Lu /* 847677b2efSYinghai Lu * check aperture_64.c allocate_aperture() for reason about 857677b2efSYinghai Lu * using 512M as goal 867677b2efSYinghai Lu */ 87116890d5SGlauber Costa align = 64ULL<<20; 881ddb5518SJoerg Roedel size = roundup(dma32_bootmem_size, align); 89116890d5SGlauber Costa dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align, 907677b2efSYinghai Lu 512ULL<<20); 91116890d5SGlauber Costa if (dma32_bootmem_ptr) 92116890d5SGlauber Costa dma32_bootmem_size = size; 93116890d5SGlauber Costa else 94116890d5SGlauber Costa dma32_bootmem_size = 0; 95116890d5SGlauber Costa } 96116890d5SGlauber Costa static void __init dma32_free_bootmem(void) 97116890d5SGlauber Costa { 98116890d5SGlauber Costa 99c987d12fSYinghai Lu if (max_pfn <= MAX_DMA32_PFN) 100116890d5SGlauber Costa return; 101116890d5SGlauber Costa 102116890d5SGlauber Costa if (!dma32_bootmem_ptr) 103116890d5SGlauber Costa return; 104116890d5SGlauber Costa 105330fce23SYinghai Lu free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size); 106116890d5SGlauber Costa 107116890d5SGlauber Costa dma32_bootmem_ptr = NULL; 108116890d5SGlauber Costa dma32_bootmem_size = 0; 109116890d5SGlauber Costa } 110cfb80c9eSJeremy Fitzhardinge #endif 111116890d5SGlauber Costa 112116890d5SGlauber Costa void __init pci_iommu_alloc(void) 113116890d5SGlauber Costa { 114cfb80c9eSJeremy Fitzhardinge #ifdef CONFIG_X86_64 115116890d5SGlauber Costa /* free the range so iommu could get some range less than 4G */ 116116890d5SGlauber Costa dma32_free_bootmem(); 117cfb80c9eSJeremy Fitzhardinge #endif 118cfb80c9eSJeremy Fitzhardinge 119116890d5SGlauber Costa /* 120116890d5SGlauber Costa * The order of these functions is important for 121116890d5SGlauber Costa * fall-back/fail-over reasons 122116890d5SGlauber Costa */ 123116890d5SGlauber Costa gart_iommu_hole_init(); 124116890d5SGlauber Costa 125116890d5SGlauber Costa detect_calgary(); 126116890d5SGlauber Costa 127116890d5SGlauber Costa detect_intel_iommu(); 128116890d5SGlauber Costa 129a69ca340SJoerg Roedel amd_iommu_detect(); 130a69ca340SJoerg Roedel 131116890d5SGlauber Costa pci_swiotlb_init(); 132116890d5SGlauber Costa } 1338978b742SFUJITA Tomonori 1349f6ac577SFUJITA Tomonori void *dma_generic_alloc_coherent(struct device *dev, size_t size, 1359f6ac577SFUJITA Tomonori dma_addr_t *dma_addr, gfp_t flag) 1369f6ac577SFUJITA Tomonori { 1379f6ac577SFUJITA Tomonori unsigned long dma_mask; 1389f6ac577SFUJITA Tomonori struct page *page; 1399f6ac577SFUJITA Tomonori dma_addr_t addr; 1409f6ac577SFUJITA Tomonori 1419f6ac577SFUJITA Tomonori dma_mask = dma_alloc_coherent_mask(dev, flag); 1429f6ac577SFUJITA Tomonori 1439f6ac577SFUJITA Tomonori flag |= __GFP_ZERO; 1449f6ac577SFUJITA Tomonori again: 1459f6ac577SFUJITA Tomonori page = alloc_pages_node(dev_to_node(dev), flag, get_order(size)); 1469f6ac577SFUJITA Tomonori if (!page) 1479f6ac577SFUJITA Tomonori return NULL; 1489f6ac577SFUJITA Tomonori 1499f6ac577SFUJITA Tomonori addr = page_to_phys(page); 1509f6ac577SFUJITA Tomonori if (!is_buffer_dma_capable(dma_mask, addr, size)) { 1519f6ac577SFUJITA Tomonori __free_pages(page, get_order(size)); 1529f6ac577SFUJITA Tomonori 153284901a9SYang Hongyang if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) { 1549f6ac577SFUJITA Tomonori flag = (flag & ~GFP_DMA32) | GFP_DMA; 1559f6ac577SFUJITA Tomonori goto again; 1569f6ac577SFUJITA Tomonori } 1579f6ac577SFUJITA Tomonori 1589f6ac577SFUJITA Tomonori return NULL; 1599f6ac577SFUJITA Tomonori } 1609f6ac577SFUJITA Tomonori 1619f6ac577SFUJITA Tomonori *dma_addr = addr; 1629f6ac577SFUJITA Tomonori return page_address(page); 1639f6ac577SFUJITA Tomonori } 1649f6ac577SFUJITA Tomonori 165fae9a0d8SGlauber Costa /* 166fae9a0d8SGlauber Costa * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter 167fae9a0d8SGlauber Costa * documentation. 168fae9a0d8SGlauber Costa */ 169fae9a0d8SGlauber Costa static __init int iommu_setup(char *p) 170fae9a0d8SGlauber Costa { 171fae9a0d8SGlauber Costa iommu_merge = 1; 172fae9a0d8SGlauber Costa 173fae9a0d8SGlauber Costa if (!p) 174fae9a0d8SGlauber Costa return -EINVAL; 175fae9a0d8SGlauber Costa 176fae9a0d8SGlauber Costa while (*p) { 177fae9a0d8SGlauber Costa if (!strncmp(p, "off", 3)) 178fae9a0d8SGlauber Costa no_iommu = 1; 179fae9a0d8SGlauber Costa /* gart_parse_options has more force support */ 180fae9a0d8SGlauber Costa if (!strncmp(p, "force", 5)) 181fae9a0d8SGlauber Costa force_iommu = 1; 182fae9a0d8SGlauber Costa if (!strncmp(p, "noforce", 7)) { 183fae9a0d8SGlauber Costa iommu_merge = 0; 184fae9a0d8SGlauber Costa force_iommu = 0; 185fae9a0d8SGlauber Costa } 186fae9a0d8SGlauber Costa 187fae9a0d8SGlauber Costa if (!strncmp(p, "biomerge", 8)) { 188fae9a0d8SGlauber Costa iommu_merge = 1; 189fae9a0d8SGlauber Costa force_iommu = 1; 190fae9a0d8SGlauber Costa } 191fae9a0d8SGlauber Costa if (!strncmp(p, "panic", 5)) 192fae9a0d8SGlauber Costa panic_on_overflow = 1; 193fae9a0d8SGlauber Costa if (!strncmp(p, "nopanic", 7)) 194fae9a0d8SGlauber Costa panic_on_overflow = 0; 195fae9a0d8SGlauber Costa if (!strncmp(p, "merge", 5)) { 196fae9a0d8SGlauber Costa iommu_merge = 1; 197fae9a0d8SGlauber Costa force_iommu = 1; 198fae9a0d8SGlauber Costa } 199fae9a0d8SGlauber Costa if (!strncmp(p, "nomerge", 7)) 200fae9a0d8SGlauber Costa iommu_merge = 0; 201fae9a0d8SGlauber Costa if (!strncmp(p, "forcesac", 8)) 202fae9a0d8SGlauber Costa iommu_sac_force = 1; 203fae9a0d8SGlauber Costa if (!strncmp(p, "allowdac", 8)) 204fae9a0d8SGlauber Costa forbid_dac = 0; 205fae9a0d8SGlauber Costa if (!strncmp(p, "nodac", 5)) 206fae9a0d8SGlauber Costa forbid_dac = -1; 207fae9a0d8SGlauber Costa if (!strncmp(p, "usedac", 6)) { 208fae9a0d8SGlauber Costa forbid_dac = -1; 209fae9a0d8SGlauber Costa return 1; 210fae9a0d8SGlauber Costa } 211fae9a0d8SGlauber Costa #ifdef CONFIG_SWIOTLB 212fae9a0d8SGlauber Costa if (!strncmp(p, "soft", 4)) 213fae9a0d8SGlauber Costa swiotlb = 1; 2143238c0c4SDavid Woodhouse #endif 215*80286879SAlex Williamson if (!strncmp(p, "pt", 2)) 2164ed0d3e6SFenghua Yu iommu_pass_through = 1; 217fae9a0d8SGlauber Costa 218fae9a0d8SGlauber Costa gart_parse_options(p); 219fae9a0d8SGlauber Costa 220fae9a0d8SGlauber Costa #ifdef CONFIG_CALGARY_IOMMU 221fae9a0d8SGlauber Costa if (!strncmp(p, "calgary", 7)) 222fae9a0d8SGlauber Costa use_calgary = 1; 223fae9a0d8SGlauber Costa #endif /* CONFIG_CALGARY_IOMMU */ 224fae9a0d8SGlauber Costa 225fae9a0d8SGlauber Costa p += strcspn(p, ","); 226fae9a0d8SGlauber Costa if (*p == ',') 227fae9a0d8SGlauber Costa ++p; 228fae9a0d8SGlauber Costa } 229fae9a0d8SGlauber Costa return 0; 230fae9a0d8SGlauber Costa } 231fae9a0d8SGlauber Costa early_param("iommu", iommu_setup); 232fae9a0d8SGlauber Costa 2338e0c3797SGlauber Costa int dma_supported(struct device *dev, u64 mask) 2348e0c3797SGlauber Costa { 235160c1d8eSFUJITA Tomonori struct dma_map_ops *ops = get_dma_ops(dev); 2368d8bb39bSFUJITA Tomonori 2378e0c3797SGlauber Costa #ifdef CONFIG_PCI 2388e0c3797SGlauber Costa if (mask > 0xffffffff && forbid_dac > 0) { 239fc3a8828SGreg Kroah-Hartman dev_info(dev, "PCI: Disallowing DAC for device\n"); 2408e0c3797SGlauber Costa return 0; 2418e0c3797SGlauber Costa } 2428e0c3797SGlauber Costa #endif 2438e0c3797SGlauber Costa 2448d8bb39bSFUJITA Tomonori if (ops->dma_supported) 2458d8bb39bSFUJITA Tomonori return ops->dma_supported(dev, mask); 2468e0c3797SGlauber Costa 2478e0c3797SGlauber Costa /* Copied from i386. Doesn't make much sense, because it will 2488e0c3797SGlauber Costa only work for pci_alloc_coherent. 2498e0c3797SGlauber Costa The caller just has to use GFP_DMA in this case. */ 2502f4f27d4SYang Hongyang if (mask < DMA_BIT_MASK(24)) 2518e0c3797SGlauber Costa return 0; 2528e0c3797SGlauber Costa 2538e0c3797SGlauber Costa /* Tell the device to use SAC when IOMMU force is on. This 2548e0c3797SGlauber Costa allows the driver to use cheaper accesses in some cases. 2558e0c3797SGlauber Costa 2568e0c3797SGlauber Costa Problem with this is that if we overflow the IOMMU area and 2578e0c3797SGlauber Costa return DAC as fallback address the device may not handle it 2588e0c3797SGlauber Costa correctly. 2598e0c3797SGlauber Costa 2608e0c3797SGlauber Costa As a special case some controllers have a 39bit address 2618e0c3797SGlauber Costa mode that is as efficient as 32bit (aic79xx). Don't force 2628e0c3797SGlauber Costa SAC for these. Assume all masks <= 40 bits are of this 2638e0c3797SGlauber Costa type. Normally this doesn't make any difference, but gives 2648e0c3797SGlauber Costa more gentle handling of IOMMU overflow. */ 26550cf156aSYang Hongyang if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) { 266fc3a8828SGreg Kroah-Hartman dev_info(dev, "Force SAC with mask %Lx\n", mask); 2678e0c3797SGlauber Costa return 0; 2688e0c3797SGlauber Costa } 2698e0c3797SGlauber Costa 2708e0c3797SGlauber Costa return 1; 2718e0c3797SGlauber Costa } 2728e0c3797SGlauber Costa EXPORT_SYMBOL(dma_supported); 2738e0c3797SGlauber Costa 274cb5867a5SGlauber Costa static int __init pci_iommu_init(void) 275cb5867a5SGlauber Costa { 2762118d0c5SJoerg Roedel dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 2772118d0c5SJoerg Roedel 27886f31952SJoerg Roedel #ifdef CONFIG_PCI 27986f31952SJoerg Roedel dma_debug_add_bus(&pci_bus_type); 28086f31952SJoerg Roedel #endif 28186f31952SJoerg Roedel 282cb5867a5SGlauber Costa calgary_iommu_init(); 283459121c9SGlauber Costa 284cb5867a5SGlauber Costa intel_iommu_init(); 285cb5867a5SGlauber Costa 286a69ca340SJoerg Roedel amd_iommu_init(); 287a69ca340SJoerg Roedel 288cb5867a5SGlauber Costa gart_iommu_init(); 289cb5867a5SGlauber Costa 290cb5867a5SGlauber Costa no_iommu_init(); 291cb5867a5SGlauber Costa return 0; 292cb5867a5SGlauber Costa } 293cb5867a5SGlauber Costa 294cb5867a5SGlauber Costa void pci_iommu_shutdown(void) 295cb5867a5SGlauber Costa { 296cb5867a5SGlauber Costa gart_iommu_shutdown(); 29709759042SJoerg Roedel 29809759042SJoerg Roedel amd_iommu_shutdown(); 299cb5867a5SGlauber Costa } 300cb5867a5SGlauber Costa /* Must execute after PCI subsystem */ 301cb5867a5SGlauber Costa fs_initcall(pci_iommu_init); 3023b15e581SFenghua Yu 3033b15e581SFenghua Yu #ifdef CONFIG_PCI 3043b15e581SFenghua Yu /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ 3053b15e581SFenghua Yu 3063b15e581SFenghua Yu static __devinit void via_no_dac(struct pci_dev *dev) 3073b15e581SFenghua Yu { 3083b15e581SFenghua Yu if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { 30913bf7576SBjorn Helgaas dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n"); 3103b15e581SFenghua Yu forbid_dac = 1; 3113b15e581SFenghua Yu } 3123b15e581SFenghua Yu } 3133b15e581SFenghua Yu DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac); 3143b15e581SFenghua Yu #endif 315