19c17d615SPaolo Bonzini /* 29c17d615SPaolo Bonzini * DMA helper functions 39c17d615SPaolo Bonzini * 49c211ad2SPhilippe Mathieu-Daudé * Copyright (c) 2009, 2020 Red Hat 59c17d615SPaolo Bonzini * 69c17d615SPaolo Bonzini * This work is licensed under the terms of the GNU General Public License 79c17d615SPaolo Bonzini * (GNU GPL), version 2 or later. 89c17d615SPaolo Bonzini */ 99c17d615SPaolo Bonzini 109c17d615SPaolo Bonzini #ifndef DMA_H 119c17d615SPaolo Bonzini #define DMA_H 129c17d615SPaolo Bonzini 139c17d615SPaolo Bonzini #include "exec/memory.h" 14df32fd1cSPaolo Bonzini #include "exec/address-spaces.h" 159c17d615SPaolo Bonzini #include "block/block.h" 165e5a94b6SBenoît Canet #include "block/accounting.h" 179c17d615SPaolo Bonzini 189c17d615SPaolo Bonzini typedef struct ScatterGatherEntry ScatterGatherEntry; 199c17d615SPaolo Bonzini 209c17d615SPaolo Bonzini typedef enum { 219c17d615SPaolo Bonzini DMA_DIRECTION_TO_DEVICE = 0, 229c17d615SPaolo Bonzini DMA_DIRECTION_FROM_DEVICE = 1, 239c17d615SPaolo Bonzini } DMADirection; 249c17d615SPaolo Bonzini 259c17d615SPaolo Bonzini struct QEMUSGList { 269c17d615SPaolo Bonzini ScatterGatherEntry *sg; 279c17d615SPaolo Bonzini int nsg; 289c17d615SPaolo Bonzini int nalloc; 299c17d615SPaolo Bonzini size_t size; 30f487b677SPaolo Bonzini DeviceState *dev; 31df32fd1cSPaolo Bonzini AddressSpace *as; 329c17d615SPaolo Bonzini }; 339c17d615SPaolo Bonzini 349c17d615SPaolo Bonzini #ifndef CONFIG_USER_ONLY 359c17d615SPaolo Bonzini 369c17d615SPaolo Bonzini /* 379c17d615SPaolo Bonzini * When an IOMMU is present, bus addresses become distinct from 389c17d615SPaolo Bonzini * CPU/memory physical addresses and may be a different size. Because 399c17d615SPaolo Bonzini * the IOVA size depends more on the bus than on the platform, we more 409c17d615SPaolo Bonzini * or less have to treat these as 64-bit always to cover all (or at 419c17d615SPaolo Bonzini * least most) cases. 429c17d615SPaolo Bonzini */ 439c17d615SPaolo Bonzini typedef uint64_t dma_addr_t; 449c17d615SPaolo Bonzini 459c17d615SPaolo Bonzini #define DMA_ADDR_BITS 64 469c17d615SPaolo Bonzini #define DMA_ADDR_FMT "%" PRIx64 479c17d615SPaolo Bonzini 48df32fd1cSPaolo Bonzini static inline void dma_barrier(AddressSpace *as, DMADirection dir) 499c17d615SPaolo Bonzini { 509c17d615SPaolo Bonzini /* 519c17d615SPaolo Bonzini * This is called before DMA read and write operations 529c17d615SPaolo Bonzini * unless the _relaxed form is used and is responsible 539c17d615SPaolo Bonzini * for providing some sane ordering of accesses vs 549c17d615SPaolo Bonzini * concurrently running VCPUs. 559c17d615SPaolo Bonzini * 569c17d615SPaolo Bonzini * Users of map(), unmap() or lower level st/ld_* 579c17d615SPaolo Bonzini * operations are responsible for providing their own 589c17d615SPaolo Bonzini * ordering via barriers. 599c17d615SPaolo Bonzini * 609c17d615SPaolo Bonzini * This primitive implementation does a simple smp_mb() 619c17d615SPaolo Bonzini * before each operation which provides pretty much full 629c17d615SPaolo Bonzini * ordering. 639c17d615SPaolo Bonzini * 649c17d615SPaolo Bonzini * A smarter implementation can be devised if needed to 659c17d615SPaolo Bonzini * use lighter barriers based on the direction of the 669c17d615SPaolo Bonzini * transfer, the DMA context, etc... 679c17d615SPaolo Bonzini */ 689c17d615SPaolo Bonzini smp_mb(); 699c17d615SPaolo Bonzini } 709c17d615SPaolo Bonzini 719c17d615SPaolo Bonzini /* Checks that the given range of addresses is valid for DMA. This is 729c17d615SPaolo Bonzini * useful for certain cases, but usually you should just use 739c17d615SPaolo Bonzini * dma_memory_{read,write}() and check for errors */ 74df32fd1cSPaolo Bonzini static inline bool dma_memory_valid(AddressSpace *as, 759c17d615SPaolo Bonzini dma_addr_t addr, dma_addr_t len, 767ccb391cSPhilippe Mathieu-Daudé DMADirection dir, MemTxAttrs attrs) 779c17d615SPaolo Bonzini { 78df32fd1cSPaolo Bonzini return address_space_access_valid(as, addr, len, 79fddffa42SPeter Maydell dir == DMA_DIRECTION_FROM_DEVICE, 807ccb391cSPhilippe Mathieu-Daudé attrs); 819c17d615SPaolo Bonzini } 829c17d615SPaolo Bonzini 839989bcd3SPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_rw_relaxed(AddressSpace *as, 849989bcd3SPhilippe Mathieu-Daudé dma_addr_t addr, 859c17d615SPaolo Bonzini void *buf, dma_addr_t len, 869c17d615SPaolo Bonzini DMADirection dir) 879c17d615SPaolo Bonzini { 889989bcd3SPhilippe Mathieu-Daudé return address_space_rw(as, addr, MEMTXATTRS_UNSPECIFIED, 895c9eb028SPeter Maydell buf, len, dir == DMA_DIRECTION_FROM_DEVICE); 909c17d615SPaolo Bonzini } 919c17d615SPaolo Bonzini 92b1f51303SPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_read_relaxed(AddressSpace *as, 93b1f51303SPhilippe Mathieu-Daudé dma_addr_t addr, 949c17d615SPaolo Bonzini void *buf, dma_addr_t len) 959c17d615SPaolo Bonzini { 96df32fd1cSPaolo Bonzini return dma_memory_rw_relaxed(as, addr, buf, len, DMA_DIRECTION_TO_DEVICE); 979c17d615SPaolo Bonzini } 989c17d615SPaolo Bonzini 9977c71d1dSPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_write_relaxed(AddressSpace *as, 10077c71d1dSPhilippe Mathieu-Daudé dma_addr_t addr, 10177c71d1dSPhilippe Mathieu-Daudé const void *buf, 10277c71d1dSPhilippe Mathieu-Daudé dma_addr_t len) 1039c17d615SPaolo Bonzini { 104df32fd1cSPaolo Bonzini return dma_memory_rw_relaxed(as, addr, (void *)buf, len, 1059c17d615SPaolo Bonzini DMA_DIRECTION_FROM_DEVICE); 1069c17d615SPaolo Bonzini } 1079c17d615SPaolo Bonzini 1089989bcd3SPhilippe Mathieu-Daudé /** 1099989bcd3SPhilippe Mathieu-Daudé * dma_memory_rw: Read from or write to an address space from DMA controller. 1109989bcd3SPhilippe Mathieu-Daudé * 1119989bcd3SPhilippe Mathieu-Daudé * Return a MemTxResult indicating whether the operation succeeded 1129989bcd3SPhilippe Mathieu-Daudé * or failed (eg unassigned memory, device rejected the transaction, 1139989bcd3SPhilippe Mathieu-Daudé * IOMMU fault). 1149989bcd3SPhilippe Mathieu-Daudé * 1159989bcd3SPhilippe Mathieu-Daudé * @as: #AddressSpace to be accessed 1169989bcd3SPhilippe Mathieu-Daudé * @addr: address within that address space 1179989bcd3SPhilippe Mathieu-Daudé * @buf: buffer with the data transferred 1189989bcd3SPhilippe Mathieu-Daudé * @len: the number of bytes to read or write 1199989bcd3SPhilippe Mathieu-Daudé * @dir: indicates the transfer direction 1209989bcd3SPhilippe Mathieu-Daudé */ 1219989bcd3SPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_rw(AddressSpace *as, dma_addr_t addr, 1229c17d615SPaolo Bonzini void *buf, dma_addr_t len, 1239c17d615SPaolo Bonzini DMADirection dir) 1249c17d615SPaolo Bonzini { 125df32fd1cSPaolo Bonzini dma_barrier(as, dir); 1269c17d615SPaolo Bonzini 127df32fd1cSPaolo Bonzini return dma_memory_rw_relaxed(as, addr, buf, len, dir); 1289c17d615SPaolo Bonzini } 1299c17d615SPaolo Bonzini 130b1f51303SPhilippe Mathieu-Daudé /** 131b1f51303SPhilippe Mathieu-Daudé * dma_memory_read: Read from an address space from DMA controller. 132b1f51303SPhilippe Mathieu-Daudé * 133b1f51303SPhilippe Mathieu-Daudé * Return a MemTxResult indicating whether the operation succeeded 134b1f51303SPhilippe Mathieu-Daudé * or failed (eg unassigned memory, device rejected the transaction, 135b1f51303SPhilippe Mathieu-Daudé * IOMMU fault). Called within RCU critical section. 136b1f51303SPhilippe Mathieu-Daudé * 137b1f51303SPhilippe Mathieu-Daudé * @as: #AddressSpace to be accessed 138b1f51303SPhilippe Mathieu-Daudé * @addr: address within that address space 139b1f51303SPhilippe Mathieu-Daudé * @buf: buffer with the data transferred 140b1f51303SPhilippe Mathieu-Daudé * @len: length of the data transferred 141b1f51303SPhilippe Mathieu-Daudé */ 142b1f51303SPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_read(AddressSpace *as, dma_addr_t addr, 1439c17d615SPaolo Bonzini void *buf, dma_addr_t len) 1449c17d615SPaolo Bonzini { 145df32fd1cSPaolo Bonzini return dma_memory_rw(as, addr, buf, len, DMA_DIRECTION_TO_DEVICE); 1469c17d615SPaolo Bonzini } 1479c17d615SPaolo Bonzini 14877c71d1dSPhilippe Mathieu-Daudé /** 14977c71d1dSPhilippe Mathieu-Daudé * address_space_write: Write to address space from DMA controller. 15077c71d1dSPhilippe Mathieu-Daudé * 15177c71d1dSPhilippe Mathieu-Daudé * Return a MemTxResult indicating whether the operation succeeded 15277c71d1dSPhilippe Mathieu-Daudé * or failed (eg unassigned memory, device rejected the transaction, 15377c71d1dSPhilippe Mathieu-Daudé * IOMMU fault). 15477c71d1dSPhilippe Mathieu-Daudé * 15577c71d1dSPhilippe Mathieu-Daudé * @as: #AddressSpace to be accessed 15677c71d1dSPhilippe Mathieu-Daudé * @addr: address within that address space 15777c71d1dSPhilippe Mathieu-Daudé * @buf: buffer with the data transferred 15877c71d1dSPhilippe Mathieu-Daudé * @len: the number of bytes to write 15977c71d1dSPhilippe Mathieu-Daudé */ 16077c71d1dSPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_write(AddressSpace *as, dma_addr_t addr, 1619c17d615SPaolo Bonzini const void *buf, dma_addr_t len) 1629c17d615SPaolo Bonzini { 163df32fd1cSPaolo Bonzini return dma_memory_rw(as, addr, (void *)buf, len, 1649c17d615SPaolo Bonzini DMA_DIRECTION_FROM_DEVICE); 1659c17d615SPaolo Bonzini } 1669c17d615SPaolo Bonzini 167bb755f52SPhilippe Mathieu-Daudé /** 168bb755f52SPhilippe Mathieu-Daudé * dma_memory_set: Fill memory with a constant byte from DMA controller. 169bb755f52SPhilippe Mathieu-Daudé * 170bb755f52SPhilippe Mathieu-Daudé * Return a MemTxResult indicating whether the operation succeeded 171bb755f52SPhilippe Mathieu-Daudé * or failed (eg unassigned memory, device rejected the transaction, 172bb755f52SPhilippe Mathieu-Daudé * IOMMU fault). 173bb755f52SPhilippe Mathieu-Daudé * 174bb755f52SPhilippe Mathieu-Daudé * @as: #AddressSpace to be accessed 175bb755f52SPhilippe Mathieu-Daudé * @addr: address within that address space 176bb755f52SPhilippe Mathieu-Daudé * @c: constant byte to fill the memory 177bb755f52SPhilippe Mathieu-Daudé * @len: the number of bytes to fill with the constant byte 178*7a36e42dSPhilippe Mathieu-Daudé * @attrs: memory transaction attributes 179bb755f52SPhilippe Mathieu-Daudé */ 180bb755f52SPhilippe Mathieu-Daudé MemTxResult dma_memory_set(AddressSpace *as, dma_addr_t addr, 181*7a36e42dSPhilippe Mathieu-Daudé uint8_t c, dma_addr_t len, MemTxAttrs attrs); 1829c17d615SPaolo Bonzini 1839c211ad2SPhilippe Mathieu-Daudé /** 1849c211ad2SPhilippe Mathieu-Daudé * address_space_map: Map a physical memory region into a host virtual address. 1859c211ad2SPhilippe Mathieu-Daudé * 1869c211ad2SPhilippe Mathieu-Daudé * May map a subset of the requested range, given by and returned in @plen. 1879c211ad2SPhilippe Mathieu-Daudé * May return %NULL and set *@plen to zero(0), if resources needed to perform 1889c211ad2SPhilippe Mathieu-Daudé * the mapping are exhausted. 1899c211ad2SPhilippe Mathieu-Daudé * Use only for reads OR writes - not for read-modify-write operations. 1909c211ad2SPhilippe Mathieu-Daudé * 1919c211ad2SPhilippe Mathieu-Daudé * @as: #AddressSpace to be accessed 1929c211ad2SPhilippe Mathieu-Daudé * @addr: address within that address space 1939c211ad2SPhilippe Mathieu-Daudé * @len: pointer to length of buffer; updated on return 1949c211ad2SPhilippe Mathieu-Daudé * @dir: indicates the transfer direction 1959c211ad2SPhilippe Mathieu-Daudé */ 196df32fd1cSPaolo Bonzini static inline void *dma_memory_map(AddressSpace *as, 1979c17d615SPaolo Bonzini dma_addr_t addr, dma_addr_t *len, 1989c17d615SPaolo Bonzini DMADirection dir) 1999c17d615SPaolo Bonzini { 2009c17d615SPaolo Bonzini hwaddr xlen = *len; 2019c17d615SPaolo Bonzini void *p; 2029c17d615SPaolo Bonzini 203f26404fbSPeter Maydell p = address_space_map(as, addr, &xlen, dir == DMA_DIRECTION_FROM_DEVICE, 204f26404fbSPeter Maydell MEMTXATTRS_UNSPECIFIED); 2059c17d615SPaolo Bonzini *len = xlen; 2069c17d615SPaolo Bonzini return p; 2079c17d615SPaolo Bonzini } 2089c17d615SPaolo Bonzini 2099c211ad2SPhilippe Mathieu-Daudé /** 2109c211ad2SPhilippe Mathieu-Daudé * address_space_unmap: Unmaps a memory region previously mapped 2119c211ad2SPhilippe Mathieu-Daudé * by dma_memory_map() 2129c211ad2SPhilippe Mathieu-Daudé * 2139c211ad2SPhilippe Mathieu-Daudé * Will also mark the memory as dirty if @dir == %DMA_DIRECTION_FROM_DEVICE. 2149c211ad2SPhilippe Mathieu-Daudé * @access_len gives the amount of memory that was actually read or written 2159c211ad2SPhilippe Mathieu-Daudé * by the caller. 2169c211ad2SPhilippe Mathieu-Daudé * 2179c211ad2SPhilippe Mathieu-Daudé * @as: #AddressSpace used 2189c211ad2SPhilippe Mathieu-Daudé * @buffer: host pointer as returned by address_space_map() 2199c211ad2SPhilippe Mathieu-Daudé * @len: buffer length as returned by address_space_map() 2209c211ad2SPhilippe Mathieu-Daudé * @dir: indicates the transfer direction 2219c211ad2SPhilippe Mathieu-Daudé * @access_len: amount of data actually transferred 2229c211ad2SPhilippe Mathieu-Daudé */ 223df32fd1cSPaolo Bonzini static inline void dma_memory_unmap(AddressSpace *as, 2249c17d615SPaolo Bonzini void *buffer, dma_addr_t len, 2259c17d615SPaolo Bonzini DMADirection dir, dma_addr_t access_len) 2269c17d615SPaolo Bonzini { 227df32fd1cSPaolo Bonzini address_space_unmap(as, buffer, (hwaddr)len, 2289c17d615SPaolo Bonzini dir == DMA_DIRECTION_FROM_DEVICE, access_len); 2299c17d615SPaolo Bonzini } 2309c17d615SPaolo Bonzini 2319c17d615SPaolo Bonzini #define DEFINE_LDST_DMA(_lname, _sname, _bits, _end) \ 232df32fd1cSPaolo Bonzini static inline uint##_bits##_t ld##_lname##_##_end##_dma(AddressSpace *as, \ 2339c17d615SPaolo Bonzini dma_addr_t addr) \ 2349c17d615SPaolo Bonzini { \ 2359c17d615SPaolo Bonzini uint##_bits##_t val; \ 236df32fd1cSPaolo Bonzini dma_memory_read(as, addr, &val, (_bits) / 8); \ 2379c17d615SPaolo Bonzini return _end##_bits##_to_cpu(val); \ 2389c17d615SPaolo Bonzini } \ 239df32fd1cSPaolo Bonzini static inline void st##_sname##_##_end##_dma(AddressSpace *as, \ 2409c17d615SPaolo Bonzini dma_addr_t addr, \ 2419c17d615SPaolo Bonzini uint##_bits##_t val) \ 2429c17d615SPaolo Bonzini { \ 2439c17d615SPaolo Bonzini val = cpu_to_##_end##_bits(val); \ 244df32fd1cSPaolo Bonzini dma_memory_write(as, addr, &val, (_bits) / 8); \ 2459c17d615SPaolo Bonzini } 2469c17d615SPaolo Bonzini 247df32fd1cSPaolo Bonzini static inline uint8_t ldub_dma(AddressSpace *as, dma_addr_t addr) 2489c17d615SPaolo Bonzini { 2499c17d615SPaolo Bonzini uint8_t val; 2509c17d615SPaolo Bonzini 251df32fd1cSPaolo Bonzini dma_memory_read(as, addr, &val, 1); 2529c17d615SPaolo Bonzini return val; 2539c17d615SPaolo Bonzini } 2549c17d615SPaolo Bonzini 255df32fd1cSPaolo Bonzini static inline void stb_dma(AddressSpace *as, dma_addr_t addr, uint8_t val) 2569c17d615SPaolo Bonzini { 257df32fd1cSPaolo Bonzini dma_memory_write(as, addr, &val, 1); 2589c17d615SPaolo Bonzini } 2599c17d615SPaolo Bonzini 2609c17d615SPaolo Bonzini DEFINE_LDST_DMA(uw, w, 16, le); 2619c17d615SPaolo Bonzini DEFINE_LDST_DMA(l, l, 32, le); 2629c17d615SPaolo Bonzini DEFINE_LDST_DMA(q, q, 64, le); 2639c17d615SPaolo Bonzini DEFINE_LDST_DMA(uw, w, 16, be); 2649c17d615SPaolo Bonzini DEFINE_LDST_DMA(l, l, 32, be); 2659c17d615SPaolo Bonzini DEFINE_LDST_DMA(q, q, 64, be); 2669c17d615SPaolo Bonzini 2679c17d615SPaolo Bonzini #undef DEFINE_LDST_DMA 2689c17d615SPaolo Bonzini 2699c17d615SPaolo Bonzini struct ScatterGatherEntry { 2709c17d615SPaolo Bonzini dma_addr_t base; 2719c17d615SPaolo Bonzini dma_addr_t len; 2729c17d615SPaolo Bonzini }; 2739c17d615SPaolo Bonzini 274f487b677SPaolo Bonzini void qemu_sglist_init(QEMUSGList *qsg, DeviceState *dev, int alloc_hint, 275f487b677SPaolo Bonzini AddressSpace *as); 2769c17d615SPaolo Bonzini void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len); 2779c17d615SPaolo Bonzini void qemu_sglist_destroy(QEMUSGList *qsg); 2789c17d615SPaolo Bonzini #endif 2799c17d615SPaolo Bonzini 2808a8e63ebSPaolo Bonzini typedef BlockAIOCB *DMAIOFunc(int64_t offset, QEMUIOVector *iov, 2818a8e63ebSPaolo Bonzini BlockCompletionFunc *cb, void *cb_opaque, 2828a8e63ebSPaolo Bonzini void *opaque); 2839c17d615SPaolo Bonzini 2848a8e63ebSPaolo Bonzini BlockAIOCB *dma_blk_io(AioContext *ctx, 28599868af3SMark Cave-Ayland QEMUSGList *sg, uint64_t offset, uint32_t align, 2868a8e63ebSPaolo Bonzini DMAIOFunc *io_func, void *io_func_opaque, 2878a8e63ebSPaolo Bonzini BlockCompletionFunc *cb, void *opaque, DMADirection dir); 2884be74634SMarkus Armbruster BlockAIOCB *dma_blk_read(BlockBackend *blk, 28999868af3SMark Cave-Ayland QEMUSGList *sg, uint64_t offset, uint32_t align, 290097310b5SMarkus Armbruster BlockCompletionFunc *cb, void *opaque); 2914be74634SMarkus Armbruster BlockAIOCB *dma_blk_write(BlockBackend *blk, 29299868af3SMark Cave-Ayland QEMUSGList *sg, uint64_t offset, uint32_t align, 293097310b5SMarkus Armbruster BlockCompletionFunc *cb, void *opaque); 2949c17d615SPaolo Bonzini uint64_t dma_buf_read(uint8_t *ptr, int32_t len, QEMUSGList *sg); 2959c17d615SPaolo Bonzini uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg); 2969c17d615SPaolo Bonzini 2974be74634SMarkus Armbruster void dma_acct_start(BlockBackend *blk, BlockAcctCookie *cookie, 2989c17d615SPaolo Bonzini QEMUSGList *sg, enum BlockAcctType type); 2999c17d615SPaolo Bonzini 300f14fb6c2SEric Auger /** 301f14fb6c2SEric Auger * dma_aligned_pow2_mask: Return the address bit mask of the largest 302f14fb6c2SEric Auger * power of 2 size less or equal than @end - @start + 1, aligned with @start, 303f14fb6c2SEric Auger * and bounded by 1 << @max_addr_bits bits. 304f14fb6c2SEric Auger * 305f14fb6c2SEric Auger * @start: range start address 306f14fb6c2SEric Auger * @end: range end address (greater than @start) 307f14fb6c2SEric Auger * @max_addr_bits: max address bits (<= 64) 308f14fb6c2SEric Auger */ 309f14fb6c2SEric Auger uint64_t dma_aligned_pow2_mask(uint64_t start, uint64_t end, 310f14fb6c2SEric Auger int max_addr_bits); 311f14fb6c2SEric Auger 3129c17d615SPaolo Bonzini #endif 313