19c17d615SPaolo Bonzini /* 29c17d615SPaolo Bonzini * DMA helper functions 39c17d615SPaolo Bonzini * 49c211ad2SPhilippe Mathieu-Daudé * Copyright (c) 2009, 2020 Red Hat 59c17d615SPaolo Bonzini * 69c17d615SPaolo Bonzini * This work is licensed under the terms of the GNU General Public License 79c17d615SPaolo Bonzini * (GNU GPL), version 2 or later. 89c17d615SPaolo Bonzini */ 99c17d615SPaolo Bonzini 109c17d615SPaolo Bonzini #ifndef DMA_H 119c17d615SPaolo Bonzini #define DMA_H 129c17d615SPaolo Bonzini 139c17d615SPaolo Bonzini #include "exec/memory.h" 14df32fd1cSPaolo Bonzini #include "exec/address-spaces.h" 159c17d615SPaolo Bonzini #include "block/block.h" 165e5a94b6SBenoît Canet #include "block/accounting.h" 179c17d615SPaolo Bonzini 189c17d615SPaolo Bonzini typedef struct ScatterGatherEntry ScatterGatherEntry; 199c17d615SPaolo Bonzini 209c17d615SPaolo Bonzini typedef enum { 219c17d615SPaolo Bonzini DMA_DIRECTION_TO_DEVICE = 0, 229c17d615SPaolo Bonzini DMA_DIRECTION_FROM_DEVICE = 1, 239c17d615SPaolo Bonzini } DMADirection; 249c17d615SPaolo Bonzini 259c17d615SPaolo Bonzini struct QEMUSGList { 269c17d615SPaolo Bonzini ScatterGatherEntry *sg; 279c17d615SPaolo Bonzini int nsg; 289c17d615SPaolo Bonzini int nalloc; 299c17d615SPaolo Bonzini size_t size; 30f487b677SPaolo Bonzini DeviceState *dev; 31df32fd1cSPaolo Bonzini AddressSpace *as; 329c17d615SPaolo Bonzini }; 339c17d615SPaolo Bonzini 349c17d615SPaolo Bonzini #ifndef CONFIG_USER_ONLY 359c17d615SPaolo Bonzini 369c17d615SPaolo Bonzini /* 379c17d615SPaolo Bonzini * When an IOMMU is present, bus addresses become distinct from 389c17d615SPaolo Bonzini * CPU/memory physical addresses and may be a different size. Because 399c17d615SPaolo Bonzini * the IOVA size depends more on the bus than on the platform, we more 409c17d615SPaolo Bonzini * or less have to treat these as 64-bit always to cover all (or at 419c17d615SPaolo Bonzini * least most) cases. 429c17d615SPaolo Bonzini */ 439c17d615SPaolo Bonzini typedef uint64_t dma_addr_t; 449c17d615SPaolo Bonzini 459c17d615SPaolo Bonzini #define DMA_ADDR_BITS 64 469c17d615SPaolo Bonzini #define DMA_ADDR_FMT "%" PRIx64 479c17d615SPaolo Bonzini 48df32fd1cSPaolo Bonzini static inline void dma_barrier(AddressSpace *as, DMADirection dir) 499c17d615SPaolo Bonzini { 509c17d615SPaolo Bonzini /* 519c17d615SPaolo Bonzini * This is called before DMA read and write operations 529c17d615SPaolo Bonzini * unless the _relaxed form is used and is responsible 539c17d615SPaolo Bonzini * for providing some sane ordering of accesses vs 549c17d615SPaolo Bonzini * concurrently running VCPUs. 559c17d615SPaolo Bonzini * 569c17d615SPaolo Bonzini * Users of map(), unmap() or lower level st/ld_* 579c17d615SPaolo Bonzini * operations are responsible for providing their own 589c17d615SPaolo Bonzini * ordering via barriers. 599c17d615SPaolo Bonzini * 609c17d615SPaolo Bonzini * This primitive implementation does a simple smp_mb() 619c17d615SPaolo Bonzini * before each operation which provides pretty much full 629c17d615SPaolo Bonzini * ordering. 639c17d615SPaolo Bonzini * 649c17d615SPaolo Bonzini * A smarter implementation can be devised if needed to 659c17d615SPaolo Bonzini * use lighter barriers based on the direction of the 669c17d615SPaolo Bonzini * transfer, the DMA context, etc... 679c17d615SPaolo Bonzini */ 689c17d615SPaolo Bonzini smp_mb(); 699c17d615SPaolo Bonzini } 709c17d615SPaolo Bonzini 719c17d615SPaolo Bonzini /* Checks that the given range of addresses is valid for DMA. This is 729c17d615SPaolo Bonzini * useful for certain cases, but usually you should just use 739c17d615SPaolo Bonzini * dma_memory_{read,write}() and check for errors */ 74df32fd1cSPaolo Bonzini static inline bool dma_memory_valid(AddressSpace *as, 759c17d615SPaolo Bonzini dma_addr_t addr, dma_addr_t len, 767ccb391cSPhilippe Mathieu-Daudé DMADirection dir, MemTxAttrs attrs) 779c17d615SPaolo Bonzini { 78df32fd1cSPaolo Bonzini return address_space_access_valid(as, addr, len, 79fddffa42SPeter Maydell dir == DMA_DIRECTION_FROM_DEVICE, 807ccb391cSPhilippe Mathieu-Daudé attrs); 819c17d615SPaolo Bonzini } 829c17d615SPaolo Bonzini 839989bcd3SPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_rw_relaxed(AddressSpace *as, 849989bcd3SPhilippe Mathieu-Daudé dma_addr_t addr, 859c17d615SPaolo Bonzini void *buf, dma_addr_t len, 86*4afd0f2fSPhilippe Mathieu-Daudé DMADirection dir, 87*4afd0f2fSPhilippe Mathieu-Daudé MemTxAttrs attrs) 889c17d615SPaolo Bonzini { 89*4afd0f2fSPhilippe Mathieu-Daudé return address_space_rw(as, addr, attrs, 905c9eb028SPeter Maydell buf, len, dir == DMA_DIRECTION_FROM_DEVICE); 919c17d615SPaolo Bonzini } 929c17d615SPaolo Bonzini 93b1f51303SPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_read_relaxed(AddressSpace *as, 94b1f51303SPhilippe Mathieu-Daudé dma_addr_t addr, 959c17d615SPaolo Bonzini void *buf, dma_addr_t len) 969c17d615SPaolo Bonzini { 97*4afd0f2fSPhilippe Mathieu-Daudé return dma_memory_rw_relaxed(as, addr, buf, len, 98*4afd0f2fSPhilippe Mathieu-Daudé DMA_DIRECTION_TO_DEVICE, 99*4afd0f2fSPhilippe Mathieu-Daudé MEMTXATTRS_UNSPECIFIED); 1009c17d615SPaolo Bonzini } 1019c17d615SPaolo Bonzini 10277c71d1dSPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_write_relaxed(AddressSpace *as, 10377c71d1dSPhilippe Mathieu-Daudé dma_addr_t addr, 10477c71d1dSPhilippe Mathieu-Daudé const void *buf, 10577c71d1dSPhilippe Mathieu-Daudé dma_addr_t len) 1069c17d615SPaolo Bonzini { 107df32fd1cSPaolo Bonzini return dma_memory_rw_relaxed(as, addr, (void *)buf, len, 108*4afd0f2fSPhilippe Mathieu-Daudé DMA_DIRECTION_FROM_DEVICE, 109*4afd0f2fSPhilippe Mathieu-Daudé MEMTXATTRS_UNSPECIFIED); 1109c17d615SPaolo Bonzini } 1119c17d615SPaolo Bonzini 1129989bcd3SPhilippe Mathieu-Daudé /** 1139989bcd3SPhilippe Mathieu-Daudé * dma_memory_rw: Read from or write to an address space from DMA controller. 1149989bcd3SPhilippe Mathieu-Daudé * 1159989bcd3SPhilippe Mathieu-Daudé * Return a MemTxResult indicating whether the operation succeeded 1169989bcd3SPhilippe Mathieu-Daudé * or failed (eg unassigned memory, device rejected the transaction, 1179989bcd3SPhilippe Mathieu-Daudé * IOMMU fault). 1189989bcd3SPhilippe Mathieu-Daudé * 1199989bcd3SPhilippe Mathieu-Daudé * @as: #AddressSpace to be accessed 1209989bcd3SPhilippe Mathieu-Daudé * @addr: address within that address space 1219989bcd3SPhilippe Mathieu-Daudé * @buf: buffer with the data transferred 1229989bcd3SPhilippe Mathieu-Daudé * @len: the number of bytes to read or write 1239989bcd3SPhilippe Mathieu-Daudé * @dir: indicates the transfer direction 1249989bcd3SPhilippe Mathieu-Daudé */ 1259989bcd3SPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_rw(AddressSpace *as, dma_addr_t addr, 1269c17d615SPaolo Bonzini void *buf, dma_addr_t len, 1279c17d615SPaolo Bonzini DMADirection dir) 1289c17d615SPaolo Bonzini { 129df32fd1cSPaolo Bonzini dma_barrier(as, dir); 1309c17d615SPaolo Bonzini 131*4afd0f2fSPhilippe Mathieu-Daudé return dma_memory_rw_relaxed(as, addr, buf, len, dir, 132*4afd0f2fSPhilippe Mathieu-Daudé MEMTXATTRS_UNSPECIFIED); 1339c17d615SPaolo Bonzini } 1349c17d615SPaolo Bonzini 135b1f51303SPhilippe Mathieu-Daudé /** 136b1f51303SPhilippe Mathieu-Daudé * dma_memory_read: Read from an address space from DMA controller. 137b1f51303SPhilippe Mathieu-Daudé * 138b1f51303SPhilippe Mathieu-Daudé * Return a MemTxResult indicating whether the operation succeeded 139b1f51303SPhilippe Mathieu-Daudé * or failed (eg unassigned memory, device rejected the transaction, 140b1f51303SPhilippe Mathieu-Daudé * IOMMU fault). Called within RCU critical section. 141b1f51303SPhilippe Mathieu-Daudé * 142b1f51303SPhilippe Mathieu-Daudé * @as: #AddressSpace to be accessed 143b1f51303SPhilippe Mathieu-Daudé * @addr: address within that address space 144b1f51303SPhilippe Mathieu-Daudé * @buf: buffer with the data transferred 145b1f51303SPhilippe Mathieu-Daudé * @len: length of the data transferred 146b1f51303SPhilippe Mathieu-Daudé */ 147b1f51303SPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_read(AddressSpace *as, dma_addr_t addr, 1489c17d615SPaolo Bonzini void *buf, dma_addr_t len) 1499c17d615SPaolo Bonzini { 150df32fd1cSPaolo Bonzini return dma_memory_rw(as, addr, buf, len, DMA_DIRECTION_TO_DEVICE); 1519c17d615SPaolo Bonzini } 1529c17d615SPaolo Bonzini 15377c71d1dSPhilippe Mathieu-Daudé /** 15477c71d1dSPhilippe Mathieu-Daudé * address_space_write: Write to address space from DMA controller. 15577c71d1dSPhilippe Mathieu-Daudé * 15677c71d1dSPhilippe Mathieu-Daudé * Return a MemTxResult indicating whether the operation succeeded 15777c71d1dSPhilippe Mathieu-Daudé * or failed (eg unassigned memory, device rejected the transaction, 15877c71d1dSPhilippe Mathieu-Daudé * IOMMU fault). 15977c71d1dSPhilippe Mathieu-Daudé * 16077c71d1dSPhilippe Mathieu-Daudé * @as: #AddressSpace to be accessed 16177c71d1dSPhilippe Mathieu-Daudé * @addr: address within that address space 16277c71d1dSPhilippe Mathieu-Daudé * @buf: buffer with the data transferred 16377c71d1dSPhilippe Mathieu-Daudé * @len: the number of bytes to write 16477c71d1dSPhilippe Mathieu-Daudé */ 16577c71d1dSPhilippe Mathieu-Daudé static inline MemTxResult dma_memory_write(AddressSpace *as, dma_addr_t addr, 1669c17d615SPaolo Bonzini const void *buf, dma_addr_t len) 1679c17d615SPaolo Bonzini { 168df32fd1cSPaolo Bonzini return dma_memory_rw(as, addr, (void *)buf, len, 1699c17d615SPaolo Bonzini DMA_DIRECTION_FROM_DEVICE); 1709c17d615SPaolo Bonzini } 1719c17d615SPaolo Bonzini 172bb755f52SPhilippe Mathieu-Daudé /** 173bb755f52SPhilippe Mathieu-Daudé * dma_memory_set: Fill memory with a constant byte from DMA controller. 174bb755f52SPhilippe Mathieu-Daudé * 175bb755f52SPhilippe Mathieu-Daudé * Return a MemTxResult indicating whether the operation succeeded 176bb755f52SPhilippe Mathieu-Daudé * or failed (eg unassigned memory, device rejected the transaction, 177bb755f52SPhilippe Mathieu-Daudé * IOMMU fault). 178bb755f52SPhilippe Mathieu-Daudé * 179bb755f52SPhilippe Mathieu-Daudé * @as: #AddressSpace to be accessed 180bb755f52SPhilippe Mathieu-Daudé * @addr: address within that address space 181bb755f52SPhilippe Mathieu-Daudé * @c: constant byte to fill the memory 182bb755f52SPhilippe Mathieu-Daudé * @len: the number of bytes to fill with the constant byte 1837a36e42dSPhilippe Mathieu-Daudé * @attrs: memory transaction attributes 184bb755f52SPhilippe Mathieu-Daudé */ 185bb755f52SPhilippe Mathieu-Daudé MemTxResult dma_memory_set(AddressSpace *as, dma_addr_t addr, 1867a36e42dSPhilippe Mathieu-Daudé uint8_t c, dma_addr_t len, MemTxAttrs attrs); 1879c17d615SPaolo Bonzini 1889c211ad2SPhilippe Mathieu-Daudé /** 1899c211ad2SPhilippe Mathieu-Daudé * address_space_map: Map a physical memory region into a host virtual address. 1909c211ad2SPhilippe Mathieu-Daudé * 1919c211ad2SPhilippe Mathieu-Daudé * May map a subset of the requested range, given by and returned in @plen. 1929c211ad2SPhilippe Mathieu-Daudé * May return %NULL and set *@plen to zero(0), if resources needed to perform 1939c211ad2SPhilippe Mathieu-Daudé * the mapping are exhausted. 1949c211ad2SPhilippe Mathieu-Daudé * Use only for reads OR writes - not for read-modify-write operations. 1959c211ad2SPhilippe Mathieu-Daudé * 1969c211ad2SPhilippe Mathieu-Daudé * @as: #AddressSpace to be accessed 1979c211ad2SPhilippe Mathieu-Daudé * @addr: address within that address space 1989c211ad2SPhilippe Mathieu-Daudé * @len: pointer to length of buffer; updated on return 1999c211ad2SPhilippe Mathieu-Daudé * @dir: indicates the transfer direction 2009c211ad2SPhilippe Mathieu-Daudé */ 201df32fd1cSPaolo Bonzini static inline void *dma_memory_map(AddressSpace *as, 2029c17d615SPaolo Bonzini dma_addr_t addr, dma_addr_t *len, 2039c17d615SPaolo Bonzini DMADirection dir) 2049c17d615SPaolo Bonzini { 2059c17d615SPaolo Bonzini hwaddr xlen = *len; 2069c17d615SPaolo Bonzini void *p; 2079c17d615SPaolo Bonzini 208f26404fbSPeter Maydell p = address_space_map(as, addr, &xlen, dir == DMA_DIRECTION_FROM_DEVICE, 209f26404fbSPeter Maydell MEMTXATTRS_UNSPECIFIED); 2109c17d615SPaolo Bonzini *len = xlen; 2119c17d615SPaolo Bonzini return p; 2129c17d615SPaolo Bonzini } 2139c17d615SPaolo Bonzini 2149c211ad2SPhilippe Mathieu-Daudé /** 2159c211ad2SPhilippe Mathieu-Daudé * address_space_unmap: Unmaps a memory region previously mapped 2169c211ad2SPhilippe Mathieu-Daudé * by dma_memory_map() 2179c211ad2SPhilippe Mathieu-Daudé * 2189c211ad2SPhilippe Mathieu-Daudé * Will also mark the memory as dirty if @dir == %DMA_DIRECTION_FROM_DEVICE. 2199c211ad2SPhilippe Mathieu-Daudé * @access_len gives the amount of memory that was actually read or written 2209c211ad2SPhilippe Mathieu-Daudé * by the caller. 2219c211ad2SPhilippe Mathieu-Daudé * 2229c211ad2SPhilippe Mathieu-Daudé * @as: #AddressSpace used 2239c211ad2SPhilippe Mathieu-Daudé * @buffer: host pointer as returned by address_space_map() 2249c211ad2SPhilippe Mathieu-Daudé * @len: buffer length as returned by address_space_map() 2259c211ad2SPhilippe Mathieu-Daudé * @dir: indicates the transfer direction 2269c211ad2SPhilippe Mathieu-Daudé * @access_len: amount of data actually transferred 2279c211ad2SPhilippe Mathieu-Daudé */ 228df32fd1cSPaolo Bonzini static inline void dma_memory_unmap(AddressSpace *as, 2299c17d615SPaolo Bonzini void *buffer, dma_addr_t len, 2309c17d615SPaolo Bonzini DMADirection dir, dma_addr_t access_len) 2319c17d615SPaolo Bonzini { 232df32fd1cSPaolo Bonzini address_space_unmap(as, buffer, (hwaddr)len, 2339c17d615SPaolo Bonzini dir == DMA_DIRECTION_FROM_DEVICE, access_len); 2349c17d615SPaolo Bonzini } 2359c17d615SPaolo Bonzini 2369c17d615SPaolo Bonzini #define DEFINE_LDST_DMA(_lname, _sname, _bits, _end) \ 237df32fd1cSPaolo Bonzini static inline uint##_bits##_t ld##_lname##_##_end##_dma(AddressSpace *as, \ 2389c17d615SPaolo Bonzini dma_addr_t addr) \ 2399c17d615SPaolo Bonzini { \ 2409c17d615SPaolo Bonzini uint##_bits##_t val; \ 241df32fd1cSPaolo Bonzini dma_memory_read(as, addr, &val, (_bits) / 8); \ 2429c17d615SPaolo Bonzini return _end##_bits##_to_cpu(val); \ 2439c17d615SPaolo Bonzini } \ 244df32fd1cSPaolo Bonzini static inline void st##_sname##_##_end##_dma(AddressSpace *as, \ 2459c17d615SPaolo Bonzini dma_addr_t addr, \ 2469c17d615SPaolo Bonzini uint##_bits##_t val) \ 2479c17d615SPaolo Bonzini { \ 2489c17d615SPaolo Bonzini val = cpu_to_##_end##_bits(val); \ 249df32fd1cSPaolo Bonzini dma_memory_write(as, addr, &val, (_bits) / 8); \ 2509c17d615SPaolo Bonzini } 2519c17d615SPaolo Bonzini 252df32fd1cSPaolo Bonzini static inline uint8_t ldub_dma(AddressSpace *as, dma_addr_t addr) 2539c17d615SPaolo Bonzini { 2549c17d615SPaolo Bonzini uint8_t val; 2559c17d615SPaolo Bonzini 256df32fd1cSPaolo Bonzini dma_memory_read(as, addr, &val, 1); 2579c17d615SPaolo Bonzini return val; 2589c17d615SPaolo Bonzini } 2599c17d615SPaolo Bonzini 260df32fd1cSPaolo Bonzini static inline void stb_dma(AddressSpace *as, dma_addr_t addr, uint8_t val) 2619c17d615SPaolo Bonzini { 262df32fd1cSPaolo Bonzini dma_memory_write(as, addr, &val, 1); 2639c17d615SPaolo Bonzini } 2649c17d615SPaolo Bonzini 2659c17d615SPaolo Bonzini DEFINE_LDST_DMA(uw, w, 16, le); 2669c17d615SPaolo Bonzini DEFINE_LDST_DMA(l, l, 32, le); 2679c17d615SPaolo Bonzini DEFINE_LDST_DMA(q, q, 64, le); 2689c17d615SPaolo Bonzini DEFINE_LDST_DMA(uw, w, 16, be); 2699c17d615SPaolo Bonzini DEFINE_LDST_DMA(l, l, 32, be); 2709c17d615SPaolo Bonzini DEFINE_LDST_DMA(q, q, 64, be); 2719c17d615SPaolo Bonzini 2729c17d615SPaolo Bonzini #undef DEFINE_LDST_DMA 2739c17d615SPaolo Bonzini 2749c17d615SPaolo Bonzini struct ScatterGatherEntry { 2759c17d615SPaolo Bonzini dma_addr_t base; 2769c17d615SPaolo Bonzini dma_addr_t len; 2779c17d615SPaolo Bonzini }; 2789c17d615SPaolo Bonzini 279f487b677SPaolo Bonzini void qemu_sglist_init(QEMUSGList *qsg, DeviceState *dev, int alloc_hint, 280f487b677SPaolo Bonzini AddressSpace *as); 2819c17d615SPaolo Bonzini void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len); 2829c17d615SPaolo Bonzini void qemu_sglist_destroy(QEMUSGList *qsg); 2839c17d615SPaolo Bonzini #endif 2849c17d615SPaolo Bonzini 2858a8e63ebSPaolo Bonzini typedef BlockAIOCB *DMAIOFunc(int64_t offset, QEMUIOVector *iov, 2868a8e63ebSPaolo Bonzini BlockCompletionFunc *cb, void *cb_opaque, 2878a8e63ebSPaolo Bonzini void *opaque); 2889c17d615SPaolo Bonzini 2898a8e63ebSPaolo Bonzini BlockAIOCB *dma_blk_io(AioContext *ctx, 29099868af3SMark Cave-Ayland QEMUSGList *sg, uint64_t offset, uint32_t align, 2918a8e63ebSPaolo Bonzini DMAIOFunc *io_func, void *io_func_opaque, 2928a8e63ebSPaolo Bonzini BlockCompletionFunc *cb, void *opaque, DMADirection dir); 2934be74634SMarkus Armbruster BlockAIOCB *dma_blk_read(BlockBackend *blk, 29499868af3SMark Cave-Ayland QEMUSGList *sg, uint64_t offset, uint32_t align, 295097310b5SMarkus Armbruster BlockCompletionFunc *cb, void *opaque); 2964be74634SMarkus Armbruster BlockAIOCB *dma_blk_write(BlockBackend *blk, 29799868af3SMark Cave-Ayland QEMUSGList *sg, uint64_t offset, uint32_t align, 298097310b5SMarkus Armbruster BlockCompletionFunc *cb, void *opaque); 2999c17d615SPaolo Bonzini uint64_t dma_buf_read(uint8_t *ptr, int32_t len, QEMUSGList *sg); 3009c17d615SPaolo Bonzini uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg); 3019c17d615SPaolo Bonzini 3024be74634SMarkus Armbruster void dma_acct_start(BlockBackend *blk, BlockAcctCookie *cookie, 3039c17d615SPaolo Bonzini QEMUSGList *sg, enum BlockAcctType type); 3049c17d615SPaolo Bonzini 305f14fb6c2SEric Auger /** 306f14fb6c2SEric Auger * dma_aligned_pow2_mask: Return the address bit mask of the largest 307f14fb6c2SEric Auger * power of 2 size less or equal than @end - @start + 1, aligned with @start, 308f14fb6c2SEric Auger * and bounded by 1 << @max_addr_bits bits. 309f14fb6c2SEric Auger * 310f14fb6c2SEric Auger * @start: range start address 311f14fb6c2SEric Auger * @end: range end address (greater than @start) 312f14fb6c2SEric Auger * @max_addr_bits: max address bits (<= 64) 313f14fb6c2SEric Auger */ 314f14fb6c2SEric Auger uint64_t dma_aligned_pow2_mask(uint64_t start, uint64_t end, 315f14fb6c2SEric Auger int max_addr_bits); 316f14fb6c2SEric Auger 3179c17d615SPaolo Bonzini #endif 318