1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 4 * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. 5 * Copyright 2019 Marvell. All rights reserved. 6 */ 7 #include <linux/xarray.h> 8 #include "uverbs.h" 9 #include "core_priv.h" 10 11 /** 12 * rdma_umap_priv_init() - Initialize the private data of a vma 13 * 14 * @priv: The already allocated private data 15 * @vma: The vm area struct that needs private data 16 * @entry: entry into the mmap_xa that needs to be linked with 17 * this vma 18 * 19 * Each time we map IO memory into user space this keeps track of the 20 * mapping. When the device is hot-unplugged we 'zap' the mmaps in user space 21 * to point to the zero page and allow the hot unplug to proceed. 22 * 23 * This is necessary for cases like PCI physical hot unplug as the actual BAR 24 * memory may vanish after this and access to it from userspace could MCE. 25 * 26 * RDMA drivers supporting disassociation must have their user space designed 27 * to cope in some way with their IO pages going to the zero page. 28 * 29 */ 30 void rdma_umap_priv_init(struct rdma_umap_priv *priv, 31 struct vm_area_struct *vma, 32 struct rdma_user_mmap_entry *entry) 33 { 34 struct ib_uverbs_file *ufile = vma->vm_file->private_data; 35 36 priv->vma = vma; 37 if (entry) { 38 kref_get(&entry->ref); 39 priv->entry = entry; 40 } 41 vma->vm_private_data = priv; 42 /* vm_ops is setup in ib_uverbs_mmap() to avoid module dependencies */ 43 44 mutex_lock(&ufile->umap_lock); 45 list_add(&priv->list, &ufile->umaps); 46 mutex_unlock(&ufile->umap_lock); 47 } 48 EXPORT_SYMBOL(rdma_umap_priv_init); 49 50 /** 51 * rdma_user_mmap_io() - Map IO memory into a process 52 * 53 * @ucontext: associated user context 54 * @vma: the vma related to the current mmap call 55 * @pfn: pfn to map 56 * @size: size to map 57 * @prot: pgprot to use in remap call 58 * @entry: mmap_entry retrieved from rdma_user_mmap_entry_get(), or NULL 59 * if mmap_entry is not used by the driver 60 * 61 * This is to be called by drivers as part of their mmap() functions if they 62 * wish to send something like PCI-E BAR memory to userspace. 63 * 64 * Return -EINVAL on wrong flags or size, -EAGAIN on failure to map. 0 on 65 * success. 66 */ 67 int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma, 68 unsigned long pfn, unsigned long size, pgprot_t prot, 69 struct rdma_user_mmap_entry *entry) 70 { 71 struct ib_uverbs_file *ufile = ucontext->ufile; 72 struct rdma_umap_priv *priv; 73 74 if (!(vma->vm_flags & VM_SHARED)) 75 return -EINVAL; 76 77 if (vma->vm_end - vma->vm_start != size) 78 return -EINVAL; 79 80 /* Driver is using this wrong, must be called by ib_uverbs_mmap */ 81 if (WARN_ON(!vma->vm_file || 82 vma->vm_file->private_data != ufile)) 83 return -EINVAL; 84 lockdep_assert_held(&ufile->device->disassociate_srcu); 85 86 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 87 if (!priv) 88 return -ENOMEM; 89 90 vma->vm_page_prot = prot; 91 if (io_remap_pfn_range(vma, vma->vm_start, pfn, size, prot)) { 92 kfree(priv); 93 return -EAGAIN; 94 } 95 96 rdma_umap_priv_init(priv, vma, entry); 97 return 0; 98 } 99 EXPORT_SYMBOL(rdma_user_mmap_io); 100 101 /** 102 * rdma_user_mmap_entry_get_pgoff() - Get an entry from the mmap_xa 103 * 104 * @ucontext: associated user context 105 * @pgoff: The mmap offset >> PAGE_SHIFT 106 * 107 * This function is called when a user tries to mmap with an offset (returned 108 * by rdma_user_mmap_get_offset()) it initially received from the driver. The 109 * rdma_user_mmap_entry was created by the function 110 * rdma_user_mmap_entry_insert(). This function increases the refcnt of the 111 * entry so that it won't be deleted from the xarray in the meantime. 112 * 113 * Return an reference to an entry if exists or NULL if there is no 114 * match. rdma_user_mmap_entry_put() must be called to put the reference. 115 */ 116 struct rdma_user_mmap_entry * 117 rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext, 118 unsigned long pgoff) 119 { 120 struct rdma_user_mmap_entry *entry; 121 122 if (pgoff > U32_MAX) 123 return NULL; 124 125 xa_lock(&ucontext->mmap_xa); 126 127 entry = xa_load(&ucontext->mmap_xa, pgoff); 128 129 /* 130 * If refcount is zero, entry is already being deleted, driver_removed 131 * indicates that the no further mmaps are possible and we waiting for 132 * the active VMAs to be closed. 133 */ 134 if (!entry || entry->start_pgoff != pgoff || entry->driver_removed || 135 !kref_get_unless_zero(&entry->ref)) 136 goto err; 137 138 xa_unlock(&ucontext->mmap_xa); 139 140 ibdev_dbg(ucontext->device, "mmap: pgoff[%#lx] npages[%#zx] returned\n", 141 pgoff, entry->npages); 142 143 return entry; 144 145 err: 146 xa_unlock(&ucontext->mmap_xa); 147 return NULL; 148 } 149 EXPORT_SYMBOL(rdma_user_mmap_entry_get_pgoff); 150 151 /** 152 * rdma_user_mmap_entry_get() - Get an entry from the mmap_xa 153 * 154 * @ucontext: associated user context 155 * @vma: the vma being mmap'd into 156 * 157 * This function is like rdma_user_mmap_entry_get_pgoff() except that it also 158 * checks that the VMA is correct. 159 */ 160 struct rdma_user_mmap_entry * 161 rdma_user_mmap_entry_get(struct ib_ucontext *ucontext, 162 struct vm_area_struct *vma) 163 { 164 struct rdma_user_mmap_entry *entry; 165 166 if (!(vma->vm_flags & VM_SHARED)) 167 return NULL; 168 entry = rdma_user_mmap_entry_get_pgoff(ucontext, vma->vm_pgoff); 169 if (!entry) 170 return NULL; 171 if (entry->npages * PAGE_SIZE != vma->vm_end - vma->vm_start) { 172 rdma_user_mmap_entry_put(entry); 173 return NULL; 174 } 175 return entry; 176 } 177 EXPORT_SYMBOL(rdma_user_mmap_entry_get); 178 179 static void rdma_user_mmap_entry_free(struct kref *kref) 180 { 181 struct rdma_user_mmap_entry *entry = 182 container_of(kref, struct rdma_user_mmap_entry, ref); 183 struct ib_ucontext *ucontext = entry->ucontext; 184 unsigned long i; 185 186 /* 187 * Erase all entries occupied by this single entry, this is deferred 188 * until all VMA are closed so that the mmap offsets remain unique. 189 */ 190 xa_lock(&ucontext->mmap_xa); 191 for (i = 0; i < entry->npages; i++) 192 __xa_erase(&ucontext->mmap_xa, entry->start_pgoff + i); 193 xa_unlock(&ucontext->mmap_xa); 194 195 ibdev_dbg(ucontext->device, "mmap: pgoff[%#lx] npages[%#zx] removed\n", 196 entry->start_pgoff, entry->npages); 197 198 if (ucontext->device->ops.mmap_free) 199 ucontext->device->ops.mmap_free(entry); 200 } 201 202 /** 203 * rdma_user_mmap_entry_put() - Drop reference to the mmap entry 204 * 205 * @entry: an entry in the mmap_xa 206 * 207 * This function is called when the mapping is closed if it was 208 * an io mapping or when the driver is done with the entry for 209 * some other reason. 210 * Should be called after rdma_user_mmap_entry_get was called 211 * and entry is no longer needed. This function will erase the 212 * entry and free it if its refcnt reaches zero. 213 */ 214 void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry) 215 { 216 kref_put(&entry->ref, rdma_user_mmap_entry_free); 217 } 218 EXPORT_SYMBOL(rdma_user_mmap_entry_put); 219 220 /** 221 * rdma_user_mmap_entry_remove() - Drop reference to entry and 222 * mark it as unmmapable 223 * 224 * @entry: the entry to insert into the mmap_xa 225 * 226 * Drivers can call this to prevent userspace from creating more mappings for 227 * entry, however existing mmaps continue to exist and ops->mmap_free() will 228 * not be called until all user mmaps are destroyed. 229 */ 230 void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry) 231 { 232 if (!entry) 233 return; 234 235 xa_lock(&entry->ucontext->mmap_xa); 236 entry->driver_removed = true; 237 xa_unlock(&entry->ucontext->mmap_xa); 238 kref_put(&entry->ref, rdma_user_mmap_entry_free); 239 } 240 EXPORT_SYMBOL(rdma_user_mmap_entry_remove); 241 242 /** 243 * rdma_user_mmap_entry_insert_range() - Insert an entry to the mmap_xa 244 * in a given range. 245 * 246 * @ucontext: associated user context. 247 * @entry: the entry to insert into the mmap_xa 248 * @length: length of the address that will be mmapped 249 * @min_pgoff: minimum pgoff to be returned 250 * @max_pgoff: maximum pgoff to be returned 251 * 252 * This function should be called by drivers that use the rdma_user_mmap 253 * interface for implementing their mmap syscall A database of mmap offsets is 254 * handled in the core and helper functions are provided to insert entries 255 * into the database and extract entries when the user calls mmap with the 256 * given offset. The function allocates a unique page offset in a given range 257 * that should be provided to user, the user will use the offset to retrieve 258 * information such as address to be mapped and how. 259 * 260 * Return: 0 on success and -ENOMEM on failure 261 */ 262 int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext, 263 struct rdma_user_mmap_entry *entry, 264 size_t length, u32 min_pgoff, 265 u32 max_pgoff) 266 { 267 struct ib_uverbs_file *ufile = ucontext->ufile; 268 XA_STATE(xas, &ucontext->mmap_xa, min_pgoff); 269 u32 xa_first, xa_last, npages; 270 int err; 271 u32 i; 272 273 if (!entry) 274 return -EINVAL; 275 276 kref_init(&entry->ref); 277 entry->ucontext = ucontext; 278 279 /* 280 * We want the whole allocation to be done without interruption from a 281 * different thread. The allocation requires finding a free range and 282 * storing. During the xa_insert the lock could be released, possibly 283 * allowing another thread to choose the same range. 284 */ 285 mutex_lock(&ufile->umap_lock); 286 287 xa_lock(&ucontext->mmap_xa); 288 289 /* We want to find an empty range */ 290 npages = (u32)DIV_ROUND_UP(length, PAGE_SIZE); 291 entry->npages = npages; 292 while (true) { 293 /* First find an empty index */ 294 xas_find_marked(&xas, max_pgoff, XA_FREE_MARK); 295 if (xas.xa_node == XAS_RESTART) 296 goto err_unlock; 297 298 xa_first = xas.xa_index; 299 300 /* Is there enough room to have the range? */ 301 if (check_add_overflow(xa_first, npages, &xa_last)) 302 goto err_unlock; 303 304 /* 305 * Now look for the next present entry. If an entry doesn't 306 * exist, we found an empty range and can proceed. 307 */ 308 xas_next_entry(&xas, xa_last - 1); 309 if (xas.xa_node == XAS_BOUNDS || xas.xa_index >= xa_last) 310 break; 311 } 312 313 for (i = xa_first; i < xa_last; i++) { 314 err = __xa_insert(&ucontext->mmap_xa, i, entry, GFP_KERNEL); 315 if (err) 316 goto err_undo; 317 } 318 319 /* 320 * Internally the kernel uses a page offset, in libc this is a byte 321 * offset. Drivers should not return pgoff to userspace. 322 */ 323 entry->start_pgoff = xa_first; 324 xa_unlock(&ucontext->mmap_xa); 325 mutex_unlock(&ufile->umap_lock); 326 327 ibdev_dbg(ucontext->device, "mmap: pgoff[%#lx] npages[%#x] inserted\n", 328 entry->start_pgoff, npages); 329 330 return 0; 331 332 err_undo: 333 for (; i > xa_first; i--) 334 __xa_erase(&ucontext->mmap_xa, i - 1); 335 336 err_unlock: 337 xa_unlock(&ucontext->mmap_xa); 338 mutex_unlock(&ufile->umap_lock); 339 return -ENOMEM; 340 } 341 EXPORT_SYMBOL(rdma_user_mmap_entry_insert_range); 342 343 /** 344 * rdma_user_mmap_entry_insert() - Insert an entry to the mmap_xa. 345 * 346 * @ucontext: associated user context. 347 * @entry: the entry to insert into the mmap_xa 348 * @length: length of the address that will be mmapped 349 * 350 * This function should be called by drivers that use the rdma_user_mmap 351 * interface for handling user mmapped addresses. The database is handled in 352 * the core and helper functions are provided to insert entries into the 353 * database and extract entries when the user calls mmap with the given offset. 354 * The function allocates a unique page offset that should be provided to user, 355 * the user will use the offset to retrieve information such as address to 356 * be mapped and how. 357 * 358 * Return: 0 on success and -ENOMEM on failure 359 */ 360 int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext, 361 struct rdma_user_mmap_entry *entry, 362 size_t length) 363 { 364 return rdma_user_mmap_entry_insert_range(ucontext, entry, length, 0, 365 U32_MAX); 366 } 367 EXPORT_SYMBOL(rdma_user_mmap_entry_insert); 368