1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/errno.h> 34 #include <linux/slab.h> 35 #include <linux/mm.h> 36 #include <linux/export.h> 37 #include <linux/bitmap.h> 38 #include <linux/dma-mapping.h> 39 #include <linux/vmalloc.h> 40 #include <linux/mlx5/driver.h> 41 42 #include "mlx5_core.h" 43 44 struct mlx5_db_pgdir { 45 struct list_head list; 46 unsigned long *bitmap; 47 __be32 *db_page; 48 dma_addr_t db_dma; 49 }; 50 51 /* Handling for queue buffers -- we allocate a bunch of memory and 52 * register it in a memory region at HCA virtual address 0. 53 */ 54 55 static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev, 56 size_t size, dma_addr_t *dma_handle, 57 int node) 58 { 59 struct mlx5_priv *priv = &dev->priv; 60 struct device *device = dev->device; 61 int original_node; 62 void *cpu_handle; 63 64 mutex_lock(&priv->alloc_mutex); 65 original_node = dev_to_node(device); 66 set_dev_node(device, node); 67 cpu_handle = dma_alloc_coherent(device, size, dma_handle, 68 GFP_KERNEL); 69 set_dev_node(device, original_node); 70 mutex_unlock(&priv->alloc_mutex); 71 return cpu_handle; 72 } 73 74 static int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, 75 struct mlx5_frag_buf *buf, int node) 76 { 77 dma_addr_t t; 78 79 buf->size = size; 80 buf->npages = 1; 81 buf->page_shift = (u8)get_order(size) + PAGE_SHIFT; 82 83 buf->frags = kzalloc(sizeof(*buf->frags), GFP_KERNEL); 84 if (!buf->frags) 85 return -ENOMEM; 86 87 buf->frags->buf = mlx5_dma_zalloc_coherent_node(dev, size, 88 &t, node); 89 if (!buf->frags->buf) 90 goto err_out; 91 92 buf->frags->map = t; 93 94 while (t & ((1 << buf->page_shift) - 1)) { 95 --buf->page_shift; 96 buf->npages *= 2; 97 } 98 99 return 0; 100 err_out: 101 kfree(buf->frags); 102 return -ENOMEM; 103 } 104 105 int mlx5_buf_alloc(struct mlx5_core_dev *dev, 106 int size, struct mlx5_frag_buf *buf) 107 { 108 return mlx5_buf_alloc_node(dev, size, buf, dev->priv.numa_node); 109 } 110 EXPORT_SYMBOL(mlx5_buf_alloc); 111 112 void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf) 113 { 114 dma_free_coherent(dev->device, buf->size, buf->frags->buf, 115 buf->frags->map); 116 117 kfree(buf->frags); 118 } 119 EXPORT_SYMBOL_GPL(mlx5_buf_free); 120 121 int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, 122 struct mlx5_frag_buf *buf, int node) 123 { 124 int i; 125 126 buf->size = size; 127 buf->npages = DIV_ROUND_UP(size, PAGE_SIZE); 128 buf->page_shift = PAGE_SHIFT; 129 buf->frags = kcalloc(buf->npages, sizeof(struct mlx5_buf_list), 130 GFP_KERNEL); 131 if (!buf->frags) 132 goto err_out; 133 134 for (i = 0; i < buf->npages; i++) { 135 struct mlx5_buf_list *frag = &buf->frags[i]; 136 int frag_sz = min_t(int, size, PAGE_SIZE); 137 138 frag->buf = mlx5_dma_zalloc_coherent_node(dev, frag_sz, 139 &frag->map, node); 140 if (!frag->buf) 141 goto err_free_buf; 142 if (frag->map & ((1 << buf->page_shift) - 1)) { 143 dma_free_coherent(dev->device, frag_sz, 144 buf->frags[i].buf, buf->frags[i].map); 145 mlx5_core_warn(dev, "unexpected map alignment: %pad, page_shift=%d\n", 146 &frag->map, buf->page_shift); 147 goto err_free_buf; 148 } 149 size -= frag_sz; 150 } 151 152 return 0; 153 154 err_free_buf: 155 while (i--) 156 dma_free_coherent(dev->device, PAGE_SIZE, buf->frags[i].buf, 157 buf->frags[i].map); 158 kfree(buf->frags); 159 err_out: 160 return -ENOMEM; 161 } 162 EXPORT_SYMBOL_GPL(mlx5_frag_buf_alloc_node); 163 164 void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf) 165 { 166 int size = buf->size; 167 int i; 168 169 for (i = 0; i < buf->npages; i++) { 170 int frag_sz = min_t(int, size, PAGE_SIZE); 171 172 dma_free_coherent(dev->device, frag_sz, buf->frags[i].buf, 173 buf->frags[i].map); 174 size -= frag_sz; 175 } 176 kfree(buf->frags); 177 } 178 EXPORT_SYMBOL_GPL(mlx5_frag_buf_free); 179 180 static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev, 181 int node) 182 { 183 u32 db_per_page = PAGE_SIZE / cache_line_size(); 184 struct mlx5_db_pgdir *pgdir; 185 186 pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); 187 if (!pgdir) 188 return NULL; 189 190 pgdir->bitmap = bitmap_zalloc(db_per_page, GFP_KERNEL); 191 if (!pgdir->bitmap) { 192 kfree(pgdir); 193 return NULL; 194 } 195 196 bitmap_fill(pgdir->bitmap, db_per_page); 197 198 pgdir->db_page = mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE, 199 &pgdir->db_dma, node); 200 if (!pgdir->db_page) { 201 bitmap_free(pgdir->bitmap); 202 kfree(pgdir); 203 return NULL; 204 } 205 206 return pgdir; 207 } 208 209 static int mlx5_alloc_db_from_pgdir(struct mlx5_db_pgdir *pgdir, 210 struct mlx5_db *db) 211 { 212 u32 db_per_page = PAGE_SIZE / cache_line_size(); 213 int offset; 214 int i; 215 216 i = find_first_bit(pgdir->bitmap, db_per_page); 217 if (i >= db_per_page) 218 return -ENOMEM; 219 220 __clear_bit(i, pgdir->bitmap); 221 222 db->u.pgdir = pgdir; 223 db->index = i; 224 offset = db->index * cache_line_size(); 225 db->db = pgdir->db_page + offset / sizeof(*pgdir->db_page); 226 db->dma = pgdir->db_dma + offset; 227 228 db->db[0] = 0; 229 db->db[1] = 0; 230 231 return 0; 232 } 233 234 int mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db, int node) 235 { 236 struct mlx5_db_pgdir *pgdir; 237 int ret = 0; 238 239 mutex_lock(&dev->priv.pgdir_mutex); 240 241 list_for_each_entry(pgdir, &dev->priv.pgdir_list, list) 242 if (!mlx5_alloc_db_from_pgdir(pgdir, db)) 243 goto out; 244 245 pgdir = mlx5_alloc_db_pgdir(dev, node); 246 if (!pgdir) { 247 ret = -ENOMEM; 248 goto out; 249 } 250 251 list_add(&pgdir->list, &dev->priv.pgdir_list); 252 253 /* This should never fail -- we just allocated an empty page: */ 254 WARN_ON(mlx5_alloc_db_from_pgdir(pgdir, db)); 255 256 out: 257 mutex_unlock(&dev->priv.pgdir_mutex); 258 259 return ret; 260 } 261 EXPORT_SYMBOL_GPL(mlx5_db_alloc_node); 262 263 int mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db) 264 { 265 return mlx5_db_alloc_node(dev, db, dev->priv.numa_node); 266 } 267 EXPORT_SYMBOL_GPL(mlx5_db_alloc); 268 269 void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db) 270 { 271 u32 db_per_page = PAGE_SIZE / cache_line_size(); 272 273 mutex_lock(&dev->priv.pgdir_mutex); 274 275 __set_bit(db->index, db->u.pgdir->bitmap); 276 277 if (bitmap_full(db->u.pgdir->bitmap, db_per_page)) { 278 dma_free_coherent(dev->device, PAGE_SIZE, 279 db->u.pgdir->db_page, db->u.pgdir->db_dma); 280 list_del(&db->u.pgdir->list); 281 bitmap_free(db->u.pgdir->bitmap); 282 kfree(db->u.pgdir); 283 } 284 285 mutex_unlock(&dev->priv.pgdir_mutex); 286 } 287 EXPORT_SYMBOL_GPL(mlx5_db_free); 288 289 void mlx5_fill_page_array(struct mlx5_frag_buf *buf, __be64 *pas) 290 { 291 u64 addr; 292 int i; 293 294 for (i = 0; i < buf->npages; i++) { 295 addr = buf->frags->map + (i << buf->page_shift); 296 297 pas[i] = cpu_to_be64(addr); 298 } 299 } 300 EXPORT_SYMBOL_GPL(mlx5_fill_page_array); 301 302 void mlx5_fill_page_frag_array(struct mlx5_frag_buf *buf, __be64 *pas) 303 { 304 int i; 305 306 for (i = 0; i < buf->npages; i++) 307 pas[i] = cpu_to_be64(buf->frags[i].map); 308 } 309 EXPORT_SYMBOL_GPL(mlx5_fill_page_frag_array); 310