1 /*- 2 * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include <linux/kernel.h> 29 #include <linux/module.h> 30 #include <linux/delay.h> 31 #include <dev/mlx5/driver.h> 32 #include "mlx5_core.h" 33 34 CTASSERT((uintptr_t)PAGE_MASK > (uintptr_t)PAGE_SIZE); 35 36 struct mlx5_pages_req { 37 struct mlx5_core_dev *dev; 38 u16 func_id; 39 s32 npages; 40 struct work_struct work; 41 }; 42 43 44 enum { 45 MAX_RECLAIM_TIME_MSECS = 5000, 46 }; 47 48 static void 49 mlx5_fwp_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 50 { 51 struct mlx5_fw_page *fwp; 52 uint8_t owned; 53 54 fwp = (struct mlx5_fw_page *)arg; 55 owned = MLX5_DMA_OWNED(fwp->dev); 56 57 if (!owned) 58 MLX5_DMA_LOCK(fwp->dev); 59 60 if (error == 0) { 61 KASSERT(nseg == 1, ("Number of segments is different from 1")); 62 fwp->dma_addr = segs->ds_addr; 63 fwp->load_done = MLX5_LOAD_ST_SUCCESS; 64 } else { 65 fwp->load_done = MLX5_LOAD_ST_FAILURE; 66 } 67 MLX5_DMA_DONE(fwp->dev); 68 69 if (!owned) 70 MLX5_DMA_UNLOCK(fwp->dev); 71 } 72 73 void 74 mlx5_fwp_flush(struct mlx5_fw_page *fwp) 75 { 76 unsigned num = fwp->numpages; 77 78 while (num--) 79 bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_PREWRITE); 80 } 81 82 void 83 mlx5_fwp_invalidate(struct mlx5_fw_page *fwp) 84 { 85 unsigned num = fwp->numpages; 86 87 while (num--) { 88 bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_POSTREAD); 89 bus_dmamap_sync(fwp[num].dev->cmd.dma_tag, fwp[num].dma_map, BUS_DMASYNC_PREREAD); 90 } 91 } 92 93 struct mlx5_fw_page * 94 mlx5_fwp_alloc(struct mlx5_core_dev *dev, gfp_t flags, unsigned num) 95 { 96 struct mlx5_fw_page *fwp; 97 unsigned x; 98 int err; 99 100 /* check for special case */ 101 if (num == 0) { 102 fwp = kzalloc(sizeof(*fwp), flags); 103 if (fwp != NULL) 104 fwp->dev = dev; 105 return (fwp); 106 } 107 108 /* we need sleeping context for this function */ 109 if (flags & M_NOWAIT) 110 return (NULL); 111 112 fwp = kzalloc(sizeof(*fwp) * num, flags); 113 114 /* serialize loading the DMA map(s) */ 115 sx_xlock(&dev->cmd.dma_sx); 116 117 for (x = 0; x != num; x++) { 118 /* store pointer to MLX5 core device */ 119 fwp[x].dev = dev; 120 /* store number of pages left from the array */ 121 fwp[x].numpages = num - x; 122 123 /* allocate memory */ 124 err = bus_dmamem_alloc(dev->cmd.dma_tag, &fwp[x].virt_addr, 125 BUS_DMA_WAITOK | BUS_DMA_COHERENT, &fwp[x].dma_map); 126 if (err != 0) 127 goto failure; 128 129 /* load memory into DMA */ 130 MLX5_DMA_LOCK(dev); 131 (void) bus_dmamap_load( 132 dev->cmd.dma_tag, fwp[x].dma_map, fwp[x].virt_addr, 133 MLX5_ADAPTER_PAGE_SIZE, &mlx5_fwp_load_mem_cb, 134 fwp + x, BUS_DMA_WAITOK | BUS_DMA_COHERENT); 135 136 while (fwp[x].load_done == MLX5_LOAD_ST_NONE) 137 MLX5_DMA_WAIT(dev); 138 MLX5_DMA_UNLOCK(dev); 139 140 /* check for error */ 141 if (fwp[x].load_done != MLX5_LOAD_ST_SUCCESS) { 142 bus_dmamem_free(dev->cmd.dma_tag, fwp[x].virt_addr, 143 fwp[x].dma_map); 144 goto failure; 145 } 146 } 147 sx_xunlock(&dev->cmd.dma_sx); 148 return (fwp); 149 150 failure: 151 while (x--) { 152 bus_dmamap_unload(dev->cmd.dma_tag, fwp[x].dma_map); 153 bus_dmamem_free(dev->cmd.dma_tag, fwp[x].virt_addr, fwp[x].dma_map); 154 } 155 sx_xunlock(&dev->cmd.dma_sx); 156 kfree(fwp); 157 return (NULL); 158 } 159 160 void 161 mlx5_fwp_free(struct mlx5_fw_page *fwp) 162 { 163 struct mlx5_core_dev *dev; 164 unsigned num; 165 166 /* be NULL safe */ 167 if (fwp == NULL) 168 return; 169 170 /* check for special case */ 171 if (fwp->numpages == 0) { 172 kfree(fwp); 173 return; 174 } 175 176 num = fwp->numpages; 177 dev = fwp->dev; 178 179 while (num--) { 180 bus_dmamap_unload(dev->cmd.dma_tag, fwp[num].dma_map); 181 bus_dmamem_free(dev->cmd.dma_tag, fwp[num].virt_addr, fwp[num].dma_map); 182 } 183 184 kfree(fwp); 185 } 186 187 u64 188 mlx5_fwp_get_dma(struct mlx5_fw_page *fwp, size_t offset) 189 { 190 size_t index = (offset / MLX5_ADAPTER_PAGE_SIZE); 191 KASSERT(index < fwp->numpages, ("Invalid offset: %lld", (long long)offset)); 192 193 return ((fwp + index)->dma_addr + (offset % MLX5_ADAPTER_PAGE_SIZE)); 194 } 195 196 void * 197 mlx5_fwp_get_virt(struct mlx5_fw_page *fwp, size_t offset) 198 { 199 size_t index = (offset / MLX5_ADAPTER_PAGE_SIZE); 200 KASSERT(index < fwp->numpages, ("Invalid offset: %lld", (long long)offset)); 201 202 return ((char *)(fwp + index)->virt_addr + (offset % MLX5_ADAPTER_PAGE_SIZE)); 203 } 204 205 static int 206 mlx5_insert_fw_page_locked(struct mlx5_core_dev *dev, struct mlx5_fw_page *nfp) 207 { 208 struct rb_root *root = &dev->priv.page_root; 209 struct rb_node **new = &root->rb_node; 210 struct rb_node *parent = NULL; 211 struct mlx5_fw_page *tfp; 212 213 while (*new) { 214 parent = *new; 215 tfp = rb_entry(parent, struct mlx5_fw_page, rb_node); 216 if (tfp->dma_addr < nfp->dma_addr) 217 new = &parent->rb_left; 218 else if (tfp->dma_addr > nfp->dma_addr) 219 new = &parent->rb_right; 220 else 221 return (-EEXIST); 222 } 223 224 rb_link_node(&nfp->rb_node, parent, new); 225 rb_insert_color(&nfp->rb_node, root); 226 return (0); 227 } 228 229 static struct mlx5_fw_page * 230 mlx5_remove_fw_page_locked(struct mlx5_core_dev *dev, bus_addr_t addr) 231 { 232 struct rb_root *root = &dev->priv.page_root; 233 struct rb_node *tmp = root->rb_node; 234 struct mlx5_fw_page *result = NULL; 235 struct mlx5_fw_page *tfp; 236 237 while (tmp) { 238 tfp = rb_entry(tmp, struct mlx5_fw_page, rb_node); 239 if (tfp->dma_addr < addr) { 240 tmp = tmp->rb_left; 241 } else if (tfp->dma_addr > addr) { 242 tmp = tmp->rb_right; 243 } else { 244 rb_erase(&tfp->rb_node, &dev->priv.page_root); 245 result = tfp; 246 break; 247 } 248 } 249 return (result); 250 } 251 252 static int 253 alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u16 func_id) 254 { 255 struct mlx5_fw_page *fwp; 256 int err; 257 258 fwp = mlx5_fwp_alloc(dev, GFP_KERNEL, 1); 259 if (fwp == NULL) 260 return (-ENOMEM); 261 262 fwp->func_id = func_id; 263 264 MLX5_DMA_LOCK(dev); 265 err = mlx5_insert_fw_page_locked(dev, fwp); 266 MLX5_DMA_UNLOCK(dev); 267 268 if (err != 0) { 269 mlx5_fwp_free(fwp); 270 } else { 271 /* make sure cached data is cleaned */ 272 mlx5_fwp_invalidate(fwp); 273 274 /* store DMA address */ 275 *addr = fwp->dma_addr; 276 } 277 return (err); 278 } 279 280 static void 281 free_4k(struct mlx5_core_dev *dev, u64 addr) 282 { 283 struct mlx5_fw_page *fwp; 284 285 MLX5_DMA_LOCK(dev); 286 fwp = mlx5_remove_fw_page_locked(dev, addr); 287 MLX5_DMA_UNLOCK(dev); 288 289 if (fwp == NULL) { 290 mlx5_core_warn(dev, "Cannot free 4K page at 0x%llx\n", (long long)addr); 291 return; 292 } 293 mlx5_fwp_free(fwp); 294 } 295 296 static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, 297 s32 *npages, int boot) 298 { 299 u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {0}; 300 u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {0}; 301 int err; 302 303 MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); 304 MLX5_SET(query_pages_in, in, op_mod, boot ? 305 MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES : 306 MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES); 307 308 err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 309 if (err) 310 return err; 311 312 *npages = MLX5_GET(query_pages_out, out, num_pages); 313 *func_id = MLX5_GET(query_pages_out, out, function_id); 314 315 return 0; 316 } 317 318 static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, 319 int notify_fail) 320 { 321 u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; 322 int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); 323 u64 addr; 324 int err; 325 u32 *in, *nin; 326 int i = 0; 327 328 inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]); 329 in = mlx5_vzalloc(inlen); 330 if (!in) { 331 mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); 332 err = -ENOMEM; 333 goto out_alloc; 334 } 335 336 for (i = 0; i < npages; i++) { 337 err = alloc_4k(dev, &addr, func_id); 338 if (err) 339 goto out_alloc; 340 MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, addr); 341 } 342 343 MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); 344 MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE); 345 MLX5_SET(manage_pages_in, in, function_id, func_id); 346 MLX5_SET(manage_pages_in, in, input_num_entries, npages); 347 348 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 349 if (err) { 350 mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", 351 func_id, npages, err); 352 goto out_alloc; 353 } 354 dev->priv.fw_pages += npages; 355 dev->priv.pages_per_func[func_id] += npages; 356 357 mlx5_core_dbg(dev, "err %d\n", err); 358 359 goto out_free; 360 361 out_alloc: 362 if (notify_fail) { 363 nin = mlx5_vzalloc(inlen); 364 if (!nin) 365 goto out_4k; 366 367 memset(&out, 0, sizeof(out)); 368 MLX5_SET(manage_pages_in, nin, opcode, MLX5_CMD_OP_MANAGE_PAGES); 369 MLX5_SET(manage_pages_in, nin, op_mod, MLX5_PAGES_CANT_GIVE); 370 MLX5_SET(manage_pages_in, nin, function_id, func_id); 371 if (mlx5_cmd_exec(dev, nin, inlen, out, sizeof(out))) 372 mlx5_core_warn(dev, "page notify failed\n"); 373 kvfree(nin); 374 } 375 376 out_4k: 377 for (i--; i >= 0; i--) 378 free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i])); 379 out_free: 380 kvfree(in); 381 return err; 382 } 383 384 static int reclaim_pages_cmd(struct mlx5_core_dev *dev, 385 u32 *in, int in_size, u32 *out, int out_size) 386 { 387 struct mlx5_fw_page *fwp; 388 struct rb_node *p; 389 u32 func_id; 390 u32 npages; 391 u32 i = 0; 392 393 if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) 394 return mlx5_cmd_exec(dev, in, in_size, out, out_size); 395 396 /* No hard feelings, we want our pages back! */ 397 npages = MLX5_GET(manage_pages_in, in, input_num_entries); 398 func_id = MLX5_GET(manage_pages_in, in, function_id); 399 400 p = rb_first(&dev->priv.page_root); 401 while (p && i < npages) { 402 fwp = rb_entry(p, struct mlx5_fw_page, rb_node); 403 p = rb_next(p); 404 if (fwp->func_id != func_id) 405 continue; 406 407 MLX5_ARRAY_SET64(manage_pages_out, out, pas, i, fwp->dma_addr); 408 i++; 409 } 410 411 MLX5_SET(manage_pages_out, out, output_num_entries, i); 412 return 0; 413 } 414 415 static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, 416 int *nclaimed) 417 { 418 int outlen = MLX5_ST_SZ_BYTES(manage_pages_out); 419 u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {0}; 420 int num_claimed; 421 u32 *out; 422 int err; 423 int i; 424 425 if (nclaimed) 426 *nclaimed = 0; 427 428 outlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); 429 out = mlx5_vzalloc(outlen); 430 if (!out) 431 return -ENOMEM; 432 433 MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); 434 MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE); 435 MLX5_SET(manage_pages_in, in, function_id, func_id); 436 MLX5_SET(manage_pages_in, in, input_num_entries, npages); 437 438 mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); 439 err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen); 440 if (err) { 441 mlx5_core_err(dev, "failed reclaiming pages\n"); 442 goto out_free; 443 } 444 445 num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries); 446 if (nclaimed) 447 *nclaimed = num_claimed; 448 449 dev->priv.fw_pages -= num_claimed; 450 dev->priv.pages_per_func[func_id] -= num_claimed; 451 for (i = 0; i < num_claimed; i++) 452 free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i])); 453 454 out_free: 455 kvfree(out); 456 return err; 457 } 458 459 static void pages_work_handler(struct work_struct *work) 460 { 461 struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); 462 struct mlx5_core_dev *dev = req->dev; 463 int err = 0; 464 465 if (req->npages < 0) 466 err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); 467 else if (req->npages > 0) 468 err = give_pages(dev, req->func_id, req->npages, 1); 469 470 if (err) 471 mlx5_core_warn(dev, "%s fail %d\n", 472 req->npages < 0 ? "reclaim" : "give", err); 473 474 kfree(req); 475 } 476 477 void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, 478 s32 npages) 479 { 480 struct mlx5_pages_req *req; 481 482 req = kzalloc(sizeof(*req), GFP_ATOMIC); 483 if (!req) { 484 mlx5_core_warn(dev, "failed to allocate pages request\n"); 485 return; 486 } 487 488 req->dev = dev; 489 req->func_id = func_id; 490 req->npages = npages; 491 INIT_WORK(&req->work, pages_work_handler); 492 if (!queue_work(dev->priv.pg_wq, &req->work)) 493 mlx5_core_warn(dev, "failed to queue pages handler work\n"); 494 } 495 496 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) 497 { 498 u16 uninitialized_var(func_id); 499 s32 uninitialized_var(npages); 500 int err; 501 502 err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); 503 if (err) 504 return err; 505 506 mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", 507 npages, boot ? "boot" : "init", func_id); 508 509 return give_pages(dev, func_id, npages, 0); 510 } 511 512 enum { 513 MLX5_BLKS_FOR_RECLAIM_PAGES = 12 514 }; 515 516 s64 mlx5_wait_for_reclaim_vfs_pages(struct mlx5_core_dev *dev) 517 { 518 int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 519 s64 prevpages = 0; 520 s64 npages = 0; 521 522 while (!time_after(jiffies, end)) { 523 /* exclude own function, VFs only */ 524 npages = dev->priv.fw_pages - dev->priv.pages_per_func[0]; 525 if (!npages) 526 break; 527 528 if (npages != prevpages) 529 end = end + msecs_to_jiffies(100); 530 531 prevpages = npages; 532 msleep(1); 533 } 534 535 if (npages) 536 mlx5_core_warn(dev, "FW did not return all VFs pages, will cause to memory leak\n"); 537 538 return -npages; 539 } 540 541 static int optimal_reclaimed_pages(void) 542 { 543 struct mlx5_cmd_prot_block *block; 544 struct mlx5_cmd_layout *lay; 545 int ret; 546 547 ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - 548 MLX5_ST_SZ_BYTES(manage_pages_out)) / 549 MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); 550 551 return ret; 552 } 553 554 int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) 555 { 556 int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 557 struct mlx5_fw_page *fwp; 558 struct rb_node *p; 559 int nclaimed = 0; 560 int err; 561 562 do { 563 p = rb_first(&dev->priv.page_root); 564 if (p) { 565 fwp = rb_entry(p, struct mlx5_fw_page, rb_node); 566 err = reclaim_pages(dev, fwp->func_id, 567 optimal_reclaimed_pages(), 568 &nclaimed); 569 if (err) { 570 mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", 571 err); 572 return err; 573 } 574 575 if (nclaimed) 576 end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 577 } 578 if (time_after(jiffies, end)) { 579 mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); 580 break; 581 } 582 } while (p); 583 584 return 0; 585 } 586 587 void mlx5_pagealloc_init(struct mlx5_core_dev *dev) 588 { 589 590 dev->priv.page_root = RB_ROOT; 591 } 592 593 void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) 594 { 595 /* nothing */ 596 } 597 598 int mlx5_pagealloc_start(struct mlx5_core_dev *dev) 599 { 600 dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); 601 if (!dev->priv.pg_wq) 602 return -ENOMEM; 603 604 return 0; 605 } 606 607 void mlx5_pagealloc_stop(struct mlx5_core_dev *dev) 608 { 609 destroy_workqueue(dev->priv.pg_wq); 610 } 611