1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/highmem.h> 34 #include <linux/kernel.h> 35 #include <linux/delay.h> 36 #include <linux/mlx5/driver.h> 37 #include <linux/xarray.h> 38 #include "mlx5_core.h" 39 #include "lib/eq.h" 40 #include "lib/tout.h" 41 42 enum { 43 MLX5_PAGES_CANT_GIVE = 0, 44 MLX5_PAGES_GIVE = 1, 45 MLX5_PAGES_TAKE = 2 46 }; 47 48 struct mlx5_pages_req { 49 struct mlx5_core_dev *dev; 50 u16 func_id; 51 u8 ec_function; 52 s32 npages; 53 struct work_struct work; 54 u8 release_all; 55 }; 56 57 struct fw_page { 58 struct rb_node rb_node; 59 u64 addr; 60 struct page *page; 61 u32 function; 62 unsigned long bitmask; 63 struct list_head list; 64 unsigned int free_count; 65 }; 66 67 enum { 68 MLX5_MAX_RECLAIM_TIME_MILI = 5000, 69 MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE, 70 }; 71 72 static u32 get_function(u16 func_id, bool ec_function) 73 { 74 return (u32)func_id | (ec_function << 16); 75 } 76 77 static struct rb_root *page_root_per_function(struct mlx5_core_dev *dev, u32 function) 78 { 79 struct rb_root *root; 80 int err; 81 82 root = xa_load(&dev->priv.page_root_xa, function); 83 if (root) 84 return root; 85 86 root = kzalloc(sizeof(*root), GFP_KERNEL); 87 if (!root) 88 return ERR_PTR(-ENOMEM); 89 90 err = xa_insert(&dev->priv.page_root_xa, function, root, GFP_KERNEL); 91 if (err) { 92 kfree(root); 93 return ERR_PTR(err); 94 } 95 96 *root = RB_ROOT; 97 98 return root; 99 } 100 101 static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u32 function) 102 { 103 struct rb_node *parent = NULL; 104 struct rb_root *root; 105 struct rb_node **new; 106 struct fw_page *nfp; 107 struct fw_page *tfp; 108 int i; 109 110 root = page_root_per_function(dev, function); 111 if (IS_ERR(root)) 112 return PTR_ERR(root); 113 114 new = &root->rb_node; 115 116 while (*new) { 117 parent = *new; 118 tfp = rb_entry(parent, struct fw_page, rb_node); 119 if (tfp->addr < addr) 120 new = &parent->rb_left; 121 else if (tfp->addr > addr) 122 new = &parent->rb_right; 123 else 124 return -EEXIST; 125 } 126 127 nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); 128 if (!nfp) 129 return -ENOMEM; 130 131 nfp->addr = addr; 132 nfp->page = page; 133 nfp->function = function; 134 nfp->free_count = MLX5_NUM_4K_IN_PAGE; 135 for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++) 136 set_bit(i, &nfp->bitmask); 137 138 rb_link_node(&nfp->rb_node, parent, new); 139 rb_insert_color(&nfp->rb_node, root); 140 list_add(&nfp->list, &dev->priv.free_list); 141 142 return 0; 143 } 144 145 static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr, 146 u32 function) 147 { 148 struct fw_page *result = NULL; 149 struct rb_root *root; 150 struct rb_node *tmp; 151 struct fw_page *tfp; 152 153 root = xa_load(&dev->priv.page_root_xa, function); 154 if (WARN_ON_ONCE(!root)) 155 return NULL; 156 157 tmp = root->rb_node; 158 159 while (tmp) { 160 tfp = rb_entry(tmp, struct fw_page, rb_node); 161 if (tfp->addr < addr) { 162 tmp = tmp->rb_left; 163 } else if (tfp->addr > addr) { 164 tmp = tmp->rb_right; 165 } else { 166 result = tfp; 167 break; 168 } 169 } 170 171 return result; 172 } 173 174 static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, 175 s32 *npages, int boot) 176 { 177 u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {}; 178 u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {}; 179 int err; 180 181 MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); 182 MLX5_SET(query_pages_in, in, op_mod, boot ? 183 MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES : 184 MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES); 185 MLX5_SET(query_pages_in, in, embedded_cpu_function, mlx5_core_is_ecpf(dev)); 186 187 err = mlx5_cmd_exec_inout(dev, query_pages, in, out); 188 if (err) 189 return err; 190 191 *npages = MLX5_GET(query_pages_out, out, num_pages); 192 *func_id = MLX5_GET(query_pages_out, out, function_id); 193 194 return err; 195 } 196 197 static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u32 function) 198 { 199 struct fw_page *fp = NULL; 200 struct fw_page *iter; 201 unsigned n; 202 203 list_for_each_entry(iter, &dev->priv.free_list, list) { 204 if (iter->function != function) 205 continue; 206 fp = iter; 207 } 208 209 if (list_empty(&dev->priv.free_list) || !fp) 210 return -ENOMEM; 211 212 n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask)); 213 if (n >= MLX5_NUM_4K_IN_PAGE) { 214 mlx5_core_warn(dev, "alloc 4k bug\n"); 215 return -ENOENT; 216 } 217 clear_bit(n, &fp->bitmask); 218 fp->free_count--; 219 if (!fp->free_count) 220 list_del(&fp->list); 221 222 *addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE; 223 224 return 0; 225 } 226 227 #define MLX5_U64_4K_PAGE_MASK ((~(u64)0U) << PAGE_SHIFT) 228 229 static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp, 230 bool in_free_list) 231 { 232 struct rb_root *root; 233 234 root = xa_load(&dev->priv.page_root_xa, fwp->function); 235 if (WARN_ON_ONCE(!root)) 236 return; 237 238 rb_erase(&fwp->rb_node, root); 239 if (in_free_list) 240 list_del(&fwp->list); 241 dma_unmap_page(mlx5_core_dma_dev(dev), fwp->addr & MLX5_U64_4K_PAGE_MASK, 242 PAGE_SIZE, DMA_BIDIRECTIONAL); 243 __free_page(fwp->page); 244 kfree(fwp); 245 } 246 247 static void free_4k(struct mlx5_core_dev *dev, u64 addr, u32 function) 248 { 249 struct fw_page *fwp; 250 int n; 251 252 fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK, function); 253 if (!fwp) { 254 mlx5_core_warn_rl(dev, "page not found\n"); 255 return; 256 } 257 n = (addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT; 258 fwp->free_count++; 259 set_bit(n, &fwp->bitmask); 260 if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) 261 free_fwp(dev, fwp, fwp->free_count != 1); 262 else if (fwp->free_count == 1) 263 list_add(&fwp->list, &dev->priv.free_list); 264 } 265 266 static int alloc_system_page(struct mlx5_core_dev *dev, u32 function) 267 { 268 struct device *device = mlx5_core_dma_dev(dev); 269 int nid = dev_to_node(device); 270 struct page *page; 271 u64 zero_addr = 1; 272 u64 addr; 273 int err; 274 275 page = alloc_pages_node(nid, GFP_HIGHUSER, 0); 276 if (!page) { 277 mlx5_core_warn(dev, "failed to allocate page\n"); 278 return -ENOMEM; 279 } 280 map: 281 addr = dma_map_page(device, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 282 if (dma_mapping_error(device, addr)) { 283 mlx5_core_warn(dev, "failed dma mapping page\n"); 284 err = -ENOMEM; 285 goto err_mapping; 286 } 287 288 /* Firmware doesn't support page with physical address 0 */ 289 if (addr == 0) { 290 zero_addr = addr; 291 goto map; 292 } 293 294 err = insert_page(dev, addr, page, function); 295 if (err) { 296 mlx5_core_err(dev, "failed to track allocated page\n"); 297 dma_unmap_page(device, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 298 } 299 300 err_mapping: 301 if (err) 302 __free_page(page); 303 304 if (zero_addr == 0) 305 dma_unmap_page(device, zero_addr, PAGE_SIZE, 306 DMA_BIDIRECTIONAL); 307 308 return err; 309 } 310 311 static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id, 312 bool ec_function) 313 { 314 u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {}; 315 int err; 316 317 MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); 318 MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_CANT_GIVE); 319 MLX5_SET(manage_pages_in, in, function_id, func_id); 320 MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function); 321 322 err = mlx5_cmd_exec_in(dev, manage_pages, in); 323 if (err) 324 mlx5_core_warn(dev, "page notify failed func_id(%d) err(%d)\n", 325 func_id, err); 326 } 327 328 static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, 329 int event, bool ec_function) 330 { 331 u32 function = get_function(func_id, ec_function); 332 u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0}; 333 int inlen = MLX5_ST_SZ_BYTES(manage_pages_in); 334 int notify_fail = event; 335 u64 addr; 336 int err; 337 u32 *in; 338 int i; 339 340 inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]); 341 in = kvzalloc(inlen, GFP_KERNEL); 342 if (!in) { 343 err = -ENOMEM; 344 mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); 345 goto out_free; 346 } 347 348 for (i = 0; i < npages; i++) { 349 retry: 350 err = alloc_4k(dev, &addr, function); 351 if (err) { 352 if (err == -ENOMEM) 353 err = alloc_system_page(dev, function); 354 if (err) { 355 dev->priv.fw_pages_alloc_failed += (npages - i); 356 goto out_4k; 357 } 358 359 goto retry; 360 } 361 MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, addr); 362 } 363 364 MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); 365 MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE); 366 MLX5_SET(manage_pages_in, in, function_id, func_id); 367 MLX5_SET(manage_pages_in, in, input_num_entries, npages); 368 MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function); 369 370 err = mlx5_cmd_do(dev, in, inlen, out, sizeof(out)); 371 if (err == -EREMOTEIO) { 372 notify_fail = 0; 373 /* if triggered by FW and failed by FW ignore */ 374 if (event) { 375 err = 0; 376 goto out_dropped; 377 } 378 } 379 err = mlx5_cmd_check(dev, err, in, out); 380 if (err) { 381 mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", 382 func_id, npages, err); 383 goto out_dropped; 384 } 385 386 dev->priv.fw_pages += npages; 387 if (func_id) 388 dev->priv.vfs_pages += npages; 389 else if (mlx5_core_is_ecpf(dev) && !ec_function) 390 dev->priv.host_pf_pages += npages; 391 392 mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x, err %d\n", 393 npages, ec_function, func_id, err); 394 395 kvfree(in); 396 return 0; 397 398 out_dropped: 399 dev->priv.give_pages_dropped += npages; 400 out_4k: 401 for (i--; i >= 0; i--) 402 free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i]), function); 403 out_free: 404 kvfree(in); 405 if (notify_fail) 406 page_notify_fail(dev, func_id, ec_function); 407 return err; 408 } 409 410 static void release_all_pages(struct mlx5_core_dev *dev, u16 func_id, 411 bool ec_function) 412 { 413 u32 function = get_function(func_id, ec_function); 414 struct rb_root *root; 415 struct rb_node *p; 416 int npages = 0; 417 418 root = xa_load(&dev->priv.page_root_xa, function); 419 if (WARN_ON_ONCE(!root)) 420 return; 421 422 p = rb_first(root); 423 while (p) { 424 struct fw_page *fwp = rb_entry(p, struct fw_page, rb_node); 425 426 p = rb_next(p); 427 npages += (MLX5_NUM_4K_IN_PAGE - fwp->free_count); 428 free_fwp(dev, fwp, fwp->free_count); 429 } 430 431 dev->priv.fw_pages -= npages; 432 if (func_id) 433 dev->priv.vfs_pages -= npages; 434 else if (mlx5_core_is_ecpf(dev) && !ec_function) 435 dev->priv.host_pf_pages -= npages; 436 437 mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x\n", 438 npages, ec_function, func_id); 439 } 440 441 static u32 fwp_fill_manage_pages_out(struct fw_page *fwp, u32 *out, u32 index, 442 u32 npages) 443 { 444 u32 pages_set = 0; 445 unsigned int n; 446 447 for_each_clear_bit(n, &fwp->bitmask, MLX5_NUM_4K_IN_PAGE) { 448 MLX5_ARRAY_SET64(manage_pages_out, out, pas, index + pages_set, 449 fwp->addr + (n * MLX5_ADAPTER_PAGE_SIZE)); 450 pages_set++; 451 452 if (!--npages) 453 break; 454 } 455 456 return pages_set; 457 } 458 459 static int reclaim_pages_cmd(struct mlx5_core_dev *dev, 460 u32 *in, int in_size, u32 *out, int out_size) 461 { 462 struct rb_root *root; 463 struct fw_page *fwp; 464 struct rb_node *p; 465 bool ec_function; 466 u32 func_id; 467 u32 npages; 468 u32 i = 0; 469 470 if (!mlx5_cmd_is_down(dev)) 471 return mlx5_cmd_do(dev, in, in_size, out, out_size); 472 473 /* No hard feelings, we want our pages back! */ 474 npages = MLX5_GET(manage_pages_in, in, input_num_entries); 475 func_id = MLX5_GET(manage_pages_in, in, function_id); 476 ec_function = MLX5_GET(manage_pages_in, in, embedded_cpu_function); 477 478 root = xa_load(&dev->priv.page_root_xa, get_function(func_id, ec_function)); 479 if (WARN_ON_ONCE(!root)) 480 return -EEXIST; 481 482 p = rb_first(root); 483 while (p && i < npages) { 484 fwp = rb_entry(p, struct fw_page, rb_node); 485 p = rb_next(p); 486 487 i += fwp_fill_manage_pages_out(fwp, out, i, npages - i); 488 } 489 490 MLX5_SET(manage_pages_out, out, output_num_entries, i); 491 return 0; 492 } 493 494 static int reclaim_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, 495 int *nclaimed, bool event, bool ec_function) 496 { 497 u32 function = get_function(func_id, ec_function); 498 int outlen = MLX5_ST_SZ_BYTES(manage_pages_out); 499 u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {}; 500 int num_claimed; 501 u32 *out; 502 int err; 503 int i; 504 505 if (nclaimed) 506 *nclaimed = 0; 507 508 outlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); 509 out = kvzalloc(outlen, GFP_KERNEL); 510 if (!out) 511 return -ENOMEM; 512 513 MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES); 514 MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE); 515 MLX5_SET(manage_pages_in, in, function_id, func_id); 516 MLX5_SET(manage_pages_in, in, input_num_entries, npages); 517 MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function); 518 519 mlx5_core_dbg(dev, "func 0x%x, npages %d, outlen %d\n", 520 func_id, npages, outlen); 521 err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen); 522 if (err) { 523 npages = MLX5_GET(manage_pages_in, in, input_num_entries); 524 dev->priv.reclaim_pages_discard += npages; 525 } 526 /* if triggered by FW event and failed by FW then ignore */ 527 if (event && err == -EREMOTEIO) { 528 err = 0; 529 goto out_free; 530 } 531 532 err = mlx5_cmd_check(dev, err, in, out); 533 if (err) { 534 mlx5_core_err(dev, "failed reclaiming pages: err %d\n", err); 535 goto out_free; 536 } 537 538 num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries); 539 if (num_claimed > npages) { 540 mlx5_core_warn(dev, "fw returned %d, driver asked %d => corruption\n", 541 num_claimed, npages); 542 err = -EINVAL; 543 goto out_free; 544 } 545 546 for (i = 0; i < num_claimed; i++) 547 free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i]), function); 548 549 if (nclaimed) 550 *nclaimed = num_claimed; 551 552 dev->priv.fw_pages -= num_claimed; 553 if (func_id) 554 dev->priv.vfs_pages -= num_claimed; 555 else if (mlx5_core_is_ecpf(dev) && !ec_function) 556 dev->priv.host_pf_pages -= num_claimed; 557 558 out_free: 559 kvfree(out); 560 return err; 561 } 562 563 static void pages_work_handler(struct work_struct *work) 564 { 565 struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); 566 struct mlx5_core_dev *dev = req->dev; 567 int err = 0; 568 569 if (req->release_all) 570 release_all_pages(dev, req->func_id, req->ec_function); 571 else if (req->npages < 0) 572 err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL, 573 true, req->ec_function); 574 else if (req->npages > 0) 575 err = give_pages(dev, req->func_id, req->npages, 1, req->ec_function); 576 577 if (err) 578 mlx5_core_warn(dev, "%s fail %d\n", 579 req->npages < 0 ? "reclaim" : "give", err); 580 581 kfree(req); 582 } 583 584 enum { 585 EC_FUNCTION_MASK = 0x8000, 586 RELEASE_ALL_PAGES_MASK = 0x4000, 587 }; 588 589 static int req_pages_handler(struct notifier_block *nb, 590 unsigned long type, void *data) 591 { 592 struct mlx5_pages_req *req; 593 struct mlx5_core_dev *dev; 594 struct mlx5_priv *priv; 595 struct mlx5_eqe *eqe; 596 bool ec_function; 597 bool release_all; 598 u16 func_id; 599 s32 npages; 600 601 priv = mlx5_nb_cof(nb, struct mlx5_priv, pg_nb); 602 dev = container_of(priv, struct mlx5_core_dev, priv); 603 eqe = data; 604 605 func_id = be16_to_cpu(eqe->data.req_pages.func_id); 606 npages = be32_to_cpu(eqe->data.req_pages.num_pages); 607 ec_function = be16_to_cpu(eqe->data.req_pages.ec_function) & EC_FUNCTION_MASK; 608 release_all = be16_to_cpu(eqe->data.req_pages.ec_function) & 609 RELEASE_ALL_PAGES_MASK; 610 mlx5_core_dbg(dev, "page request for func 0x%x, npages %d, release_all %d\n", 611 func_id, npages, release_all); 612 req = kzalloc(sizeof(*req), GFP_ATOMIC); 613 if (!req) { 614 mlx5_core_warn(dev, "failed to allocate pages request\n"); 615 return NOTIFY_DONE; 616 } 617 618 req->dev = dev; 619 req->func_id = func_id; 620 req->npages = npages; 621 req->ec_function = ec_function; 622 req->release_all = release_all; 623 INIT_WORK(&req->work, pages_work_handler); 624 queue_work(dev->priv.pg_wq, &req->work); 625 return NOTIFY_OK; 626 } 627 628 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) 629 { 630 u16 func_id; 631 s32 npages; 632 int err; 633 634 err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); 635 if (err) 636 return err; 637 638 mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", 639 npages, boot ? "boot" : "init", func_id); 640 641 return give_pages(dev, func_id, npages, 0, mlx5_core_is_ecpf(dev)); 642 } 643 644 enum { 645 MLX5_BLKS_FOR_RECLAIM_PAGES = 12 646 }; 647 648 static int optimal_reclaimed_pages(void) 649 { 650 struct mlx5_cmd_prot_block *block; 651 struct mlx5_cmd_layout *lay; 652 int ret; 653 654 ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - 655 MLX5_ST_SZ_BYTES(manage_pages_out)) / 656 MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]); 657 658 return ret; 659 } 660 661 static int mlx5_reclaim_root_pages(struct mlx5_core_dev *dev, 662 struct rb_root *root, u16 func_id) 663 { 664 u64 recl_pages_to_jiffies = msecs_to_jiffies(mlx5_tout_ms(dev, RECLAIM_PAGES)); 665 unsigned long end = jiffies + recl_pages_to_jiffies; 666 667 while (!RB_EMPTY_ROOT(root)) { 668 int nclaimed; 669 int err; 670 671 err = reclaim_pages(dev, func_id, optimal_reclaimed_pages(), 672 &nclaimed, false, mlx5_core_is_ecpf(dev)); 673 if (err) { 674 mlx5_core_warn(dev, "failed reclaiming pages (%d) for func id 0x%x\n", 675 err, func_id); 676 return err; 677 } 678 679 if (nclaimed) 680 end = jiffies + recl_pages_to_jiffies; 681 682 if (time_after(jiffies, end)) { 683 mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); 684 break; 685 } 686 } 687 688 return 0; 689 } 690 691 int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) 692 { 693 struct rb_root *root; 694 unsigned long id; 695 void *entry; 696 697 xa_for_each(&dev->priv.page_root_xa, id, entry) { 698 root = entry; 699 mlx5_reclaim_root_pages(dev, root, id); 700 xa_erase(&dev->priv.page_root_xa, id); 701 kfree(root); 702 } 703 704 WARN_ON(!xa_empty(&dev->priv.page_root_xa)); 705 706 WARN(dev->priv.fw_pages, 707 "FW pages counter is %d after reclaiming all pages\n", 708 dev->priv.fw_pages); 709 WARN(dev->priv.vfs_pages, 710 "VFs FW pages counter is %d after reclaiming all pages\n", 711 dev->priv.vfs_pages); 712 WARN(dev->priv.host_pf_pages, 713 "External host PF FW pages counter is %d after reclaiming all pages\n", 714 dev->priv.host_pf_pages); 715 716 return 0; 717 } 718 719 int mlx5_pagealloc_init(struct mlx5_core_dev *dev) 720 { 721 INIT_LIST_HEAD(&dev->priv.free_list); 722 dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); 723 if (!dev->priv.pg_wq) 724 return -ENOMEM; 725 726 xa_init(&dev->priv.page_root_xa); 727 mlx5_pages_debugfs_init(dev); 728 729 return 0; 730 } 731 732 void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) 733 { 734 mlx5_pages_debugfs_cleanup(dev); 735 xa_destroy(&dev->priv.page_root_xa); 736 destroy_workqueue(dev->priv.pg_wq); 737 } 738 739 void mlx5_pagealloc_start(struct mlx5_core_dev *dev) 740 { 741 MLX5_NB_INIT(&dev->priv.pg_nb, req_pages_handler, PAGE_REQUEST); 742 mlx5_eq_notifier_register(dev, &dev->priv.pg_nb); 743 } 744 745 void mlx5_pagealloc_stop(struct mlx5_core_dev *dev) 746 { 747 mlx5_eq_notifier_unregister(dev, &dev->priv.pg_nb); 748 flush_workqueue(dev->priv.pg_wq); 749 } 750 751 int mlx5_wait_for_pages(struct mlx5_core_dev *dev, int *pages) 752 { 753 u64 recl_vf_pages_to_jiffies = msecs_to_jiffies(mlx5_tout_ms(dev, RECLAIM_VFS_PAGES)); 754 unsigned long end = jiffies + recl_vf_pages_to_jiffies; 755 int prev_pages = *pages; 756 757 /* In case of internal error we will free the pages manually later */ 758 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 759 mlx5_core_warn(dev, "Skipping wait for vf pages stage"); 760 return 0; 761 } 762 763 mlx5_core_dbg(dev, "Waiting for %d pages\n", prev_pages); 764 while (*pages) { 765 if (time_after(jiffies, end)) { 766 mlx5_core_warn(dev, "aborting while there are %d pending pages\n", *pages); 767 return -ETIMEDOUT; 768 } 769 if (*pages < prev_pages) { 770 end = jiffies + recl_vf_pages_to_jiffies; 771 prev_pages = *pages; 772 } 773 msleep(50); 774 } 775 776 mlx5_core_dbg(dev, "All pages received\n"); 777 return 0; 778 } 779