1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/pci.h> 10 #include <linux/device.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/slab.h> 13 #include <linux/atomic.h> 14 #include <linux/semaphore.h> 15 #include <linux/errno.h> 16 #include <linux/vmalloc.h> 17 #include <linux/err.h> 18 #include <asm/byteorder.h> 19 20 #include "hinic_hw_if.h" 21 #include "hinic_hw_wqe.h" 22 #include "hinic_hw_wq.h" 23 #include "hinic_hw_cmdq.h" 24 25 #define WQS_BLOCKS_PER_PAGE 4 26 27 #define WQ_BLOCK_SIZE 4096 28 #define WQS_PAGE_SIZE (WQS_BLOCKS_PER_PAGE * WQ_BLOCK_SIZE) 29 30 #define WQS_MAX_NUM_BLOCKS 128 31 #define WQS_FREE_BLOCKS_SIZE(wqs) (WQS_MAX_NUM_BLOCKS * \ 32 sizeof((wqs)->free_blocks[0])) 33 34 #define WQ_SIZE(wq) ((wq)->q_depth * (wq)->wqebb_size) 35 36 #define WQ_PAGE_ADDR_SIZE sizeof(u64) 37 #define WQ_MAX_PAGES (WQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) 38 39 #define CMDQ_BLOCK_SIZE 512 40 #define CMDQ_PAGE_SIZE 4096 41 42 #define CMDQ_WQ_MAX_PAGES (CMDQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE) 43 44 #define WQ_BASE_VADDR(wqs, wq) \ 45 ((void *)((wqs)->page_vaddr[(wq)->page_idx]) \ 46 + (wq)->block_idx * WQ_BLOCK_SIZE) 47 48 #define WQ_BASE_PADDR(wqs, wq) \ 49 ((wqs)->page_paddr[(wq)->page_idx] \ 50 + (wq)->block_idx * WQ_BLOCK_SIZE) 51 52 #define WQ_BASE_ADDR(wqs, wq) \ 53 ((void *)((wqs)->shadow_page_vaddr[(wq)->page_idx]) \ 54 + (wq)->block_idx * WQ_BLOCK_SIZE) 55 56 #define CMDQ_BASE_VADDR(cmdq_pages, wq) \ 57 ((void *)((cmdq_pages)->page_vaddr) \ 58 + (wq)->block_idx * CMDQ_BLOCK_SIZE) 59 60 #define CMDQ_BASE_PADDR(cmdq_pages, wq) \ 61 ((cmdq_pages)->page_paddr \ 62 + (wq)->block_idx * CMDQ_BLOCK_SIZE) 63 64 #define CMDQ_BASE_ADDR(cmdq_pages, wq) \ 65 ((void *)((cmdq_pages)->shadow_page_vaddr) \ 66 + (wq)->block_idx * CMDQ_BLOCK_SIZE) 67 68 #define WQ_PAGE_ADDR(wq, idx) \ 69 ((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)]) 70 71 #define MASKED_WQE_IDX(wq, idx) ((idx) & (wq)->mask) 72 73 #define WQE_IN_RANGE(wqe, start, end) \ 74 (((unsigned long)(wqe) >= (unsigned long)(start)) && \ 75 ((unsigned long)(wqe) < (unsigned long)(end))) 76 77 #define WQE_SHADOW_PAGE(wq, wqe) \ 78 (((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \ 79 / (wq)->max_wqe_size) 80 81 static inline int WQE_PAGE_OFF(struct hinic_wq *wq, u16 idx) 82 { 83 return (((idx) & ((wq)->num_wqebbs_per_page - 1)) 84 << (wq)->wqebb_size_shift); 85 } 86 87 static inline int WQE_PAGE_NUM(struct hinic_wq *wq, u16 idx) 88 { 89 return (((idx) >> ((wq)->wqebbs_per_page_shift)) 90 & ((wq)->num_q_pages - 1)); 91 } 92 93 /** 94 * queue_alloc_page - allocate page for Queue 95 * @hwif: HW interface for allocating DMA 96 * @vaddr: virtual address will be returned in this address 97 * @paddr: physical address will be returned in this address 98 * @shadow_vaddr: VM area will be return here for holding WQ page addresses 99 * @page_sz: page size of each WQ page 100 * 101 * Return 0 - Success, negative - Failure 102 **/ 103 static int queue_alloc_page(struct hinic_hwif *hwif, u64 **vaddr, u64 *paddr, 104 void ***shadow_vaddr, size_t page_sz) 105 { 106 struct pci_dev *pdev = hwif->pdev; 107 dma_addr_t dma_addr; 108 109 *vaddr = dma_alloc_coherent(&pdev->dev, page_sz, &dma_addr, 110 GFP_KERNEL); 111 if (!*vaddr) { 112 dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n"); 113 return -ENOMEM; 114 } 115 116 *paddr = (u64)dma_addr; 117 118 /* use vzalloc for big mem */ 119 *shadow_vaddr = vzalloc(page_sz); 120 if (!*shadow_vaddr) 121 goto err_shadow_vaddr; 122 123 return 0; 124 125 err_shadow_vaddr: 126 dma_free_coherent(&pdev->dev, page_sz, *vaddr, dma_addr); 127 return -ENOMEM; 128 } 129 130 /** 131 * wqs_allocate_page - allocate page for WQ set 132 * @wqs: Work Queue Set 133 * @page_idx: the page index of the page will be allocated 134 * 135 * Return 0 - Success, negative - Failure 136 **/ 137 static int wqs_allocate_page(struct hinic_wqs *wqs, int page_idx) 138 { 139 return queue_alloc_page(wqs->hwif, &wqs->page_vaddr[page_idx], 140 &wqs->page_paddr[page_idx], 141 &wqs->shadow_page_vaddr[page_idx], 142 WQS_PAGE_SIZE); 143 } 144 145 /** 146 * wqs_free_page - free page of WQ set 147 * @wqs: Work Queue Set 148 * @page_idx: the page index of the page will be freed 149 **/ 150 static void wqs_free_page(struct hinic_wqs *wqs, int page_idx) 151 { 152 struct hinic_hwif *hwif = wqs->hwif; 153 struct pci_dev *pdev = hwif->pdev; 154 155 dma_free_coherent(&pdev->dev, WQS_PAGE_SIZE, 156 wqs->page_vaddr[page_idx], 157 (dma_addr_t)wqs->page_paddr[page_idx]); 158 vfree(wqs->shadow_page_vaddr[page_idx]); 159 } 160 161 /** 162 * cmdq_allocate_page - allocate page for cmdq 163 * @cmdq_pages: the pages of the cmdq queue struct to hold the page 164 * 165 * Return 0 - Success, negative - Failure 166 **/ 167 static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages) 168 { 169 return queue_alloc_page(cmdq_pages->hwif, &cmdq_pages->page_vaddr, 170 &cmdq_pages->page_paddr, 171 &cmdq_pages->shadow_page_vaddr, 172 CMDQ_PAGE_SIZE); 173 } 174 175 /** 176 * cmdq_free_page - free page from cmdq 177 * @cmdq_pages: the pages of the cmdq queue struct that hold the page 178 **/ 179 static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages) 180 { 181 struct hinic_hwif *hwif = cmdq_pages->hwif; 182 struct pci_dev *pdev = hwif->pdev; 183 184 dma_free_coherent(&pdev->dev, CMDQ_PAGE_SIZE, 185 cmdq_pages->page_vaddr, 186 (dma_addr_t)cmdq_pages->page_paddr); 187 vfree(cmdq_pages->shadow_page_vaddr); 188 } 189 190 static int alloc_page_arrays(struct hinic_wqs *wqs) 191 { 192 struct hinic_hwif *hwif = wqs->hwif; 193 struct pci_dev *pdev = hwif->pdev; 194 195 wqs->page_paddr = devm_kcalloc(&pdev->dev, wqs->num_pages, 196 sizeof(*wqs->page_paddr), GFP_KERNEL); 197 if (!wqs->page_paddr) 198 return -ENOMEM; 199 200 wqs->page_vaddr = devm_kcalloc(&pdev->dev, wqs->num_pages, 201 sizeof(*wqs->page_vaddr), GFP_KERNEL); 202 if (!wqs->page_vaddr) 203 goto err_page_vaddr; 204 205 wqs->shadow_page_vaddr = devm_kcalloc(&pdev->dev, wqs->num_pages, 206 sizeof(*wqs->shadow_page_vaddr), 207 GFP_KERNEL); 208 if (!wqs->shadow_page_vaddr) 209 goto err_page_shadow_vaddr; 210 211 return 0; 212 213 err_page_shadow_vaddr: 214 devm_kfree(&pdev->dev, wqs->page_vaddr); 215 216 err_page_vaddr: 217 devm_kfree(&pdev->dev, wqs->page_paddr); 218 return -ENOMEM; 219 } 220 221 static void free_page_arrays(struct hinic_wqs *wqs) 222 { 223 struct hinic_hwif *hwif = wqs->hwif; 224 struct pci_dev *pdev = hwif->pdev; 225 226 devm_kfree(&pdev->dev, wqs->shadow_page_vaddr); 227 devm_kfree(&pdev->dev, wqs->page_vaddr); 228 devm_kfree(&pdev->dev, wqs->page_paddr); 229 } 230 231 static int wqs_next_block(struct hinic_wqs *wqs, int *page_idx, 232 int *block_idx) 233 { 234 int pos; 235 236 down(&wqs->alloc_blocks_lock); 237 238 wqs->num_free_blks--; 239 240 if (wqs->num_free_blks < 0) { 241 wqs->num_free_blks++; 242 up(&wqs->alloc_blocks_lock); 243 return -ENOMEM; 244 } 245 246 pos = wqs->alloc_blk_pos++; 247 pos &= WQS_MAX_NUM_BLOCKS - 1; 248 249 *page_idx = wqs->free_blocks[pos].page_idx; 250 *block_idx = wqs->free_blocks[pos].block_idx; 251 252 wqs->free_blocks[pos].page_idx = -1; 253 wqs->free_blocks[pos].block_idx = -1; 254 255 up(&wqs->alloc_blocks_lock); 256 return 0; 257 } 258 259 static void wqs_return_block(struct hinic_wqs *wqs, int page_idx, 260 int block_idx) 261 { 262 int pos; 263 264 down(&wqs->alloc_blocks_lock); 265 266 pos = wqs->return_blk_pos++; 267 pos &= WQS_MAX_NUM_BLOCKS - 1; 268 269 wqs->free_blocks[pos].page_idx = page_idx; 270 wqs->free_blocks[pos].block_idx = block_idx; 271 272 wqs->num_free_blks++; 273 274 up(&wqs->alloc_blocks_lock); 275 } 276 277 static void init_wqs_blocks_arr(struct hinic_wqs *wqs) 278 { 279 int page_idx, blk_idx, pos = 0; 280 281 for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { 282 for (blk_idx = 0; blk_idx < WQS_BLOCKS_PER_PAGE; blk_idx++) { 283 wqs->free_blocks[pos].page_idx = page_idx; 284 wqs->free_blocks[pos].block_idx = blk_idx; 285 pos++; 286 } 287 } 288 289 wqs->alloc_blk_pos = 0; 290 wqs->return_blk_pos = pos; 291 wqs->num_free_blks = pos; 292 293 sema_init(&wqs->alloc_blocks_lock, 1); 294 } 295 296 /** 297 * hinic_wqs_alloc - allocate Work Queues set 298 * @wqs: Work Queue Set 299 * @max_wqs: maximum wqs to allocate 300 * @hwif: HW interface for use for the allocation 301 * 302 * Return 0 - Success, negative - Failure 303 **/ 304 int hinic_wqs_alloc(struct hinic_wqs *wqs, int max_wqs, 305 struct hinic_hwif *hwif) 306 { 307 struct pci_dev *pdev = hwif->pdev; 308 int err, i, page_idx; 309 310 max_wqs = ALIGN(max_wqs, WQS_BLOCKS_PER_PAGE); 311 if (max_wqs > WQS_MAX_NUM_BLOCKS) { 312 dev_err(&pdev->dev, "Invalid max_wqs = %d\n", max_wqs); 313 return -EINVAL; 314 } 315 316 wqs->hwif = hwif; 317 wqs->num_pages = max_wqs / WQS_BLOCKS_PER_PAGE; 318 319 if (alloc_page_arrays(wqs)) { 320 dev_err(&pdev->dev, 321 "Failed to allocate mem for page addresses\n"); 322 return -ENOMEM; 323 } 324 325 for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) { 326 err = wqs_allocate_page(wqs, page_idx); 327 if (err) { 328 dev_err(&pdev->dev, "Failed wq page allocation\n"); 329 goto err_wq_allocate_page; 330 } 331 } 332 333 wqs->free_blocks = devm_kzalloc(&pdev->dev, WQS_FREE_BLOCKS_SIZE(wqs), 334 GFP_KERNEL); 335 if (!wqs->free_blocks) { 336 err = -ENOMEM; 337 goto err_alloc_blocks; 338 } 339 340 init_wqs_blocks_arr(wqs); 341 return 0; 342 343 err_alloc_blocks: 344 err_wq_allocate_page: 345 for (i = 0; i < page_idx; i++) 346 wqs_free_page(wqs, i); 347 348 free_page_arrays(wqs); 349 return err; 350 } 351 352 /** 353 * hinic_wqs_free - free Work Queues set 354 * @wqs: Work Queue Set 355 **/ 356 void hinic_wqs_free(struct hinic_wqs *wqs) 357 { 358 struct hinic_hwif *hwif = wqs->hwif; 359 struct pci_dev *pdev = hwif->pdev; 360 int page_idx; 361 362 devm_kfree(&pdev->dev, wqs->free_blocks); 363 364 for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) 365 wqs_free_page(wqs, page_idx); 366 367 free_page_arrays(wqs); 368 } 369 370 /** 371 * alloc_wqes_shadow - allocate WQE shadows for WQ 372 * @wq: WQ to allocate shadows for 373 * 374 * Return 0 - Success, negative - Failure 375 **/ 376 static int alloc_wqes_shadow(struct hinic_wq *wq) 377 { 378 struct hinic_hwif *hwif = wq->hwif; 379 struct pci_dev *pdev = hwif->pdev; 380 381 wq->shadow_wqe = devm_kcalloc(&pdev->dev, wq->num_q_pages, 382 wq->max_wqe_size, GFP_KERNEL); 383 if (!wq->shadow_wqe) 384 return -ENOMEM; 385 386 wq->shadow_idx = devm_kcalloc(&pdev->dev, wq->num_q_pages, 387 sizeof(*wq->shadow_idx), GFP_KERNEL); 388 if (!wq->shadow_idx) 389 goto err_shadow_idx; 390 391 return 0; 392 393 err_shadow_idx: 394 devm_kfree(&pdev->dev, wq->shadow_wqe); 395 return -ENOMEM; 396 } 397 398 /** 399 * free_wqes_shadow - free WQE shadows of WQ 400 * @wq: WQ to free shadows from 401 **/ 402 static void free_wqes_shadow(struct hinic_wq *wq) 403 { 404 struct hinic_hwif *hwif = wq->hwif; 405 struct pci_dev *pdev = hwif->pdev; 406 407 devm_kfree(&pdev->dev, wq->shadow_idx); 408 devm_kfree(&pdev->dev, wq->shadow_wqe); 409 } 410 411 /** 412 * free_wq_pages - free pages of WQ 413 * @hwif: HW interface for releasing dma addresses 414 * @wq: WQ to free pages from 415 * @num_q_pages: number pages to free 416 **/ 417 static void free_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, 418 int num_q_pages) 419 { 420 struct pci_dev *pdev = hwif->pdev; 421 int i; 422 423 for (i = 0; i < num_q_pages; i++) { 424 void **vaddr = &wq->shadow_block_vaddr[i]; 425 u64 *paddr = &wq->block_vaddr[i]; 426 dma_addr_t dma_addr; 427 428 dma_addr = (dma_addr_t)be64_to_cpu(*paddr); 429 dma_free_coherent(&pdev->dev, wq->wq_page_size, *vaddr, 430 dma_addr); 431 } 432 433 free_wqes_shadow(wq); 434 } 435 436 /** 437 * alloc_wq_pages - alloc pages for WQ 438 * @hwif: HW interface for allocating dma addresses 439 * @wq: WQ to allocate pages for 440 * @max_pages: maximum pages allowed 441 * 442 * Return 0 - Success, negative - Failure 443 **/ 444 static int alloc_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif, 445 int max_pages) 446 { 447 struct pci_dev *pdev = hwif->pdev; 448 int i, err, num_q_pages; 449 450 num_q_pages = ALIGN(WQ_SIZE(wq), wq->wq_page_size) / wq->wq_page_size; 451 if (num_q_pages > max_pages) { 452 dev_err(&pdev->dev, "Number wq pages exceeds the limit\n"); 453 return -EINVAL; 454 } 455 456 if (num_q_pages & (num_q_pages - 1)) { 457 dev_err(&pdev->dev, "Number wq pages must be power of 2\n"); 458 return -EINVAL; 459 } 460 461 wq->num_q_pages = num_q_pages; 462 463 err = alloc_wqes_shadow(wq); 464 if (err) { 465 dev_err(&pdev->dev, "Failed to allocate wqe shadow\n"); 466 return err; 467 } 468 469 for (i = 0; i < num_q_pages; i++) { 470 void **vaddr = &wq->shadow_block_vaddr[i]; 471 u64 *paddr = &wq->block_vaddr[i]; 472 dma_addr_t dma_addr; 473 474 *vaddr = dma_alloc_coherent(&pdev->dev, wq->wq_page_size, 475 &dma_addr, GFP_KERNEL); 476 if (!*vaddr) { 477 dev_err(&pdev->dev, "Failed to allocate wq page\n"); 478 goto err_alloc_wq_pages; 479 } 480 481 /* HW uses Big Endian Format */ 482 *paddr = cpu_to_be64(dma_addr); 483 } 484 485 return 0; 486 487 err_alloc_wq_pages: 488 free_wq_pages(wq, hwif, i); 489 return -ENOMEM; 490 } 491 492 /** 493 * hinic_wq_allocate - Allocate the WQ resources from the WQS 494 * @wqs: WQ set from which to allocate the WQ resources 495 * @wq: WQ to allocate resources for it from the WQ set 496 * @wqebb_size: Work Queue Block Byte Size 497 * @wq_page_size: the page size in the Work Queue 498 * @q_depth: number of wqebbs in WQ 499 * @max_wqe_size: maximum WQE size that will be used in the WQ 500 * 501 * Return 0 - Success, negative - Failure 502 **/ 503 int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq, 504 u16 wqebb_size, u32 wq_page_size, u16 q_depth, 505 u16 max_wqe_size) 506 { 507 struct hinic_hwif *hwif = wqs->hwif; 508 struct pci_dev *pdev = hwif->pdev; 509 u16 num_wqebbs_per_page; 510 u16 wqebb_size_shift; 511 int err; 512 513 if (!is_power_of_2(wqebb_size)) { 514 dev_err(&pdev->dev, "wqebb_size must be power of 2\n"); 515 return -EINVAL; 516 } 517 518 if (wq_page_size == 0) { 519 dev_err(&pdev->dev, "wq_page_size must be > 0\n"); 520 return -EINVAL; 521 } 522 523 if (q_depth & (q_depth - 1)) { 524 dev_err(&pdev->dev, "WQ q_depth must be power of 2\n"); 525 return -EINVAL; 526 } 527 528 wqebb_size_shift = ilog2(wqebb_size); 529 num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) 530 >> wqebb_size_shift; 531 532 if (!is_power_of_2(num_wqebbs_per_page)) { 533 dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n"); 534 return -EINVAL; 535 } 536 537 wq->hwif = hwif; 538 539 err = wqs_next_block(wqs, &wq->page_idx, &wq->block_idx); 540 if (err) { 541 dev_err(&pdev->dev, "Failed to get free wqs next block\n"); 542 return err; 543 } 544 545 wq->wqebb_size = wqebb_size; 546 wq->wq_page_size = wq_page_size; 547 wq->q_depth = q_depth; 548 wq->max_wqe_size = max_wqe_size; 549 wq->num_wqebbs_per_page = num_wqebbs_per_page; 550 wq->wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); 551 wq->wqebb_size_shift = wqebb_size_shift; 552 wq->block_vaddr = WQ_BASE_VADDR(wqs, wq); 553 wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq); 554 wq->block_paddr = WQ_BASE_PADDR(wqs, wq); 555 556 err = alloc_wq_pages(wq, wqs->hwif, WQ_MAX_PAGES); 557 if (err) { 558 dev_err(&pdev->dev, "Failed to allocate wq pages\n"); 559 goto err_alloc_wq_pages; 560 } 561 562 atomic_set(&wq->cons_idx, 0); 563 atomic_set(&wq->prod_idx, 0); 564 atomic_set(&wq->delta, q_depth); 565 wq->mask = q_depth - 1; 566 567 return 0; 568 569 err_alloc_wq_pages: 570 wqs_return_block(wqs, wq->page_idx, wq->block_idx); 571 return err; 572 } 573 574 /** 575 * hinic_wq_free - Free the WQ resources to the WQS 576 * @wqs: WQ set to free the WQ resources to it 577 * @wq: WQ to free its resources to the WQ set resources 578 **/ 579 void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq) 580 { 581 free_wq_pages(wq, wqs->hwif, wq->num_q_pages); 582 583 wqs_return_block(wqs, wq->page_idx, wq->block_idx); 584 } 585 586 /** 587 * hinic_wqs_cmdq_alloc - Allocate wqs for cmdqs 588 * @cmdq_pages: will hold the pages of the cmdq 589 * @wq: returned wqs 590 * @hwif: HW interface 591 * @cmdq_blocks: number of cmdq blocks/wq to allocate 592 * @wqebb_size: Work Queue Block Byte Size 593 * @wq_page_size: the page size in the Work Queue 594 * @q_depth: number of wqebbs in WQ 595 * @max_wqe_size: maximum WQE size that will be used in the WQ 596 * 597 * Return 0 - Success, negative - Failure 598 **/ 599 int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages, 600 struct hinic_wq *wq, struct hinic_hwif *hwif, 601 int cmdq_blocks, u16 wqebb_size, u32 wq_page_size, 602 u16 q_depth, u16 max_wqe_size) 603 { 604 struct pci_dev *pdev = hwif->pdev; 605 u16 num_wqebbs_per_page_shift; 606 u16 num_wqebbs_per_page; 607 u16 wqebb_size_shift; 608 int i, j, err = -ENOMEM; 609 610 if (!is_power_of_2(wqebb_size)) { 611 dev_err(&pdev->dev, "wqebb_size must be power of 2\n"); 612 return -EINVAL; 613 } 614 615 if (wq_page_size == 0) { 616 dev_err(&pdev->dev, "wq_page_size must be > 0\n"); 617 return -EINVAL; 618 } 619 620 if (q_depth & (q_depth - 1)) { 621 dev_err(&pdev->dev, "WQ q_depth must be power of 2\n"); 622 return -EINVAL; 623 } 624 625 wqebb_size_shift = ilog2(wqebb_size); 626 num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size) 627 >> wqebb_size_shift; 628 629 if (!is_power_of_2(num_wqebbs_per_page)) { 630 dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n"); 631 return -EINVAL; 632 } 633 634 cmdq_pages->hwif = hwif; 635 636 err = cmdq_allocate_page(cmdq_pages); 637 if (err) { 638 dev_err(&pdev->dev, "Failed to allocate CMDQ page\n"); 639 return err; 640 } 641 num_wqebbs_per_page_shift = ilog2(num_wqebbs_per_page); 642 643 for (i = 0; i < cmdq_blocks; i++) { 644 wq[i].hwif = hwif; 645 wq[i].page_idx = 0; 646 wq[i].block_idx = i; 647 648 wq[i].wqebb_size = wqebb_size; 649 wq[i].wq_page_size = wq_page_size; 650 wq[i].q_depth = q_depth; 651 wq[i].max_wqe_size = max_wqe_size; 652 wq[i].num_wqebbs_per_page = num_wqebbs_per_page; 653 wq[i].wqebbs_per_page_shift = num_wqebbs_per_page_shift; 654 wq[i].wqebb_size_shift = wqebb_size_shift; 655 wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]); 656 wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]); 657 wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]); 658 659 err = alloc_wq_pages(&wq[i], cmdq_pages->hwif, 660 CMDQ_WQ_MAX_PAGES); 661 if (err) { 662 dev_err(&pdev->dev, "Failed to alloc CMDQ blocks\n"); 663 goto err_cmdq_block; 664 } 665 666 atomic_set(&wq[i].cons_idx, 0); 667 atomic_set(&wq[i].prod_idx, 0); 668 atomic_set(&wq[i].delta, q_depth); 669 wq[i].mask = q_depth - 1; 670 } 671 672 return 0; 673 674 err_cmdq_block: 675 for (j = 0; j < i; j++) 676 free_wq_pages(&wq[j], cmdq_pages->hwif, wq[j].num_q_pages); 677 678 cmdq_free_page(cmdq_pages); 679 return err; 680 } 681 682 /** 683 * hinic_wqs_cmdq_free - Free wqs from cmdqs 684 * @cmdq_pages: hold the pages of the cmdq 685 * @wq: wqs to free 686 * @cmdq_blocks: number of wqs to free 687 **/ 688 void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages, 689 struct hinic_wq *wq, int cmdq_blocks) 690 { 691 int i; 692 693 for (i = 0; i < cmdq_blocks; i++) 694 free_wq_pages(&wq[i], cmdq_pages->hwif, wq[i].num_q_pages); 695 696 cmdq_free_page(cmdq_pages); 697 } 698 699 static void copy_wqe_to_shadow(struct hinic_wq *wq, void *shadow_addr, 700 int num_wqebbs, u16 idx) 701 { 702 void *wqebb_addr; 703 int i; 704 705 for (i = 0; i < num_wqebbs; i++, idx++) { 706 idx = MASKED_WQE_IDX(wq, idx); 707 wqebb_addr = WQ_PAGE_ADDR(wq, idx) + 708 WQE_PAGE_OFF(wq, idx); 709 710 memcpy(shadow_addr, wqebb_addr, wq->wqebb_size); 711 712 shadow_addr += wq->wqebb_size; 713 } 714 } 715 716 static void copy_wqe_from_shadow(struct hinic_wq *wq, void *shadow_addr, 717 int num_wqebbs, u16 idx) 718 { 719 void *wqebb_addr; 720 int i; 721 722 for (i = 0; i < num_wqebbs; i++, idx++) { 723 idx = MASKED_WQE_IDX(wq, idx); 724 wqebb_addr = WQ_PAGE_ADDR(wq, idx) + 725 WQE_PAGE_OFF(wq, idx); 726 727 memcpy(wqebb_addr, shadow_addr, wq->wqebb_size); 728 shadow_addr += wq->wqebb_size; 729 } 730 } 731 732 /** 733 * hinic_get_wqe - get wqe ptr in the current pi and update the pi 734 * @wq: wq to get wqe from 735 * @wqe_size: wqe size 736 * @prod_idx: returned pi 737 * 738 * Return wqe pointer 739 **/ 740 struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size, 741 u16 *prod_idx) 742 { 743 int curr_pg, end_pg, num_wqebbs; 744 u16 curr_prod_idx, end_prod_idx; 745 746 *prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx)); 747 748 num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) >> wq->wqebb_size_shift; 749 750 if (atomic_sub_return(num_wqebbs, &wq->delta) <= 0) { 751 atomic_add(num_wqebbs, &wq->delta); 752 return ERR_PTR(-EBUSY); 753 } 754 755 end_prod_idx = atomic_add_return(num_wqebbs, &wq->prod_idx); 756 757 end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx); 758 curr_prod_idx = end_prod_idx - num_wqebbs; 759 curr_prod_idx = MASKED_WQE_IDX(wq, curr_prod_idx); 760 761 /* end prod index points to the next wqebb, therefore minus 1 */ 762 end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx - 1); 763 764 curr_pg = WQE_PAGE_NUM(wq, curr_prod_idx); 765 end_pg = WQE_PAGE_NUM(wq, end_prod_idx); 766 767 *prod_idx = curr_prod_idx; 768 769 /* If we only have one page, still need to get shadown wqe when 770 * wqe rolling-over page 771 */ 772 if (curr_pg != end_pg || end_prod_idx < *prod_idx) { 773 void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; 774 775 copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx); 776 777 wq->shadow_idx[curr_pg] = *prod_idx; 778 return shadow_addr; 779 } 780 781 return WQ_PAGE_ADDR(wq, *prod_idx) + WQE_PAGE_OFF(wq, *prod_idx); 782 } 783 784 /** 785 * hinic_return_wqe - return the wqe when transmit failed 786 * @wq: wq to return wqe 787 * @wqe_size: wqe size 788 **/ 789 void hinic_return_wqe(struct hinic_wq *wq, unsigned int wqe_size) 790 { 791 int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; 792 793 atomic_sub(num_wqebbs, &wq->prod_idx); 794 795 atomic_add(num_wqebbs, &wq->delta); 796 } 797 798 /** 799 * hinic_put_wqe - return the wqe place to use for a new wqe 800 * @wq: wq to return wqe 801 * @wqe_size: wqe size 802 **/ 803 void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size) 804 { 805 int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) 806 >> wq->wqebb_size_shift; 807 808 atomic_add(num_wqebbs, &wq->cons_idx); 809 810 atomic_add(num_wqebbs, &wq->delta); 811 } 812 813 /** 814 * hinic_read_wqe - read wqe ptr in the current ci 815 * @wq: wq to get read from 816 * @wqe_size: wqe size 817 * @cons_idx: returned ci 818 * 819 * Return wqe pointer 820 **/ 821 struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size, 822 u16 *cons_idx) 823 { 824 int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) 825 >> wq->wqebb_size_shift; 826 u16 curr_cons_idx, end_cons_idx; 827 int curr_pg, end_pg; 828 829 if ((atomic_read(&wq->delta) + num_wqebbs) > wq->q_depth) 830 return ERR_PTR(-EBUSY); 831 832 curr_cons_idx = atomic_read(&wq->cons_idx); 833 834 curr_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx); 835 end_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx + num_wqebbs - 1); 836 837 curr_pg = WQE_PAGE_NUM(wq, curr_cons_idx); 838 end_pg = WQE_PAGE_NUM(wq, end_cons_idx); 839 840 *cons_idx = curr_cons_idx; 841 842 /* If we only have one page, still need to get shadown wqe when 843 * wqe rolling-over page 844 */ 845 if (curr_pg != end_pg || end_cons_idx < curr_cons_idx) { 846 void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; 847 848 copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx); 849 return shadow_addr; 850 } 851 852 return WQ_PAGE_ADDR(wq, *cons_idx) + WQE_PAGE_OFF(wq, *cons_idx); 853 } 854 855 /** 856 * hinic_read_wqe_direct - read wqe directly from ci position 857 * @wq: wq 858 * @cons_idx: ci position 859 * 860 * Return wqe 861 **/ 862 struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx) 863 { 864 return WQ_PAGE_ADDR(wq, cons_idx) + WQE_PAGE_OFF(wq, cons_idx); 865 } 866 867 /** 868 * wqe_shadow - check if a wqe is shadow 869 * @wq: wq of the wqe 870 * @wqe: the wqe for shadow checking 871 * 872 * Return true - shadow, false - Not shadow 873 **/ 874 static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe) 875 { 876 size_t wqe_shadow_size = wq->num_q_pages * wq->max_wqe_size; 877 878 return WQE_IN_RANGE(wqe, wq->shadow_wqe, 879 &wq->shadow_wqe[wqe_shadow_size]); 880 } 881 882 /** 883 * hinic_write_wqe - write the wqe to the wq 884 * @wq: wq to write wqe to 885 * @wqe: wqe to write 886 * @wqe_size: wqe size 887 **/ 888 void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe, 889 unsigned int wqe_size) 890 { 891 int curr_pg, num_wqebbs; 892 void *shadow_addr; 893 u16 prod_idx; 894 895 if (wqe_shadow(wq, wqe)) { 896 curr_pg = WQE_SHADOW_PAGE(wq, wqe); 897 898 prod_idx = wq->shadow_idx[curr_pg]; 899 num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size; 900 shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size]; 901 902 copy_wqe_from_shadow(wq, shadow_addr, num_wqebbs, prod_idx); 903 } 904 } 905