1 // SPDX-License-Identifier: MIT 2 3 /* 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include <linux/delay.h> 8 #include <linux/dma-fence.h> 9 #include <linux/dma-fence-chain.h> 10 #include <linux/kernel.h> 11 #include <linux/kthread.h> 12 #include <linux/mm.h> 13 #include <linux/sched/signal.h> 14 #include <linux/slab.h> 15 #include <linux/spinlock.h> 16 #include <linux/random.h> 17 18 #include "selftest.h" 19 20 #define CHAIN_SZ (4 << 10) 21 22 static struct kmem_cache *slab_fences; 23 24 static inline struct mock_fence { 25 struct dma_fence base; 26 spinlock_t lock; 27 } *to_mock_fence(struct dma_fence *f) { 28 return container_of(f, struct mock_fence, base); 29 } 30 31 static const char *mock_name(struct dma_fence *f) 32 { 33 return "mock"; 34 } 35 36 static void mock_fence_release(struct dma_fence *f) 37 { 38 kmem_cache_free(slab_fences, to_mock_fence(f)); 39 } 40 41 static const struct dma_fence_ops mock_ops = { 42 .get_driver_name = mock_name, 43 .get_timeline_name = mock_name, 44 .release = mock_fence_release, 45 }; 46 47 static struct dma_fence *mock_fence(void) 48 { 49 struct mock_fence *f; 50 51 f = kmem_cache_alloc(slab_fences, GFP_KERNEL); 52 if (!f) 53 return NULL; 54 55 spin_lock_init(&f->lock); 56 dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0); 57 58 return &f->base; 59 } 60 61 static struct dma_fence *mock_chain(struct dma_fence *prev, 62 struct dma_fence *fence, 63 u64 seqno) 64 { 65 struct dma_fence_chain *f; 66 67 f = dma_fence_chain_alloc(); 68 if (!f) 69 return NULL; 70 71 dma_fence_chain_init(f, dma_fence_get(prev), dma_fence_get(fence), 72 seqno); 73 74 return &f->base; 75 } 76 77 static int sanitycheck(void *arg) 78 { 79 struct dma_fence *f, *chain; 80 int err = 0; 81 82 f = mock_fence(); 83 if (!f) 84 return -ENOMEM; 85 86 chain = mock_chain(NULL, f, 1); 87 if (!chain) 88 err = -ENOMEM; 89 90 dma_fence_enable_sw_signaling(chain); 91 92 dma_fence_signal(f); 93 dma_fence_put(f); 94 95 dma_fence_put(chain); 96 97 return err; 98 } 99 100 struct fence_chains { 101 unsigned int chain_length; 102 struct dma_fence **fences; 103 struct dma_fence **chains; 104 105 struct dma_fence *tail; 106 }; 107 108 static uint64_t seqno_inc(unsigned int i) 109 { 110 return i + 1; 111 } 112 113 static int fence_chains_init(struct fence_chains *fc, unsigned int count, 114 uint64_t (*seqno_fn)(unsigned int)) 115 { 116 unsigned int i; 117 int err = 0; 118 119 fc->chains = kvmalloc_array(count, sizeof(*fc->chains), 120 GFP_KERNEL | __GFP_ZERO); 121 if (!fc->chains) 122 return -ENOMEM; 123 124 fc->fences = kvmalloc_array(count, sizeof(*fc->fences), 125 GFP_KERNEL | __GFP_ZERO); 126 if (!fc->fences) { 127 err = -ENOMEM; 128 goto err_chains; 129 } 130 131 fc->tail = NULL; 132 for (i = 0; i < count; i++) { 133 fc->fences[i] = mock_fence(); 134 if (!fc->fences[i]) { 135 err = -ENOMEM; 136 goto unwind; 137 } 138 139 fc->chains[i] = mock_chain(fc->tail, 140 fc->fences[i], 141 seqno_fn(i)); 142 if (!fc->chains[i]) { 143 err = -ENOMEM; 144 goto unwind; 145 } 146 147 fc->tail = fc->chains[i]; 148 149 dma_fence_enable_sw_signaling(fc->chains[i]); 150 } 151 152 fc->chain_length = i; 153 return 0; 154 155 unwind: 156 for (i = 0; i < count; i++) { 157 dma_fence_put(fc->fences[i]); 158 dma_fence_put(fc->chains[i]); 159 } 160 kvfree(fc->fences); 161 err_chains: 162 kvfree(fc->chains); 163 return err; 164 } 165 166 static void fence_chains_fini(struct fence_chains *fc) 167 { 168 unsigned int i; 169 170 for (i = 0; i < fc->chain_length; i++) { 171 dma_fence_signal(fc->fences[i]); 172 dma_fence_put(fc->fences[i]); 173 } 174 kvfree(fc->fences); 175 176 for (i = 0; i < fc->chain_length; i++) 177 dma_fence_put(fc->chains[i]); 178 kvfree(fc->chains); 179 } 180 181 static int find_seqno(void *arg) 182 { 183 struct fence_chains fc; 184 struct dma_fence *fence; 185 int err; 186 int i; 187 188 err = fence_chains_init(&fc, 64, seqno_inc); 189 if (err) 190 return err; 191 192 fence = dma_fence_get(fc.tail); 193 err = dma_fence_chain_find_seqno(&fence, 0); 194 dma_fence_put(fence); 195 if (err) { 196 pr_err("Reported %d for find_seqno(0)!\n", err); 197 goto err; 198 } 199 200 for (i = 0; i < fc.chain_length; i++) { 201 fence = dma_fence_get(fc.tail); 202 err = dma_fence_chain_find_seqno(&fence, i + 1); 203 dma_fence_put(fence); 204 if (err) { 205 pr_err("Reported %d for find_seqno(%d:%d)!\n", 206 err, fc.chain_length + 1, i + 1); 207 goto err; 208 } 209 if (fence != fc.chains[i]) { 210 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 211 fc.chain_length + 1, i + 1); 212 err = -EINVAL; 213 goto err; 214 } 215 216 dma_fence_get(fence); 217 err = dma_fence_chain_find_seqno(&fence, i + 1); 218 dma_fence_put(fence); 219 if (err) { 220 pr_err("Error reported for finding self\n"); 221 goto err; 222 } 223 if (fence != fc.chains[i]) { 224 pr_err("Incorrect fence reported by find self\n"); 225 err = -EINVAL; 226 goto err; 227 } 228 229 dma_fence_get(fence); 230 err = dma_fence_chain_find_seqno(&fence, i + 2); 231 dma_fence_put(fence); 232 if (!err) { 233 pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n", 234 i + 1, i + 2); 235 err = -EINVAL; 236 goto err; 237 } 238 239 dma_fence_get(fence); 240 err = dma_fence_chain_find_seqno(&fence, i); 241 dma_fence_put(fence); 242 if (err) { 243 pr_err("Error reported for previous fence!\n"); 244 goto err; 245 } 246 if (i > 0 && fence != fc.chains[i - 1]) { 247 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 248 i + 1, i); 249 err = -EINVAL; 250 goto err; 251 } 252 } 253 254 err: 255 fence_chains_fini(&fc); 256 return err; 257 } 258 259 static int find_signaled(void *arg) 260 { 261 struct fence_chains fc; 262 struct dma_fence *fence; 263 int err; 264 265 err = fence_chains_init(&fc, 2, seqno_inc); 266 if (err) 267 return err; 268 269 dma_fence_signal(fc.fences[0]); 270 271 fence = dma_fence_get(fc.tail); 272 err = dma_fence_chain_find_seqno(&fence, 1); 273 dma_fence_put(fence); 274 if (err) { 275 pr_err("Reported %d for find_seqno()!\n", err); 276 goto err; 277 } 278 279 if (fence && fence != fc.chains[0]) { 280 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n", 281 fence->seqno); 282 283 dma_fence_get(fence); 284 err = dma_fence_chain_find_seqno(&fence, 1); 285 dma_fence_put(fence); 286 if (err) 287 pr_err("Reported %d for finding self!\n", err); 288 289 err = -EINVAL; 290 } 291 292 err: 293 fence_chains_fini(&fc); 294 return err; 295 } 296 297 static int find_out_of_order(void *arg) 298 { 299 struct fence_chains fc; 300 struct dma_fence *fence; 301 int err; 302 303 err = fence_chains_init(&fc, 3, seqno_inc); 304 if (err) 305 return err; 306 307 dma_fence_signal(fc.fences[1]); 308 309 fence = dma_fence_get(fc.tail); 310 err = dma_fence_chain_find_seqno(&fence, 2); 311 dma_fence_put(fence); 312 if (err) { 313 pr_err("Reported %d for find_seqno()!\n", err); 314 goto err; 315 } 316 317 /* 318 * We signaled the middle fence (2) of the 1-2-3 chain. The behavior 319 * of the dma-fence-chain is to make us wait for all the fences up to 320 * the point we want. Since fence 1 is still not signaled, this what 321 * we should get as fence to wait upon (fence 2 being garbage 322 * collected during the traversal of the chain). 323 */ 324 if (fence != fc.chains[0]) { 325 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n", 326 fence ? fence->seqno : 0); 327 328 err = -EINVAL; 329 } 330 331 err: 332 fence_chains_fini(&fc); 333 return err; 334 } 335 336 static uint64_t seqno_inc2(unsigned int i) 337 { 338 return 2 * i + 2; 339 } 340 341 static int find_gap(void *arg) 342 { 343 struct fence_chains fc; 344 struct dma_fence *fence; 345 int err; 346 int i; 347 348 err = fence_chains_init(&fc, 64, seqno_inc2); 349 if (err) 350 return err; 351 352 for (i = 0; i < fc.chain_length; i++) { 353 fence = dma_fence_get(fc.tail); 354 err = dma_fence_chain_find_seqno(&fence, 2 * i + 1); 355 dma_fence_put(fence); 356 if (err) { 357 pr_err("Reported %d for find_seqno(%d:%d)!\n", 358 err, fc.chain_length + 1, 2 * i + 1); 359 goto err; 360 } 361 if (fence != fc.chains[i]) { 362 pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n", 363 fence->seqno, 364 fc.chain_length + 1, 365 2 * i + 1); 366 err = -EINVAL; 367 goto err; 368 } 369 370 dma_fence_get(fence); 371 err = dma_fence_chain_find_seqno(&fence, 2 * i + 2); 372 dma_fence_put(fence); 373 if (err) { 374 pr_err("Error reported for finding self\n"); 375 goto err; 376 } 377 if (fence != fc.chains[i]) { 378 pr_err("Incorrect fence reported by find self\n"); 379 err = -EINVAL; 380 goto err; 381 } 382 } 383 384 err: 385 fence_chains_fini(&fc); 386 return err; 387 } 388 389 struct find_race { 390 struct fence_chains fc; 391 atomic_t children; 392 }; 393 394 static int __find_race(void *arg) 395 { 396 struct find_race *data = arg; 397 int err = 0; 398 399 while (!kthread_should_stop()) { 400 struct dma_fence *fence = dma_fence_get(data->fc.tail); 401 int seqno; 402 403 seqno = prandom_u32_max(data->fc.chain_length) + 1; 404 405 err = dma_fence_chain_find_seqno(&fence, seqno); 406 if (err) { 407 pr_err("Failed to find fence seqno:%d\n", 408 seqno); 409 dma_fence_put(fence); 410 break; 411 } 412 if (!fence) 413 goto signal; 414 415 /* 416 * We can only find ourselves if we are on fence we were 417 * looking for. 418 */ 419 if (fence->seqno == seqno) { 420 err = dma_fence_chain_find_seqno(&fence, seqno); 421 if (err) { 422 pr_err("Reported an invalid fence for find-self:%d\n", 423 seqno); 424 dma_fence_put(fence); 425 break; 426 } 427 } 428 429 dma_fence_put(fence); 430 431 signal: 432 seqno = prandom_u32_max(data->fc.chain_length - 1); 433 dma_fence_signal(data->fc.fences[seqno]); 434 cond_resched(); 435 } 436 437 if (atomic_dec_and_test(&data->children)) 438 wake_up_var(&data->children); 439 return err; 440 } 441 442 static int find_race(void *arg) 443 { 444 struct find_race data; 445 int ncpus = num_online_cpus(); 446 struct task_struct **threads; 447 unsigned long count; 448 int err; 449 int i; 450 451 err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc); 452 if (err) 453 return err; 454 455 threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL); 456 if (!threads) { 457 err = -ENOMEM; 458 goto err; 459 } 460 461 atomic_set(&data.children, 0); 462 for (i = 0; i < ncpus; i++) { 463 threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i); 464 if (IS_ERR(threads[i])) { 465 ncpus = i; 466 break; 467 } 468 atomic_inc(&data.children); 469 get_task_struct(threads[i]); 470 } 471 472 wait_var_event_timeout(&data.children, 473 !atomic_read(&data.children), 474 5 * HZ); 475 476 for (i = 0; i < ncpus; i++) { 477 int ret; 478 479 ret = kthread_stop(threads[i]); 480 if (ret && !err) 481 err = ret; 482 put_task_struct(threads[i]); 483 } 484 kfree(threads); 485 486 count = 0; 487 for (i = 0; i < data.fc.chain_length; i++) 488 if (dma_fence_is_signaled(data.fc.fences[i])) 489 count++; 490 pr_info("Completed %lu cycles\n", count); 491 492 err: 493 fence_chains_fini(&data.fc); 494 return err; 495 } 496 497 static int signal_forward(void *arg) 498 { 499 struct fence_chains fc; 500 int err; 501 int i; 502 503 err = fence_chains_init(&fc, 64, seqno_inc); 504 if (err) 505 return err; 506 507 for (i = 0; i < fc.chain_length; i++) { 508 dma_fence_signal(fc.fences[i]); 509 510 if (!dma_fence_is_signaled(fc.chains[i])) { 511 pr_err("chain[%d] not signaled!\n", i); 512 err = -EINVAL; 513 goto err; 514 } 515 516 if (i + 1 < fc.chain_length && 517 dma_fence_is_signaled(fc.chains[i + 1])) { 518 pr_err("chain[%d] is signaled!\n", i); 519 err = -EINVAL; 520 goto err; 521 } 522 } 523 524 err: 525 fence_chains_fini(&fc); 526 return err; 527 } 528 529 static int signal_backward(void *arg) 530 { 531 struct fence_chains fc; 532 int err; 533 int i; 534 535 err = fence_chains_init(&fc, 64, seqno_inc); 536 if (err) 537 return err; 538 539 for (i = fc.chain_length; i--; ) { 540 dma_fence_signal(fc.fences[i]); 541 542 if (i > 0 && dma_fence_is_signaled(fc.chains[i])) { 543 pr_err("chain[%d] is signaled!\n", i); 544 err = -EINVAL; 545 goto err; 546 } 547 } 548 549 for (i = 0; i < fc.chain_length; i++) { 550 if (!dma_fence_is_signaled(fc.chains[i])) { 551 pr_err("chain[%d] was not signaled!\n", i); 552 err = -EINVAL; 553 goto err; 554 } 555 } 556 557 err: 558 fence_chains_fini(&fc); 559 return err; 560 } 561 562 static int __wait_fence_chains(void *arg) 563 { 564 struct fence_chains *fc = arg; 565 566 if (dma_fence_wait(fc->tail, false)) 567 return -EIO; 568 569 return 0; 570 } 571 572 static int wait_forward(void *arg) 573 { 574 struct fence_chains fc; 575 struct task_struct *tsk; 576 int err; 577 int i; 578 579 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 580 if (err) 581 return err; 582 583 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 584 if (IS_ERR(tsk)) { 585 err = PTR_ERR(tsk); 586 goto err; 587 } 588 get_task_struct(tsk); 589 yield_to(tsk, true); 590 591 for (i = 0; i < fc.chain_length; i++) 592 dma_fence_signal(fc.fences[i]); 593 594 err = kthread_stop(tsk); 595 put_task_struct(tsk); 596 597 err: 598 fence_chains_fini(&fc); 599 return err; 600 } 601 602 static int wait_backward(void *arg) 603 { 604 struct fence_chains fc; 605 struct task_struct *tsk; 606 int err; 607 int i; 608 609 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 610 if (err) 611 return err; 612 613 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 614 if (IS_ERR(tsk)) { 615 err = PTR_ERR(tsk); 616 goto err; 617 } 618 get_task_struct(tsk); 619 yield_to(tsk, true); 620 621 for (i = fc.chain_length; i--; ) 622 dma_fence_signal(fc.fences[i]); 623 624 err = kthread_stop(tsk); 625 put_task_struct(tsk); 626 627 err: 628 fence_chains_fini(&fc); 629 return err; 630 } 631 632 static void randomise_fences(struct fence_chains *fc) 633 { 634 unsigned int count = fc->chain_length; 635 636 /* Fisher-Yates shuffle courtesy of Knuth */ 637 while (--count) { 638 unsigned int swp; 639 640 swp = prandom_u32_max(count + 1); 641 if (swp == count) 642 continue; 643 644 swap(fc->fences[count], fc->fences[swp]); 645 } 646 } 647 648 static int wait_random(void *arg) 649 { 650 struct fence_chains fc; 651 struct task_struct *tsk; 652 int err; 653 int i; 654 655 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 656 if (err) 657 return err; 658 659 randomise_fences(&fc); 660 661 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 662 if (IS_ERR(tsk)) { 663 err = PTR_ERR(tsk); 664 goto err; 665 } 666 get_task_struct(tsk); 667 yield_to(tsk, true); 668 669 for (i = 0; i < fc.chain_length; i++) 670 dma_fence_signal(fc.fences[i]); 671 672 err = kthread_stop(tsk); 673 put_task_struct(tsk); 674 675 err: 676 fence_chains_fini(&fc); 677 return err; 678 } 679 680 int dma_fence_chain(void) 681 { 682 static const struct subtest tests[] = { 683 SUBTEST(sanitycheck), 684 SUBTEST(find_seqno), 685 SUBTEST(find_signaled), 686 SUBTEST(find_out_of_order), 687 SUBTEST(find_gap), 688 SUBTEST(find_race), 689 SUBTEST(signal_forward), 690 SUBTEST(signal_backward), 691 SUBTEST(wait_forward), 692 SUBTEST(wait_backward), 693 SUBTEST(wait_random), 694 }; 695 int ret; 696 697 pr_info("sizeof(dma_fence_chain)=%zu\n", 698 sizeof(struct dma_fence_chain)); 699 700 slab_fences = KMEM_CACHE(mock_fence, 701 SLAB_TYPESAFE_BY_RCU | 702 SLAB_HWCACHE_ALIGN); 703 if (!slab_fences) 704 return -ENOMEM; 705 706 ret = subtests(tests, NULL); 707 708 kmem_cache_destroy(slab_fences); 709 return ret; 710 } 711