drivers/dma-buf/st-dma-fence-chain.c

   1 // SPDX-License-Identifier: MIT
   2
   3 /*
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include <linux/delay.h>
   8 #include <linux/dma-fence.h>
   9 #include <linux/dma-fence-chain.h>
  10 #include <linux/kernel.h>
  11 #include <linux/kthread.h>
  12 #include <linux/mm.h>
  13 #include <linux/sched/signal.h>
  14 #include <linux/slab.h>
  15 #include <linux/spinlock.h>
  16 #include <linux/random.h>
  17
  18 #include "selftest.h"
  19
  20 #define CHAIN_SZ (4 << 10)
  21
  22 static struct kmem_cache *slab_fences;
  23
  24 static inline struct mock_fence {
  25         struct dma_fence base;
  26         spinlock_t lock;
  27 } *to_mock_fence(struct dma_fence *f) {
  28         return container_of(f, struct mock_fence, base);
  29 }
  30
  31 static const char *mock_name(struct dma_fence *f)
  32 {
  33         return "mock";
  34 }
  35
  36 static void mock_fence_release(struct dma_fence *f)
  37 {
  38         kmem_cache_free(slab_fences, to_mock_fence(f));
  39 }
  40
  41 static const struct dma_fence_ops mock_ops = {
  42         .get_driver_name = mock_name,
  43         .get_timeline_name = mock_name,
  44         .release = mock_fence_release,
  45 };
  46
  47 static struct dma_fence *mock_fence(void)
  48 {
  49         struct mock_fence *f;
  50
  51         f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
  52         if (!f)
  53                 return NULL;
  54
  55         spin_lock_init(&f->lock);
  56         dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
  57
  58         return &f->base;
  59 }
  60
  61 static inline struct mock_chain {
  62         struct dma_fence_chain base;
  63 } *to_mock_chain(struct dma_fence *f) {
  64         return container_of(f, struct mock_chain, base.base);
  65 }
  66
  67 static struct dma_fence *mock_chain(struct dma_fence *prev,
  68                                     struct dma_fence *fence,
  69                                     u64 seqno)
  70 {
  71         struct mock_chain *f;
  72
  73         f = kmalloc(sizeof(*f), GFP_KERNEL);
  74         if (!f)
  75                 return NULL;
  76
  77         dma_fence_chain_init(&f->base,
  78                              dma_fence_get(prev),
  79                              dma_fence_get(fence),
  80                              seqno);
  81
  82         return &f->base.base;
  83 }
  84
  85 static int sanitycheck(void *arg)
  86 {
  87         struct dma_fence *f, *chain;
  88         int err = 0;
  89
  90         f = mock_fence();
  91         if (!f)
  92                 return -ENOMEM;
  93
  94         chain = mock_chain(NULL, f, 1);
  95         if (!chain)
  96                 err = -ENOMEM;
  97
  98         dma_fence_signal(f);
  99         dma_fence_put(f);
 100
 101         dma_fence_put(chain);
 102
 103         return err;
 104 }
 105
 106 struct fence_chains {
 107         unsigned int chain_length;
 108         struct dma_fence **fences;
 109         struct dma_fence **chains;
 110
 111         struct dma_fence *tail;
 112 };
 113
 114 static uint64_t seqno_inc(unsigned int i)
 115 {
 116         return i + 1;
 117 }
 118
 119 static int fence_chains_init(struct fence_chains *fc, unsigned int count,
 120                              uint64_t (*seqno_fn)(unsigned int))
 121 {
 122         unsigned int i;
 123         int err = 0;
 124
 125         fc->chains = kvmalloc_array(count, sizeof(*fc->chains),
 126                                     GFP_KERNEL | __GFP_ZERO);
 127         if (!fc->chains)
 128                 return -ENOMEM;
 129
 130         fc->fences = kvmalloc_array(count, sizeof(*fc->fences),
 131                                     GFP_KERNEL | __GFP_ZERO);
 132         if (!fc->fences) {
 133                 err = -ENOMEM;
 134                 goto err_chains;
 135         }
 136
 137         fc->tail = NULL;
 138         for (i = 0; i < count; i++) {
 139                 fc->fences[i] = mock_fence();
 140                 if (!fc->fences[i]) {
 141                         err = -ENOMEM;
 142                         goto unwind;
 143                 }
 144
 145                 fc->chains[i] = mock_chain(fc->tail,
 146                                            fc->fences[i],
 147                                            seqno_fn(i));
 148                 if (!fc->chains[i]) {
 149                         err = -ENOMEM;
 150                         goto unwind;
 151                 }
 152
 153                 fc->tail = fc->chains[i];
 154         }
 155
 156         fc->chain_length = i;
 157         return 0;
 158
 159 unwind:
 160         for (i = 0; i < count; i++) {
 161                 dma_fence_put(fc->fences[i]);
 162                 dma_fence_put(fc->chains[i]);
 163         }
 164         kvfree(fc->fences);
 165 err_chains:
 166         kvfree(fc->chains);
 167         return err;
 168 }
 169
 170 static void fence_chains_fini(struct fence_chains *fc)
 171 {
 172         unsigned int i;
 173
 174         for (i = 0; i < fc->chain_length; i++) {
 175                 dma_fence_signal(fc->fences[i]);
 176                 dma_fence_put(fc->fences[i]);
 177         }
 178         kvfree(fc->fences);
 179
 180         for (i = 0; i < fc->chain_length; i++)
 181                 dma_fence_put(fc->chains[i]);
 182         kvfree(fc->chains);
 183 }
 184
 185 static int find_seqno(void *arg)
 186 {
 187         struct fence_chains fc;
 188         struct dma_fence *fence;
 189         int err;
 190         int i;
 191
 192         err = fence_chains_init(&fc, 64, seqno_inc);
 193         if (err)
 194                 return err;
 195
 196         fence = dma_fence_get(fc.tail);
 197         err = dma_fence_chain_find_seqno(&fence, 0);
 198         dma_fence_put(fence);
 199         if (err) {
 200                 pr_err("Reported %d for find_seqno(0)!\n", err);
 201                 goto err;
 202         }
 203
 204         for (i = 0; i < fc.chain_length; i++) {
 205                 fence = dma_fence_get(fc.tail);
 206                 err = dma_fence_chain_find_seqno(&fence, i + 1);
 207                 dma_fence_put(fence);
 208                 if (err) {
 209                         pr_err("Reported %d for find_seqno(%d:%d)!\n",
 210                                err, fc.chain_length + 1, i + 1);
 211                         goto err;
 212                 }
 213                 if (fence != fc.chains[i]) {
 214                         pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
 215                                fc.chain_length + 1, i + 1);
 216                         err = -EINVAL;
 217                         goto err;
 218                 }
 219
 220                 dma_fence_get(fence);
 221                 err = dma_fence_chain_find_seqno(&fence, i + 1);
 222                 dma_fence_put(fence);
 223                 if (err) {
 224                         pr_err("Error reported for finding self\n");
 225                         goto err;
 226                 }
 227                 if (fence != fc.chains[i]) {
 228                         pr_err("Incorrect fence reported by find self\n");
 229                         err = -EINVAL;
 230                         goto err;
 231                 }
 232
 233                 dma_fence_get(fence);
 234                 err = dma_fence_chain_find_seqno(&fence, i + 2);
 235                 dma_fence_put(fence);
 236                 if (!err) {
 237                         pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n",
 238                                i + 1, i + 2);
 239                         err = -EINVAL;
 240                         goto err;
 241                 }
 242
 243                 dma_fence_get(fence);
 244                 err = dma_fence_chain_find_seqno(&fence, i);
 245                 dma_fence_put(fence);
 246                 if (err) {
 247                         pr_err("Error reported for previous fence!\n");
 248                         goto err;
 249                 }
 250                 if (i > 0 && fence != fc.chains[i - 1]) {
 251                         pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
 252                                i + 1, i);
 253                         err = -EINVAL;
 254                         goto err;
 255                 }
 256         }
 257
 258 err:
 259         fence_chains_fini(&fc);
 260         return err;
 261 }
 262
 263 static int find_signaled(void *arg)
 264 {
 265         struct fence_chains fc;
 266         struct dma_fence *fence;
 267         int err;
 268
 269         err = fence_chains_init(&fc, 2, seqno_inc);
 270         if (err)
 271                 return err;
 272
 273         dma_fence_signal(fc.fences[0]);
 274
 275         fence = dma_fence_get(fc.tail);
 276         err = dma_fence_chain_find_seqno(&fence, 1);
 277         dma_fence_put(fence);
 278         if (err) {
 279                 pr_err("Reported %d for find_seqno()!\n", err);
 280                 goto err;
 281         }
 282
 283         if (fence && fence != fc.chains[0]) {
 284                 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n",
 285                        fence->seqno);
 286
 287                 dma_fence_get(fence);
 288                 err = dma_fence_chain_find_seqno(&fence, 1);
 289                 dma_fence_put(fence);
 290                 if (err)
 291                         pr_err("Reported %d for finding self!\n", err);
 292
 293                 err = -EINVAL;
 294         }
 295
 296 err:
 297         fence_chains_fini(&fc);
 298         return err;
 299 }
 300
 301 static int find_out_of_order(void *arg)
 302 {
 303         struct fence_chains fc;
 304         struct dma_fence *fence;
 305         int err;
 306
 307         err = fence_chains_init(&fc, 3, seqno_inc);
 308         if (err)
 309                 return err;
 310
 311         dma_fence_signal(fc.fences[1]);
 312
 313         fence = dma_fence_get(fc.tail);
 314         err = dma_fence_chain_find_seqno(&fence, 2);
 315         dma_fence_put(fence);
 316         if (err) {
 317                 pr_err("Reported %d for find_seqno()!\n", err);
 318                 goto err;
 319         }
 320
 321         if (fence && fence != fc.chains[1]) {
 322                 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n",
 323                        fence->seqno);
 324
 325                 dma_fence_get(fence);
 326                 err = dma_fence_chain_find_seqno(&fence, 2);
 327                 dma_fence_put(fence);
 328                 if (err)
 329                         pr_err("Reported %d for finding self!\n", err);
 330
 331                 err = -EINVAL;
 332         }
 333
 334 err:
 335         fence_chains_fini(&fc);
 336         return err;
 337 }
 338
 339 static uint64_t seqno_inc2(unsigned int i)
 340 {
 341         return 2 * i + 2;
 342 }
 343
 344 static int find_gap(void *arg)
 345 {
 346         struct fence_chains fc;
 347         struct dma_fence *fence;
 348         int err;
 349         int i;
 350
 351         err = fence_chains_init(&fc, 64, seqno_inc2);
 352         if (err)
 353                 return err;
 354
 355         for (i = 0; i < fc.chain_length; i++) {
 356                 fence = dma_fence_get(fc.tail);
 357                 err = dma_fence_chain_find_seqno(&fence, 2 * i + 1);
 358                 dma_fence_put(fence);
 359                 if (err) {
 360                         pr_err("Reported %d for find_seqno(%d:%d)!\n",
 361                                err, fc.chain_length + 1, 2 * i + 1);
 362                         goto err;
 363                 }
 364                 if (fence != fc.chains[i]) {
 365                         pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n",
 366                                fence->seqno,
 367                                fc.chain_length + 1,
 368                                2 * i + 1);
 369                         err = -EINVAL;
 370                         goto err;
 371                 }
 372
 373                 dma_fence_get(fence);
 374                 err = dma_fence_chain_find_seqno(&fence, 2 * i + 2);
 375                 dma_fence_put(fence);
 376                 if (err) {
 377                         pr_err("Error reported for finding self\n");
 378                         goto err;
 379                 }
 380                 if (fence != fc.chains[i]) {
 381                         pr_err("Incorrect fence reported by find self\n");
 382                         err = -EINVAL;
 383                         goto err;
 384                 }
 385         }
 386
 387 err:
 388         fence_chains_fini(&fc);
 389         return err;
 390 }
 391
 392 struct find_race {
 393         struct fence_chains fc;
 394         atomic_t children;
 395 };
 396
 397 static int __find_race(void *arg)
 398 {
 399         struct find_race *data = arg;
 400         int err = 0;
 401
 402         while (!kthread_should_stop()) {
 403                 struct dma_fence *fence = dma_fence_get(data->fc.tail);
 404                 int seqno;
 405
 406                 seqno = prandom_u32_max(data->fc.chain_length) + 1;
 407
 408                 err = dma_fence_chain_find_seqno(&fence, seqno);
 409                 if (err) {
 410                         pr_err("Failed to find fence seqno:%d\n",
 411                                seqno);
 412                         dma_fence_put(fence);
 413                         break;
 414                 }
 415                 if (!fence)
 416                         goto signal;
 417
 418                 err = dma_fence_chain_find_seqno(&fence, seqno);
 419                 if (err) {
 420                         pr_err("Reported an invalid fence for find-self:%d\n",
 421                                seqno);
 422                         dma_fence_put(fence);
 423                         break;
 424                 }
 425
 426                 if (fence->seqno < seqno) {
 427                         pr_err("Reported an earlier fence.seqno:%lld for seqno:%d\n",
 428                                fence->seqno, seqno);
 429                         err = -EINVAL;
 430                         dma_fence_put(fence);
 431                         break;
 432                 }
 433
 434                 dma_fence_put(fence);
 435
 436 signal:
 437                 seqno = prandom_u32_max(data->fc.chain_length - 1);
 438                 dma_fence_signal(data->fc.fences[seqno]);
 439                 cond_resched();
 440         }
 441
 442         if (atomic_dec_and_test(&data->children))
 443                 wake_up_var(&data->children);
 444         return err;
 445 }
 446
 447 static int find_race(void *arg)
 448 {
 449         struct find_race data;
 450         int ncpus = num_online_cpus();
 451         struct task_struct **threads;
 452         unsigned long count;
 453         int err;
 454         int i;
 455
 456         err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc);
 457         if (err)
 458                 return err;
 459
 460         threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL);
 461         if (!threads) {
 462                 err = -ENOMEM;
 463                 goto err;
 464         }
 465
 466         atomic_set(&data.children, 0);
 467         for (i = 0; i < ncpus; i++) {
 468                 threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i);
 469                 if (IS_ERR(threads[i])) {
 470                         ncpus = i;
 471                         break;
 472                 }
 473                 atomic_inc(&data.children);
 474                 get_task_struct(threads[i]);
 475         }
 476
 477         wait_var_event_timeout(&data.children,
 478                                !atomic_read(&data.children),
 479                                5 * HZ);
 480
 481         for (i = 0; i < ncpus; i++) {
 482                 int ret;
 483
 484                 ret = kthread_stop(threads[i]);
 485                 if (ret && !err)
 486                         err = ret;
 487                 put_task_struct(threads[i]);
 488         }
 489         kfree(threads);
 490
 491         count = 0;
 492         for (i = 0; i < data.fc.chain_length; i++)
 493                 if (dma_fence_is_signaled(data.fc.fences[i]))
 494                         count++;
 495         pr_info("Completed %lu cycles\n", count);
 496
 497 err:
 498         fence_chains_fini(&data.fc);
 499         return err;
 500 }
 501
 502 static int signal_forward(void *arg)
 503 {
 504         struct fence_chains fc;
 505         int err;
 506         int i;
 507
 508         err = fence_chains_init(&fc, 64, seqno_inc);
 509         if (err)
 510                 return err;
 511
 512         for (i = 0; i < fc.chain_length; i++) {
 513                 dma_fence_signal(fc.fences[i]);
 514
 515                 if (!dma_fence_is_signaled(fc.chains[i])) {
 516                         pr_err("chain[%d] not signaled!\n", i);
 517                         err = -EINVAL;
 518                         goto err;
 519                 }
 520
 521                 if (i + 1 < fc.chain_length &&
 522                     dma_fence_is_signaled(fc.chains[i + 1])) {
 523                         pr_err("chain[%d] is signaled!\n", i);
 524                         err = -EINVAL;
 525                         goto err;
 526                 }
 527         }
 528
 529 err:
 530         fence_chains_fini(&fc);
 531         return err;
 532 }
 533
 534 static int signal_backward(void *arg)
 535 {
 536         struct fence_chains fc;
 537         int err;
 538         int i;
 539
 540         err = fence_chains_init(&fc, 64, seqno_inc);
 541         if (err)
 542                 return err;
 543
 544         for (i = fc.chain_length; i--; ) {
 545                 dma_fence_signal(fc.fences[i]);
 546
 547                 if (i > 0 && dma_fence_is_signaled(fc.chains[i])) {
 548                         pr_err("chain[%d] is signaled!\n", i);
 549                         err = -EINVAL;
 550                         goto err;
 551                 }
 552         }
 553
 554         for (i = 0; i < fc.chain_length; i++) {
 555                 if (!dma_fence_is_signaled(fc.chains[i])) {
 556                         pr_err("chain[%d] was not signaled!\n", i);
 557                         err = -EINVAL;
 558                         goto err;
 559                 }
 560         }
 561
 562 err:
 563         fence_chains_fini(&fc);
 564         return err;
 565 }
 566
 567 static int __wait_fence_chains(void *arg)
 568 {
 569         struct fence_chains *fc = arg;
 570
 571         if (dma_fence_wait(fc->tail, false))
 572                 return -EIO;
 573
 574         return 0;
 575 }
 576
 577 static int wait_forward(void *arg)
 578 {
 579         struct fence_chains fc;
 580         struct task_struct *tsk;
 581         int err;
 582         int i;
 583
 584         err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
 585         if (err)
 586                 return err;
 587
 588         tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
 589         if (IS_ERR(tsk)) {
 590                 err = PTR_ERR(tsk);
 591                 goto err;
 592         }
 593         get_task_struct(tsk);
 594         yield_to(tsk, true);
 595
 596         for (i = 0; i < fc.chain_length; i++)
 597                 dma_fence_signal(fc.fences[i]);
 598
 599         err = kthread_stop(tsk);
 600         put_task_struct(tsk);
 601
 602 err:
 603         fence_chains_fini(&fc);
 604         return err;
 605 }
 606
 607 static int wait_backward(void *arg)
 608 {
 609         struct fence_chains fc;
 610         struct task_struct *tsk;
 611         int err;
 612         int i;
 613
 614         err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
 615         if (err)
 616                 return err;
 617
 618         tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
 619         if (IS_ERR(tsk)) {
 620                 err = PTR_ERR(tsk);
 621                 goto err;
 622         }
 623         get_task_struct(tsk);
 624         yield_to(tsk, true);
 625
 626         for (i = fc.chain_length; i--; )
 627                 dma_fence_signal(fc.fences[i]);
 628
 629         err = kthread_stop(tsk);
 630         put_task_struct(tsk);
 631
 632 err:
 633         fence_chains_fini(&fc);
 634         return err;
 635 }
 636
 637 static void randomise_fences(struct fence_chains *fc)
 638 {
 639         unsigned int count = fc->chain_length;
 640
 641         /* Fisher-Yates shuffle courtesy of Knuth */
 642         while (--count) {
 643                 unsigned int swp;
 644
 645                 swp = prandom_u32_max(count + 1);
 646                 if (swp == count)
 647                         continue;
 648
 649                 swap(fc->fences[count], fc->fences[swp]);
 650         }
 651 }
 652
 653 static int wait_random(void *arg)
 654 {
 655         struct fence_chains fc;
 656         struct task_struct *tsk;
 657         int err;
 658         int i;
 659
 660         err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
 661         if (err)
 662                 return err;
 663
 664         randomise_fences(&fc);
 665
 666         tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
 667         if (IS_ERR(tsk)) {
 668                 err = PTR_ERR(tsk);
 669                 goto err;
 670         }
 671         get_task_struct(tsk);
 672         yield_to(tsk, true);
 673
 674         for (i = 0; i < fc.chain_length; i++)
 675                 dma_fence_signal(fc.fences[i]);
 676
 677         err = kthread_stop(tsk);
 678         put_task_struct(tsk);
 679
 680 err:
 681         fence_chains_fini(&fc);
 682         return err;
 683 }
 684
 685 int dma_fence_chain(void)
 686 {
 687         static const struct subtest tests[] = {
 688                 SUBTEST(sanitycheck),
 689                 SUBTEST(find_seqno),
 690                 SUBTEST(find_signaled),
 691                 SUBTEST(find_out_of_order),
 692                 SUBTEST(find_gap),
 693                 SUBTEST(find_race),
 694                 SUBTEST(signal_forward),
 695                 SUBTEST(signal_backward),
 696                 SUBTEST(wait_forward),
 697                 SUBTEST(wait_backward),
 698                 SUBTEST(wait_random),
 699         };
 700         int ret;
 701
 702         pr_info("sizeof(dma_fence_chain)=%zu\n",
 703                 sizeof(struct dma_fence_chain));
 704
 705         slab_fences = KMEM_CACHE(mock_fence,
 706                                  SLAB_TYPESAFE_BY_RCU |
 707                                  SLAB_HWCACHE_ALIGN);
 708         if (!slab_fences)
 709                 return -ENOMEM;
 710
 711         ret = subtests(tests, NULL);
 712
 713         kmem_cache_destroy(slab_fences);
 714         return ret;
 715 }