linux/drivers/dma-buf/st-dma-fence.c
<<
>>
Prefs
   1/* SPDX-License-Identifier: MIT */
   2
   3/*
   4 * Copyright \xC2\xA9 2019 Intel Corporation
   5 */
   6
   7#include <linux/delay.h>
   8#include <linux/dma-fence.h>
   9#include <linux/kernel.h>
  10#include <linux/kthread.h>
  11#include <linux/sched/signal.h>
  12#include <linux/slab.h>
  13#include <linux/spinlock.h>
  14
  15#include "selftest.h"
  16
  17static struct kmem_cache *slab_fences;
  18
  19static struct mock_fence {
  20        struct dma_fence base;
  21        struct spinlock lock;
  22} *to_mock_fence(struct dma_fence *f) {
  23        return container_of(f, struct mock_fence, base);
  24}
  25
  26static const char *mock_name(struct dma_fence *f)
  27{
  28        return "mock";
  29}
  30
  31static void mock_fence_release(struct dma_fence *f)
  32{
  33        kmem_cache_free(slab_fences, to_mock_fence(f));
  34}
  35
  36struct wait_cb {
  37        struct dma_fence_cb cb;
  38        struct task_struct *task;
  39};
  40
  41static void mock_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
  42{
  43        wake_up_process(container_of(cb, struct wait_cb, cb)->task);
  44}
  45
  46static long mock_wait(struct dma_fence *f, bool intr, long timeout)
  47{
  48        const int state = intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
  49        struct wait_cb cb = { .task = current };
  50
  51        if (dma_fence_add_callback(f, &cb.cb, mock_wakeup))
  52                return timeout;
  53
  54        while (timeout) {
  55                set_current_state(state);
  56
  57                if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
  58                        break;
  59
  60                if (signal_pending_state(state, current))
  61                        break;
  62
  63                timeout = schedule_timeout(timeout);
  64        }
  65        __set_current_state(TASK_RUNNING);
  66
  67        if (!dma_fence_remove_callback(f, &cb.cb))
  68                return timeout;
  69
  70        if (signal_pending_state(state, current))
  71                return -ERESTARTSYS;
  72
  73        return -ETIME;
  74}
  75
  76static const struct dma_fence_ops mock_ops = {
  77        .get_driver_name = mock_name,
  78        .get_timeline_name = mock_name,
  79        .wait = mock_wait,
  80        .release = mock_fence_release,
  81};
  82
  83static struct dma_fence *mock_fence(void)
  84{
  85        struct mock_fence *f;
  86
  87        f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
  88        if (!f)
  89                return NULL;
  90
  91        spin_lock_init(&f->lock);
  92        dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
  93
  94        return &f->base;
  95}
  96
  97static int sanitycheck(void *arg)
  98{
  99        struct dma_fence *f;
 100
 101        f = mock_fence();
 102        if (!f)
 103                return -ENOMEM;
 104
 105        dma_fence_signal(f);
 106        dma_fence_put(f);
 107
 108        return 0;
 109}
 110
 111static int test_signaling(void *arg)
 112{
 113        struct dma_fence *f;
 114        int err = -EINVAL;
 115
 116        f = mock_fence();
 117        if (!f)
 118                return -ENOMEM;
 119
 120        if (dma_fence_is_signaled(f)) {
 121                pr_err("Fence unexpectedly signaled on creation\n");
 122                goto err_free;
 123        }
 124
 125        if (dma_fence_signal(f)) {
 126                pr_err("Fence reported being already signaled\n");
 127                goto err_free;
 128        }
 129
 130        if (!dma_fence_is_signaled(f)) {
 131                pr_err("Fence not reporting signaled\n");
 132                goto err_free;
 133        }
 134
 135        if (!dma_fence_signal(f)) {
 136                pr_err("Fence reported not being already signaled\n");
 137                goto err_free;
 138        }
 139
 140        err = 0;
 141err_free:
 142        dma_fence_put(f);
 143        return err;
 144}
 145
 146struct simple_cb {
 147        struct dma_fence_cb cb;
 148        bool seen;
 149};
 150
 151static void simple_callback(struct dma_fence *f, struct dma_fence_cb *cb)
 152{
 153        smp_store_mb(container_of(cb, struct simple_cb, cb)->seen, true);
 154}
 155
 156static int test_add_callback(void *arg)
 157{
 158        struct simple_cb cb = {};
 159        struct dma_fence *f;
 160        int err = -EINVAL;
 161
 162        f = mock_fence();
 163        if (!f)
 164                return -ENOMEM;
 165
 166        if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
 167                pr_err("Failed to add callback, fence already signaled!\n");
 168                goto err_free;
 169        }
 170
 171        dma_fence_signal(f);
 172        if (!cb.seen) {
 173                pr_err("Callback failed!\n");
 174                goto err_free;
 175        }
 176
 177        err = 0;
 178err_free:
 179        dma_fence_put(f);
 180        return err;
 181}
 182
 183static int test_late_add_callback(void *arg)
 184{
 185        struct simple_cb cb = {};
 186        struct dma_fence *f;
 187        int err = -EINVAL;
 188
 189        f = mock_fence();
 190        if (!f)
 191                return -ENOMEM;
 192
 193        dma_fence_signal(f);
 194
 195        if (!dma_fence_add_callback(f, &cb.cb, simple_callback)) {
 196                pr_err("Added callback, but fence was already signaled!\n");
 197                goto err_free;
 198        }
 199
 200        dma_fence_signal(f);
 201        if (cb.seen) {
 202                pr_err("Callback called after failed attachment !\n");
 203                goto err_free;
 204        }
 205
 206        err = 0;
 207err_free:
 208        dma_fence_put(f);
 209        return err;
 210}
 211
 212static int test_rm_callback(void *arg)
 213{
 214        struct simple_cb cb = {};
 215        struct dma_fence *f;
 216        int err = -EINVAL;
 217
 218        f = mock_fence();
 219        if (!f)
 220                return -ENOMEM;
 221
 222        if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
 223                pr_err("Failed to add callback, fence already signaled!\n");
 224                goto err_free;
 225        }
 226
 227        if (!dma_fence_remove_callback(f, &cb.cb)) {
 228                pr_err("Failed to remove callback!\n");
 229                goto err_free;
 230        }
 231
 232        dma_fence_signal(f);
 233        if (cb.seen) {
 234                pr_err("Callback still signaled after removal!\n");
 235                goto err_free;
 236        }
 237
 238        err = 0;
 239err_free:
 240        dma_fence_put(f);
 241        return err;
 242}
 243
 244static int test_late_rm_callback(void *arg)
 245{
 246        struct simple_cb cb = {};
 247        struct dma_fence *f;
 248        int err = -EINVAL;
 249
 250        f = mock_fence();
 251        if (!f)
 252                return -ENOMEM;
 253
 254        if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
 255                pr_err("Failed to add callback, fence already signaled!\n");
 256                goto err_free;
 257        }
 258
 259        dma_fence_signal(f);
 260        if (!cb.seen) {
 261                pr_err("Callback failed!\n");
 262                goto err_free;
 263        }
 264
 265        if (dma_fence_remove_callback(f, &cb.cb)) {
 266                pr_err("Callback removal succeed after being executed!\n");
 267                goto err_free;
 268        }
 269
 270        err = 0;
 271err_free:
 272        dma_fence_put(f);
 273        return err;
 274}
 275
 276static int test_status(void *arg)
 277{
 278        struct dma_fence *f;
 279        int err = -EINVAL;
 280
 281        f = mock_fence();
 282        if (!f)
 283                return -ENOMEM;
 284
 285        if (dma_fence_get_status(f)) {
 286                pr_err("Fence unexpectedly has signaled status on creation\n");
 287                goto err_free;
 288        }
 289
 290        dma_fence_signal(f);
 291        if (!dma_fence_get_status(f)) {
 292                pr_err("Fence not reporting signaled status\n");
 293                goto err_free;
 294        }
 295
 296        err = 0;
 297err_free:
 298        dma_fence_put(f);
 299        return err;
 300}
 301
 302static int test_error(void *arg)
 303{
 304        struct dma_fence *f;
 305        int err = -EINVAL;
 306
 307        f = mock_fence();
 308        if (!f)
 309                return -ENOMEM;
 310
 311        dma_fence_set_error(f, -EIO);
 312
 313        if (dma_fence_get_status(f)) {
 314                pr_err("Fence unexpectedly has error status before signal\n");
 315                goto err_free;
 316        }
 317
 318        dma_fence_signal(f);
 319        if (dma_fence_get_status(f) != -EIO) {
 320                pr_err("Fence not reporting error status, got %d\n",
 321                       dma_fence_get_status(f));
 322                goto err_free;
 323        }
 324
 325        err = 0;
 326err_free:
 327        dma_fence_put(f);
 328        return err;
 329}
 330
 331static int test_wait(void *arg)
 332{
 333        struct dma_fence *f;
 334        int err = -EINVAL;
 335
 336        f = mock_fence();
 337        if (!f)
 338                return -ENOMEM;
 339
 340        if (dma_fence_wait_timeout(f, false, 0) != -ETIME) {
 341                pr_err("Wait reported complete before being signaled\n");
 342                goto err_free;
 343        }
 344
 345        dma_fence_signal(f);
 346
 347        if (dma_fence_wait_timeout(f, false, 0) != 0) {
 348                pr_err("Wait reported incomplete after being signaled\n");
 349                goto err_free;
 350        }
 351
 352        err = 0;
 353err_free:
 354        dma_fence_signal(f);
 355        dma_fence_put(f);
 356        return err;
 357}
 358
 359struct wait_timer {
 360        struct timer_list timer;
 361        struct dma_fence *f;
 362};
 363
 364static void wait_timer(struct timer_list *timer)
 365{
 366        struct wait_timer *wt = from_timer(wt, timer, timer);
 367
 368        dma_fence_signal(wt->f);
 369}
 370
 371static int test_wait_timeout(void *arg)
 372{
 373        struct wait_timer wt;
 374        int err = -EINVAL;
 375
 376        timer_setup_on_stack(&wt.timer, wait_timer, 0);
 377
 378        wt.f = mock_fence();
 379        if (!wt.f)
 380                return -ENOMEM;
 381
 382        if (dma_fence_wait_timeout(wt.f, false, 1) != -ETIME) {
 383                pr_err("Wait reported complete before being signaled\n");
 384                goto err_free;
 385        }
 386
 387        mod_timer(&wt.timer, jiffies + 1);
 388
 389        if (dma_fence_wait_timeout(wt.f, false, 2) == -ETIME) {
 390                if (timer_pending(&wt.timer)) {
 391                        pr_notice("Timer did not fire within the jiffie!\n");
 392                        err = 0; /* not our fault! */
 393                } else {
 394                        pr_err("Wait reported incomplete after timeout\n");
 395                }
 396                goto err_free;
 397        }
 398
 399        err = 0;
 400err_free:
 401        del_timer_sync(&wt.timer);
 402        destroy_timer_on_stack(&wt.timer);
 403        dma_fence_signal(wt.f);
 404        dma_fence_put(wt.f);
 405        return err;
 406}
 407
 408static int test_stub(void *arg)
 409{
 410        struct dma_fence *f[64];
 411        int err = -EINVAL;
 412        int i;
 413
 414        for (i = 0; i < ARRAY_SIZE(f); i++) {
 415                f[i] = dma_fence_get_stub();
 416                if (!dma_fence_is_signaled(f[i])) {
 417                        pr_err("Obtained unsignaled stub fence!\n");
 418                        goto err;
 419                }
 420        }
 421
 422        err = 0;
 423err:
 424        while (i--)
 425                dma_fence_put(f[i]);
 426        return err;
 427}
 428
 429/* Now off to the races! */
 430
 431struct race_thread {
 432        struct dma_fence __rcu **fences;
 433        struct task_struct *task;
 434        bool before;
 435        int id;
 436};
 437
 438static void __wait_for_callbacks(struct dma_fence *f)
 439{
 440        spin_lock_irq(f->lock);
 441        spin_unlock_irq(f->lock);
 442}
 443
 444static int thread_signal_callback(void *arg)
 445{
 446        const struct race_thread *t = arg;
 447        unsigned long pass = 0;
 448        unsigned long miss = 0;
 449        int err = 0;
 450
 451        while (!err && !kthread_should_stop()) {
 452                struct dma_fence *f1, *f2;
 453                struct simple_cb cb;
 454
 455                f1 = mock_fence();
 456                if (!f1) {
 457                        err = -ENOMEM;
 458                        break;
 459                }
 460
 461                rcu_assign_pointer(t->fences[t->id], f1);
 462                smp_wmb();
 463
 464                rcu_read_lock();
 465                do {
 466                        f2 = dma_fence_get_rcu_safe(&t->fences[!t->id]);
 467                } while (!f2 && !kthread_should_stop());
 468                rcu_read_unlock();
 469
 470                if (t->before)
 471                        dma_fence_signal(f1);
 472
 473                smp_store_mb(cb.seen, false);
 474                if (!f2 ||
 475                    dma_fence_add_callback(f2, &cb.cb, simple_callback)) {
 476                        miss++;
 477                        cb.seen = true;
 478                }
 479
 480                if (!t->before)
 481                        dma_fence_signal(f1);
 482
 483                if (!cb.seen) {
 484                        dma_fence_wait(f2, false);
 485                        __wait_for_callbacks(f2);
 486                }
 487
 488                if (!READ_ONCE(cb.seen)) {
 489                        pr_err("Callback not seen on thread %d, pass %lu (%lu misses), signaling %s add_callback; fence signaled? %s\n",
 490                               t->id, pass, miss,
 491                               t->before ? "before" : "after",
 492                               dma_fence_is_signaled(f2) ? "yes" : "no");
 493                        err = -EINVAL;
 494                }
 495
 496                dma_fence_put(f2);
 497
 498                rcu_assign_pointer(t->fences[t->id], NULL);
 499                smp_wmb();
 500
 501                dma_fence_put(f1);
 502
 503                pass++;
 504        }
 505
 506        pr_info("%s[%d] completed %lu passes, %lu misses\n",
 507                __func__, t->id, pass, miss);
 508        return err;
 509}
 510
 511static int race_signal_callback(void *arg)
 512{
 513        struct dma_fence __rcu *f[2] = {};
 514        int ret = 0;
 515        int pass;
 516
 517        for (pass = 0; !ret && pass <= 1; pass++) {
 518                struct race_thread t[2];
 519                int i;
 520
 521                for (i = 0; i < ARRAY_SIZE(t); i++) {
 522                        t[i].fences = f;
 523                        t[i].id = i;
 524                        t[i].before = pass;
 525                        t[i].task = kthread_run(thread_signal_callback, &t[i],
 526                                                "dma-fence:%d", i);
 527                        get_task_struct(t[i].task);
 528                }
 529
 530                msleep(50);
 531
 532                for (i = 0; i < ARRAY_SIZE(t); i++) {
 533                        int err;
 534
 535                        err = kthread_stop(t[i].task);
 536                        if (err && !ret)
 537                                ret = err;
 538
 539                        put_task_struct(t[i].task);
 540                }
 541        }
 542
 543        return ret;
 544}
 545
 546int dma_fence(void)
 547{
 548        static const struct subtest tests[] = {
 549                SUBTEST(sanitycheck),
 550                SUBTEST(test_signaling),
 551                SUBTEST(test_add_callback),
 552                SUBTEST(test_late_add_callback),
 553                SUBTEST(test_rm_callback),
 554                SUBTEST(test_late_rm_callback),
 555                SUBTEST(test_status),
 556                SUBTEST(test_error),
 557                SUBTEST(test_wait),
 558                SUBTEST(test_wait_timeout),
 559                SUBTEST(test_stub),
 560                SUBTEST(race_signal_callback),
 561        };
 562        int ret;
 563
 564        pr_info("sizeof(dma_fence)=%zu\n", sizeof(struct dma_fence));
 565
 566        slab_fences = KMEM_CACHE(mock_fence,
 567                                 SLAB_TYPESAFE_BY_RCU |
 568                                 SLAB_HWCACHE_ALIGN);
 569        if (!slab_fences)
 570                return -ENOMEM;
 571
 572        ret = subtests(tests, NULL);
 573
 574        kmem_cache_destroy(slab_fences);
 575
 576        return ret;
 577}
 578