linux/drivers/dma-buf/dma-fence.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Fence mechanism for dma-buf and to allow for asynchronous dma access
   4 *
   5 * Copyright (C) 2012 Canonical Ltd
   6 * Copyright (C) 2012 Texas Instruments
   7 *
   8 * Authors:
   9 * Rob Clark <robdclark@gmail.com>
  10 * Maarten Lankhorst <maarten.lankhorst@canonical.com>
  11 */
  12
  13#include <linux/slab.h>
  14#include <linux/export.h>
  15#include <linux/atomic.h>
  16#include <linux/dma-fence.h>
  17#include <linux/sched/signal.h>
  18
  19#define CREATE_TRACE_POINTS
  20#include <trace/events/dma_fence.h>
  21
  22EXPORT_TRACEPOINT_SYMBOL(dma_fence_emit);
  23EXPORT_TRACEPOINT_SYMBOL(dma_fence_enable_signal);
  24EXPORT_TRACEPOINT_SYMBOL(dma_fence_signaled);
  25
  26static DEFINE_SPINLOCK(dma_fence_stub_lock);
  27static struct dma_fence dma_fence_stub;
  28
  29/*
  30 * fence context counter: each execution context should have its own
  31 * fence context, this allows checking if fences belong to the same
  32 * context or not. One device can have multiple separate contexts,
  33 * and they're used if some engine can run independently of another.
  34 */
  35static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(1);
  36
  37/**
  38 * DOC: DMA fences overview
  39 *
  40 * DMA fences, represented by &struct dma_fence, are the kernel internal
  41 * synchronization primitive for DMA operations like GPU rendering, video
  42 * encoding/decoding, or displaying buffers on a screen.
  43 *
  44 * A fence is initialized using dma_fence_init() and completed using
  45 * dma_fence_signal(). Fences are associated with a context, allocated through
  46 * dma_fence_context_alloc(), and all fences on the same context are
  47 * fully ordered.
  48 *
  49 * Since the purposes of fences is to facilitate cross-device and
  50 * cross-application synchronization, there's multiple ways to use one:
  51 *
  52 * - Individual fences can be exposed as a &sync_file, accessed as a file
  53 *   descriptor from userspace, created by calling sync_file_create(). This is
  54 *   called explicit fencing, since userspace passes around explicit
  55 *   synchronization points.
  56 *
  57 * - Some subsystems also have their own explicit fencing primitives, like
  58 *   &drm_syncobj. Compared to &sync_file, a &drm_syncobj allows the underlying
  59 *   fence to be updated.
  60 *
  61 * - Then there's also implicit fencing, where the synchronization points are
  62 *   implicitly passed around as part of shared &dma_buf instances. Such
  63 *   implicit fences are stored in &struct dma_resv through the
  64 *   &dma_buf.resv pointer.
  65 */
  66
  67/**
  68 * DOC: fence cross-driver contract
  69 *
  70 * Since &dma_fence provide a cross driver contract, all drivers must follow the
  71 * same rules:
  72 *
  73 * * Fences must complete in a reasonable time. Fences which represent kernels
  74 *   and shaders submitted by userspace, which could run forever, must be backed
  75 *   up by timeout and gpu hang recovery code. Minimally that code must prevent
  76 *   further command submission and force complete all in-flight fences, e.g.
  77 *   when the driver or hardware do not support gpu reset, or if the gpu reset
  78 *   failed for some reason. Ideally the driver supports gpu recovery which only
  79 *   affects the offending userspace context, and no other userspace
  80 *   submissions.
  81 *
  82 * * Drivers may have different ideas of what completion within a reasonable
  83 *   time means. Some hang recovery code uses a fixed timeout, others a mix
  84 *   between observing forward progress and increasingly strict timeouts.
  85 *   Drivers should not try to second guess timeout handling of fences from
  86 *   other drivers.
  87 *
  88 * * To ensure there's no deadlocks of dma_fence_wait() against other locks
  89 *   drivers should annotate all code required to reach dma_fence_signal(),
  90 *   which completes the fences, with dma_fence_begin_signalling() and
  91 *   dma_fence_end_signalling().
  92 *
  93 * * Drivers are allowed to call dma_fence_wait() while holding dma_resv_lock().
  94 *   This means any code required for fence completion cannot acquire a
  95 *   &dma_resv lock. Note that this also pulls in the entire established
  96 *   locking hierarchy around dma_resv_lock() and dma_resv_unlock().
  97 *
  98 * * Drivers are allowed to call dma_fence_wait() from their &shrinker
  99 *   callbacks. This means any code required for fence completion cannot
 100 *   allocate memory with GFP_KERNEL.
 101 *
 102 * * Drivers are allowed to call dma_fence_wait() from their &mmu_notifier
 103 *   respectively &mmu_interval_notifier callbacks. This means any code required
 104 *   for fence completeion cannot allocate memory with GFP_NOFS or GFP_NOIO.
 105 *   Only GFP_ATOMIC is permissible, which might fail.
 106 *
 107 * Note that only GPU drivers have a reasonable excuse for both requiring
 108 * &mmu_interval_notifier and &shrinker callbacks at the same time as having to
 109 * track asynchronous compute work using &dma_fence. No driver outside of
 110 * drivers/gpu should ever call dma_fence_wait() in such contexts.
 111 */
 112
 113static const char *dma_fence_stub_get_name(struct dma_fence *fence)
 114{
 115        return "stub";
 116}
 117
 118static const struct dma_fence_ops dma_fence_stub_ops = {
 119        .get_driver_name = dma_fence_stub_get_name,
 120        .get_timeline_name = dma_fence_stub_get_name,
 121};
 122
 123/**
 124 * dma_fence_get_stub - return a signaled fence
 125 *
 126 * Return a stub fence which is already signaled. The fence's
 127 * timestamp corresponds to the first time after boot this
 128 * function is called.
 129 */
 130struct dma_fence *dma_fence_get_stub(void)
 131{
 132        spin_lock(&dma_fence_stub_lock);
 133        if (!dma_fence_stub.ops) {
 134                dma_fence_init(&dma_fence_stub,
 135                               &dma_fence_stub_ops,
 136                               &dma_fence_stub_lock,
 137                               0, 0);
 138                dma_fence_signal_locked(&dma_fence_stub);
 139        }
 140        spin_unlock(&dma_fence_stub_lock);
 141
 142        return dma_fence_get(&dma_fence_stub);
 143}
 144EXPORT_SYMBOL(dma_fence_get_stub);
 145
 146/**
 147 * dma_fence_allocate_private_stub - return a private, signaled fence
 148 *
 149 * Return a newly allocated and signaled stub fence.
 150 */
 151struct dma_fence *dma_fence_allocate_private_stub(void)
 152{
 153        struct dma_fence *fence;
 154
 155        fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 156        if (fence == NULL)
 157                return ERR_PTR(-ENOMEM);
 158
 159        dma_fence_init(fence,
 160                       &dma_fence_stub_ops,
 161                       &dma_fence_stub_lock,
 162                       0, 0);
 163        dma_fence_signal(fence);
 164
 165        return fence;
 166}
 167EXPORT_SYMBOL(dma_fence_allocate_private_stub);
 168
 169/**
 170 * dma_fence_context_alloc - allocate an array of fence contexts
 171 * @num: amount of contexts to allocate
 172 *
 173 * This function will return the first index of the number of fence contexts
 174 * allocated.  The fence context is used for setting &dma_fence.context to a
 175 * unique number by passing the context to dma_fence_init().
 176 */
 177u64 dma_fence_context_alloc(unsigned num)
 178{
 179        WARN_ON(!num);
 180        return atomic64_fetch_add(num, &dma_fence_context_counter);
 181}
 182EXPORT_SYMBOL(dma_fence_context_alloc);
 183
 184/**
 185 * DOC: fence signalling annotation
 186 *
 187 * Proving correctness of all the kernel code around &dma_fence through code
 188 * review and testing is tricky for a few reasons:
 189 *
 190 * * It is a cross-driver contract, and therefore all drivers must follow the
 191 *   same rules for lock nesting order, calling contexts for various functions
 192 *   and anything else significant for in-kernel interfaces. But it is also
 193 *   impossible to test all drivers in a single machine, hence brute-force N vs.
 194 *   N testing of all combinations is impossible. Even just limiting to the
 195 *   possible combinations is infeasible.
 196 *
 197 * * There is an enormous amount of driver code involved. For render drivers
 198 *   there's the tail of command submission, after fences are published,
 199 *   scheduler code, interrupt and workers to process job completion,
 200 *   and timeout, gpu reset and gpu hang recovery code. Plus for integration
 201 *   with core mm with have &mmu_notifier, respectively &mmu_interval_notifier,
 202 *   and &shrinker. For modesetting drivers there's the commit tail functions
 203 *   between when fences for an atomic modeset are published, and when the
 204 *   corresponding vblank completes, including any interrupt processing and
 205 *   related workers. Auditing all that code, across all drivers, is not
 206 *   feasible.
 207 *
 208 * * Due to how many other subsystems are involved and the locking hierarchies
 209 *   this pulls in there is extremely thin wiggle-room for driver-specific
 210 *   differences. &dma_fence interacts with almost all of the core memory
 211 *   handling through page fault handlers via &dma_resv, dma_resv_lock() and
 212 *   dma_resv_unlock(). On the other side it also interacts through all
 213 *   allocation sites through &mmu_notifier and &shrinker.
 214 *
 215 * Furthermore lockdep does not handle cross-release dependencies, which means
 216 * any deadlocks between dma_fence_wait() and dma_fence_signal() can't be caught
 217 * at runtime with some quick testing. The simplest example is one thread
 218 * waiting on a &dma_fence while holding a lock::
 219 *
 220 *     lock(A);
 221 *     dma_fence_wait(B);
 222 *     unlock(A);
 223 *
 224 * while the other thread is stuck trying to acquire the same lock, which
 225 * prevents it from signalling the fence the previous thread is stuck waiting
 226 * on::
 227 *
 228 *     lock(A);
 229 *     unlock(A);
 230 *     dma_fence_signal(B);
 231 *
 232 * By manually annotating all code relevant to signalling a &dma_fence we can
 233 * teach lockdep about these dependencies, which also helps with the validation
 234 * headache since now lockdep can check all the rules for us::
 235 *
 236 *    cookie = dma_fence_begin_signalling();
 237 *    lock(A);
 238 *    unlock(A);
 239 *    dma_fence_signal(B);
 240 *    dma_fence_end_signalling(cookie);
 241 *
 242 * For using dma_fence_begin_signalling() and dma_fence_end_signalling() to
 243 * annotate critical sections the following rules need to be observed:
 244 *
 245 * * All code necessary to complete a &dma_fence must be annotated, from the
 246 *   point where a fence is accessible to other threads, to the point where
 247 *   dma_fence_signal() is called. Un-annotated code can contain deadlock issues,
 248 *   and due to the very strict rules and many corner cases it is infeasible to
 249 *   catch these just with review or normal stress testing.
 250 *
 251 * * &struct dma_resv deserves a special note, since the readers are only
 252 *   protected by rcu. This means the signalling critical section starts as soon
 253 *   as the new fences are installed, even before dma_resv_unlock() is called.
 254 *
 255 * * The only exception are fast paths and opportunistic signalling code, which
 256 *   calls dma_fence_signal() purely as an optimization, but is not required to
 257 *   guarantee completion of a &dma_fence. The usual example is a wait IOCTL
 258 *   which calls dma_fence_signal(), while the mandatory completion path goes
 259 *   through a hardware interrupt and possible job completion worker.
 260 *
 261 * * To aid composability of code, the annotations can be freely nested, as long
 262 *   as the overall locking hierarchy is consistent. The annotations also work
 263 *   both in interrupt and process context. Due to implementation details this
 264 *   requires that callers pass an opaque cookie from
 265 *   dma_fence_begin_signalling() to dma_fence_end_signalling().
 266 *
 267 * * Validation against the cross driver contract is implemented by priming
 268 *   lockdep with the relevant hierarchy at boot-up. This means even just
 269 *   testing with a single device is enough to validate a driver, at least as
 270 *   far as deadlocks with dma_fence_wait() against dma_fence_signal() are
 271 *   concerned.
 272 */
 273#ifdef CONFIG_LOCKDEP
 274static struct lockdep_map dma_fence_lockdep_map = {
 275        .name = "dma_fence_map"
 276};
 277
 278/**
 279 * dma_fence_begin_signalling - begin a critical DMA fence signalling section
 280 *
 281 * Drivers should use this to annotate the beginning of any code section
 282 * required to eventually complete &dma_fence by calling dma_fence_signal().
 283 *
 284 * The end of these critical sections are annotated with
 285 * dma_fence_end_signalling().
 286 *
 287 * Returns:
 288 *
 289 * Opaque cookie needed by the implementation, which needs to be passed to
 290 * dma_fence_end_signalling().
 291 */
 292bool dma_fence_begin_signalling(void)
 293{
 294        /* explicitly nesting ... */
 295        if (lock_is_held_type(&dma_fence_lockdep_map, 1))
 296                return true;
 297
 298        /* rely on might_sleep check for soft/hardirq locks */
 299        if (in_atomic())
 300                return true;
 301
 302        /* ... and non-recursive readlock */
 303        lock_acquire(&dma_fence_lockdep_map, 0, 0, 1, 1, NULL, _RET_IP_);
 304
 305        return false;
 306}
 307EXPORT_SYMBOL(dma_fence_begin_signalling);
 308
 309/**
 310 * dma_fence_end_signalling - end a critical DMA fence signalling section
 311 * @cookie: opaque cookie from dma_fence_begin_signalling()
 312 *
 313 * Closes a critical section annotation opened by dma_fence_begin_signalling().
 314 */
 315void dma_fence_end_signalling(bool cookie)
 316{
 317        if (cookie)
 318                return;
 319
 320        lock_release(&dma_fence_lockdep_map, _RET_IP_);
 321}
 322EXPORT_SYMBOL(dma_fence_end_signalling);
 323
 324void __dma_fence_might_wait(void)
 325{
 326        bool tmp;
 327
 328        tmp = lock_is_held_type(&dma_fence_lockdep_map, 1);
 329        if (tmp)
 330                lock_release(&dma_fence_lockdep_map, _THIS_IP_);
 331        lock_map_acquire(&dma_fence_lockdep_map);
 332        lock_map_release(&dma_fence_lockdep_map);
 333        if (tmp)
 334                lock_acquire(&dma_fence_lockdep_map, 0, 0, 1, 1, NULL, _THIS_IP_);
 335}
 336#endif
 337
 338
 339/**
 340 * dma_fence_signal_timestamp_locked - signal completion of a fence
 341 * @fence: the fence to signal
 342 * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain
 343 *
 344 * Signal completion for software callbacks on a fence, this will unblock
 345 * dma_fence_wait() calls and run all the callbacks added with
 346 * dma_fence_add_callback(). Can be called multiple times, but since a fence
 347 * can only go from the unsignaled to the signaled state and not back, it will
 348 * only be effective the first time. Set the timestamp provided as the fence
 349 * signal timestamp.
 350 *
 351 * Unlike dma_fence_signal_timestamp(), this function must be called with
 352 * &dma_fence.lock held.
 353 *
 354 * Returns 0 on success and a negative error value when @fence has been
 355 * signalled already.
 356 */
 357int dma_fence_signal_timestamp_locked(struct dma_fence *fence,
 358                                      ktime_t timestamp)
 359{
 360        struct dma_fence_cb *cur, *tmp;
 361        struct list_head cb_list;
 362
 363        lockdep_assert_held(fence->lock);
 364
 365        if (unlikely(test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
 366                                      &fence->flags)))
 367                return -EINVAL;
 368
 369        /* Stash the cb_list before replacing it with the timestamp */
 370        list_replace(&fence->cb_list, &cb_list);
 371
 372        fence->timestamp = timestamp;
 373        set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
 374        trace_dma_fence_signaled(fence);
 375
 376        list_for_each_entry_safe(cur, tmp, &cb_list, node) {
 377                INIT_LIST_HEAD(&cur->node);
 378                cur->func(fence, cur);
 379        }
 380
 381        return 0;
 382}
 383EXPORT_SYMBOL(dma_fence_signal_timestamp_locked);
 384
 385/**
 386 * dma_fence_signal_timestamp - signal completion of a fence
 387 * @fence: the fence to signal
 388 * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain
 389 *
 390 * Signal completion for software callbacks on a fence, this will unblock
 391 * dma_fence_wait() calls and run all the callbacks added with
 392 * dma_fence_add_callback(). Can be called multiple times, but since a fence
 393 * can only go from the unsignaled to the signaled state and not back, it will
 394 * only be effective the first time. Set the timestamp provided as the fence
 395 * signal timestamp.
 396 *
 397 * Returns 0 on success and a negative error value when @fence has been
 398 * signalled already.
 399 */
 400int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp)
 401{
 402        unsigned long flags;
 403        int ret;
 404
 405        if (!fence)
 406                return -EINVAL;
 407
 408        spin_lock_irqsave(fence->lock, flags);
 409        ret = dma_fence_signal_timestamp_locked(fence, timestamp);
 410        spin_unlock_irqrestore(fence->lock, flags);
 411
 412        return ret;
 413}
 414EXPORT_SYMBOL(dma_fence_signal_timestamp);
 415
 416/**
 417 * dma_fence_signal_locked - signal completion of a fence
 418 * @fence: the fence to signal
 419 *
 420 * Signal completion for software callbacks on a fence, this will unblock
 421 * dma_fence_wait() calls and run all the callbacks added with
 422 * dma_fence_add_callback(). Can be called multiple times, but since a fence
 423 * can only go from the unsignaled to the signaled state and not back, it will
 424 * only be effective the first time.
 425 *
 426 * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock
 427 * held.
 428 *
 429 * Returns 0 on success and a negative error value when @fence has been
 430 * signalled already.
 431 */
 432int dma_fence_signal_locked(struct dma_fence *fence)
 433{
 434        return dma_fence_signal_timestamp_locked(fence, ktime_get());
 435}
 436EXPORT_SYMBOL(dma_fence_signal_locked);
 437
 438/**
 439 * dma_fence_signal - signal completion of a fence
 440 * @fence: the fence to signal
 441 *
 442 * Signal completion for software callbacks on a fence, this will unblock
 443 * dma_fence_wait() calls and run all the callbacks added with
 444 * dma_fence_add_callback(). Can be called multiple times, but since a fence
 445 * can only go from the unsignaled to the signaled state and not back, it will
 446 * only be effective the first time.
 447 *
 448 * Returns 0 on success and a negative error value when @fence has been
 449 * signalled already.
 450 */
 451int dma_fence_signal(struct dma_fence *fence)
 452{
 453        unsigned long flags;
 454        int ret;
 455        bool tmp;
 456
 457        if (!fence)
 458                return -EINVAL;
 459
 460        tmp = dma_fence_begin_signalling();
 461
 462        spin_lock_irqsave(fence->lock, flags);
 463        ret = dma_fence_signal_timestamp_locked(fence, ktime_get());
 464        spin_unlock_irqrestore(fence->lock, flags);
 465
 466        dma_fence_end_signalling(tmp);
 467
 468        return ret;
 469}
 470EXPORT_SYMBOL(dma_fence_signal);
 471
 472/**
 473 * dma_fence_wait_timeout - sleep until the fence gets signaled
 474 * or until timeout elapses
 475 * @fence: the fence to wait on
 476 * @intr: if true, do an interruptible wait
 477 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
 478 *
 479 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
 480 * remaining timeout in jiffies on success. Other error values may be
 481 * returned on custom implementations.
 482 *
 483 * Performs a synchronous wait on this fence. It is assumed the caller
 484 * directly or indirectly (buf-mgr between reservation and committing)
 485 * holds a reference to the fence, otherwise the fence might be
 486 * freed before return, resulting in undefined behavior.
 487 *
 488 * See also dma_fence_wait() and dma_fence_wait_any_timeout().
 489 */
 490signed long
 491dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
 492{
 493        signed long ret;
 494
 495        if (WARN_ON(timeout < 0))
 496                return -EINVAL;
 497
 498        might_sleep();
 499
 500        __dma_fence_might_wait();
 501
 502        trace_dma_fence_wait_start(fence);
 503        if (fence->ops->wait)
 504                ret = fence->ops->wait(fence, intr, timeout);
 505        else
 506                ret = dma_fence_default_wait(fence, intr, timeout);
 507        trace_dma_fence_wait_end(fence);
 508        return ret;
 509}
 510EXPORT_SYMBOL(dma_fence_wait_timeout);
 511
 512/**
 513 * dma_fence_release - default relese function for fences
 514 * @kref: &dma_fence.recfount
 515 *
 516 * This is the default release functions for &dma_fence. Drivers shouldn't call
 517 * this directly, but instead call dma_fence_put().
 518 */
 519void dma_fence_release(struct kref *kref)
 520{
 521        struct dma_fence *fence =
 522                container_of(kref, struct dma_fence, refcount);
 523
 524        trace_dma_fence_destroy(fence);
 525
 526        if (WARN(!list_empty(&fence->cb_list) &&
 527                 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags),
 528                 "Fence %s:%s:%llx:%llx released with pending signals!\n",
 529                 fence->ops->get_driver_name(fence),
 530                 fence->ops->get_timeline_name(fence),
 531                 fence->context, fence->seqno)) {
 532                unsigned long flags;
 533
 534                /*
 535                 * Failed to signal before release, likely a refcounting issue.
 536                 *
 537                 * This should never happen, but if it does make sure that we
 538                 * don't leave chains dangling. We set the error flag first
 539                 * so that the callbacks know this signal is due to an error.
 540                 */
 541                spin_lock_irqsave(fence->lock, flags);
 542                fence->error = -EDEADLK;
 543                dma_fence_signal_locked(fence);
 544                spin_unlock_irqrestore(fence->lock, flags);
 545        }
 546
 547        if (fence->ops->release)
 548                fence->ops->release(fence);
 549        else
 550                dma_fence_free(fence);
 551}
 552EXPORT_SYMBOL(dma_fence_release);
 553
 554/**
 555 * dma_fence_free - default release function for &dma_fence.
 556 * @fence: fence to release
 557 *
 558 * This is the default implementation for &dma_fence_ops.release. It calls
 559 * kfree_rcu() on @fence.
 560 */
 561void dma_fence_free(struct dma_fence *fence)
 562{
 563        kfree_rcu(fence, rcu);
 564}
 565EXPORT_SYMBOL(dma_fence_free);
 566
 567static bool __dma_fence_enable_signaling(struct dma_fence *fence)
 568{
 569        bool was_set;
 570
 571        lockdep_assert_held(fence->lock);
 572
 573        was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 574                                   &fence->flags);
 575
 576        if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 577                return false;
 578
 579        if (!was_set && fence->ops->enable_signaling) {
 580                trace_dma_fence_enable_signal(fence);
 581
 582                if (!fence->ops->enable_signaling(fence)) {
 583                        dma_fence_signal_locked(fence);
 584                        return false;
 585                }
 586        }
 587
 588        return true;
 589}
 590
 591/**
 592 * dma_fence_enable_sw_signaling - enable signaling on fence
 593 * @fence: the fence to enable
 594 *
 595 * This will request for sw signaling to be enabled, to make the fence
 596 * complete as soon as possible. This calls &dma_fence_ops.enable_signaling
 597 * internally.
 598 */
 599void dma_fence_enable_sw_signaling(struct dma_fence *fence)
 600{
 601        unsigned long flags;
 602
 603        if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 604                return;
 605
 606        spin_lock_irqsave(fence->lock, flags);
 607        __dma_fence_enable_signaling(fence);
 608        spin_unlock_irqrestore(fence->lock, flags);
 609}
 610EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
 611
 612/**
 613 * dma_fence_add_callback - add a callback to be called when the fence
 614 * is signaled
 615 * @fence: the fence to wait on
 616 * @cb: the callback to register
 617 * @func: the function to call
 618 *
 619 * @cb will be initialized by dma_fence_add_callback(), no initialization
 620 * by the caller is required. Any number of callbacks can be registered
 621 * to a fence, but a callback can only be registered to one fence at a time.
 622 *
 623 * Note that the callback can be called from an atomic context.  If
 624 * fence is already signaled, this function will return -ENOENT (and
 625 * *not* call the callback).
 626 *
 627 * Add a software callback to the fence. Same restrictions apply to
 628 * refcount as it does to dma_fence_wait(), however the caller doesn't need to
 629 * keep a refcount to fence afterward dma_fence_add_callback() has returned:
 630 * when software access is enabled, the creator of the fence is required to keep
 631 * the fence alive until after it signals with dma_fence_signal(). The callback
 632 * itself can be called from irq context.
 633 *
 634 * Returns 0 in case of success, -ENOENT if the fence is already signaled
 635 * and -EINVAL in case of error.
 636 */
 637int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
 638                           dma_fence_func_t func)
 639{
 640        unsigned long flags;
 641        int ret = 0;
 642
 643        if (WARN_ON(!fence || !func))
 644                return -EINVAL;
 645
 646        if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
 647                INIT_LIST_HEAD(&cb->node);
 648                return -ENOENT;
 649        }
 650
 651        spin_lock_irqsave(fence->lock, flags);
 652
 653        if (__dma_fence_enable_signaling(fence)) {
 654                cb->func = func;
 655                list_add_tail(&cb->node, &fence->cb_list);
 656        } else {
 657                INIT_LIST_HEAD(&cb->node);
 658                ret = -ENOENT;
 659        }
 660
 661        spin_unlock_irqrestore(fence->lock, flags);
 662
 663        return ret;
 664}
 665EXPORT_SYMBOL(dma_fence_add_callback);
 666
 667/**
 668 * dma_fence_get_status - returns the status upon completion
 669 * @fence: the dma_fence to query
 670 *
 671 * This wraps dma_fence_get_status_locked() to return the error status
 672 * condition on a signaled fence. See dma_fence_get_status_locked() for more
 673 * details.
 674 *
 675 * Returns 0 if the fence has not yet been signaled, 1 if the fence has
 676 * been signaled without an error condition, or a negative error code
 677 * if the fence has been completed in err.
 678 */
 679int dma_fence_get_status(struct dma_fence *fence)
 680{
 681        unsigned long flags;
 682        int status;
 683
 684        spin_lock_irqsave(fence->lock, flags);
 685        status = dma_fence_get_status_locked(fence);
 686        spin_unlock_irqrestore(fence->lock, flags);
 687
 688        return status;
 689}
 690EXPORT_SYMBOL(dma_fence_get_status);
 691
 692/**
 693 * dma_fence_remove_callback - remove a callback from the signaling list
 694 * @fence: the fence to wait on
 695 * @cb: the callback to remove
 696 *
 697 * Remove a previously queued callback from the fence. This function returns
 698 * true if the callback is successfully removed, or false if the fence has
 699 * already been signaled.
 700 *
 701 * *WARNING*:
 702 * Cancelling a callback should only be done if you really know what you're
 703 * doing, since deadlocks and race conditions could occur all too easily. For
 704 * this reason, it should only ever be done on hardware lockup recovery,
 705 * with a reference held to the fence.
 706 *
 707 * Behaviour is undefined if @cb has not been added to @fence using
 708 * dma_fence_add_callback() beforehand.
 709 */
 710bool
 711dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
 712{
 713        unsigned long flags;
 714        bool ret;
 715
 716        spin_lock_irqsave(fence->lock, flags);
 717
 718        ret = !list_empty(&cb->node);
 719        if (ret)
 720                list_del_init(&cb->node);
 721
 722        spin_unlock_irqrestore(fence->lock, flags);
 723
 724        return ret;
 725}
 726EXPORT_SYMBOL(dma_fence_remove_callback);
 727
 728struct default_wait_cb {
 729        struct dma_fence_cb base;
 730        struct task_struct *task;
 731};
 732
 733static void
 734dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 735{
 736        struct default_wait_cb *wait =
 737                container_of(cb, struct default_wait_cb, base);
 738
 739        wake_up_state(wait->task, TASK_NORMAL);
 740}
 741
 742/**
 743 * dma_fence_default_wait - default sleep until the fence gets signaled
 744 * or until timeout elapses
 745 * @fence: the fence to wait on
 746 * @intr: if true, do an interruptible wait
 747 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
 748 *
 749 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
 750 * remaining timeout in jiffies on success. If timeout is zero the value one is
 751 * returned if the fence is already signaled for consistency with other
 752 * functions taking a jiffies timeout.
 753 */
 754signed long
 755dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
 756{
 757        struct default_wait_cb cb;
 758        unsigned long flags;
 759        signed long ret = timeout ? timeout : 1;
 760
 761        if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 762                return ret;
 763
 764        spin_lock_irqsave(fence->lock, flags);
 765
 766        if (intr && signal_pending(current)) {
 767                ret = -ERESTARTSYS;
 768                goto out;
 769        }
 770
 771        if (!__dma_fence_enable_signaling(fence))
 772                goto out;
 773
 774        if (!timeout) {
 775                ret = 0;
 776                goto out;
 777        }
 778
 779        cb.base.func = dma_fence_default_wait_cb;
 780        cb.task = current;
 781        list_add(&cb.base.node, &fence->cb_list);
 782
 783        while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
 784                if (intr)
 785                        __set_current_state(TASK_INTERRUPTIBLE);
 786                else
 787                        __set_current_state(TASK_UNINTERRUPTIBLE);
 788                spin_unlock_irqrestore(fence->lock, flags);
 789
 790                ret = schedule_timeout(ret);
 791
 792                spin_lock_irqsave(fence->lock, flags);
 793                if (ret > 0 && intr && signal_pending(current))
 794                        ret = -ERESTARTSYS;
 795        }
 796
 797        if (!list_empty(&cb.base.node))
 798                list_del(&cb.base.node);
 799        __set_current_state(TASK_RUNNING);
 800
 801out:
 802        spin_unlock_irqrestore(fence->lock, flags);
 803        return ret;
 804}
 805EXPORT_SYMBOL(dma_fence_default_wait);
 806
 807static bool
 808dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
 809                            uint32_t *idx)
 810{
 811        int i;
 812
 813        for (i = 0; i < count; ++i) {
 814                struct dma_fence *fence = fences[i];
 815                if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
 816                        if (idx)
 817                                *idx = i;
 818                        return true;
 819                }
 820        }
 821        return false;
 822}
 823
 824/**
 825 * dma_fence_wait_any_timeout - sleep until any fence gets signaled
 826 * or until timeout elapses
 827 * @fences: array of fences to wait on
 828 * @count: number of fences to wait on
 829 * @intr: if true, do an interruptible wait
 830 * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
 831 * @idx: used to store the first signaled fence index, meaningful only on
 832 *      positive return
 833 *
 834 * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
 835 * interrupted, 0 if the wait timed out, or the remaining timeout in jiffies
 836 * on success.
 837 *
 838 * Synchronous waits for the first fence in the array to be signaled. The
 839 * caller needs to hold a reference to all fences in the array, otherwise a
 840 * fence might be freed before return, resulting in undefined behavior.
 841 *
 842 * See also dma_fence_wait() and dma_fence_wait_timeout().
 843 */
 844signed long
 845dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count,
 846                           bool intr, signed long timeout, uint32_t *idx)
 847{
 848        struct default_wait_cb *cb;
 849        signed long ret = timeout;
 850        unsigned i;
 851
 852        if (WARN_ON(!fences || !count || timeout < 0))
 853                return -EINVAL;
 854
 855        if (timeout == 0) {
 856                for (i = 0; i < count; ++i)
 857                        if (dma_fence_is_signaled(fences[i])) {
 858                                if (idx)
 859                                        *idx = i;
 860                                return 1;
 861                        }
 862
 863                return 0;
 864        }
 865
 866        cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL);
 867        if (cb == NULL) {
 868                ret = -ENOMEM;
 869                goto err_free_cb;
 870        }
 871
 872        for (i = 0; i < count; ++i) {
 873                struct dma_fence *fence = fences[i];
 874
 875                cb[i].task = current;
 876                if (dma_fence_add_callback(fence, &cb[i].base,
 877                                           dma_fence_default_wait_cb)) {
 878                        /* This fence is already signaled */
 879                        if (idx)
 880                                *idx = i;
 881                        goto fence_rm_cb;
 882                }
 883        }
 884
 885        while (ret > 0) {
 886                if (intr)
 887                        set_current_state(TASK_INTERRUPTIBLE);
 888                else
 889                        set_current_state(TASK_UNINTERRUPTIBLE);
 890
 891                if (dma_fence_test_signaled_any(fences, count, idx))
 892                        break;
 893
 894                ret = schedule_timeout(ret);
 895
 896                if (ret > 0 && intr && signal_pending(current))
 897                        ret = -ERESTARTSYS;
 898        }
 899
 900        __set_current_state(TASK_RUNNING);
 901
 902fence_rm_cb:
 903        while (i-- > 0)
 904                dma_fence_remove_callback(fences[i], &cb[i].base);
 905
 906err_free_cb:
 907        kfree(cb);
 908
 909        return ret;
 910}
 911EXPORT_SYMBOL(dma_fence_wait_any_timeout);
 912
 913/**
 914 * dma_fence_init - Initialize a custom fence.
 915 * @fence: the fence to initialize
 916 * @ops: the dma_fence_ops for operations on this fence
 917 * @lock: the irqsafe spinlock to use for locking this fence
 918 * @context: the execution context this fence is run on
 919 * @seqno: a linear increasing sequence number for this context
 920 *
 921 * Initializes an allocated fence, the caller doesn't have to keep its
 922 * refcount after committing with this fence, but it will need to hold a
 923 * refcount again if &dma_fence_ops.enable_signaling gets called.
 924 *
 925 * context and seqno are used for easy comparison between fences, allowing
 926 * to check which fence is later by simply using dma_fence_later().
 927 */
 928void
 929dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
 930               spinlock_t *lock, u64 context, u64 seqno)
 931{
 932        BUG_ON(!lock);
 933        BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);
 934
 935        kref_init(&fence->refcount);
 936        fence->ops = ops;
 937        INIT_LIST_HEAD(&fence->cb_list);
 938        fence->lock = lock;
 939        fence->context = context;
 940        fence->seqno = seqno;
 941        fence->flags = 0UL;
 942        fence->error = 0;
 943
 944        trace_dma_fence_init(fence);
 945}
 946EXPORT_SYMBOL(dma_fence_init);
 947