linux/drivers/dma-buf/dma-fence-chain.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * fence-chain: chain fences together in a timeline
   4 *
   5 * Copyright (C) 2018 Advanced Micro Devices, Inc.
   6 * Authors:
   7 *      Christian K\xC3\xB6nig <christian.koenig@amd.com>
   8 */
   9
  10#include <linux/dma-fence-chain.h>
  11
  12static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
  13
  14/**
  15 * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
  16 * @chain: chain node to get the previous node from
  17 *
  18 * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
  19 * chain node.
  20 */
  21static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
  22{
  23        struct dma_fence *prev;
  24
  25        rcu_read_lock();
  26        prev = dma_fence_get_rcu_safe(&chain->prev);
  27        rcu_read_unlock();
  28        return prev;
  29}
  30
  31/**
  32 * dma_fence_chain_walk - chain walking function
  33 * @fence: current chain node
  34 *
  35 * Walk the chain to the next node. Returns the next fence or NULL if we are at
  36 * the end of the chain. Garbage collects chain nodes which are already
  37 * signaled.
  38 */
  39struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
  40{
  41        struct dma_fence_chain *chain, *prev_chain;
  42        struct dma_fence *prev, *replacement, *tmp;
  43
  44        chain = to_dma_fence_chain(fence);
  45        if (!chain) {
  46                dma_fence_put(fence);
  47                return NULL;
  48        }
  49
  50        while ((prev = dma_fence_chain_get_prev(chain))) {
  51
  52                prev_chain = to_dma_fence_chain(prev);
  53                if (prev_chain) {
  54                        if (!dma_fence_is_signaled(prev_chain->fence))
  55                                break;
  56
  57                        replacement = dma_fence_chain_get_prev(prev_chain);
  58                } else {
  59                        if (!dma_fence_is_signaled(prev))
  60                                break;
  61
  62                        replacement = NULL;
  63                }
  64
  65                tmp = cmpxchg((struct dma_fence __force **)&chain->prev,
  66                              prev, replacement);
  67                if (tmp == prev)
  68                        dma_fence_put(tmp);
  69                else
  70                        dma_fence_put(replacement);
  71                dma_fence_put(prev);
  72        }
  73
  74        dma_fence_put(fence);
  75        return prev;
  76}
  77EXPORT_SYMBOL(dma_fence_chain_walk);
  78
  79/**
  80 * dma_fence_chain_find_seqno - find fence chain node by seqno
  81 * @pfence: pointer to the chain node where to start
  82 * @seqno: the sequence number to search for
  83 *
  84 * Advance the fence pointer to the chain node which will signal this sequence
  85 * number. If no sequence number is provided then this is a no-op.
  86 *
  87 * Returns EINVAL if the fence is not a chain node or the sequence number has
  88 * not yet advanced far enough.
  89 */
  90int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
  91{
  92        struct dma_fence_chain *chain;
  93
  94        if (!seqno)
  95                return 0;
  96
  97        chain = to_dma_fence_chain(*pfence);
  98        if (!chain || chain->base.seqno < seqno)
  99                return -EINVAL;
 100
 101        dma_fence_chain_for_each(*pfence, &chain->base) {
 102                if ((*pfence)->context != chain->base.context ||
 103                    to_dma_fence_chain(*pfence)->prev_seqno < seqno)
 104                        break;
 105        }
 106        dma_fence_put(&chain->base);
 107
 108        return 0;
 109}
 110EXPORT_SYMBOL(dma_fence_chain_find_seqno);
 111
 112static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
 113{
 114        return "dma_fence_chain";
 115}
 116
 117static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
 118{
 119        return "unbound";
 120}
 121
 122static void dma_fence_chain_irq_work(struct irq_work *work)
 123{
 124        struct dma_fence_chain *chain;
 125
 126        chain = container_of(work, typeof(*chain), work);
 127
 128        /* Try to rearm the callback */
 129        if (!dma_fence_chain_enable_signaling(&chain->base))
 130                /* Ok, we are done. No more unsignaled fences left */
 131                dma_fence_signal(&chain->base);
 132        dma_fence_put(&chain->base);
 133}
 134
 135static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
 136{
 137        struct dma_fence_chain *chain;
 138
 139        chain = container_of(cb, typeof(*chain), cb);
 140        irq_work_queue(&chain->work);
 141        dma_fence_put(f);
 142}
 143
 144static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
 145{
 146        struct dma_fence_chain *head = to_dma_fence_chain(fence);
 147
 148        dma_fence_get(&head->base);
 149        dma_fence_chain_for_each(fence, &head->base) {
 150                struct dma_fence_chain *chain = to_dma_fence_chain(fence);
 151                struct dma_fence *f = chain ? chain->fence : fence;
 152
 153                dma_fence_get(f);
 154                if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
 155                        dma_fence_put(fence);
 156                        return true;
 157                }
 158                dma_fence_put(f);
 159        }
 160        dma_fence_put(&head->base);
 161        return false;
 162}
 163
 164static bool dma_fence_chain_signaled(struct dma_fence *fence)
 165{
 166        dma_fence_chain_for_each(fence, fence) {
 167                struct dma_fence_chain *chain = to_dma_fence_chain(fence);
 168                struct dma_fence *f = chain ? chain->fence : fence;
 169
 170                if (!dma_fence_is_signaled(f)) {
 171                        dma_fence_put(fence);
 172                        return false;
 173                }
 174        }
 175
 176        return true;
 177}
 178
 179static void dma_fence_chain_release(struct dma_fence *fence)
 180{
 181        struct dma_fence_chain *chain = to_dma_fence_chain(fence);
 182        struct dma_fence *prev;
 183
 184        /* Manually unlink the chain as much as possible to avoid recursion
 185         * and potential stack overflow.
 186         */
 187        while ((prev = rcu_dereference_protected(chain->prev, true))) {
 188                struct dma_fence_chain *prev_chain;
 189
 190                if (kref_read(&prev->refcount) > 1)
 191                       break;
 192
 193                prev_chain = to_dma_fence_chain(prev);
 194                if (!prev_chain)
 195                        break;
 196
 197                /* No need for atomic operations since we hold the last
 198                 * reference to prev_chain.
 199                 */
 200                chain->prev = prev_chain->prev;
 201                RCU_INIT_POINTER(prev_chain->prev, NULL);
 202                dma_fence_put(prev);
 203        }
 204        dma_fence_put(prev);
 205
 206        dma_fence_put(chain->fence);
 207        dma_fence_free(fence);
 208}
 209
 210const struct dma_fence_ops dma_fence_chain_ops = {
 211        .use_64bit_seqno = true,
 212        .get_driver_name = dma_fence_chain_get_driver_name,
 213        .get_timeline_name = dma_fence_chain_get_timeline_name,
 214        .enable_signaling = dma_fence_chain_enable_signaling,
 215        .signaled = dma_fence_chain_signaled,
 216        .release = dma_fence_chain_release,
 217};
 218EXPORT_SYMBOL(dma_fence_chain_ops);
 219
 220/**
 221 * dma_fence_chain_init - initialize a fence chain
 222 * @chain: the chain node to initialize
 223 * @prev: the previous fence
 224 * @fence: the current fence
 225 * @seqno: the sequence number to use for the fence chain
 226 *
 227 * Initialize a new chain node and either start a new chain or add the node to
 228 * the existing chain of the previous fence.
 229 */
 230void dma_fence_chain_init(struct dma_fence_chain *chain,
 231                          struct dma_fence *prev,
 232                          struct dma_fence *fence,
 233                          uint64_t seqno)
 234{
 235        struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
 236        uint64_t context;
 237
 238        spin_lock_init(&chain->lock);
 239        rcu_assign_pointer(chain->prev, prev);
 240        chain->fence = fence;
 241        chain->prev_seqno = 0;
 242        init_irq_work(&chain->work, dma_fence_chain_irq_work);
 243
 244        /* Try to reuse the context of the previous chain node. */
 245        if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
 246                context = prev->context;
 247                chain->prev_seqno = prev->seqno;
 248        } else {
 249                context = dma_fence_context_alloc(1);
 250                /* Make sure that we always have a valid sequence number. */
 251                if (prev_chain)
 252                        seqno = max(prev->seqno, seqno);
 253        }
 254
 255        dma_fence_init(&chain->base, &dma_fence_chain_ops,
 256                       &chain->lock, context, seqno);
 257}
 258EXPORT_SYMBOL(dma_fence_chain_init);
 259