linux/arch/tile/include/gxio/dma_queue.h
<<
>>
Prefs
   1/*
   2 * Copyright 2012 Tilera Corporation. All Rights Reserved.
   3 *
   4 *   This program is free software; you can redistribute it and/or
   5 *   modify it under the terms of the GNU General Public License
   6 *   as published by the Free Software Foundation, version 2.
   7 *
   8 *   This program is distributed in the hope that it will be useful, but
   9 *   WITHOUT ANY WARRANTY; without even the implied warranty of
  10 *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  11 *   NON INFRINGEMENT.  See the GNU General Public License for
  12 *   more details.
  13 */
  14
  15#ifndef _GXIO_DMA_QUEUE_H_
  16#define _GXIO_DMA_QUEUE_H_
  17
  18/*
  19 * DMA queue management APIs shared between TRIO and mPIPE.
  20 */
  21
  22#include <gxio/common.h>
  23
  24/* The credit counter lives in the high 32 bits. */
  25#define DMA_QUEUE_CREDIT_SHIFT 32
  26
  27/*
  28 * State object that tracks a DMA queue's head and tail indices, as
  29 * well as the number of commands posted and completed.  The
  30 * structure is accessed via a thread-safe, lock-free algorithm.
  31 */
  32typedef struct {
  33        /*
  34         * Address of a MPIPE_EDMA_POST_REGION_VAL_t,
  35         * TRIO_PUSH_DMA_REGION_VAL_t, or TRIO_PULL_DMA_REGION_VAL_t
  36         * register.  These register have identical encodings and provide
  37         * information about how many commands have been processed.
  38         */
  39        void *post_region_addr;
  40
  41        /*
  42         * A lazily-updated count of how many edescs the hardware has
  43         * completed.
  44         */
  45        uint64_t hw_complete_count __attribute__ ((aligned(64)));
  46
  47        /*
  48         * High 32 bits are a count of available egress command credits,
  49         * low 24 bits are the next egress "slot".
  50         */
  51        int64_t credits_and_next_index;
  52
  53} __gxio_dma_queue_t;
  54
  55/* Initialize a dma queue. */
  56extern void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue,
  57                                  void *post_region_addr,
  58                                  unsigned int num_entries);
  59
  60/*
  61 * Update the "credits_and_next_index" and "hw_complete_count" fields
  62 * based on pending hardware completions.  Note that some other thread
  63 * may have already done this and, importantly, may still be in the
  64 * process of updating "credits_and_next_index".
  65 */
  66extern void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue);
  67
  68/* Wait for credits to become available. */
  69extern int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue,
  70                                                 int64_t modifier);
  71
  72/* Reserve slots in the queue, optionally waiting for slots to become
  73 * available, and optionally returning a "completion_slot" suitable for
  74 * direct comparison to "hw_complete_count".
  75 */
  76static inline int64_t __gxio_dma_queue_reserve(__gxio_dma_queue_t *dma_queue,
  77                                               unsigned int num, bool wait,
  78                                               bool completion)
  79{
  80        uint64_t slot;
  81
  82        /*
  83         * Try to reserve 'num' egress command slots.  We do this by
  84         * constructing a constant that subtracts N credits and adds N to
  85         * the index, and using fetchaddgez to only apply it if the credits
  86         * count doesn't go negative.
  87         */
  88        int64_t modifier = (((int64_t)(-num)) << DMA_QUEUE_CREDIT_SHIFT) | num;
  89        int64_t old =
  90                __insn_fetchaddgez(&dma_queue->credits_and_next_index,
  91                                   modifier);
  92
  93        if (unlikely(old + modifier < 0)) {
  94                /*
  95                 * We're out of credits.  Try once to get more by checking for
  96                 * completed egress commands.  If that fails, wait or fail.
  97                 */
  98                __gxio_dma_queue_update_credits(dma_queue);
  99                old = __insn_fetchaddgez(&dma_queue->credits_and_next_index,
 100                                         modifier);
 101                if (old + modifier < 0) {
 102                        if (wait)
 103                                old = __gxio_dma_queue_wait_for_credits
 104                                        (dma_queue, modifier);
 105                        else
 106                                return GXIO_ERR_DMA_CREDITS;
 107                }
 108        }
 109
 110        /* The bottom 24 bits of old encode the "slot". */
 111        slot = (old & 0xffffff);
 112
 113        if (completion) {
 114                /*
 115                 * A "completion_slot" is a "slot" which can be compared to
 116                 * "hw_complete_count" at any time in the future.  To convert
 117                 * "slot" into a "completion_slot", we access "hw_complete_count"
 118                 * once (knowing that we have reserved a slot, and thus, it will
 119                 * be "basically" accurate), and combine its high 40 bits with
 120                 * the 24 bit "slot", and handle "wrapping" by adding "1 << 24"
 121                 * if the result is LESS than "hw_complete_count".
 122                 */
 123                uint64_t complete;
 124                complete = ACCESS_ONCE(dma_queue->hw_complete_count);
 125                slot |= (complete & 0xffffffffff000000);
 126                if (slot < complete)
 127                        slot += 0x1000000;
 128        }
 129
 130        /*
 131         * If any of our slots mod 256 were equivalent to 0, go ahead and
 132         * collect some egress credits, and update "hw_complete_count", and
 133         * make sure the index doesn't overflow into the credits.
 134         */
 135        if (unlikely(((old + num) & 0xff) < num)) {
 136                __gxio_dma_queue_update_credits(dma_queue);
 137
 138                /* Make sure the index doesn't overflow into the credits. */
 139#ifdef __BIG_ENDIAN__
 140                *(((uint8_t *)&dma_queue->credits_and_next_index) + 4) = 0;
 141#else
 142                *(((uint8_t *)&dma_queue->credits_and_next_index) + 3) = 0;
 143#endif
 144        }
 145
 146        return slot;
 147}
 148
 149/* Non-inlinable "__gxio_dma_queue_reserve(..., true)". */
 150extern int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue,
 151                                            unsigned int num, int wait);
 152
 153/* Check whether a particular "completion slot" has completed.
 154 *
 155 * Note that this function requires a "completion slot", and thus
 156 * cannot be used with the result of any "reserve_fast" function.
 157 */
 158extern int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue,
 159                                        int64_t completion_slot, int update);
 160
 161#endif /* !_GXIO_DMA_QUEUE_H_ */
 162