linux/drivers/crypto/caam/jr.c
<<
>>
Prefs
   1/*
   2 * CAAM/SEC 4.x transport/backend driver
   3 * JobR backend functionality
   4 *
   5 * Copyright 2008-2012 Freescale Semiconductor, Inc.
   6 */
   7
   8#include <linux/of_irq.h>
   9#include <linux/of_address.h>
  10
  11#include "compat.h"
  12#include "regs.h"
  13#include "jr.h"
  14#include "desc.h"
  15#include "intern.h"
  16
  17struct jr_driver_data {
  18        /* List of Physical JobR's with the Driver */
  19        struct list_head        jr_list;
  20        spinlock_t              jr_alloc_lock;  /* jr_list lock */
  21} ____cacheline_aligned;
  22
  23static struct jr_driver_data driver_data;
  24
  25static int caam_reset_hw_jr(struct device *dev)
  26{
  27        struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
  28        unsigned int timeout = 100000;
  29
  30        /*
  31         * mask interrupts since we are going to poll
  32         * for reset completion status
  33         */
  34        setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
  35
  36        /* initiate flush (required prior to reset) */
  37        wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
  38        while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
  39                JRINT_ERR_HALT_INPROGRESS) && --timeout)
  40                cpu_relax();
  41
  42        if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
  43            JRINT_ERR_HALT_COMPLETE || timeout == 0) {
  44                dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
  45                return -EIO;
  46        }
  47
  48        /* initiate reset */
  49        timeout = 100000;
  50        wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
  51        while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
  52                cpu_relax();
  53
  54        if (timeout == 0) {
  55                dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
  56                return -EIO;
  57        }
  58
  59        /* unmask interrupts */
  60        clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
  61
  62        return 0;
  63}
  64
  65/*
  66 * Shutdown JobR independent of platform property code
  67 */
  68int caam_jr_shutdown(struct device *dev)
  69{
  70        struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
  71        dma_addr_t inpbusaddr, outbusaddr;
  72        int ret;
  73
  74        ret = caam_reset_hw_jr(dev);
  75
  76        tasklet_kill(&jrp->irqtask);
  77
  78        /* Release interrupt */
  79        free_irq(jrp->irq, dev);
  80
  81        /* Free rings */
  82        inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
  83        outbusaddr = rd_reg64(&jrp->rregs->outring_base);
  84        dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
  85                          jrp->inpring, inpbusaddr);
  86        dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
  87                          jrp->outring, outbusaddr);
  88        kfree(jrp->entinfo);
  89
  90        return ret;
  91}
  92
  93static int caam_jr_remove(struct platform_device *pdev)
  94{
  95        int ret;
  96        struct device *jrdev;
  97        struct caam_drv_private_jr *jrpriv;
  98
  99        jrdev = &pdev->dev;
 100        jrpriv = dev_get_drvdata(jrdev);
 101
 102        /*
 103         * Return EBUSY if job ring already allocated.
 104         */
 105        if (atomic_read(&jrpriv->tfm_count)) {
 106                dev_err(jrdev, "Device is busy\n");
 107                return -EBUSY;
 108        }
 109
 110        /* Remove the node from Physical JobR list maintained by driver */
 111        spin_lock(&driver_data.jr_alloc_lock);
 112        list_del(&jrpriv->list_node);
 113        spin_unlock(&driver_data.jr_alloc_lock);
 114
 115        /* Release ring */
 116        ret = caam_jr_shutdown(jrdev);
 117        if (ret)
 118                dev_err(jrdev, "Failed to shut down job ring\n");
 119        irq_dispose_mapping(jrpriv->irq);
 120
 121        return ret;
 122}
 123
 124/* Main per-ring interrupt handler */
 125static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
 126{
 127        struct device *dev = st_dev;
 128        struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 129        u32 irqstate;
 130
 131        /*
 132         * Check the output ring for ready responses, kick
 133         * tasklet if jobs done.
 134         */
 135        irqstate = rd_reg32(&jrp->rregs->jrintstatus);
 136        if (!irqstate)
 137                return IRQ_NONE;
 138
 139        /*
 140         * If JobR error, we got more development work to do
 141         * Flag a bug now, but we really need to shut down and
 142         * restart the queue (and fix code).
 143         */
 144        if (irqstate & JRINT_JR_ERROR) {
 145                dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
 146                BUG();
 147        }
 148
 149        /* mask valid interrupts */
 150        setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 151
 152        /* Have valid interrupt at this point, just ACK and trigger */
 153        wr_reg32(&jrp->rregs->jrintstatus, irqstate);
 154
 155        preempt_disable();
 156        tasklet_schedule(&jrp->irqtask);
 157        preempt_enable();
 158
 159        return IRQ_HANDLED;
 160}
 161
 162/* Deferred service handler, run as interrupt-fired tasklet */
 163static void caam_jr_dequeue(unsigned long devarg)
 164{
 165        int hw_idx, sw_idx, i, head, tail;
 166        struct device *dev = (struct device *)devarg;
 167        struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 168        void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
 169        u32 *userdesc, userstatus;
 170        void *userarg;
 171
 172        while (rd_reg32(&jrp->rregs->outring_used)) {
 173
 174                head = ACCESS_ONCE(jrp->head);
 175
 176                spin_lock(&jrp->outlock);
 177
 178                sw_idx = tail = jrp->tail;
 179                hw_idx = jrp->out_ring_read_index;
 180
 181                for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
 182                        sw_idx = (tail + i) & (JOBR_DEPTH - 1);
 183
 184                        if (jrp->outring[hw_idx].desc ==
 185                            jrp->entinfo[sw_idx].desc_addr_dma)
 186                                break; /* found */
 187                }
 188                /* we should never fail to find a matching descriptor */
 189                BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
 190
 191                /* Unmap just-run descriptor so we can post-process */
 192                dma_unmap_single(dev, jrp->outring[hw_idx].desc,
 193                                 jrp->entinfo[sw_idx].desc_size,
 194                                 DMA_TO_DEVICE);
 195
 196                /* mark completed, avoid matching on a recycled desc addr */
 197                jrp->entinfo[sw_idx].desc_addr_dma = 0;
 198
 199                /* Stash callback params for use outside of lock */
 200                usercall = jrp->entinfo[sw_idx].callbk;
 201                userarg = jrp->entinfo[sw_idx].cbkarg;
 202                userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
 203                userstatus = jrp->outring[hw_idx].jrstatus;
 204
 205                /* set done */
 206                wr_reg32(&jrp->rregs->outring_rmvd, 1);
 207
 208                jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
 209                                           (JOBR_DEPTH - 1);
 210
 211                /*
 212                 * if this job completed out-of-order, do not increment
 213                 * the tail.  Otherwise, increment tail by 1 plus the
 214                 * number of subsequent jobs already completed out-of-order
 215                 */
 216                if (sw_idx == tail) {
 217                        do {
 218                                tail = (tail + 1) & (JOBR_DEPTH - 1);
 219                        } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
 220                                 jrp->entinfo[tail].desc_addr_dma == 0);
 221
 222                        jrp->tail = tail;
 223                }
 224
 225                spin_unlock(&jrp->outlock);
 226
 227                /* Finally, execute user's callback */
 228                usercall(dev, userdesc, userstatus, userarg);
 229        }
 230
 231        /* reenable / unmask IRQs */
 232        clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 233}
 234
 235/**
 236 * caam_jr_alloc() - Alloc a job ring for someone to use as needed.
 237 *
 238 * returns :  pointer to the newly allocated physical
 239 *            JobR dev can be written to if successful.
 240 **/
 241struct device *caam_jr_alloc(void)
 242{
 243        struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
 244        struct device *dev = NULL;
 245        int min_tfm_cnt = INT_MAX;
 246        int tfm_cnt;
 247
 248        spin_lock(&driver_data.jr_alloc_lock);
 249
 250        if (list_empty(&driver_data.jr_list)) {
 251                spin_unlock(&driver_data.jr_alloc_lock);
 252                return ERR_PTR(-ENODEV);
 253        }
 254
 255        list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
 256                tfm_cnt = atomic_read(&jrpriv->tfm_count);
 257                if (tfm_cnt < min_tfm_cnt) {
 258                        min_tfm_cnt = tfm_cnt;
 259                        min_jrpriv = jrpriv;
 260                }
 261                if (!min_tfm_cnt)
 262                        break;
 263        }
 264
 265        if (min_jrpriv) {
 266                atomic_inc(&min_jrpriv->tfm_count);
 267                dev = min_jrpriv->dev;
 268        }
 269        spin_unlock(&driver_data.jr_alloc_lock);
 270
 271        return dev;
 272}
 273EXPORT_SYMBOL(caam_jr_alloc);
 274
 275/**
 276 * caam_jr_free() - Free the Job Ring
 277 * @rdev     - points to the dev that identifies the Job ring to
 278 *             be released.
 279 **/
 280void caam_jr_free(struct device *rdev)
 281{
 282        struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
 283
 284        atomic_dec(&jrpriv->tfm_count);
 285}
 286EXPORT_SYMBOL(caam_jr_free);
 287
 288/**
 289 * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
 290 * -EBUSY if the queue is full, -EIO if it cannot map the caller's
 291 * descriptor.
 292 * @dev:  device of the job ring to be used. This device should have
 293 *        been assigned prior by caam_jr_register().
 294 * @desc: points to a job descriptor that execute our request. All
 295 *        descriptors (and all referenced data) must be in a DMAable
 296 *        region, and all data references must be physical addresses
 297 *        accessible to CAAM (i.e. within a PAMU window granted
 298 *        to it).
 299 * @cbk:  pointer to a callback function to be invoked upon completion
 300 *        of this request. This has the form:
 301 *        callback(struct device *dev, u32 *desc, u32 stat, void *arg)
 302 *        where:
 303 *        @dev:    contains the job ring device that processed this
 304 *                 response.
 305 *        @desc:   descriptor that initiated the request, same as
 306 *                 "desc" being argued to caam_jr_enqueue().
 307 *        @status: untranslated status received from CAAM. See the
 308 *                 reference manual for a detailed description of
 309 *                 error meaning, or see the JRSTA definitions in the
 310 *                 register header file
 311 *        @areq:   optional pointer to an argument passed with the
 312 *                 original request
 313 * @areq: optional pointer to a user argument for use at callback
 314 *        time.
 315 **/
 316int caam_jr_enqueue(struct device *dev, u32 *desc,
 317                    void (*cbk)(struct device *dev, u32 *desc,
 318                                u32 status, void *areq),
 319                    void *areq)
 320{
 321        struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 322        struct caam_jrentry_info *head_entry;
 323        int head, tail, desc_size;
 324        dma_addr_t desc_dma;
 325
 326        desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32);
 327        desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
 328        if (dma_mapping_error(dev, desc_dma)) {
 329                dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
 330                return -EIO;
 331        }
 332
 333        spin_lock_bh(&jrp->inplock);
 334
 335        head = jrp->head;
 336        tail = ACCESS_ONCE(jrp->tail);
 337
 338        if (!rd_reg32(&jrp->rregs->inpring_avail) ||
 339            CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
 340                spin_unlock_bh(&jrp->inplock);
 341                dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
 342                return -EBUSY;
 343        }
 344
 345        head_entry = &jrp->entinfo[head];
 346        head_entry->desc_addr_virt = desc;
 347        head_entry->desc_size = desc_size;
 348        head_entry->callbk = (void *)cbk;
 349        head_entry->cbkarg = areq;
 350        head_entry->desc_addr_dma = desc_dma;
 351
 352        jrp->inpring[jrp->inp_ring_write_index] = desc_dma;
 353
 354        smp_wmb();
 355
 356        jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) &
 357                                    (JOBR_DEPTH - 1);
 358        jrp->head = (head + 1) & (JOBR_DEPTH - 1);
 359
 360        wr_reg32(&jrp->rregs->inpring_jobadd, 1);
 361
 362        spin_unlock_bh(&jrp->inplock);
 363
 364        return 0;
 365}
 366EXPORT_SYMBOL(caam_jr_enqueue);
 367
 368/*
 369 * Init JobR independent of platform property detection
 370 */
 371static int caam_jr_init(struct device *dev)
 372{
 373        struct caam_drv_private_jr *jrp;
 374        dma_addr_t inpbusaddr, outbusaddr;
 375        int i, error;
 376
 377        jrp = dev_get_drvdata(dev);
 378
 379        tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
 380
 381        /* Connect job ring interrupt handler. */
 382        error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
 383                            dev_name(dev), dev);
 384        if (error) {
 385                dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
 386                        jrp->ridx, jrp->irq);
 387                irq_dispose_mapping(jrp->irq);
 388                jrp->irq = 0;
 389                return -EINVAL;
 390        }
 391
 392        error = caam_reset_hw_jr(dev);
 393        if (error)
 394                return error;
 395
 396        jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
 397                                          &inpbusaddr, GFP_KERNEL);
 398
 399        jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) *
 400                                          JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
 401
 402        jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH,
 403                               GFP_KERNEL);
 404
 405        if ((jrp->inpring == NULL) || (jrp->outring == NULL) ||
 406            (jrp->entinfo == NULL)) {
 407                dev_err(dev, "can't allocate job rings for %d\n",
 408                        jrp->ridx);
 409                return -ENOMEM;
 410        }
 411
 412        for (i = 0; i < JOBR_DEPTH; i++)
 413                jrp->entinfo[i].desc_addr_dma = !0;
 414
 415        /* Setup rings */
 416        jrp->inp_ring_write_index = 0;
 417        jrp->out_ring_read_index = 0;
 418        jrp->head = 0;
 419        jrp->tail = 0;
 420
 421        wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
 422        wr_reg64(&jrp->rregs->outring_base, outbusaddr);
 423        wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
 424        wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
 425
 426        jrp->ringsize = JOBR_DEPTH;
 427
 428        spin_lock_init(&jrp->inplock);
 429        spin_lock_init(&jrp->outlock);
 430
 431        /* Select interrupt coalescing parameters */
 432        setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC |
 433                  (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
 434                  (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
 435
 436        return 0;
 437}
 438
 439
 440/*
 441 * Probe routine for each detected JobR subsystem.
 442 */
 443static int caam_jr_probe(struct platform_device *pdev)
 444{
 445        struct device *jrdev;
 446        struct device_node *nprop;
 447        struct caam_job_ring __iomem *ctrl;
 448        struct caam_drv_private_jr *jrpriv;
 449        static int total_jobrs;
 450        int error;
 451
 452        jrdev = &pdev->dev;
 453        jrpriv = devm_kmalloc(jrdev, sizeof(struct caam_drv_private_jr),
 454                              GFP_KERNEL);
 455        if (!jrpriv)
 456                return -ENOMEM;
 457
 458        dev_set_drvdata(jrdev, jrpriv);
 459
 460        /* save ring identity relative to detection */
 461        jrpriv->ridx = total_jobrs++;
 462
 463        nprop = pdev->dev.of_node;
 464        /* Get configuration properties from device tree */
 465        /* First, get register page */
 466        ctrl = of_iomap(nprop, 0);
 467        if (!ctrl) {
 468                dev_err(jrdev, "of_iomap() failed\n");
 469                return -ENOMEM;
 470        }
 471
 472        jrpriv->rregs = (struct caam_job_ring __force *)ctrl;
 473
 474        if (sizeof(dma_addr_t) == sizeof(u64))
 475                if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
 476                        dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40));
 477                else
 478                        dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36));
 479        else
 480                dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32));
 481
 482        /* Identify the interrupt */
 483        jrpriv->irq = irq_of_parse_and_map(nprop, 0);
 484
 485        /* Now do the platform independent part */
 486        error = caam_jr_init(jrdev); /* now turn on hardware */
 487        if (error)
 488                return error;
 489
 490        jrpriv->dev = jrdev;
 491        spin_lock(&driver_data.jr_alloc_lock);
 492        list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
 493        spin_unlock(&driver_data.jr_alloc_lock);
 494
 495        atomic_set(&jrpriv->tfm_count, 0);
 496
 497        return 0;
 498}
 499
 500static struct of_device_id caam_jr_match[] = {
 501        {
 502                .compatible = "fsl,sec-v4.0-job-ring",
 503        },
 504        {
 505                .compatible = "fsl,sec4.0-job-ring",
 506        },
 507        {},
 508};
 509MODULE_DEVICE_TABLE(of, caam_jr_match);
 510
 511static struct platform_driver caam_jr_driver = {
 512        .driver = {
 513                .name = "caam_jr",
 514                .of_match_table = caam_jr_match,
 515        },
 516        .probe       = caam_jr_probe,
 517        .remove      = caam_jr_remove,
 518};
 519
 520static int __init jr_driver_init(void)
 521{
 522        spin_lock_init(&driver_data.jr_alloc_lock);
 523        INIT_LIST_HEAD(&driver_data.jr_list);
 524        return platform_driver_register(&caam_jr_driver);
 525}
 526
 527static void __exit jr_driver_exit(void)
 528{
 529        platform_driver_unregister(&caam_jr_driver);
 530}
 531
 532module_init(jr_driver_init);
 533module_exit(jr_driver_exit);
 534
 535MODULE_LICENSE("GPL");
 536MODULE_DESCRIPTION("FSL CAAM JR request backend");
 537MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
 538
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.