linux/drivers/crypto/sahara.c
<<
>>
Prefs
   1/*
   2 * Cryptographic API.
   3 *
   4 * Support for SAHARA cryptographic accelerator.
   5 *
   6 * Copyright (c) 2013 Vista Silicon S.L.
   7 * Author: Javier Martin <javier.martin@vista-silicon.com>
   8 *
   9 * This program is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License version 2 as published
  11 * by the Free Software Foundation.
  12 *
  13 * Based on omap-aes.c and tegra-aes.c
  14 */
  15
  16#include <crypto/algapi.h>
  17#include <crypto/aes.h>
  18
  19#include <linux/clk.h>
  20#include <linux/crypto.h>
  21#include <linux/interrupt.h>
  22#include <linux/io.h>
  23#include <linux/irq.h>
  24#include <linux/kernel.h>
  25#include <linux/module.h>
  26#include <linux/of.h>
  27#include <linux/platform_device.h>
  28
  29#define SAHARA_NAME "sahara"
  30#define SAHARA_VERSION_3        3
  31#define SAHARA_TIMEOUT_MS       1000
  32#define SAHARA_MAX_HW_DESC      2
  33#define SAHARA_MAX_HW_LINK      20
  34
  35#define FLAGS_MODE_MASK         0x000f
  36#define FLAGS_ENCRYPT           BIT(0)
  37#define FLAGS_CBC               BIT(1)
  38#define FLAGS_NEW_KEY           BIT(3)
  39#define FLAGS_BUSY              4
  40
  41#define SAHARA_HDR_BASE                 0x00800000
  42#define SAHARA_HDR_SKHA_ALG_AES 0
  43#define SAHARA_HDR_SKHA_OP_ENC          (1 << 2)
  44#define SAHARA_HDR_SKHA_MODE_ECB        (0 << 3)
  45#define SAHARA_HDR_SKHA_MODE_CBC        (1 << 3)
  46#define SAHARA_HDR_FORM_DATA            (5 << 16)
  47#define SAHARA_HDR_FORM_KEY             (8 << 16)
  48#define SAHARA_HDR_LLO                  (1 << 24)
  49#define SAHARA_HDR_CHA_SKHA             (1 << 28)
  50#define SAHARA_HDR_CHA_MDHA             (2 << 28)
  51#define SAHARA_HDR_PARITY_BIT           (1 << 31)
  52
  53/* SAHARA can only process one request at a time */
  54#define SAHARA_QUEUE_LENGTH     1
  55
  56#define SAHARA_REG_VERSION      0x00
  57#define SAHARA_REG_DAR          0x04
  58#define SAHARA_REG_CONTROL      0x08
  59#define         SAHARA_CONTROL_SET_THROTTLE(x)  (((x) & 0xff) << 24)
  60#define         SAHARA_CONTROL_SET_MAXBURST(x)  (((x) & 0xff) << 16)
  61#define         SAHARA_CONTROL_RNG_AUTORSD      (1 << 7)
  62#define         SAHARA_CONTROL_ENABLE_INT       (1 << 4)
  63#define SAHARA_REG_CMD          0x0C
  64#define         SAHARA_CMD_RESET                (1 << 0)
  65#define         SAHARA_CMD_CLEAR_INT            (1 << 8)
  66#define         SAHARA_CMD_CLEAR_ERR            (1 << 9)
  67#define         SAHARA_CMD_SINGLE_STEP          (1 << 10)
  68#define         SAHARA_CMD_MODE_BATCH           (1 << 16)
  69#define         SAHARA_CMD_MODE_DEBUG           (1 << 18)
  70#define SAHARA_REG_STATUS       0x10
  71#define         SAHARA_STATUS_GET_STATE(x)      ((x) & 0x7)
  72#define                 SAHARA_STATE_IDLE       0
  73#define                 SAHARA_STATE_BUSY       1
  74#define                 SAHARA_STATE_ERR        2
  75#define                 SAHARA_STATE_FAULT      3
  76#define                 SAHARA_STATE_COMPLETE   4
  77#define                 SAHARA_STATE_COMP_FLAG  (1 << 2)
  78#define         SAHARA_STATUS_DAR_FULL          (1 << 3)
  79#define         SAHARA_STATUS_ERROR             (1 << 4)
  80#define         SAHARA_STATUS_SECURE            (1 << 5)
  81#define         SAHARA_STATUS_FAIL              (1 << 6)
  82#define         SAHARA_STATUS_INIT              (1 << 7)
  83#define         SAHARA_STATUS_RNG_RESEED        (1 << 8)
  84#define         SAHARA_STATUS_ACTIVE_RNG        (1 << 9)
  85#define         SAHARA_STATUS_ACTIVE_MDHA       (1 << 10)
  86#define         SAHARA_STATUS_ACTIVE_SKHA       (1 << 11)
  87#define         SAHARA_STATUS_MODE_BATCH        (1 << 16)
  88#define         SAHARA_STATUS_MODE_DEDICATED    (1 << 17)
  89#define         SAHARA_STATUS_MODE_DEBUG        (1 << 18)
  90#define         SAHARA_STATUS_GET_ISTATE(x)     (((x) >> 24) & 0xff)
  91#define SAHARA_REG_ERRSTATUS    0x14
  92#define         SAHARA_ERRSTATUS_GET_SOURCE(x)  ((x) & 0xf)
  93#define                 SAHARA_ERRSOURCE_CHA    14
  94#define                 SAHARA_ERRSOURCE_DMA    15
  95#define         SAHARA_ERRSTATUS_DMA_DIR        (1 << 8)
  96#define         SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
  97#define         SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
  98#define         SAHARA_ERRSTATUS_GET_CHASRC(x)  (((x) >> 16) & 0xfff)
  99#define         SAHARA_ERRSTATUS_GET_CHAERR(x)  (((x) >> 28) & 0x3)
 100#define SAHARA_REG_FADDR        0x18
 101#define SAHARA_REG_CDAR         0x1C
 102#define SAHARA_REG_IDAR         0x20
 103
 104struct sahara_hw_desc {
 105        u32             hdr;
 106        u32             len1;
 107        dma_addr_t      p1;
 108        u32             len2;
 109        dma_addr_t      p2;
 110        dma_addr_t      next;
 111};
 112
 113struct sahara_hw_link {
 114        u32             len;
 115        dma_addr_t      p;
 116        dma_addr_t      next;
 117};
 118
 119struct sahara_ctx {
 120        struct sahara_dev *dev;
 121        unsigned long flags;
 122        int keylen;
 123        u8 key[AES_KEYSIZE_128];
 124        struct crypto_ablkcipher *fallback;
 125};
 126
 127struct sahara_aes_reqctx {
 128        unsigned long mode;
 129};
 130
 131struct sahara_dev {
 132        struct device           *device;
 133        void __iomem            *regs_base;
 134        struct clk              *clk_ipg;
 135        struct clk              *clk_ahb;
 136
 137        struct sahara_ctx       *ctx;
 138        spinlock_t              lock;
 139        struct crypto_queue     queue;
 140        unsigned long           flags;
 141
 142        struct tasklet_struct   done_task;
 143        struct tasklet_struct   queue_task;
 144
 145        struct sahara_hw_desc   *hw_desc[SAHARA_MAX_HW_DESC];
 146        dma_addr_t              hw_phys_desc[SAHARA_MAX_HW_DESC];
 147
 148        u8                      *key_base;
 149        dma_addr_t              key_phys_base;
 150
 151        u8                      *iv_base;
 152        dma_addr_t              iv_phys_base;
 153
 154        struct sahara_hw_link   *hw_link[SAHARA_MAX_HW_LINK];
 155        dma_addr_t              hw_phys_link[SAHARA_MAX_HW_LINK];
 156
 157        struct ablkcipher_request *req;
 158        size_t                  total;
 159        struct scatterlist      *in_sg;
 160        unsigned int            nb_in_sg;
 161        struct scatterlist      *out_sg;
 162        unsigned int            nb_out_sg;
 163
 164        u32                     error;
 165        struct timer_list       watchdog;
 166};
 167
 168static struct sahara_dev *dev_ptr;
 169
 170static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg)
 171{
 172        writel(data, dev->regs_base + reg);
 173}
 174
 175static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg)
 176{
 177        return readl(dev->regs_base + reg);
 178}
 179
 180static u32 sahara_aes_key_hdr(struct sahara_dev *dev)
 181{
 182        u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES |
 183                        SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO |
 184                        SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
 185
 186        if (dev->flags & FLAGS_CBC) {
 187                hdr |= SAHARA_HDR_SKHA_MODE_CBC;
 188                hdr ^= SAHARA_HDR_PARITY_BIT;
 189        }
 190
 191        if (dev->flags & FLAGS_ENCRYPT) {
 192                hdr |= SAHARA_HDR_SKHA_OP_ENC;
 193                hdr ^= SAHARA_HDR_PARITY_BIT;
 194        }
 195
 196        return hdr;
 197}
 198
 199static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev)
 200{
 201        return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA |
 202                        SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
 203}
 204
 205static int sahara_sg_length(struct scatterlist *sg,
 206                            unsigned int total)
 207{
 208        int sg_nb;
 209        unsigned int len;
 210        struct scatterlist *sg_list;
 211
 212        sg_nb = 0;
 213        sg_list = sg;
 214
 215        while (total) {
 216                len = min(sg_list->length, total);
 217
 218                sg_nb++;
 219                total -= len;
 220
 221                sg_list = sg_next(sg_list);
 222                if (!sg_list)
 223                        total = 0;
 224        }
 225
 226        return sg_nb;
 227}
 228
 229static char *sahara_err_src[16] = {
 230        "No error",
 231        "Header error",
 232        "Descriptor length error",
 233        "Descriptor length or pointer error",
 234        "Link length error",
 235        "Link pointer error",
 236        "Input buffer error",
 237        "Output buffer error",
 238        "Output buffer starvation",
 239        "Internal state fault",
 240        "General descriptor problem",
 241        "Reserved",
 242        "Descriptor address error",
 243        "Link address error",
 244        "CHA error",
 245        "DMA error"
 246};
 247
 248static char *sahara_err_dmasize[4] = {
 249        "Byte transfer",
 250        "Half-word transfer",
 251        "Word transfer",
 252        "Reserved"
 253};
 254
 255static char *sahara_err_dmasrc[8] = {
 256        "No error",
 257        "AHB bus error",
 258        "Internal IP bus error",
 259        "Parity error",
 260        "DMA crosses 256 byte boundary",
 261        "DMA is busy",
 262        "Reserved",
 263        "DMA HW error"
 264};
 265
 266static char *sahara_cha_errsrc[12] = {
 267        "Input buffer non-empty",
 268        "Illegal address",
 269        "Illegal mode",
 270        "Illegal data size",
 271        "Illegal key size",
 272        "Write during processing",
 273        "CTX read during processing",
 274        "HW error",
 275        "Input buffer disabled/underflow",
 276        "Output buffer disabled/overflow",
 277        "DES key parity error",
 278        "Reserved"
 279};
 280
 281static char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" };
 282
 283static void sahara_decode_error(struct sahara_dev *dev, unsigned int error)
 284{
 285        u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error);
 286        u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error));
 287
 288        dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);
 289
 290        dev_err(dev->device, "  - %s.\n", sahara_err_src[source]);
 291
 292        if (source == SAHARA_ERRSOURCE_DMA) {
 293                if (error & SAHARA_ERRSTATUS_DMA_DIR)
 294                        dev_err(dev->device, "          * DMA read.\n");
 295                else
 296                        dev_err(dev->device, "          * DMA write.\n");
 297
 298                dev_err(dev->device, "          * %s.\n",
 299                       sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]);
 300                dev_err(dev->device, "          * %s.\n",
 301                       sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]);
 302        } else if (source == SAHARA_ERRSOURCE_CHA) {
 303                dev_err(dev->device, "          * %s.\n",
 304                        sahara_cha_errsrc[chasrc]);
 305                dev_err(dev->device, "          * %s.\n",
 306                       sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]);
 307        }
 308        dev_err(dev->device, "\n");
 309}
 310
 311static char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" };
 312
 313static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
 314{
 315        u8 state;
 316
 317        if (!IS_ENABLED(DEBUG))
 318                return;
 319
 320        state = SAHARA_STATUS_GET_STATE(status);
 321
 322        dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",
 323                __func__, status);
 324
 325        dev_dbg(dev->device, "  - State = %d:\n", state);
 326        if (state & SAHARA_STATE_COMP_FLAG)
 327                dev_dbg(dev->device, "          * Descriptor completed. IRQ pending.\n");
 328
 329        dev_dbg(dev->device, "          * %s.\n",
 330               sahara_state[state & ~SAHARA_STATE_COMP_FLAG]);
 331
 332        if (status & SAHARA_STATUS_DAR_FULL)
 333                dev_dbg(dev->device, "  - DAR Full.\n");
 334        if (status & SAHARA_STATUS_ERROR)
 335                dev_dbg(dev->device, "  - Error.\n");
 336        if (status & SAHARA_STATUS_SECURE)
 337                dev_dbg(dev->device, "  - Secure.\n");
 338        if (status & SAHARA_STATUS_FAIL)
 339                dev_dbg(dev->device, "  - Fail.\n");
 340        if (status & SAHARA_STATUS_RNG_RESEED)
 341                dev_dbg(dev->device, "  - RNG Reseed Request.\n");
 342        if (status & SAHARA_STATUS_ACTIVE_RNG)
 343                dev_dbg(dev->device, "  - RNG Active.\n");
 344        if (status & SAHARA_STATUS_ACTIVE_MDHA)
 345                dev_dbg(dev->device, "  - MDHA Active.\n");
 346        if (status & SAHARA_STATUS_ACTIVE_SKHA)
 347                dev_dbg(dev->device, "  - SKHA Active.\n");
 348
 349        if (status & SAHARA_STATUS_MODE_BATCH)
 350                dev_dbg(dev->device, "  - Batch Mode.\n");
 351        else if (status & SAHARA_STATUS_MODE_DEDICATED)
 352                dev_dbg(dev->device, "  - Decidated Mode.\n");
 353        else if (status & SAHARA_STATUS_MODE_DEBUG)
 354                dev_dbg(dev->device, "  - Debug Mode.\n");
 355
 356        dev_dbg(dev->device, "  - Internal state = 0x%02x\n",
 357               SAHARA_STATUS_GET_ISTATE(status));
 358
 359        dev_dbg(dev->device, "Current DAR: 0x%08x\n",
 360                sahara_read(dev, SAHARA_REG_CDAR));
 361        dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",
 362                sahara_read(dev, SAHARA_REG_IDAR));
 363}
 364
 365static void sahara_dump_descriptors(struct sahara_dev *dev)
 366{
 367        int i;
 368
 369        if (!IS_ENABLED(DEBUG))
 370                return;
 371
 372        for (i = 0; i < SAHARA_MAX_HW_DESC; i++) {
 373                dev_dbg(dev->device, "Descriptor (%d) (0x%08x):\n",
 374                        i, dev->hw_phys_desc[i]);
 375                dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);
 376                dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);
 377                dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);
 378                dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);
 379                dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);
 380                dev_dbg(dev->device, "\tnext = 0x%08x\n",
 381                        dev->hw_desc[i]->next);
 382        }
 383        dev_dbg(dev->device, "\n");
 384}
 385
 386static void sahara_dump_links(struct sahara_dev *dev)
 387{
 388        int i;
 389
 390        if (!IS_ENABLED(DEBUG))
 391                return;
 392
 393        for (i = 0; i < SAHARA_MAX_HW_LINK; i++) {
 394                dev_dbg(dev->device, "Link (%d) (0x%08x):\n",
 395                        i, dev->hw_phys_link[i]);
 396                dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);
 397                dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);
 398                dev_dbg(dev->device, "\tnext = 0x%08x\n",
 399                        dev->hw_link[i]->next);
 400        }
 401        dev_dbg(dev->device, "\n");
 402}
 403
 404static void sahara_aes_done_task(unsigned long data)
 405{
 406        struct sahara_dev *dev = (struct sahara_dev *)data;
 407
 408        dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
 409                DMA_TO_DEVICE);
 410        dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
 411                DMA_FROM_DEVICE);
 412
 413        spin_lock(&dev->lock);
 414        clear_bit(FLAGS_BUSY, &dev->flags);
 415        spin_unlock(&dev->lock);
 416
 417        dev->req->base.complete(&dev->req->base, dev->error);
 418}
 419
 420static void sahara_watchdog(unsigned long data)
 421{
 422        struct sahara_dev *dev = (struct sahara_dev *)data;
 423        unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
 424        unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
 425
 426        sahara_decode_status(dev, stat);
 427        sahara_decode_error(dev, err);
 428        dev->error = -ETIMEDOUT;
 429        sahara_aes_done_task(data);
 430}
 431
 432static int sahara_hw_descriptor_create(struct sahara_dev *dev)
 433{
 434        struct sahara_ctx *ctx = dev->ctx;
 435        struct scatterlist *sg;
 436        int ret;
 437        int i, j;
 438
 439        /* Copy new key if necessary */
 440        if (ctx->flags & FLAGS_NEW_KEY) {
 441                memcpy(dev->key_base, ctx->key, ctx->keylen);
 442                ctx->flags &= ~FLAGS_NEW_KEY;
 443
 444                if (dev->flags & FLAGS_CBC) {
 445                        dev->hw_desc[0]->len1 = AES_BLOCK_SIZE;
 446                        dev->hw_desc[0]->p1 = dev->iv_phys_base;
 447                } else {
 448                        dev->hw_desc[0]->len1 = 0;
 449                        dev->hw_desc[0]->p1 = 0;
 450                }
 451                dev->hw_desc[0]->len2 = ctx->keylen;
 452                dev->hw_desc[0]->p2 = dev->key_phys_base;
 453                dev->hw_desc[0]->next = dev->hw_phys_desc[1];
 454        }
 455        dev->hw_desc[0]->hdr = sahara_aes_key_hdr(dev);
 456
 457        dev->nb_in_sg = sahara_sg_length(dev->in_sg, dev->total);
 458        dev->nb_out_sg = sahara_sg_length(dev->out_sg, dev->total);
 459        if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {
 460                dev_err(dev->device, "not enough hw links (%d)\n",
 461                        dev->nb_in_sg + dev->nb_out_sg);
 462                return -EINVAL;
 463        }
 464
 465        ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
 466                         DMA_TO_DEVICE);
 467        if (ret != dev->nb_in_sg) {
 468                dev_err(dev->device, "couldn't map in sg\n");
 469                goto unmap_in;
 470        }
 471        ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,
 472                         DMA_FROM_DEVICE);
 473        if (ret != dev->nb_out_sg) {
 474                dev_err(dev->device, "couldn't map out sg\n");
 475                goto unmap_out;
 476        }
 477
 478        /* Create input links */
 479        dev->hw_desc[1]->p1 = dev->hw_phys_link[0];
 480        sg = dev->in_sg;
 481        for (i = 0; i < dev->nb_in_sg; i++) {
 482                dev->hw_link[i]->len = sg->length;
 483                dev->hw_link[i]->p = sg->dma_address;
 484                if (i == (dev->nb_in_sg - 1)) {
 485                        dev->hw_link[i]->next = 0;
 486                } else {
 487                        dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
 488                        sg = sg_next(sg);
 489                }
 490        }
 491
 492        /* Create output links */
 493        dev->hw_desc[1]->p2 = dev->hw_phys_link[i];
 494        sg = dev->out_sg;
 495        for (j = i; j < dev->nb_out_sg + i; j++) {
 496                dev->hw_link[j]->len = sg->length;
 497                dev->hw_link[j]->p = sg->dma_address;
 498                if (j == (dev->nb_out_sg + i - 1)) {
 499                        dev->hw_link[j]->next = 0;
 500                } else {
 501                        dev->hw_link[j]->next = dev->hw_phys_link[j + 1];
 502                        sg = sg_next(sg);
 503                }
 504        }
 505
 506        /* Fill remaining fields of hw_desc[1] */
 507        dev->hw_desc[1]->hdr = sahara_aes_data_link_hdr(dev);
 508        dev->hw_desc[1]->len1 = dev->total;
 509        dev->hw_desc[1]->len2 = dev->total;
 510        dev->hw_desc[1]->next = 0;
 511
 512        sahara_dump_descriptors(dev);
 513        sahara_dump_links(dev);
 514
 515        /* Start processing descriptor chain. */
 516        mod_timer(&dev->watchdog,
 517                  jiffies + msecs_to_jiffies(SAHARA_TIMEOUT_MS));
 518        sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
 519
 520        return 0;
 521
 522unmap_out:
 523        dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
 524                DMA_TO_DEVICE);
 525unmap_in:
 526        dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
 527                DMA_FROM_DEVICE);
 528
 529        return -EINVAL;
 530}
 531
 532static void sahara_aes_queue_task(unsigned long data)
 533{
 534        struct sahara_dev *dev = (struct sahara_dev *)data;
 535        struct crypto_async_request *async_req, *backlog;
 536        struct sahara_ctx *ctx;
 537        struct sahara_aes_reqctx *rctx;
 538        struct ablkcipher_request *req;
 539        int ret;
 540
 541        spin_lock(&dev->lock);
 542        backlog = crypto_get_backlog(&dev->queue);
 543        async_req = crypto_dequeue_request(&dev->queue);
 544        if (!async_req)
 545                clear_bit(FLAGS_BUSY, &dev->flags);
 546        spin_unlock(&dev->lock);
 547
 548        if (!async_req)
 549                return;
 550
 551        if (backlog)
 552                backlog->complete(backlog, -EINPROGRESS);
 553
 554        req = ablkcipher_request_cast(async_req);
 555
 556        /* Request is ready to be dispatched by the device */
 557        dev_dbg(dev->device,
 558                "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
 559                req->nbytes, req->src, req->dst);
 560
 561        /* assign new request to device */
 562        dev->req = req;
 563        dev->total = req->nbytes;
 564        dev->in_sg = req->src;
 565        dev->out_sg = req->dst;
 566
 567        rctx = ablkcipher_request_ctx(req);
 568        ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
 569        rctx->mode &= FLAGS_MODE_MASK;
 570        dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 571
 572        if ((dev->flags & FLAGS_CBC) && req->info)
 573                memcpy(dev->iv_base, req->info, AES_KEYSIZE_128);
 574
 575        /* assign new context to device */
 576        ctx->dev = dev;
 577        dev->ctx = ctx;
 578
 579        ret = sahara_hw_descriptor_create(dev);
 580        if (ret < 0) {
 581                spin_lock(&dev->lock);
 582                clear_bit(FLAGS_BUSY, &dev->flags);
 583                spin_unlock(&dev->lock);
 584                dev->req->base.complete(&dev->req->base, ret);
 585        }
 586}
 587
 588static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 589                             unsigned int keylen)
 590{
 591        struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 592        int ret;
 593
 594        ctx->keylen = keylen;
 595
 596        /* SAHARA only supports 128bit keys */
 597        if (keylen == AES_KEYSIZE_128) {
 598                memcpy(ctx->key, key, keylen);
 599                ctx->flags |= FLAGS_NEW_KEY;
 600                return 0;
 601        }
 602
 603        if (keylen != AES_KEYSIZE_128 &&
 604            keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
 605                return -EINVAL;
 606
 607        /*
 608         * The requested key size is not supported by HW, do a fallback.
 609         */
 610        ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
 611        ctx->fallback->base.crt_flags |=
 612                (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
 613
 614        ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen);
 615        if (ret) {
 616                struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm);
 617
 618                tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK;
 619                tfm_aux->crt_flags |=
 620                        (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK);
 621        }
 622        return ret;
 623}
 624
 625static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
 626{
 627        struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
 628                crypto_ablkcipher_reqtfm(req));
 629        struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req);
 630        struct sahara_dev *dev = dev_ptr;
 631        int err = 0;
 632        int busy;
 633
 634        dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",
 635                req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));
 636
 637        if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
 638                dev_err(dev->device,
 639                        "request size is not exact amount of AES blocks\n");
 640                return -EINVAL;
 641        }
 642
 643        ctx->dev = dev;
 644
 645        rctx->mode = mode;
 646        spin_lock_bh(&dev->lock);
 647        err = ablkcipher_enqueue_request(&dev->queue, req);
 648        busy = test_and_set_bit(FLAGS_BUSY, &dev->flags);
 649        spin_unlock_bh(&dev->lock);
 650
 651        if (!busy)
 652                tasklet_schedule(&dev->queue_task);
 653
 654        return err;
 655}
 656
 657static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req)
 658{
 659        struct crypto_tfm *tfm =
 660                crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
 661        struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
 662                crypto_ablkcipher_reqtfm(req));
 663        int err;
 664
 665        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
 666                ablkcipher_request_set_tfm(req, ctx->fallback);
 667                err = crypto_ablkcipher_encrypt(req);
 668                ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
 669                return err;
 670        }
 671
 672        return sahara_aes_crypt(req, FLAGS_ENCRYPT);
 673}
 674
 675static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req)
 676{
 677        struct crypto_tfm *tfm =
 678                crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
 679        struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
 680                crypto_ablkcipher_reqtfm(req));
 681        int err;
 682
 683        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
 684                ablkcipher_request_set_tfm(req, ctx->fallback);
 685                err = crypto_ablkcipher_decrypt(req);
 686                ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
 687                return err;
 688        }
 689
 690        return sahara_aes_crypt(req, 0);
 691}
 692
 693static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req)
 694{
 695        struct crypto_tfm *tfm =
 696                crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
 697        struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
 698                crypto_ablkcipher_reqtfm(req));
 699        int err;
 700
 701        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
 702                ablkcipher_request_set_tfm(req, ctx->fallback);
 703                err = crypto_ablkcipher_encrypt(req);
 704                ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
 705                return err;
 706        }
 707
 708        return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
 709}
 710
 711static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req)
 712{
 713        struct crypto_tfm *tfm =
 714                crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
 715        struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
 716                crypto_ablkcipher_reqtfm(req));
 717        int err;
 718
 719        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
 720                ablkcipher_request_set_tfm(req, ctx->fallback);
 721                err = crypto_ablkcipher_decrypt(req);
 722                ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
 723                return err;
 724        }
 725
 726        return sahara_aes_crypt(req, FLAGS_CBC);
 727}
 728
 729static int sahara_aes_cra_init(struct crypto_tfm *tfm)
 730{
 731        const char *name = tfm->__crt_alg->cra_name;
 732        struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
 733
 734        ctx->fallback = crypto_alloc_ablkcipher(name, 0,
 735                                CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
 736        if (IS_ERR(ctx->fallback)) {
 737                pr_err("Error allocating fallback algo %s\n", name);
 738                return PTR_ERR(ctx->fallback);
 739        }
 740
 741        tfm->crt_ablkcipher.reqsize = sizeof(struct sahara_aes_reqctx);
 742
 743        return 0;
 744}
 745
 746static void sahara_aes_cra_exit(struct crypto_tfm *tfm)
 747{
 748        struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
 749
 750        if (ctx->fallback)
 751                crypto_free_ablkcipher(ctx->fallback);
 752        ctx->fallback = NULL;
 753}
 754
 755static struct crypto_alg aes_algs[] = {
 756{
 757        .cra_name               = "ecb(aes)",
 758        .cra_driver_name        = "sahara-ecb-aes",
 759        .cra_priority           = 300,
 760        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 761                        CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
 762        .cra_blocksize          = AES_BLOCK_SIZE,
 763        .cra_ctxsize            = sizeof(struct sahara_ctx),
 764        .cra_alignmask          = 0x0,
 765        .cra_type               = &crypto_ablkcipher_type,
 766        .cra_module             = THIS_MODULE,
 767        .cra_init               = sahara_aes_cra_init,
 768        .cra_exit               = sahara_aes_cra_exit,
 769        .cra_u.ablkcipher = {
 770                .min_keysize    = AES_MIN_KEY_SIZE ,
 771                .max_keysize    = AES_MAX_KEY_SIZE,
 772                .setkey         = sahara_aes_setkey,
 773                .encrypt        = sahara_aes_ecb_encrypt,
 774                .decrypt        = sahara_aes_ecb_decrypt,
 775        }
 776}, {
 777        .cra_name               = "cbc(aes)",
 778        .cra_driver_name        = "sahara-cbc-aes",
 779        .cra_priority           = 300,
 780        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 781                        CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
 782        .cra_blocksize          = AES_BLOCK_SIZE,
 783        .cra_ctxsize            = sizeof(struct sahara_ctx),
 784        .cra_alignmask          = 0x0,
 785        .cra_type               = &crypto_ablkcipher_type,
 786        .cra_module             = THIS_MODULE,
 787        .cra_init               = sahara_aes_cra_init,
 788        .cra_exit               = sahara_aes_cra_exit,
 789        .cra_u.ablkcipher = {
 790                .min_keysize    = AES_MIN_KEY_SIZE ,
 791                .max_keysize    = AES_MAX_KEY_SIZE,
 792                .ivsize         = AES_BLOCK_SIZE,
 793                .setkey         = sahara_aes_setkey,
 794                .encrypt        = sahara_aes_cbc_encrypt,
 795                .decrypt        = sahara_aes_cbc_decrypt,
 796        }
 797}
 798};
 799
 800static irqreturn_t sahara_irq_handler(int irq, void *data)
 801{
 802        struct sahara_dev *dev = (struct sahara_dev *)data;
 803        unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
 804        unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
 805
 806        del_timer(&dev->watchdog);
 807
 808        sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR,
 809                     SAHARA_REG_CMD);
 810
 811        sahara_decode_status(dev, stat);
 812
 813        if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) {
 814                return IRQ_NONE;
 815        } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) {
 816                dev->error = 0;
 817        } else {
 818                sahara_decode_error(dev, err);
 819                dev->error = -EINVAL;
 820        }
 821
 822        tasklet_schedule(&dev->done_task);
 823
 824        return IRQ_HANDLED;
 825}
 826
 827
 828static int sahara_register_algs(struct sahara_dev *dev)
 829{
 830        int err, i, j;
 831
 832        for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
 833                INIT_LIST_HEAD(&aes_algs[i].cra_list);
 834                err = crypto_register_alg(&aes_algs[i]);
 835                if (err)
 836                        goto err_aes_algs;
 837        }
 838
 839        return 0;
 840
 841err_aes_algs:
 842        for (j = 0; j < i; j++)
 843                crypto_unregister_alg(&aes_algs[j]);
 844
 845        return err;
 846}
 847
 848static void sahara_unregister_algs(struct sahara_dev *dev)
 849{
 850        int i;
 851
 852        for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
 853                crypto_unregister_alg(&aes_algs[i]);
 854}
 855
 856static struct platform_device_id sahara_platform_ids[] = {
 857        { .name = "sahara-imx27" },
 858        { /* sentinel */ }
 859};
 860MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
 861
 862static struct of_device_id sahara_dt_ids[] = {
 863        { .compatible = "fsl,imx27-sahara" },
 864        { /* sentinel */ }
 865};
 866MODULE_DEVICE_TABLE(of, sahara_dt_ids);
 867
 868static int sahara_probe(struct platform_device *pdev)
 869{
 870        struct sahara_dev *dev;
 871        struct resource *res;
 872        u32 version;
 873        int irq;
 874        int err;
 875        int i;
 876
 877        dev = devm_kzalloc(&pdev->dev, sizeof(struct sahara_dev), GFP_KERNEL);
 878        if (dev == NULL) {
 879                dev_err(&pdev->dev, "unable to alloc data struct.\n");
 880                return -ENOMEM;
 881        }
 882
 883        dev->device = &pdev->dev;
 884        platform_set_drvdata(pdev, dev);
 885
 886        /* Get the base address */
 887        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 888        if (!res) {
 889                dev_err(&pdev->dev, "failed to get memory region resource\n");
 890                return -ENODEV;
 891        }
 892
 893        if (devm_request_mem_region(&pdev->dev, res->start,
 894                        resource_size(res), SAHARA_NAME) == NULL) {
 895                dev_err(&pdev->dev, "failed to request memory region\n");
 896                return -ENOENT;
 897        }
 898        dev->regs_base = devm_ioremap(&pdev->dev, res->start,
 899                                      resource_size(res));
 900        if (!dev->regs_base) {
 901                dev_err(&pdev->dev, "failed to ioremap address region\n");
 902                return -ENOENT;
 903        }
 904
 905        /* Get the IRQ */
 906        irq = platform_get_irq(pdev,  0);
 907        if (irq < 0) {
 908                dev_err(&pdev->dev, "failed to get irq resource\n");
 909                return irq;
 910        }
 911
 912        if (devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
 913                0, SAHARA_NAME, dev) < 0) {
 914                dev_err(&pdev->dev, "failed to request irq\n");
 915                return -ENOENT;
 916        }
 917
 918        /* clocks */
 919        dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
 920        if (IS_ERR(dev->clk_ipg)) {
 921                dev_err(&pdev->dev, "Could not get ipg clock\n");
 922                return PTR_ERR(dev->clk_ipg);
 923        }
 924
 925        dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
 926        if (IS_ERR(dev->clk_ahb)) {
 927                dev_err(&pdev->dev, "Could not get ahb clock\n");
 928                return PTR_ERR(dev->clk_ahb);
 929        }
 930
 931        /* Allocate HW descriptors */
 932        dev->hw_desc[0] = dma_alloc_coherent(&pdev->dev,
 933                        SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
 934                        &dev->hw_phys_desc[0], GFP_KERNEL);
 935        if (!dev->hw_desc[0]) {
 936                dev_err(&pdev->dev, "Could not allocate hw descriptors\n");
 937                return -ENOMEM;
 938        }
 939        dev->hw_desc[1] = dev->hw_desc[0] + 1;
 940        dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +
 941                                sizeof(struct sahara_hw_desc);
 942
 943        /* Allocate space for iv and key */
 944        dev->key_base = dma_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,
 945                                &dev->key_phys_base, GFP_KERNEL);
 946        if (!dev->key_base) {
 947                dev_err(&pdev->dev, "Could not allocate memory for key\n");
 948                err = -ENOMEM;
 949                goto err_key;
 950        }
 951        dev->iv_base = dev->key_base + AES_KEYSIZE_128;
 952        dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;
 953
 954        /* Allocate space for HW links */
 955        dev->hw_link[0] = dma_alloc_coherent(&pdev->dev,
 956                        SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
 957                        &dev->hw_phys_link[0], GFP_KERNEL);
 958        if (!dev->hw_link[0]) {
 959                dev_err(&pdev->dev, "Could not allocate hw links\n");
 960                err = -ENOMEM;
 961                goto err_link;
 962        }
 963        for (i = 1; i < SAHARA_MAX_HW_LINK; i++) {
 964                dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +
 965                                        sizeof(struct sahara_hw_link);
 966                dev->hw_link[i] = dev->hw_link[i - 1] + 1;
 967        }
 968
 969        crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
 970
 971        dev_ptr = dev;
 972
 973        tasklet_init(&dev->queue_task, sahara_aes_queue_task,
 974                     (unsigned long)dev);
 975        tasklet_init(&dev->done_task, sahara_aes_done_task,
 976                     (unsigned long)dev);
 977
 978        init_timer(&dev->watchdog);
 979        dev->watchdog.function = &sahara_watchdog;
 980        dev->watchdog.data = (unsigned long)dev;
 981
 982        clk_prepare_enable(dev->clk_ipg);
 983        clk_prepare_enable(dev->clk_ahb);
 984
 985        version = sahara_read(dev, SAHARA_REG_VERSION);
 986        if (version != SAHARA_VERSION_3) {
 987                dev_err(&pdev->dev, "SAHARA version %d not supported\n",
 988                        version);
 989                err = -ENODEV;
 990                goto err_algs;
 991        }
 992
 993        sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH,
 994                     SAHARA_REG_CMD);
 995        sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) |
 996                        SAHARA_CONTROL_SET_MAXBURST(8) |
 997                        SAHARA_CONTROL_RNG_AUTORSD |
 998                        SAHARA_CONTROL_ENABLE_INT,
 999                        SAHARA_REG_CONTROL);
1000
1001        err = sahara_register_algs(dev);
1002        if (err)
1003                goto err_algs;
1004
1005        dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);
1006
1007        return 0;
1008
1009err_algs:
1010        dma_free_coherent(&pdev->dev,
1011                          SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1012                          dev->hw_link[0], dev->hw_phys_link[0]);
1013        clk_disable_unprepare(dev->clk_ipg);
1014        clk_disable_unprepare(dev->clk_ahb);
1015        dev_ptr = NULL;
1016err_link:
1017        dma_free_coherent(&pdev->dev,
1018                          2 * AES_KEYSIZE_128,
1019                          dev->key_base, dev->key_phys_base);
1020err_key:
1021        dma_free_coherent(&pdev->dev,
1022                          SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1023                          dev->hw_desc[0], dev->hw_phys_desc[0]);
1024
1025        return err;
1026}
1027
1028static int sahara_remove(struct platform_device *pdev)
1029{
1030        struct sahara_dev *dev = platform_get_drvdata(pdev);
1031
1032        dma_free_coherent(&pdev->dev,
1033                          SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1034                          dev->hw_link[0], dev->hw_phys_link[0]);
1035        dma_free_coherent(&pdev->dev,
1036                          2 * AES_KEYSIZE_128,
1037                          dev->key_base, dev->key_phys_base);
1038        dma_free_coherent(&pdev->dev,
1039                          SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1040                          dev->hw_desc[0], dev->hw_phys_desc[0]);
1041
1042        tasklet_kill(&dev->done_task);
1043        tasklet_kill(&dev->queue_task);
1044
1045        sahara_unregister_algs(dev);
1046
1047        clk_disable_unprepare(dev->clk_ipg);
1048        clk_disable_unprepare(dev->clk_ahb);
1049
1050        dev_ptr = NULL;
1051
1052        return 0;
1053}
1054
1055static struct platform_driver sahara_driver = {
1056        .probe          = sahara_probe,
1057        .remove         = sahara_remove,
1058        .driver         = {
1059                .name   = SAHARA_NAME,
1060                .owner  = THIS_MODULE,
1061                .of_match_table = of_match_ptr(sahara_dt_ids),
1062        },
1063        .id_table = sahara_platform_ids,
1064};
1065
1066module_platform_driver(sahara_driver);
1067
1068MODULE_LICENSE("GPL");
1069MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1070MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");
1071
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.