linux/drivers/crypto/sahara.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Cryptographic API.
   4 *
   5 * Support for SAHARA cryptographic accelerator.
   6 *
   7 * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
   8 * Copyright (c) 2013 Vista Silicon S.L.
   9 * Author: Javier Martin <javier.martin@vista-silicon.com>
  10 *
  11 * Based on omap-aes.c and tegra-aes.c
  12 */
  13
  14#include <crypto/aes.h>
  15#include <crypto/internal/hash.h>
  16#include <crypto/internal/skcipher.h>
  17#include <crypto/scatterwalk.h>
  18#include <crypto/sha1.h>
  19#include <crypto/sha2.h>
  20
  21#include <linux/clk.h>
  22#include <linux/dma-mapping.h>
  23#include <linux/interrupt.h>
  24#include <linux/io.h>
  25#include <linux/irq.h>
  26#include <linux/kernel.h>
  27#include <linux/kthread.h>
  28#include <linux/module.h>
  29#include <linux/mutex.h>
  30#include <linux/of.h>
  31#include <linux/of_device.h>
  32#include <linux/platform_device.h>
  33
  34#define SHA_BUFFER_LEN          PAGE_SIZE
  35#define SAHARA_MAX_SHA_BLOCK_SIZE       SHA256_BLOCK_SIZE
  36
  37#define SAHARA_NAME "sahara"
  38#define SAHARA_VERSION_3        3
  39#define SAHARA_VERSION_4        4
  40#define SAHARA_TIMEOUT_MS       1000
  41#define SAHARA_MAX_HW_DESC      2
  42#define SAHARA_MAX_HW_LINK      20
  43
  44#define FLAGS_MODE_MASK         0x000f
  45#define FLAGS_ENCRYPT           BIT(0)
  46#define FLAGS_CBC               BIT(1)
  47#define FLAGS_NEW_KEY           BIT(3)
  48
  49#define SAHARA_HDR_BASE                 0x00800000
  50#define SAHARA_HDR_SKHA_ALG_AES 0
  51#define SAHARA_HDR_SKHA_OP_ENC          (1 << 2)
  52#define SAHARA_HDR_SKHA_MODE_ECB        (0 << 3)
  53#define SAHARA_HDR_SKHA_MODE_CBC        (1 << 3)
  54#define SAHARA_HDR_FORM_DATA            (5 << 16)
  55#define SAHARA_HDR_FORM_KEY             (8 << 16)
  56#define SAHARA_HDR_LLO                  (1 << 24)
  57#define SAHARA_HDR_CHA_SKHA             (1 << 28)
  58#define SAHARA_HDR_CHA_MDHA             (2 << 28)
  59#define SAHARA_HDR_PARITY_BIT           (1 << 31)
  60
  61#define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000
  62#define SAHARA_HDR_MDHA_SET_MODE_HASH   0x208D0000
  63#define SAHARA_HDR_MDHA_HASH            0xA0850000
  64#define SAHARA_HDR_MDHA_STORE_DIGEST    0x20820000
  65#define SAHARA_HDR_MDHA_ALG_SHA1        0
  66#define SAHARA_HDR_MDHA_ALG_MD5         1
  67#define SAHARA_HDR_MDHA_ALG_SHA256      2
  68#define SAHARA_HDR_MDHA_ALG_SHA224      3
  69#define SAHARA_HDR_MDHA_PDATA           (1 << 2)
  70#define SAHARA_HDR_MDHA_HMAC            (1 << 3)
  71#define SAHARA_HDR_MDHA_INIT            (1 << 5)
  72#define SAHARA_HDR_MDHA_IPAD            (1 << 6)
  73#define SAHARA_HDR_MDHA_OPAD            (1 << 7)
  74#define SAHARA_HDR_MDHA_SWAP            (1 << 8)
  75#define SAHARA_HDR_MDHA_MAC_FULL        (1 << 9)
  76#define SAHARA_HDR_MDHA_SSL             (1 << 10)
  77
  78/* SAHARA can only process one request at a time */
  79#define SAHARA_QUEUE_LENGTH     1
  80
  81#define SAHARA_REG_VERSION      0x00
  82#define SAHARA_REG_DAR          0x04
  83#define SAHARA_REG_CONTROL      0x08
  84#define         SAHARA_CONTROL_SET_THROTTLE(x)  (((x) & 0xff) << 24)
  85#define         SAHARA_CONTROL_SET_MAXBURST(x)  (((x) & 0xff) << 16)
  86#define         SAHARA_CONTROL_RNG_AUTORSD      (1 << 7)
  87#define         SAHARA_CONTROL_ENABLE_INT       (1 << 4)
  88#define SAHARA_REG_CMD          0x0C
  89#define         SAHARA_CMD_RESET                (1 << 0)
  90#define         SAHARA_CMD_CLEAR_INT            (1 << 8)
  91#define         SAHARA_CMD_CLEAR_ERR            (1 << 9)
  92#define         SAHARA_CMD_SINGLE_STEP          (1 << 10)
  93#define         SAHARA_CMD_MODE_BATCH           (1 << 16)
  94#define         SAHARA_CMD_MODE_DEBUG           (1 << 18)
  95#define SAHARA_REG_STATUS       0x10
  96#define         SAHARA_STATUS_GET_STATE(x)      ((x) & 0x7)
  97#define                 SAHARA_STATE_IDLE       0
  98#define                 SAHARA_STATE_BUSY       1
  99#define                 SAHARA_STATE_ERR        2
 100#define                 SAHARA_STATE_FAULT      3
 101#define                 SAHARA_STATE_COMPLETE   4
 102#define                 SAHARA_STATE_COMP_FLAG  (1 << 2)
 103#define         SAHARA_STATUS_DAR_FULL          (1 << 3)
 104#define         SAHARA_STATUS_ERROR             (1 << 4)
 105#define         SAHARA_STATUS_SECURE            (1 << 5)
 106#define         SAHARA_STATUS_FAIL              (1 << 6)
 107#define         SAHARA_STATUS_INIT              (1 << 7)
 108#define         SAHARA_STATUS_RNG_RESEED        (1 << 8)
 109#define         SAHARA_STATUS_ACTIVE_RNG        (1 << 9)
 110#define         SAHARA_STATUS_ACTIVE_MDHA       (1 << 10)
 111#define         SAHARA_STATUS_ACTIVE_SKHA       (1 << 11)
 112#define         SAHARA_STATUS_MODE_BATCH        (1 << 16)
 113#define         SAHARA_STATUS_MODE_DEDICATED    (1 << 17)
 114#define         SAHARA_STATUS_MODE_DEBUG        (1 << 18)
 115#define         SAHARA_STATUS_GET_ISTATE(x)     (((x) >> 24) & 0xff)
 116#define SAHARA_REG_ERRSTATUS    0x14
 117#define         SAHARA_ERRSTATUS_GET_SOURCE(x)  ((x) & 0xf)
 118#define                 SAHARA_ERRSOURCE_CHA    14
 119#define                 SAHARA_ERRSOURCE_DMA    15
 120#define         SAHARA_ERRSTATUS_DMA_DIR        (1 << 8)
 121#define         SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
 122#define         SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
 123#define         SAHARA_ERRSTATUS_GET_CHASRC(x)  (((x) >> 16) & 0xfff)
 124#define         SAHARA_ERRSTATUS_GET_CHAERR(x)  (((x) >> 28) & 0x3)
 125#define SAHARA_REG_FADDR        0x18
 126#define SAHARA_REG_CDAR         0x1C
 127#define SAHARA_REG_IDAR         0x20
 128
 129struct sahara_hw_desc {
 130        u32     hdr;
 131        u32     len1;
 132        u32     p1;
 133        u32     len2;
 134        u32     p2;
 135        u32     next;
 136};
 137
 138struct sahara_hw_link {
 139        u32     len;
 140        u32     p;
 141        u32     next;
 142};
 143
 144struct sahara_ctx {
 145        unsigned long flags;
 146
 147        /* AES-specific context */
 148        int keylen;
 149        u8 key[AES_KEYSIZE_128];
 150        struct crypto_skcipher *fallback;
 151};
 152
 153struct sahara_aes_reqctx {
 154        unsigned long mode;
 155        struct skcipher_request fallback_req;   // keep at the end
 156};
 157
 158/*
 159 * struct sahara_sha_reqctx - private data per request
 160 * @buf: holds data for requests smaller than block_size
 161 * @rembuf: used to prepare one block_size-aligned request
 162 * @context: hw-specific context for request. Digest is extracted from this
 163 * @mode: specifies what type of hw-descriptor needs to be built
 164 * @digest_size: length of digest for this request
 165 * @context_size: length of hw-context for this request.
 166 *                Always digest_size + 4
 167 * @buf_cnt: number of bytes saved in buf
 168 * @sg_in_idx: number of hw links
 169 * @in_sg: scatterlist for input data
 170 * @in_sg_chain: scatterlists for chained input data
 171 * @total: total number of bytes for transfer
 172 * @last: is this the last block
 173 * @first: is this the first block
 174 * @active: inside a transfer
 175 */
 176struct sahara_sha_reqctx {
 177        u8                      buf[SAHARA_MAX_SHA_BLOCK_SIZE];
 178        u8                      rembuf[SAHARA_MAX_SHA_BLOCK_SIZE];
 179        u8                      context[SHA256_DIGEST_SIZE + 4];
 180        unsigned int            mode;
 181        unsigned int            digest_size;
 182        unsigned int            context_size;
 183        unsigned int            buf_cnt;
 184        unsigned int            sg_in_idx;
 185        struct scatterlist      *in_sg;
 186        struct scatterlist      in_sg_chain[2];
 187        size_t                  total;
 188        unsigned int            last;
 189        unsigned int            first;
 190        unsigned int            active;
 191};
 192
 193struct sahara_dev {
 194        struct device           *device;
 195        unsigned int            version;
 196        void __iomem            *regs_base;
 197        struct clk              *clk_ipg;
 198        struct clk              *clk_ahb;
 199        struct mutex            queue_mutex;
 200        struct task_struct      *kthread;
 201        struct completion       dma_completion;
 202
 203        struct sahara_ctx       *ctx;
 204        struct crypto_queue     queue;
 205        unsigned long           flags;
 206
 207        struct sahara_hw_desc   *hw_desc[SAHARA_MAX_HW_DESC];
 208        dma_addr_t              hw_phys_desc[SAHARA_MAX_HW_DESC];
 209
 210        u8                      *key_base;
 211        dma_addr_t              key_phys_base;
 212
 213        u8                      *iv_base;
 214        dma_addr_t              iv_phys_base;
 215
 216        u8                      *context_base;
 217        dma_addr_t              context_phys_base;
 218
 219        struct sahara_hw_link   *hw_link[SAHARA_MAX_HW_LINK];
 220        dma_addr_t              hw_phys_link[SAHARA_MAX_HW_LINK];
 221
 222        size_t                  total;
 223        struct scatterlist      *in_sg;
 224        int             nb_in_sg;
 225        struct scatterlist      *out_sg;
 226        int             nb_out_sg;
 227
 228        u32                     error;
 229};
 230
 231static struct sahara_dev *dev_ptr;
 232
 233static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg)
 234{
 235        writel(data, dev->regs_base + reg);
 236}
 237
 238static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg)
 239{
 240        return readl(dev->regs_base + reg);
 241}
 242
 243static u32 sahara_aes_key_hdr(struct sahara_dev *dev)
 244{
 245        u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES |
 246                        SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO |
 247                        SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
 248
 249        if (dev->flags & FLAGS_CBC) {
 250                hdr |= SAHARA_HDR_SKHA_MODE_CBC;
 251                hdr ^= SAHARA_HDR_PARITY_BIT;
 252        }
 253
 254        if (dev->flags & FLAGS_ENCRYPT) {
 255                hdr |= SAHARA_HDR_SKHA_OP_ENC;
 256                hdr ^= SAHARA_HDR_PARITY_BIT;
 257        }
 258
 259        return hdr;
 260}
 261
 262static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev)
 263{
 264        return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA |
 265                        SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
 266}
 267
 268static const char *sahara_err_src[16] = {
 269        "No error",
 270        "Header error",
 271        "Descriptor length error",
 272        "Descriptor length or pointer error",
 273        "Link length error",
 274        "Link pointer error",
 275        "Input buffer error",
 276        "Output buffer error",
 277        "Output buffer starvation",
 278        "Internal state fault",
 279        "General descriptor problem",
 280        "Reserved",
 281        "Descriptor address error",
 282        "Link address error",
 283        "CHA error",
 284        "DMA error"
 285};
 286
 287static const char *sahara_err_dmasize[4] = {
 288        "Byte transfer",
 289        "Half-word transfer",
 290        "Word transfer",
 291        "Reserved"
 292};
 293
 294static const char *sahara_err_dmasrc[8] = {
 295        "No error",
 296        "AHB bus error",
 297        "Internal IP bus error",
 298        "Parity error",
 299        "DMA crosses 256 byte boundary",
 300        "DMA is busy",
 301        "Reserved",
 302        "DMA HW error"
 303};
 304
 305static const char *sahara_cha_errsrc[12] = {
 306        "Input buffer non-empty",
 307        "Illegal address",
 308        "Illegal mode",
 309        "Illegal data size",
 310        "Illegal key size",
 311        "Write during processing",
 312        "CTX read during processing",
 313        "HW error",
 314        "Input buffer disabled/underflow",
 315        "Output buffer disabled/overflow",
 316        "DES key parity error",
 317        "Reserved"
 318};
 319
 320static const char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" };
 321
 322static void sahara_decode_error(struct sahara_dev *dev, unsigned int error)
 323{
 324        u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error);
 325        u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error));
 326
 327        dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);
 328
 329        dev_err(dev->device, "  - %s.\n", sahara_err_src[source]);
 330
 331        if (source == SAHARA_ERRSOURCE_DMA) {
 332                if (error & SAHARA_ERRSTATUS_DMA_DIR)
 333                        dev_err(dev->device, "          * DMA read.\n");
 334                else
 335                        dev_err(dev->device, "          * DMA write.\n");
 336
 337                dev_err(dev->device, "          * %s.\n",
 338                       sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]);
 339                dev_err(dev->device, "          * %s.\n",
 340                       sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]);
 341        } else if (source == SAHARA_ERRSOURCE_CHA) {
 342                dev_err(dev->device, "          * %s.\n",
 343                        sahara_cha_errsrc[chasrc]);
 344                dev_err(dev->device, "          * %s.\n",
 345                       sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]);
 346        }
 347        dev_err(dev->device, "\n");
 348}
 349
 350static const char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" };
 351
 352static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
 353{
 354        u8 state;
 355
 356        if (!__is_defined(DEBUG))
 357                return;
 358
 359        state = SAHARA_STATUS_GET_STATE(status);
 360
 361        dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",
 362                __func__, status);
 363
 364        dev_dbg(dev->device, "  - State = %d:\n", state);
 365        if (state & SAHARA_STATE_COMP_FLAG)
 366                dev_dbg(dev->device, "          * Descriptor completed. IRQ pending.\n");
 367
 368        dev_dbg(dev->device, "          * %s.\n",
 369               sahara_state[state & ~SAHARA_STATE_COMP_FLAG]);
 370
 371        if (status & SAHARA_STATUS_DAR_FULL)
 372                dev_dbg(dev->device, "  - DAR Full.\n");
 373        if (status & SAHARA_STATUS_ERROR)
 374                dev_dbg(dev->device, "  - Error.\n");
 375        if (status & SAHARA_STATUS_SECURE)
 376                dev_dbg(dev->device, "  - Secure.\n");
 377        if (status & SAHARA_STATUS_FAIL)
 378                dev_dbg(dev->device, "  - Fail.\n");
 379        if (status & SAHARA_STATUS_RNG_RESEED)
 380                dev_dbg(dev->device, "  - RNG Reseed Request.\n");
 381        if (status & SAHARA_STATUS_ACTIVE_RNG)
 382                dev_dbg(dev->device, "  - RNG Active.\n");
 383        if (status & SAHARA_STATUS_ACTIVE_MDHA)
 384                dev_dbg(dev->device, "  - MDHA Active.\n");
 385        if (status & SAHARA_STATUS_ACTIVE_SKHA)
 386                dev_dbg(dev->device, "  - SKHA Active.\n");
 387
 388        if (status & SAHARA_STATUS_MODE_BATCH)
 389                dev_dbg(dev->device, "  - Batch Mode.\n");
 390        else if (status & SAHARA_STATUS_MODE_DEDICATED)
 391                dev_dbg(dev->device, "  - Dedicated Mode.\n");
 392        else if (status & SAHARA_STATUS_MODE_DEBUG)
 393                dev_dbg(dev->device, "  - Debug Mode.\n");
 394
 395        dev_dbg(dev->device, "  - Internal state = 0x%02x\n",
 396               SAHARA_STATUS_GET_ISTATE(status));
 397
 398        dev_dbg(dev->device, "Current DAR: 0x%08x\n",
 399                sahara_read(dev, SAHARA_REG_CDAR));
 400        dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",
 401                sahara_read(dev, SAHARA_REG_IDAR));
 402}
 403
 404static void sahara_dump_descriptors(struct sahara_dev *dev)
 405{
 406        int i;
 407
 408        if (!__is_defined(DEBUG))
 409                return;
 410
 411        for (i = 0; i < SAHARA_MAX_HW_DESC; i++) {
 412                dev_dbg(dev->device, "Descriptor (%d) (%pad):\n",
 413                        i, &dev->hw_phys_desc[i]);
 414                dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);
 415                dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);
 416                dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);
 417                dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);
 418                dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);
 419                dev_dbg(dev->device, "\tnext = 0x%08x\n",
 420                        dev->hw_desc[i]->next);
 421        }
 422        dev_dbg(dev->device, "\n");
 423}
 424
 425static void sahara_dump_links(struct sahara_dev *dev)
 426{
 427        int i;
 428
 429        if (!__is_defined(DEBUG))
 430                return;
 431
 432        for (i = 0; i < SAHARA_MAX_HW_LINK; i++) {
 433                dev_dbg(dev->device, "Link (%d) (%pad):\n",
 434                        i, &dev->hw_phys_link[i]);
 435                dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);
 436                dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);
 437                dev_dbg(dev->device, "\tnext = 0x%08x\n",
 438                        dev->hw_link[i]->next);
 439        }
 440        dev_dbg(dev->device, "\n");
 441}
 442
 443static int sahara_hw_descriptor_create(struct sahara_dev *dev)
 444{
 445        struct sahara_ctx *ctx = dev->ctx;
 446        struct scatterlist *sg;
 447        int ret;
 448        int i, j;
 449        int idx = 0;
 450
 451        /* Copy new key if necessary */
 452        if (ctx->flags & FLAGS_NEW_KEY) {
 453                memcpy(dev->key_base, ctx->key, ctx->keylen);
 454                ctx->flags &= ~FLAGS_NEW_KEY;
 455
 456                if (dev->flags & FLAGS_CBC) {
 457                        dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE;
 458                        dev->hw_desc[idx]->p1 = dev->iv_phys_base;
 459                } else {
 460                        dev->hw_desc[idx]->len1 = 0;
 461                        dev->hw_desc[idx]->p1 = 0;
 462                }
 463                dev->hw_desc[idx]->len2 = ctx->keylen;
 464                dev->hw_desc[idx]->p2 = dev->key_phys_base;
 465                dev->hw_desc[idx]->next = dev->hw_phys_desc[1];
 466
 467                dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev);
 468
 469                idx++;
 470        }
 471
 472        dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total);
 473        if (dev->nb_in_sg < 0) {
 474                dev_err(dev->device, "Invalid numbers of src SG.\n");
 475                return dev->nb_in_sg;
 476        }
 477        dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total);
 478        if (dev->nb_out_sg < 0) {
 479                dev_err(dev->device, "Invalid numbers of dst SG.\n");
 480                return dev->nb_out_sg;
 481        }
 482        if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {
 483                dev_err(dev->device, "not enough hw links (%d)\n",
 484                        dev->nb_in_sg + dev->nb_out_sg);
 485                return -EINVAL;
 486        }
 487
 488        ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
 489                         DMA_TO_DEVICE);
 490        if (ret != dev->nb_in_sg) {
 491                dev_err(dev->device, "couldn't map in sg\n");
 492                goto unmap_in;
 493        }
 494        ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,
 495                         DMA_FROM_DEVICE);
 496        if (ret != dev->nb_out_sg) {
 497                dev_err(dev->device, "couldn't map out sg\n");
 498                goto unmap_out;
 499        }
 500
 501        /* Create input links */
 502        dev->hw_desc[idx]->p1 = dev->hw_phys_link[0];
 503        sg = dev->in_sg;
 504        for (i = 0; i < dev->nb_in_sg; i++) {
 505                dev->hw_link[i]->len = sg->length;
 506                dev->hw_link[i]->p = sg->dma_address;
 507                if (i == (dev->nb_in_sg - 1)) {
 508                        dev->hw_link[i]->next = 0;
 509                } else {
 510                        dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
 511                        sg = sg_next(sg);
 512                }
 513        }
 514
 515        /* Create output links */
 516        dev->hw_desc[idx]->p2 = dev->hw_phys_link[i];
 517        sg = dev->out_sg;
 518        for (j = i; j < dev->nb_out_sg + i; j++) {
 519                dev->hw_link[j]->len = sg->length;
 520                dev->hw_link[j]->p = sg->dma_address;
 521                if (j == (dev->nb_out_sg + i - 1)) {
 522                        dev->hw_link[j]->next = 0;
 523                } else {
 524                        dev->hw_link[j]->next = dev->hw_phys_link[j + 1];
 525                        sg = sg_next(sg);
 526                }
 527        }
 528
 529        /* Fill remaining fields of hw_desc[1] */
 530        dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev);
 531        dev->hw_desc[idx]->len1 = dev->total;
 532        dev->hw_desc[idx]->len2 = dev->total;
 533        dev->hw_desc[idx]->next = 0;
 534
 535        sahara_dump_descriptors(dev);
 536        sahara_dump_links(dev);
 537
 538        sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
 539
 540        return 0;
 541
 542unmap_out:
 543        dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
 544                DMA_FROM_DEVICE);
 545unmap_in:
 546        dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
 547                DMA_TO_DEVICE);
 548
 549        return -EINVAL;
 550}
 551
 552static int sahara_aes_process(struct skcipher_request *req)
 553{
 554        struct sahara_dev *dev = dev_ptr;
 555        struct sahara_ctx *ctx;
 556        struct sahara_aes_reqctx *rctx;
 557        int ret;
 558        unsigned long timeout;
 559
 560        /* Request is ready to be dispatched by the device */
 561        dev_dbg(dev->device,
 562                "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
 563                req->cryptlen, req->src, req->dst);
 564
 565        /* assign new request to device */
 566        dev->total = req->cryptlen;
 567        dev->in_sg = req->src;
 568        dev->out_sg = req->dst;
 569
 570        rctx = skcipher_request_ctx(req);
 571        ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
 572        rctx->mode &= FLAGS_MODE_MASK;
 573        dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 574
 575        if ((dev->flags & FLAGS_CBC) && req->iv)
 576                memcpy(dev->iv_base, req->iv, AES_KEYSIZE_128);
 577
 578        /* assign new context to device */
 579        dev->ctx = ctx;
 580
 581        reinit_completion(&dev->dma_completion);
 582
 583        ret = sahara_hw_descriptor_create(dev);
 584        if (ret)
 585                return -EINVAL;
 586
 587        timeout = wait_for_completion_timeout(&dev->dma_completion,
 588                                msecs_to_jiffies(SAHARA_TIMEOUT_MS));
 589        if (!timeout) {
 590                dev_err(dev->device, "AES timeout\n");
 591                return -ETIMEDOUT;
 592        }
 593
 594        dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
 595                DMA_FROM_DEVICE);
 596        dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
 597                DMA_TO_DEVICE);
 598
 599        return 0;
 600}
 601
 602static int sahara_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
 603                             unsigned int keylen)
 604{
 605        struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm);
 606
 607        ctx->keylen = keylen;
 608
 609        /* SAHARA only supports 128bit keys */
 610        if (keylen == AES_KEYSIZE_128) {
 611                memcpy(ctx->key, key, keylen);
 612                ctx->flags |= FLAGS_NEW_KEY;
 613                return 0;
 614        }
 615
 616        if (keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
 617                return -EINVAL;
 618
 619        /*
 620         * The requested key size is not supported by HW, do a fallback.
 621         */
 622        crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
 623        crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
 624                                                 CRYPTO_TFM_REQ_MASK);
 625        return crypto_skcipher_setkey(ctx->fallback, key, keylen);
 626}
 627
 628static int sahara_aes_crypt(struct skcipher_request *req, unsigned long mode)
 629{
 630        struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req);
 631        struct sahara_dev *dev = dev_ptr;
 632        int err = 0;
 633
 634        dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",
 635                req->cryptlen, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));
 636
 637        if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) {
 638                dev_err(dev->device,
 639                        "request size is not exact amount of AES blocks\n");
 640                return -EINVAL;
 641        }
 642
 643        rctx->mode = mode;
 644
 645        mutex_lock(&dev->queue_mutex);
 646        err = crypto_enqueue_request(&dev->queue, &req->base);
 647        mutex_unlock(&dev->queue_mutex);
 648
 649        wake_up_process(dev->kthread);
 650
 651        return err;
 652}
 653
 654static int sahara_aes_ecb_encrypt(struct skcipher_request *req)
 655{
 656        struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req);
 657        struct sahara_ctx *ctx = crypto_skcipher_ctx(
 658                crypto_skcipher_reqtfm(req));
 659
 660        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
 661                skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 662                skcipher_request_set_callback(&rctx->fallback_req,
 663                                              req->base.flags,
 664                                              req->base.complete,
 665                                              req->base.data);
 666                skcipher_request_set_crypt(&rctx->fallback_req, req->src,
 667                                           req->dst, req->cryptlen, req->iv);
 668                return crypto_skcipher_encrypt(&rctx->fallback_req);
 669        }
 670
 671        return sahara_aes_crypt(req, FLAGS_ENCRYPT);
 672}
 673
 674static int sahara_aes_ecb_decrypt(struct skcipher_request *req)
 675{
 676        struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req);
 677        struct sahara_ctx *ctx = crypto_skcipher_ctx(
 678                crypto_skcipher_reqtfm(req));
 679
 680        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
 681                skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 682                skcipher_request_set_callback(&rctx->fallback_req,
 683                                              req->base.flags,
 684                                              req->base.complete,
 685                                              req->base.data);
 686                skcipher_request_set_crypt(&rctx->fallback_req, req->src,
 687                                           req->dst, req->cryptlen, req->iv);
 688                return crypto_skcipher_decrypt(&rctx->fallback_req);
 689        }
 690
 691        return sahara_aes_crypt(req, 0);
 692}
 693
 694static int sahara_aes_cbc_encrypt(struct skcipher_request *req)
 695{
 696        struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req);
 697        struct sahara_ctx *ctx = crypto_skcipher_ctx(
 698                crypto_skcipher_reqtfm(req));
 699
 700        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
 701                skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 702                skcipher_request_set_callback(&rctx->fallback_req,
 703                                              req->base.flags,
 704                                              req->base.complete,
 705                                              req->base.data);
 706                skcipher_request_set_crypt(&rctx->fallback_req, req->src,
 707                                           req->dst, req->cryptlen, req->iv);
 708                return crypto_skcipher_encrypt(&rctx->fallback_req);
 709        }
 710
 711        return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
 712}
 713
 714static int sahara_aes_cbc_decrypt(struct skcipher_request *req)
 715{
 716        struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req);
 717        struct sahara_ctx *ctx = crypto_skcipher_ctx(
 718                crypto_skcipher_reqtfm(req));
 719
 720        if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
 721                skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 722                skcipher_request_set_callback(&rctx->fallback_req,
 723                                              req->base.flags,
 724                                              req->base.complete,
 725                                              req->base.data);
 726                skcipher_request_set_crypt(&rctx->fallback_req, req->src,
 727                                           req->dst, req->cryptlen, req->iv);
 728                return crypto_skcipher_decrypt(&rctx->fallback_req);
 729        }
 730
 731        return sahara_aes_crypt(req, FLAGS_CBC);
 732}
 733
 734static int sahara_aes_init_tfm(struct crypto_skcipher *tfm)
 735{
 736        const char *name = crypto_tfm_alg_name(&tfm->base);
 737        struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm);
 738
 739        ctx->fallback = crypto_alloc_skcipher(name, 0,
 740                                              CRYPTO_ALG_NEED_FALLBACK);
 741        if (IS_ERR(ctx->fallback)) {
 742                pr_err("Error allocating fallback algo %s\n", name);
 743                return PTR_ERR(ctx->fallback);
 744        }
 745
 746        crypto_skcipher_set_reqsize(tfm, sizeof(struct sahara_aes_reqctx) +
 747                                         crypto_skcipher_reqsize(ctx->fallback));
 748
 749        return 0;
 750}
 751
 752static void sahara_aes_exit_tfm(struct crypto_skcipher *tfm)
 753{
 754        struct sahara_ctx *ctx = crypto_skcipher_ctx(tfm);
 755
 756        crypto_free_skcipher(ctx->fallback);
 757}
 758
 759static u32 sahara_sha_init_hdr(struct sahara_dev *dev,
 760                              struct sahara_sha_reqctx *rctx)
 761{
 762        u32 hdr = 0;
 763
 764        hdr = rctx->mode;
 765
 766        if (rctx->first) {
 767                hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH;
 768                hdr |= SAHARA_HDR_MDHA_INIT;
 769        } else {
 770                hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY;
 771        }
 772
 773        if (rctx->last)
 774                hdr |= SAHARA_HDR_MDHA_PDATA;
 775
 776        if (hweight_long(hdr) % 2 == 0)
 777                hdr |= SAHARA_HDR_PARITY_BIT;
 778
 779        return hdr;
 780}
 781
 782static int sahara_sha_hw_links_create(struct sahara_dev *dev,
 783                                       struct sahara_sha_reqctx *rctx,
 784                                       int start)
 785{
 786        struct scatterlist *sg;
 787        unsigned int i;
 788        int ret;
 789
 790        dev->in_sg = rctx->in_sg;
 791
 792        dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total);
 793        if (dev->nb_in_sg < 0) {
 794                dev_err(dev->device, "Invalid numbers of src SG.\n");
 795                return dev->nb_in_sg;
 796        }
 797        if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) {
 798                dev_err(dev->device, "not enough hw links (%d)\n",
 799                        dev->nb_in_sg + dev->nb_out_sg);
 800                return -EINVAL;
 801        }
 802
 803        sg = dev->in_sg;
 804        ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE);
 805        if (!ret)
 806                return -EFAULT;
 807
 808        for (i = start; i < dev->nb_in_sg + start; i++) {
 809                dev->hw_link[i]->len = sg->length;
 810                dev->hw_link[i]->p = sg->dma_address;
 811                if (i == (dev->nb_in_sg + start - 1)) {
 812                        dev->hw_link[i]->next = 0;
 813                } else {
 814                        dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
 815                        sg = sg_next(sg);
 816                }
 817        }
 818
 819        return i;
 820}
 821
 822static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev,
 823                                                struct sahara_sha_reqctx *rctx,
 824                                                struct ahash_request *req,
 825                                                int index)
 826{
 827        unsigned result_len;
 828        int i = index;
 829
 830        if (rctx->first)
 831                /* Create initial descriptor: #8*/
 832                dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
 833        else
 834                /* Create hash descriptor: #10. Must follow #6. */
 835                dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH;
 836
 837        dev->hw_desc[index]->len1 = rctx->total;
 838        if (dev->hw_desc[index]->len1 == 0) {
 839                /* if len1 is 0, p1 must be 0, too */
 840                dev->hw_desc[index]->p1 = 0;
 841                rctx->sg_in_idx = 0;
 842        } else {
 843                /* Create input links */
 844                dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
 845                i = sahara_sha_hw_links_create(dev, rctx, index);
 846
 847                rctx->sg_in_idx = index;
 848                if (i < 0)
 849                        return i;
 850        }
 851
 852        dev->hw_desc[index]->p2 = dev->hw_phys_link[i];
 853
 854        /* Save the context for the next operation */
 855        result_len = rctx->context_size;
 856        dev->hw_link[i]->p = dev->context_phys_base;
 857
 858        dev->hw_link[i]->len = result_len;
 859        dev->hw_desc[index]->len2 = result_len;
 860
 861        dev->hw_link[i]->next = 0;
 862
 863        return 0;
 864}
 865
 866/*
 867 * Load descriptor aka #6
 868 *
 869 * To load a previously saved context back to the MDHA unit
 870 *
 871 * p1: Saved Context
 872 * p2: NULL
 873 *
 874 */
 875static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev,
 876                                                struct sahara_sha_reqctx *rctx,
 877                                                struct ahash_request *req,
 878                                                int index)
 879{
 880        dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
 881
 882        dev->hw_desc[index]->len1 = rctx->context_size;
 883        dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
 884        dev->hw_desc[index]->len2 = 0;
 885        dev->hw_desc[index]->p2 = 0;
 886
 887        dev->hw_link[index]->len = rctx->context_size;
 888        dev->hw_link[index]->p = dev->context_phys_base;
 889        dev->hw_link[index]->next = 0;
 890
 891        return 0;
 892}
 893
 894static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes)
 895{
 896        if (!sg || !sg->length)
 897                return nbytes;
 898
 899        while (nbytes && sg) {
 900                if (nbytes <= sg->length) {
 901                        sg->length = nbytes;
 902                        sg_mark_end(sg);
 903                        break;
 904                }
 905                nbytes -= sg->length;
 906                sg = sg_next(sg);
 907        }
 908
 909        return nbytes;
 910}
 911
 912static int sahara_sha_prepare_request(struct ahash_request *req)
 913{
 914        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 915        struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
 916        unsigned int hash_later;
 917        unsigned int block_size;
 918        unsigned int len;
 919
 920        block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
 921
 922        /* append bytes from previous operation */
 923        len = rctx->buf_cnt + req->nbytes;
 924
 925        /* only the last transfer can be padded in hardware */
 926        if (!rctx->last && (len < block_size)) {
 927                /* to few data, save for next operation */
 928                scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src,
 929                                         0, req->nbytes, 0);
 930                rctx->buf_cnt += req->nbytes;
 931
 932                return 0;
 933        }
 934
 935        /* add data from previous operation first */
 936        if (rctx->buf_cnt)
 937                memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt);
 938
 939        /* data must always be a multiple of block_size */
 940        hash_later = rctx->last ? 0 : len & (block_size - 1);
 941        if (hash_later) {
 942                unsigned int offset = req->nbytes - hash_later;
 943                /* Save remaining bytes for later use */
 944                scatterwalk_map_and_copy(rctx->buf, req->src, offset,
 945                                        hash_later, 0);
 946        }
 947
 948        /* nbytes should now be multiple of blocksize */
 949        req->nbytes = req->nbytes - hash_later;
 950
 951        sahara_walk_and_recalc(req->src, req->nbytes);
 952
 953        /* have data from previous operation and current */
 954        if (rctx->buf_cnt && req->nbytes) {
 955                sg_init_table(rctx->in_sg_chain, 2);
 956                sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt);
 957
 958                sg_chain(rctx->in_sg_chain, 2, req->src);
 959
 960                rctx->total = req->nbytes + rctx->buf_cnt;
 961                rctx->in_sg = rctx->in_sg_chain;
 962
 963                req->src = rctx->in_sg_chain;
 964        /* only data from previous operation */
 965        } else if (rctx->buf_cnt) {
 966                if (req->src)
 967                        rctx->in_sg = req->src;
 968                else
 969                        rctx->in_sg = rctx->in_sg_chain;
 970                /* buf was copied into rembuf above */
 971                sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt);
 972                rctx->total = rctx->buf_cnt;
 973        /* no data from previous operation */
 974        } else {
 975                rctx->in_sg = req->src;
 976                rctx->total = req->nbytes;
 977                req->src = rctx->in_sg;
 978        }
 979
 980        /* on next call, we only have the remaining data in the buffer */
 981        rctx->buf_cnt = hash_later;
 982
 983        return -EINPROGRESS;
 984}
 985
 986static int sahara_sha_process(struct ahash_request *req)
 987{
 988        struct sahara_dev *dev = dev_ptr;
 989        struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
 990        int ret;
 991        unsigned long timeout;
 992
 993        ret = sahara_sha_prepare_request(req);
 994        if (!ret)
 995                return ret;
 996
 997        if (rctx->first) {
 998                sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0);
 999                dev->hw_desc[0]->next = 0;
1000                rctx->first = 0;
1001        } else {
1002                memcpy(dev->context_base, rctx->context, rctx->context_size);
1003
1004                sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0);
1005                dev->hw_desc[0]->next = dev->hw_phys_desc[1];
1006                sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1);
1007                dev->hw_desc[1]->next = 0;
1008        }
1009
1010        sahara_dump_descriptors(dev);
1011        sahara_dump_links(dev);
1012
1013        reinit_completion(&dev->dma_completion);
1014
1015        sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
1016
1017        timeout = wait_for_completion_timeout(&dev->dma_completion,
1018                                msecs_to_jiffies(SAHARA_TIMEOUT_MS));
1019        if (!timeout) {
1020                dev_err(dev->device, "SHA timeout\n");
1021                return -ETIMEDOUT;
1022        }
1023
1024        if (rctx->sg_in_idx)
1025                dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
1026                             DMA_TO_DEVICE);
1027
1028        memcpy(rctx->context, dev->context_base, rctx->context_size);
1029
1030        if (req->result)
1031                memcpy(req->result, rctx->context, rctx->digest_size);
1032
1033        return 0;
1034}
1035
1036static int sahara_queue_manage(void *data)
1037{
1038        struct sahara_dev *dev = (struct sahara_dev *)data;
1039        struct crypto_async_request *async_req;
1040        struct crypto_async_request *backlog;
1041        int ret = 0;
1042
1043        do {
1044                __set_current_state(TASK_INTERRUPTIBLE);
1045
1046                mutex_lock(&dev->queue_mutex);
1047                backlog = crypto_get_backlog(&dev->queue);
1048                async_req = crypto_dequeue_request(&dev->queue);
1049                mutex_unlock(&dev->queue_mutex);
1050
1051                if (backlog)
1052                        backlog->complete(backlog, -EINPROGRESS);
1053
1054                if (async_req) {
1055                        if (crypto_tfm_alg_type(async_req->tfm) ==
1056                            CRYPTO_ALG_TYPE_AHASH) {
1057                                struct ahash_request *req =
1058                                        ahash_request_cast(async_req);
1059
1060                                ret = sahara_sha_process(req);
1061                        } else {
1062                                struct skcipher_request *req =
1063                                        skcipher_request_cast(async_req);
1064
1065                                ret = sahara_aes_process(req);
1066                        }
1067
1068                        async_req->complete(async_req, ret);
1069
1070                        continue;
1071                }
1072
1073                schedule();
1074        } while (!kthread_should_stop());
1075
1076        return 0;
1077}
1078
1079static int sahara_sha_enqueue(struct ahash_request *req, int last)
1080{
1081        struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1082        struct sahara_dev *dev = dev_ptr;
1083        int ret;
1084
1085        if (!req->nbytes && !last)
1086                return 0;
1087
1088        rctx->last = last;
1089
1090        if (!rctx->active) {
1091                rctx->active = 1;
1092                rctx->first = 1;
1093        }
1094
1095        mutex_lock(&dev->queue_mutex);
1096        ret = crypto_enqueue_request(&dev->queue, &req->base);
1097        mutex_unlock(&dev->queue_mutex);
1098
1099        wake_up_process(dev->kthread);
1100
1101        return ret;
1102}
1103
1104static int sahara_sha_init(struct ahash_request *req)
1105{
1106        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1107        struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1108
1109        memset(rctx, 0, sizeof(*rctx));
1110
1111        switch (crypto_ahash_digestsize(tfm)) {
1112        case SHA1_DIGEST_SIZE:
1113                rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1;
1114                rctx->digest_size = SHA1_DIGEST_SIZE;
1115                break;
1116        case SHA256_DIGEST_SIZE:
1117                rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256;
1118                rctx->digest_size = SHA256_DIGEST_SIZE;
1119                break;
1120        default:
1121                return -EINVAL;
1122        }
1123
1124        rctx->context_size = rctx->digest_size + 4;
1125        rctx->active = 0;
1126
1127        return 0;
1128}
1129
1130static int sahara_sha_update(struct ahash_request *req)
1131{
1132        return sahara_sha_enqueue(req, 0);
1133}
1134
1135static int sahara_sha_final(struct ahash_request *req)
1136{
1137        req->nbytes = 0;
1138        return sahara_sha_enqueue(req, 1);
1139}
1140
1141static int sahara_sha_finup(struct ahash_request *req)
1142{
1143        return sahara_sha_enqueue(req, 1);
1144}
1145
1146static int sahara_sha_digest(struct ahash_request *req)
1147{
1148        sahara_sha_init(req);
1149
1150        return sahara_sha_finup(req);
1151}
1152
1153static int sahara_sha_export(struct ahash_request *req, void *out)
1154{
1155        struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1156
1157        memcpy(out, rctx, sizeof(struct sahara_sha_reqctx));
1158
1159        return 0;
1160}
1161
1162static int sahara_sha_import(struct ahash_request *req, const void *in)
1163{
1164        struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1165
1166        memcpy(rctx, in, sizeof(struct sahara_sha_reqctx));
1167
1168        return 0;
1169}
1170
1171static int sahara_sha_cra_init(struct crypto_tfm *tfm)
1172{
1173        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1174                                 sizeof(struct sahara_sha_reqctx) +
1175                                 SHA_BUFFER_LEN + SHA256_BLOCK_SIZE);
1176
1177        return 0;
1178}
1179
1180static struct skcipher_alg aes_algs[] = {
1181{
1182        .base.cra_name          = "ecb(aes)",
1183        .base.cra_driver_name   = "sahara-ecb-aes",
1184        .base.cra_priority      = 300,
1185        .base.cra_flags         = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1186        .base.cra_blocksize     = AES_BLOCK_SIZE,
1187        .base.cra_ctxsize       = sizeof(struct sahara_ctx),
1188        .base.cra_alignmask     = 0x0,
1189        .base.cra_module        = THIS_MODULE,
1190
1191        .init                   = sahara_aes_init_tfm,
1192        .exit                   = sahara_aes_exit_tfm,
1193        .min_keysize            = AES_MIN_KEY_SIZE ,
1194        .max_keysize            = AES_MAX_KEY_SIZE,
1195        .setkey                 = sahara_aes_setkey,
1196        .encrypt                = sahara_aes_ecb_encrypt,
1197        .decrypt                = sahara_aes_ecb_decrypt,
1198}, {
1199        .base.cra_name          = "cbc(aes)",
1200        .base.cra_driver_name   = "sahara-cbc-aes",
1201        .base.cra_priority      = 300,
1202        .base.cra_flags         = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1203        .base.cra_blocksize     = AES_BLOCK_SIZE,
1204        .base.cra_ctxsize       = sizeof(struct sahara_ctx),
1205        .base.cra_alignmask     = 0x0,
1206        .base.cra_module        = THIS_MODULE,
1207
1208        .init                   = sahara_aes_init_tfm,
1209        .exit                   = sahara_aes_exit_tfm,
1210        .min_keysize            = AES_MIN_KEY_SIZE ,
1211        .max_keysize            = AES_MAX_KEY_SIZE,
1212        .ivsize                 = AES_BLOCK_SIZE,
1213        .setkey                 = sahara_aes_setkey,
1214        .encrypt                = sahara_aes_cbc_encrypt,
1215        .decrypt                = sahara_aes_cbc_decrypt,
1216}
1217};
1218
1219static struct ahash_alg sha_v3_algs[] = {
1220{
1221        .init           = sahara_sha_init,
1222        .update         = sahara_sha_update,
1223        .final          = sahara_sha_final,
1224        .finup          = sahara_sha_finup,
1225        .digest         = sahara_sha_digest,
1226        .export         = sahara_sha_export,
1227        .import         = sahara_sha_import,
1228        .halg.digestsize        = SHA1_DIGEST_SIZE,
1229        .halg.statesize         = sizeof(struct sahara_sha_reqctx),
1230        .halg.base      = {
1231                .cra_name               = "sha1",
1232                .cra_driver_name        = "sahara-sha1",
1233                .cra_priority           = 300,
1234                .cra_flags              = CRYPTO_ALG_ASYNC |
1235                                                CRYPTO_ALG_NEED_FALLBACK,
1236                .cra_blocksize          = SHA1_BLOCK_SIZE,
1237                .cra_ctxsize            = sizeof(struct sahara_ctx),
1238                .cra_alignmask          = 0,
1239                .cra_module             = THIS_MODULE,
1240                .cra_init               = sahara_sha_cra_init,
1241        }
1242},
1243};
1244
1245static struct ahash_alg sha_v4_algs[] = {
1246{
1247        .init           = sahara_sha_init,
1248        .update         = sahara_sha_update,
1249        .final          = sahara_sha_final,
1250        .finup          = sahara_sha_finup,
1251        .digest         = sahara_sha_digest,
1252        .export         = sahara_sha_export,
1253        .import         = sahara_sha_import,
1254        .halg.digestsize        = SHA256_DIGEST_SIZE,
1255        .halg.statesize         = sizeof(struct sahara_sha_reqctx),
1256        .halg.base      = {
1257                .cra_name               = "sha256",
1258                .cra_driver_name        = "sahara-sha256",
1259                .cra_priority           = 300,
1260                .cra_flags              = CRYPTO_ALG_ASYNC |
1261                                                CRYPTO_ALG_NEED_FALLBACK,
1262                .cra_blocksize          = SHA256_BLOCK_SIZE,
1263                .cra_ctxsize            = sizeof(struct sahara_ctx),
1264                .cra_alignmask          = 0,
1265                .cra_module             = THIS_MODULE,
1266                .cra_init               = sahara_sha_cra_init,
1267        }
1268},
1269};
1270
1271static irqreturn_t sahara_irq_handler(int irq, void *data)
1272{
1273        struct sahara_dev *dev = (struct sahara_dev *)data;
1274        unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
1275        unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
1276
1277        sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR,
1278                     SAHARA_REG_CMD);
1279
1280        sahara_decode_status(dev, stat);
1281
1282        if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) {
1283                return IRQ_NONE;
1284        } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) {
1285                dev->error = 0;
1286        } else {
1287                sahara_decode_error(dev, err);
1288                dev->error = -EINVAL;
1289        }
1290
1291        complete(&dev->dma_completion);
1292
1293        return IRQ_HANDLED;
1294}
1295
1296
1297static int sahara_register_algs(struct sahara_dev *dev)
1298{
1299        int err;
1300        unsigned int i, j, k, l;
1301
1302        for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1303                err = crypto_register_skcipher(&aes_algs[i]);
1304                if (err)
1305                        goto err_aes_algs;
1306        }
1307
1308        for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) {
1309                err = crypto_register_ahash(&sha_v3_algs[k]);
1310                if (err)
1311                        goto err_sha_v3_algs;
1312        }
1313
1314        if (dev->version > SAHARA_VERSION_3)
1315                for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) {
1316                        err = crypto_register_ahash(&sha_v4_algs[l]);
1317                        if (err)
1318                                goto err_sha_v4_algs;
1319                }
1320
1321        return 0;
1322
1323err_sha_v4_algs:
1324        for (j = 0; j < l; j++)
1325                crypto_unregister_ahash(&sha_v4_algs[j]);
1326
1327err_sha_v3_algs:
1328        for (j = 0; j < k; j++)
1329                crypto_unregister_ahash(&sha_v3_algs[j]);
1330
1331err_aes_algs:
1332        for (j = 0; j < i; j++)
1333                crypto_unregister_skcipher(&aes_algs[j]);
1334
1335        return err;
1336}
1337
1338static void sahara_unregister_algs(struct sahara_dev *dev)
1339{
1340        unsigned int i;
1341
1342        for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1343                crypto_unregister_skcipher(&aes_algs[i]);
1344
1345        for (i = 0; i < ARRAY_SIZE(sha_v3_algs); i++)
1346                crypto_unregister_ahash(&sha_v3_algs[i]);
1347
1348        if (dev->version > SAHARA_VERSION_3)
1349                for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1350                        crypto_unregister_ahash(&sha_v4_algs[i]);
1351}
1352
1353static const struct of_device_id sahara_dt_ids[] = {
1354        { .compatible = "fsl,imx53-sahara" },
1355        { .compatible = "fsl,imx27-sahara" },
1356        { /* sentinel */ }
1357};
1358MODULE_DEVICE_TABLE(of, sahara_dt_ids);
1359
1360static int sahara_probe(struct platform_device *pdev)
1361{
1362        struct sahara_dev *dev;
1363        u32 version;
1364        int irq;
1365        int err;
1366        int i;
1367
1368        dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
1369        if (!dev)
1370                return -ENOMEM;
1371
1372        dev->device = &pdev->dev;
1373        platform_set_drvdata(pdev, dev);
1374
1375        /* Get the base address */
1376        dev->regs_base = devm_platform_ioremap_resource(pdev, 0);
1377        if (IS_ERR(dev->regs_base))
1378                return PTR_ERR(dev->regs_base);
1379
1380        /* Get the IRQ */
1381        irq = platform_get_irq(pdev,  0);
1382        if (irq < 0)
1383                return irq;
1384
1385        err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
1386                               0, dev_name(&pdev->dev), dev);
1387        if (err) {
1388                dev_err(&pdev->dev, "failed to request irq\n");
1389                return err;
1390        }
1391
1392        /* clocks */
1393        dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1394        if (IS_ERR(dev->clk_ipg)) {
1395                dev_err(&pdev->dev, "Could not get ipg clock\n");
1396                return PTR_ERR(dev->clk_ipg);
1397        }
1398
1399        dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
1400        if (IS_ERR(dev->clk_ahb)) {
1401                dev_err(&pdev->dev, "Could not get ahb clock\n");
1402                return PTR_ERR(dev->clk_ahb);
1403        }
1404
1405        /* Allocate HW descriptors */
1406        dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev,
1407                        SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1408                        &dev->hw_phys_desc[0], GFP_KERNEL);
1409        if (!dev->hw_desc[0]) {
1410                dev_err(&pdev->dev, "Could not allocate hw descriptors\n");
1411                return -ENOMEM;
1412        }
1413        dev->hw_desc[1] = dev->hw_desc[0] + 1;
1414        dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +
1415                                sizeof(struct sahara_hw_desc);
1416
1417        /* Allocate space for iv and key */
1418        dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,
1419                                &dev->key_phys_base, GFP_KERNEL);
1420        if (!dev->key_base) {
1421                dev_err(&pdev->dev, "Could not allocate memory for key\n");
1422                return -ENOMEM;
1423        }
1424        dev->iv_base = dev->key_base + AES_KEYSIZE_128;
1425        dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;
1426
1427        /* Allocate space for context: largest digest + message length field */
1428        dev->context_base = dmam_alloc_coherent(&pdev->dev,
1429                                        SHA256_DIGEST_SIZE + 4,
1430                                        &dev->context_phys_base, GFP_KERNEL);
1431        if (!dev->context_base) {
1432                dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n");
1433                return -ENOMEM;
1434        }
1435
1436        /* Allocate space for HW links */
1437        dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev,
1438                        SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1439                        &dev->hw_phys_link[0], GFP_KERNEL);
1440        if (!dev->hw_link[0]) {
1441                dev_err(&pdev->dev, "Could not allocate hw links\n");
1442                return -ENOMEM;
1443        }
1444        for (i = 1; i < SAHARA_MAX_HW_LINK; i++) {
1445                dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +
1446                                        sizeof(struct sahara_hw_link);
1447                dev->hw_link[i] = dev->hw_link[i - 1] + 1;
1448        }
1449
1450        crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
1451
1452        mutex_init(&dev->queue_mutex);
1453
1454        dev_ptr = dev;
1455
1456        dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto");
1457        if (IS_ERR(dev->kthread)) {
1458                return PTR_ERR(dev->kthread);
1459        }
1460
1461        init_completion(&dev->dma_completion);
1462
1463        err = clk_prepare_enable(dev->clk_ipg);
1464        if (err)
1465                return err;
1466        err = clk_prepare_enable(dev->clk_ahb);
1467        if (err)
1468                goto clk_ipg_disable;
1469
1470        version = sahara_read(dev, SAHARA_REG_VERSION);
1471        if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) {
1472                if (version != SAHARA_VERSION_3)
1473                        err = -ENODEV;
1474        } else if (of_device_is_compatible(pdev->dev.of_node,
1475                        "fsl,imx53-sahara")) {
1476                if (((version >> 8) & 0xff) != SAHARA_VERSION_4)
1477                        err = -ENODEV;
1478                version = (version >> 8) & 0xff;
1479        }
1480        if (err == -ENODEV) {
1481                dev_err(&pdev->dev, "SAHARA version %d not supported\n",
1482                                version);
1483                goto err_algs;
1484        }
1485
1486        dev->version = version;
1487
1488        sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH,
1489                     SAHARA_REG_CMD);
1490        sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) |
1491                        SAHARA_CONTROL_SET_MAXBURST(8) |
1492                        SAHARA_CONTROL_RNG_AUTORSD |
1493                        SAHARA_CONTROL_ENABLE_INT,
1494                        SAHARA_REG_CONTROL);
1495
1496        err = sahara_register_algs(dev);
1497        if (err)
1498                goto err_algs;
1499
1500        dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);
1501
1502        return 0;
1503
1504err_algs:
1505        kthread_stop(dev->kthread);
1506        dev_ptr = NULL;
1507        clk_disable_unprepare(dev->clk_ahb);
1508clk_ipg_disable:
1509        clk_disable_unprepare(dev->clk_ipg);
1510
1511        return err;
1512}
1513
1514static int sahara_remove(struct platform_device *pdev)
1515{
1516        struct sahara_dev *dev = platform_get_drvdata(pdev);
1517
1518        kthread_stop(dev->kthread);
1519
1520        sahara_unregister_algs(dev);
1521
1522        clk_disable_unprepare(dev->clk_ipg);
1523        clk_disable_unprepare(dev->clk_ahb);
1524
1525        dev_ptr = NULL;
1526
1527        return 0;
1528}
1529
1530static struct platform_driver sahara_driver = {
1531        .probe          = sahara_probe,
1532        .remove         = sahara_remove,
1533        .driver         = {
1534                .name   = SAHARA_NAME,
1535                .of_match_table = sahara_dt_ids,
1536        },
1537};
1538
1539module_platform_driver(sahara_driver);
1540
1541MODULE_LICENSE("GPL");
1542MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1543MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>");
1544MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");
1545
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.