linux/drivers/crypto/atmel-sha.c
<<
>>
Prefs
   1/*
   2 * Cryptographic API.
   3 *
   4 * Support for ATMEL SHA1/SHA256 HW acceleration.
   5 *
   6 * Copyright (c) 2012 Eukréa Electromatique - ATMEL
   7 * Author: Nicolas Royer <nicolas@eukrea.com>
   8 *
   9 * This program is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License version 2 as published
  11 * by the Free Software Foundation.
  12 *
  13 * Some ideas are from omap-sham.c drivers.
  14 */
  15
  16
  17#include <linux/kernel.h>
  18#include <linux/module.h>
  19#include <linux/slab.h>
  20#include <linux/err.h>
  21#include <linux/clk.h>
  22#include <linux/io.h>
  23#include <linux/hw_random.h>
  24#include <linux/platform_device.h>
  25
  26#include <linux/device.h>
  27#include <linux/init.h>
  28#include <linux/errno.h>
  29#include <linux/interrupt.h>
  30#include <linux/irq.h>
  31#include <linux/scatterlist.h>
  32#include <linux/dma-mapping.h>
  33#include <linux/delay.h>
  34#include <linux/crypto.h>
  35#include <linux/cryptohash.h>
  36#include <crypto/scatterwalk.h>
  37#include <crypto/algapi.h>
  38#include <crypto/sha.h>
  39#include <crypto/hash.h>
  40#include <crypto/internal/hash.h>
  41#include <linux/platform_data/crypto-atmel.h>
  42#include "atmel-sha-regs.h"
  43
  44/* SHA flags */
  45#define SHA_FLAGS_BUSY                  BIT(0)
  46#define SHA_FLAGS_FINAL                 BIT(1)
  47#define SHA_FLAGS_DMA_ACTIVE    BIT(2)
  48#define SHA_FLAGS_OUTPUT_READY  BIT(3)
  49#define SHA_FLAGS_INIT                  BIT(4)
  50#define SHA_FLAGS_CPU                   BIT(5)
  51#define SHA_FLAGS_DMA_READY             BIT(6)
  52
  53#define SHA_FLAGS_FINUP         BIT(16)
  54#define SHA_FLAGS_SG            BIT(17)
  55#define SHA_FLAGS_SHA1          BIT(18)
  56#define SHA_FLAGS_SHA224        BIT(19)
  57#define SHA_FLAGS_SHA256        BIT(20)
  58#define SHA_FLAGS_SHA384        BIT(21)
  59#define SHA_FLAGS_SHA512        BIT(22)
  60#define SHA_FLAGS_ERROR         BIT(23)
  61#define SHA_FLAGS_PAD           BIT(24)
  62
  63#define SHA_OP_UPDATE   1
  64#define SHA_OP_FINAL    2
  65
  66#define SHA_BUFFER_LEN          PAGE_SIZE
  67
  68#define ATMEL_SHA_DMA_THRESHOLD         56
  69
  70struct atmel_sha_caps {
  71        bool    has_dma;
  72        bool    has_dualbuff;
  73        bool    has_sha224;
  74        bool    has_sha_384_512;
  75};
  76
  77struct atmel_sha_dev;
  78
  79struct atmel_sha_reqctx {
  80        struct atmel_sha_dev    *dd;
  81        unsigned long   flags;
  82        unsigned long   op;
  83
  84        u8      digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32));
  85        u64     digcnt[2];
  86        size_t  bufcnt;
  87        size_t  buflen;
  88        dma_addr_t      dma_addr;
  89
  90        /* walk state */
  91        struct scatterlist      *sg;
  92        unsigned int    offset; /* offset in current sg */
  93        unsigned int    total;  /* total request */
  94
  95        size_t block_size;
  96
  97        u8      buffer[0] __aligned(sizeof(u32));
  98};
  99
 100struct atmel_sha_ctx {
 101        struct atmel_sha_dev    *dd;
 102
 103        unsigned long           flags;
 104
 105        /* fallback stuff */
 106        struct crypto_shash     *fallback;
 107
 108};
 109
 110#define ATMEL_SHA_QUEUE_LENGTH  50
 111
 112struct atmel_sha_dma {
 113        struct dma_chan                 *chan;
 114        struct dma_slave_config dma_conf;
 115};
 116
 117struct atmel_sha_dev {
 118        struct list_head        list;
 119        unsigned long           phys_base;
 120        struct device           *dev;
 121        struct clk                      *iclk;
 122        int                                     irq;
 123        void __iomem            *io_base;
 124
 125        spinlock_t              lock;
 126        int                     err;
 127        struct tasklet_struct   done_task;
 128
 129        unsigned long           flags;
 130        struct crypto_queue     queue;
 131        struct ahash_request    *req;
 132
 133        struct atmel_sha_dma    dma_lch_in;
 134
 135        struct atmel_sha_caps   caps;
 136
 137        u32     hw_version;
 138};
 139
 140struct atmel_sha_drv {
 141        struct list_head        dev_list;
 142        spinlock_t              lock;
 143};
 144
 145static struct atmel_sha_drv atmel_sha = {
 146        .dev_list = LIST_HEAD_INIT(atmel_sha.dev_list),
 147        .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock),
 148};
 149
 150static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset)
 151{
 152        return readl_relaxed(dd->io_base + offset);
 153}
 154
 155static inline void atmel_sha_write(struct atmel_sha_dev *dd,
 156                                        u32 offset, u32 value)
 157{
 158        writel_relaxed(value, dd->io_base + offset);
 159}
 160
 161static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx)
 162{
 163        size_t count;
 164
 165        while ((ctx->bufcnt < ctx->buflen) && ctx->total) {
 166                count = min(ctx->sg->length - ctx->offset, ctx->total);
 167                count = min(count, ctx->buflen - ctx->bufcnt);
 168
 169                if (count <= 0)
 170                        break;
 171
 172                scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg,
 173                        ctx->offset, count, 0);
 174
 175                ctx->bufcnt += count;
 176                ctx->offset += count;
 177                ctx->total -= count;
 178
 179                if (ctx->offset == ctx->sg->length) {
 180                        ctx->sg = sg_next(ctx->sg);
 181                        if (ctx->sg)
 182                                ctx->offset = 0;
 183                        else
 184                                ctx->total = 0;
 185                }
 186        }
 187
 188        return 0;
 189}
 190
 191/*
 192 * The purpose of this padding is to ensure that the padded message is a
 193 * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512).
 194 * The bit "1" is appended at the end of the message followed by
 195 * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or
 196 * 128 bits block (SHA384/SHA512) equals to the message length in bits
 197 * is appended.
 198 *
 199 * For SHA1/SHA224/SHA256, padlen is calculated as followed:
 200 *  - if message length < 56 bytes then padlen = 56 - message length
 201 *  - else padlen = 64 + 56 - message length
 202 *
 203 * For SHA384/SHA512, padlen is calculated as followed:
 204 *  - if message length < 112 bytes then padlen = 112 - message length
 205 *  - else padlen = 128 + 112 - message length
 206 */
 207static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)
 208{
 209        unsigned int index, padlen;
 210        u64 bits[2];
 211        u64 size[2];
 212
 213        size[0] = ctx->digcnt[0];
 214        size[1] = ctx->digcnt[1];
 215
 216        size[0] += ctx->bufcnt;
 217        if (size[0] < ctx->bufcnt)
 218                size[1]++;
 219
 220        size[0] += length;
 221        if (size[0]  < length)
 222                size[1]++;
 223
 224        bits[1] = cpu_to_be64(size[0] << 3);
 225        bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61);
 226
 227        if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) {
 228                index = ctx->bufcnt & 0x7f;
 229                padlen = (index < 112) ? (112 - index) : ((128+112) - index);
 230                *(ctx->buffer + ctx->bufcnt) = 0x80;
 231                memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1);
 232                memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16);
 233                ctx->bufcnt += padlen + 16;
 234                ctx->flags |= SHA_FLAGS_PAD;
 235        } else {
 236                index = ctx->bufcnt & 0x3f;
 237                padlen = (index < 56) ? (56 - index) : ((64+56) - index);
 238                *(ctx->buffer + ctx->bufcnt) = 0x80;
 239                memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1);
 240                memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8);
 241                ctx->bufcnt += padlen + 8;
 242                ctx->flags |= SHA_FLAGS_PAD;
 243        }
 244}
 245
 246static int atmel_sha_init(struct ahash_request *req)
 247{
 248        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 249        struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm);
 250        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 251        struct atmel_sha_dev *dd = NULL;
 252        struct atmel_sha_dev *tmp;
 253
 254        spin_lock_bh(&atmel_sha.lock);
 255        if (!tctx->dd) {
 256                list_for_each_entry(tmp, &atmel_sha.dev_list, list) {
 257                        dd = tmp;
 258                        break;
 259                }
 260                tctx->dd = dd;
 261        } else {
 262                dd = tctx->dd;
 263        }
 264
 265        spin_unlock_bh(&atmel_sha.lock);
 266
 267        ctx->dd = dd;
 268
 269        ctx->flags = 0;
 270
 271        dev_dbg(dd->dev, "init: digest size: %d\n",
 272                crypto_ahash_digestsize(tfm));
 273
 274        switch (crypto_ahash_digestsize(tfm)) {
 275        case SHA1_DIGEST_SIZE:
 276                ctx->flags |= SHA_FLAGS_SHA1;
 277                ctx->block_size = SHA1_BLOCK_SIZE;
 278                break;
 279        case SHA224_DIGEST_SIZE:
 280                ctx->flags |= SHA_FLAGS_SHA224;
 281                ctx->block_size = SHA224_BLOCK_SIZE;
 282                break;
 283        case SHA256_DIGEST_SIZE:
 284                ctx->flags |= SHA_FLAGS_SHA256;
 285                ctx->block_size = SHA256_BLOCK_SIZE;
 286                break;
 287        case SHA384_DIGEST_SIZE:
 288                ctx->flags |= SHA_FLAGS_SHA384;
 289                ctx->block_size = SHA384_BLOCK_SIZE;
 290                break;
 291        case SHA512_DIGEST_SIZE:
 292                ctx->flags |= SHA_FLAGS_SHA512;
 293                ctx->block_size = SHA512_BLOCK_SIZE;
 294                break;
 295        default:
 296                return -EINVAL;
 297                break;
 298        }
 299
 300        ctx->bufcnt = 0;
 301        ctx->digcnt[0] = 0;
 302        ctx->digcnt[1] = 0;
 303        ctx->buflen = SHA_BUFFER_LEN;
 304
 305        return 0;
 306}
 307
 308static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma)
 309{
 310        struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 311        u32 valcr = 0, valmr = SHA_MR_MODE_AUTO;
 312
 313        if (likely(dma)) {
 314                if (!dd->caps.has_dma)
 315                        atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE);
 316                valmr = SHA_MR_MODE_PDC;
 317                if (dd->caps.has_dualbuff)
 318                        valmr |= SHA_MR_DUALBUFF;
 319        } else {
 320                atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
 321        }
 322
 323        if (ctx->flags & SHA_FLAGS_SHA1)
 324                valmr |= SHA_MR_ALGO_SHA1;
 325        else if (ctx->flags & SHA_FLAGS_SHA224)
 326                valmr |= SHA_MR_ALGO_SHA224;
 327        else if (ctx->flags & SHA_FLAGS_SHA256)
 328                valmr |= SHA_MR_ALGO_SHA256;
 329        else if (ctx->flags & SHA_FLAGS_SHA384)
 330                valmr |= SHA_MR_ALGO_SHA384;
 331        else if (ctx->flags & SHA_FLAGS_SHA512)
 332                valmr |= SHA_MR_ALGO_SHA512;
 333
 334        /* Setting CR_FIRST only for the first iteration */
 335        if (!(ctx->digcnt[0] || ctx->digcnt[1]))
 336                valcr = SHA_CR_FIRST;
 337
 338        atmel_sha_write(dd, SHA_CR, valcr);
 339        atmel_sha_write(dd, SHA_MR, valmr);
 340}
 341
 342static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf,
 343                              size_t length, int final)
 344{
 345        struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 346        int count, len32;
 347        const u32 *buffer = (const u32 *)buf;
 348
 349        dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
 350                ctx->digcnt[1], ctx->digcnt[0], length, final);
 351
 352        atmel_sha_write_ctrl(dd, 0);
 353
 354        /* should be non-zero before next lines to disable clocks later */
 355        ctx->digcnt[0] += length;
 356        if (ctx->digcnt[0] < length)
 357                ctx->digcnt[1]++;
 358
 359        if (final)
 360                dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
 361
 362        len32 = DIV_ROUND_UP(length, sizeof(u32));
 363
 364        dd->flags |= SHA_FLAGS_CPU;
 365
 366        for (count = 0; count < len32; count++)
 367                atmel_sha_write(dd, SHA_REG_DIN(count), buffer[count]);
 368
 369        return -EINPROGRESS;
 370}
 371
 372static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
 373                size_t length1, dma_addr_t dma_addr2, size_t length2, int final)
 374{
 375        struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 376        int len32;
 377
 378        dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
 379                ctx->digcnt[1], ctx->digcnt[0], length1, final);
 380
 381        len32 = DIV_ROUND_UP(length1, sizeof(u32));
 382        atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTDIS);
 383        atmel_sha_write(dd, SHA_TPR, dma_addr1);
 384        atmel_sha_write(dd, SHA_TCR, len32);
 385
 386        len32 = DIV_ROUND_UP(length2, sizeof(u32));
 387        atmel_sha_write(dd, SHA_TNPR, dma_addr2);
 388        atmel_sha_write(dd, SHA_TNCR, len32);
 389
 390        atmel_sha_write_ctrl(dd, 1);
 391
 392        /* should be non-zero before next lines to disable clocks later */
 393        ctx->digcnt[0] += length1;
 394        if (ctx->digcnt[0] < length1)
 395                ctx->digcnt[1]++;
 396
 397        if (final)
 398                dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
 399
 400        dd->flags |=  SHA_FLAGS_DMA_ACTIVE;
 401
 402        /* Start DMA transfer */
 403        atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTEN);
 404
 405        return -EINPROGRESS;
 406}
 407
 408static void atmel_sha_dma_callback(void *data)
 409{
 410        struct atmel_sha_dev *dd = data;
 411
 412        /* dma_lch_in - completed - wait DATRDY */
 413        atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
 414}
 415
 416static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
 417                size_t length1, dma_addr_t dma_addr2, size_t length2, int final)
 418{
 419        struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 420        struct dma_async_tx_descriptor  *in_desc;
 421        struct scatterlist sg[2];
 422
 423        dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
 424                ctx->digcnt[1], ctx->digcnt[0], length1, final);
 425
 426        if (ctx->flags & (SHA_FLAGS_SHA1 | SHA_FLAGS_SHA224 |
 427                        SHA_FLAGS_SHA256)) {
 428                dd->dma_lch_in.dma_conf.src_maxburst = 16;
 429                dd->dma_lch_in.dma_conf.dst_maxburst = 16;
 430        } else {
 431                dd->dma_lch_in.dma_conf.src_maxburst = 32;
 432                dd->dma_lch_in.dma_conf.dst_maxburst = 32;
 433        }
 434
 435        dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);
 436
 437        if (length2) {
 438                sg_init_table(sg, 2);
 439                sg_dma_address(&sg[0]) = dma_addr1;
 440                sg_dma_len(&sg[0]) = length1;
 441                sg_dma_address(&sg[1]) = dma_addr2;
 442                sg_dma_len(&sg[1]) = length2;
 443                in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2,
 444                        DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 445        } else {
 446                sg_init_table(sg, 1);
 447                sg_dma_address(&sg[0]) = dma_addr1;
 448                sg_dma_len(&sg[0]) = length1;
 449                in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1,
 450                        DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 451        }
 452        if (!in_desc)
 453                return -EINVAL;
 454
 455        in_desc->callback = atmel_sha_dma_callback;
 456        in_desc->callback_param = dd;
 457
 458        atmel_sha_write_ctrl(dd, 1);
 459
 460        /* should be non-zero before next lines to disable clocks later */
 461        ctx->digcnt[0] += length1;
 462        if (ctx->digcnt[0] < length1)
 463                ctx->digcnt[1]++;
 464
 465        if (final)
 466                dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
 467
 468        dd->flags |=  SHA_FLAGS_DMA_ACTIVE;
 469
 470        /* Start DMA transfer */
 471        dmaengine_submit(in_desc);
 472        dma_async_issue_pending(dd->dma_lch_in.chan);
 473
 474        return -EINPROGRESS;
 475}
 476
 477static int atmel_sha_xmit_start(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
 478                size_t length1, dma_addr_t dma_addr2, size_t length2, int final)
 479{
 480        if (dd->caps.has_dma)
 481                return atmel_sha_xmit_dma(dd, dma_addr1, length1,
 482                                dma_addr2, length2, final);
 483        else
 484                return atmel_sha_xmit_pdc(dd, dma_addr1, length1,
 485                                dma_addr2, length2, final);
 486}
 487
 488static int atmel_sha_update_cpu(struct atmel_sha_dev *dd)
 489{
 490        struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 491        int bufcnt;
 492
 493        atmel_sha_append_sg(ctx);
 494        atmel_sha_fill_padding(ctx, 0);
 495        bufcnt = ctx->bufcnt;
 496        ctx->bufcnt = 0;
 497
 498        return atmel_sha_xmit_cpu(dd, ctx->buffer, bufcnt, 1);
 499}
 500
 501static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd,
 502                                        struct atmel_sha_reqctx *ctx,
 503                                        size_t length, int final)
 504{
 505        ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,
 506                                ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
 507        if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
 508                dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen +
 509                                ctx->block_size);
 510                return -EINVAL;
 511        }
 512
 513        ctx->flags &= ~SHA_FLAGS_SG;
 514
 515        /* next call does not fail... so no unmap in the case of error */
 516        return atmel_sha_xmit_start(dd, ctx->dma_addr, length, 0, 0, final);
 517}
 518
 519static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd)
 520{
 521        struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 522        unsigned int final;
 523        size_t count;
 524
 525        atmel_sha_append_sg(ctx);
 526
 527        final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
 528
 529        dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: 0x%llx 0x%llx, final: %d\n",
 530                 ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final);
 531
 532        if (final)
 533                atmel_sha_fill_padding(ctx, 0);
 534
 535        if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {
 536                count = ctx->bufcnt;
 537                ctx->bufcnt = 0;
 538                return atmel_sha_xmit_dma_map(dd, ctx, count, final);
 539        }
 540
 541        return 0;
 542}
 543
 544static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
 545{
 546        struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 547        unsigned int length, final, tail;
 548        struct scatterlist *sg;
 549        unsigned int count;
 550
 551        if (!ctx->total)
 552                return 0;
 553
 554        if (ctx->bufcnt || ctx->offset)
 555                return atmel_sha_update_dma_slow(dd);
 556
 557        dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %u, total: %u\n",
 558                ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total);
 559
 560        sg = ctx->sg;
 561
 562        if (!IS_ALIGNED(sg->offset, sizeof(u32)))
 563                return atmel_sha_update_dma_slow(dd);
 564
 565        if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->block_size))
 566                /* size is not ctx->block_size aligned */
 567                return atmel_sha_update_dma_slow(dd);
 568
 569        length = min(ctx->total, sg->length);
 570
 571        if (sg_is_last(sg)) {
 572                if (!(ctx->flags & SHA_FLAGS_FINUP)) {
 573                        /* not last sg must be ctx->block_size aligned */
 574                        tail = length & (ctx->block_size - 1);
 575                        length -= tail;
 576                }
 577        }
 578
 579        ctx->total -= length;
 580        ctx->offset = length; /* offset where to start slow */
 581
 582        final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
 583
 584        /* Add padding */
 585        if (final) {
 586                tail = length & (ctx->block_size - 1);
 587                length -= tail;
 588                ctx->total += tail;
 589                ctx->offset = length; /* offset where to start slow */
 590
 591                sg = ctx->sg;
 592                atmel_sha_append_sg(ctx);
 593
 594                atmel_sha_fill_padding(ctx, length);
 595
 596                ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,
 597                        ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
 598                if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
 599                        dev_err(dd->dev, "dma %u bytes error\n",
 600                                ctx->buflen + ctx->block_size);
 601                        return -EINVAL;
 602                }
 603
 604                if (length == 0) {
 605                        ctx->flags &= ~SHA_FLAGS_SG;
 606                        count = ctx->bufcnt;
 607                        ctx->bufcnt = 0;
 608                        return atmel_sha_xmit_start(dd, ctx->dma_addr, count, 0,
 609                                        0, final);
 610                } else {
 611                        ctx->sg = sg;
 612                        if (!dma_map_sg(dd->dev, ctx->sg, 1,
 613                                DMA_TO_DEVICE)) {
 614                                        dev_err(dd->dev, "dma_map_sg  error\n");
 615                                        return -EINVAL;
 616                        }
 617
 618                        ctx->flags |= SHA_FLAGS_SG;
 619
 620                        count = ctx->bufcnt;
 621                        ctx->bufcnt = 0;
 622                        return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg),
 623                                        length, ctx->dma_addr, count, final);
 624                }
 625        }
 626
 627        if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
 628                dev_err(dd->dev, "dma_map_sg  error\n");
 629                return -EINVAL;
 630        }
 631
 632        ctx->flags |= SHA_FLAGS_SG;
 633
 634        /* next call does not fail... so no unmap in the case of error */
 635        return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, 0,
 636                                                                0, final);
 637}
 638
 639static int atmel_sha_update_dma_stop(struct atmel_sha_dev *dd)
 640{
 641        struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
 642
 643        if (ctx->flags & SHA_FLAGS_SG) {
 644                dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
 645                if (ctx->sg->length == ctx->offset) {
 646                        ctx->sg = sg_next(ctx->sg);
 647                        if (ctx->sg)
 648                                ctx->offset = 0;
 649                }
 650                if (ctx->flags & SHA_FLAGS_PAD) {
 651                        dma_unmap_single(dd->dev, ctx->dma_addr,
 652                                ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
 653                }
 654        } else {
 655                dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen +
 656                                                ctx->block_size, DMA_TO_DEVICE);
 657        }
 658
 659        return 0;
 660}
 661
 662static int atmel_sha_update_req(struct atmel_sha_dev *dd)
 663{
 664        struct ahash_request *req = dd->req;
 665        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 666        int err;
 667
 668        dev_dbg(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n",
 669                ctx->total, ctx->digcnt[1], ctx->digcnt[0]);
 670
 671        if (ctx->flags & SHA_FLAGS_CPU)
 672                err = atmel_sha_update_cpu(dd);
 673        else
 674                err = atmel_sha_update_dma_start(dd);
 675
 676        /* wait for dma completion before can take more data */
 677        dev_dbg(dd->dev, "update: err: %d, digcnt: 0x%llx 0%llx\n",
 678                        err, ctx->digcnt[1], ctx->digcnt[0]);
 679
 680        return err;
 681}
 682
 683static int atmel_sha_final_req(struct atmel_sha_dev *dd)
 684{
 685        struct ahash_request *req = dd->req;
 686        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 687        int err = 0;
 688        int count;
 689
 690        if (ctx->bufcnt >= ATMEL_SHA_DMA_THRESHOLD) {
 691                atmel_sha_fill_padding(ctx, 0);
 692                count = ctx->bufcnt;
 693                ctx->bufcnt = 0;
 694                err = atmel_sha_xmit_dma_map(dd, ctx, count, 1);
 695        }
 696        /* faster to handle last block with cpu */
 697        else {
 698                atmel_sha_fill_padding(ctx, 0);
 699                count = ctx->bufcnt;
 700                ctx->bufcnt = 0;
 701                err = atmel_sha_xmit_cpu(dd, ctx->buffer, count, 1);
 702        }
 703
 704        dev_dbg(dd->dev, "final_req: err: %d\n", err);
 705
 706        return err;
 707}
 708
 709static void atmel_sha_copy_hash(struct ahash_request *req)
 710{
 711        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 712        u32 *hash = (u32 *)ctx->digest;
 713        int i;
 714
 715        if (ctx->flags & SHA_FLAGS_SHA1)
 716                for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++)
 717                        hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i));
 718        else if (ctx->flags & SHA_FLAGS_SHA224)
 719                for (i = 0; i < SHA224_DIGEST_SIZE / sizeof(u32); i++)
 720                        hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i));
 721        else if (ctx->flags & SHA_FLAGS_SHA256)
 722                for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(u32); i++)
 723                        hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i));
 724        else if (ctx->flags & SHA_FLAGS_SHA384)
 725                for (i = 0; i < SHA384_DIGEST_SIZE / sizeof(u32); i++)
 726                        hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i));
 727        else
 728                for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(u32); i++)
 729                        hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i));
 730}
 731
 732static void atmel_sha_copy_ready_hash(struct ahash_request *req)
 733{
 734        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 735
 736        if (!req->result)
 737                return;
 738
 739        if (ctx->flags & SHA_FLAGS_SHA1)
 740                memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE);
 741        else if (ctx->flags & SHA_FLAGS_SHA224)
 742                memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE);
 743        else if (ctx->flags & SHA_FLAGS_SHA256)
 744                memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE);
 745        else if (ctx->flags & SHA_FLAGS_SHA384)
 746                memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE);
 747        else
 748                memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE);
 749}
 750
 751static int atmel_sha_finish(struct ahash_request *req)
 752{
 753        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 754        struct atmel_sha_dev *dd = ctx->dd;
 755        int err = 0;
 756
 757        if (ctx->digcnt[0] || ctx->digcnt[1])
 758                atmel_sha_copy_ready_hash(req);
 759
 760        dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %d\n", ctx->digcnt[1],
 761                ctx->digcnt[0], ctx->bufcnt);
 762
 763        return err;
 764}
 765
 766static void atmel_sha_finish_req(struct ahash_request *req, int err)
 767{
 768        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 769        struct atmel_sha_dev *dd = ctx->dd;
 770
 771        if (!err) {
 772                atmel_sha_copy_hash(req);
 773                if (SHA_FLAGS_FINAL & dd->flags)
 774                        err = atmel_sha_finish(req);
 775        } else {
 776                ctx->flags |= SHA_FLAGS_ERROR;
 777        }
 778
 779        /* atomic operation is not needed here */
 780        dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU |
 781                        SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY);
 782
 783        clk_disable_unprepare(dd->iclk);
 784
 785        if (req->base.complete)
 786                req->base.complete(&req->base, err);
 787
 788        /* handle new request */
 789        tasklet_schedule(&dd->done_task);
 790}
 791
 792static int atmel_sha_hw_init(struct atmel_sha_dev *dd)
 793{
 794        clk_prepare_enable(dd->iclk);
 795
 796        if (!(SHA_FLAGS_INIT & dd->flags)) {
 797                atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST);
 798                dd->flags |= SHA_FLAGS_INIT;
 799                dd->err = 0;
 800        }
 801
 802        return 0;
 803}
 804
 805static inline unsigned int atmel_sha_get_version(struct atmel_sha_dev *dd)
 806{
 807        return atmel_sha_read(dd, SHA_HW_VERSION) & 0x00000fff;
 808}
 809
 810static void atmel_sha_hw_version_init(struct atmel_sha_dev *dd)
 811{
 812        atmel_sha_hw_init(dd);
 813
 814        dd->hw_version = atmel_sha_get_version(dd);
 815
 816        dev_info(dd->dev,
 817                        "version: 0x%x\n", dd->hw_version);
 818
 819        clk_disable_unprepare(dd->iclk);
 820}
 821
 822static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,
 823                                  struct ahash_request *req)
 824{
 825        struct crypto_async_request *async_req, *backlog;
 826        struct atmel_sha_reqctx *ctx;
 827        unsigned long flags;
 828        int err = 0, ret = 0;
 829
 830        spin_lock_irqsave(&dd->lock, flags);
 831        if (req)
 832                ret = ahash_enqueue_request(&dd->queue, req);
 833
 834        if (SHA_FLAGS_BUSY & dd->flags) {
 835                spin_unlock_irqrestore(&dd->lock, flags);
 836                return ret;
 837        }
 838
 839        backlog = crypto_get_backlog(&dd->queue);
 840        async_req = crypto_dequeue_request(&dd->queue);
 841        if (async_req)
 842                dd->flags |= SHA_FLAGS_BUSY;
 843
 844        spin_unlock_irqrestore(&dd->lock, flags);
 845
 846        if (!async_req)
 847                return ret;
 848
 849        if (backlog)
 850                backlog->complete(backlog, -EINPROGRESS);
 851
 852        req = ahash_request_cast(async_req);
 853        dd->req = req;
 854        ctx = ahash_request_ctx(req);
 855
 856        dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
 857                                                ctx->op, req->nbytes);
 858
 859        err = atmel_sha_hw_init(dd);
 860
 861        if (err)
 862                goto err1;
 863
 864        if (ctx->op == SHA_OP_UPDATE) {
 865                err = atmel_sha_update_req(dd);
 866                if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP))
 867                        /* no final() after finup() */
 868                        err = atmel_sha_final_req(dd);
 869        } else if (ctx->op == SHA_OP_FINAL) {
 870                err = atmel_sha_final_req(dd);
 871        }
 872
 873err1:
 874        if (err != -EINPROGRESS)
 875                /* done_task will not finish it, so do it here */
 876                atmel_sha_finish_req(req, err);
 877
 878        dev_dbg(dd->dev, "exit, err: %d\n", err);
 879
 880        return ret;
 881}
 882
 883static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op)
 884{
 885        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 886        struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
 887        struct atmel_sha_dev *dd = tctx->dd;
 888
 889        ctx->op = op;
 890
 891        return atmel_sha_handle_queue(dd, req);
 892}
 893
 894static int atmel_sha_update(struct ahash_request *req)
 895{
 896        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 897
 898        if (!req->nbytes)
 899                return 0;
 900
 901        ctx->total = req->nbytes;
 902        ctx->sg = req->src;
 903        ctx->offset = 0;
 904
 905        if (ctx->flags & SHA_FLAGS_FINUP) {
 906                if (ctx->bufcnt + ctx->total < ATMEL_SHA_DMA_THRESHOLD)
 907                        /* faster to use CPU for short transfers */
 908                        ctx->flags |= SHA_FLAGS_CPU;
 909        } else if (ctx->bufcnt + ctx->total < ctx->buflen) {
 910                atmel_sha_append_sg(ctx);
 911                return 0;
 912        }
 913        return atmel_sha_enqueue(req, SHA_OP_UPDATE);
 914}
 915
 916static int atmel_sha_final(struct ahash_request *req)
 917{
 918        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 919        struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
 920        struct atmel_sha_dev *dd = tctx->dd;
 921
 922        int err = 0;
 923
 924        ctx->flags |= SHA_FLAGS_FINUP;
 925
 926        if (ctx->flags & SHA_FLAGS_ERROR)
 927                return 0; /* uncompleted hash is not needed */
 928
 929        if (ctx->bufcnt) {
 930                return atmel_sha_enqueue(req, SHA_OP_FINAL);
 931        } else if (!(ctx->flags & SHA_FLAGS_PAD)) { /* add padding */
 932                err = atmel_sha_hw_init(dd);
 933                if (err)
 934                        goto err1;
 935
 936                dd->flags |= SHA_FLAGS_BUSY;
 937                err = atmel_sha_final_req(dd);
 938        } else {
 939                /* copy ready hash (+ finalize hmac) */
 940                return atmel_sha_finish(req);
 941        }
 942
 943err1:
 944        if (err != -EINPROGRESS)
 945                /* done_task will not finish it, so do it here */
 946                atmel_sha_finish_req(req, err);
 947
 948        return err;
 949}
 950
 951static int atmel_sha_finup(struct ahash_request *req)
 952{
 953        struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
 954        int err1, err2;
 955
 956        ctx->flags |= SHA_FLAGS_FINUP;
 957
 958        err1 = atmel_sha_update(req);
 959        if (err1 == -EINPROGRESS || err1 == -EBUSY)
 960                return err1;
 961
 962        /*
 963         * final() has to be always called to cleanup resources
 964         * even if udpate() failed, except EINPROGRESS
 965         */
 966        err2 = atmel_sha_final(req);
 967
 968        return err1 ?: err2;
 969}
 970
 971static int atmel_sha_digest(struct ahash_request *req)
 972{
 973        return atmel_sha_init(req) ?: atmel_sha_finup(req);
 974}
 975
 976static int atmel_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
 977{
 978        struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm);
 979        const char *alg_name = crypto_tfm_alg_name(tfm);
 980
 981        /* Allocate a fallback and abort if it failed. */
 982        tctx->fallback = crypto_alloc_shash(alg_name, 0,
 983                                            CRYPTO_ALG_NEED_FALLBACK);
 984        if (IS_ERR(tctx->fallback)) {
 985                pr_err("atmel-sha: fallback driver '%s' could not be loaded.\n",
 986                                alg_name);
 987                return PTR_ERR(tctx->fallback);
 988        }
 989        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 990                                 sizeof(struct atmel_sha_reqctx) +
 991                                 SHA_BUFFER_LEN + SHA512_BLOCK_SIZE);
 992
 993        return 0;
 994}
 995
 996static int atmel_sha_cra_init(struct crypto_tfm *tfm)
 997{
 998        return atmel_sha_cra_init_alg(tfm, NULL);
 999}
1000
1001static void atmel_sha_cra_exit(struct crypto_tfm *tfm)
1002{
1003        struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm);
1004
1005        crypto_free_shash(tctx->fallback);
1006        tctx->fallback = NULL;
1007}
1008
1009static struct ahash_alg sha_1_256_algs[] = {
1010{
1011        .init           = atmel_sha_init,
1012        .update         = atmel_sha_update,
1013        .final          = atmel_sha_final,
1014        .finup          = atmel_sha_finup,
1015        .digest         = atmel_sha_digest,
1016        .halg = {
1017                .digestsize     = SHA1_DIGEST_SIZE,
1018                .base   = {
1019                        .cra_name               = "sha1",
1020                        .cra_driver_name        = "atmel-sha1",
1021                        .cra_priority           = 100,
1022                        .cra_flags              = CRYPTO_ALG_ASYNC |
1023                                                CRYPTO_ALG_NEED_FALLBACK,
1024                        .cra_blocksize          = SHA1_BLOCK_SIZE,
1025                        .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1026                        .cra_alignmask          = 0,
1027                        .cra_module             = THIS_MODULE,
1028                        .cra_init               = atmel_sha_cra_init,
1029                        .cra_exit               = atmel_sha_cra_exit,
1030                }
1031        }
1032},
1033{
1034        .init           = atmel_sha_init,
1035        .update         = atmel_sha_update,
1036        .final          = atmel_sha_final,
1037        .finup          = atmel_sha_finup,
1038        .digest         = atmel_sha_digest,
1039        .halg = {
1040                .digestsize     = SHA256_DIGEST_SIZE,
1041                .base   = {
1042                        .cra_name               = "sha256",
1043                        .cra_driver_name        = "atmel-sha256",
1044                        .cra_priority           = 100,
1045                        .cra_flags              = CRYPTO_ALG_ASYNC |
1046                                                CRYPTO_ALG_NEED_FALLBACK,
1047                        .cra_blocksize          = SHA256_BLOCK_SIZE,
1048                        .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1049                        .cra_alignmask          = 0,
1050                        .cra_module             = THIS_MODULE,
1051                        .cra_init               = atmel_sha_cra_init,
1052                        .cra_exit               = atmel_sha_cra_exit,
1053                }
1054        }
1055},
1056};
1057
1058static struct ahash_alg sha_224_alg = {
1059        .init           = atmel_sha_init,
1060        .update         = atmel_sha_update,
1061        .final          = atmel_sha_final,
1062        .finup          = atmel_sha_finup,
1063        .digest         = atmel_sha_digest,
1064        .halg = {
1065                .digestsize     = SHA224_DIGEST_SIZE,
1066                .base   = {
1067                        .cra_name               = "sha224",
1068                        .cra_driver_name        = "atmel-sha224",
1069                        .cra_priority           = 100,
1070                        .cra_flags              = CRYPTO_ALG_ASYNC |
1071                                                CRYPTO_ALG_NEED_FALLBACK,
1072                        .cra_blocksize          = SHA224_BLOCK_SIZE,
1073                        .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1074                        .cra_alignmask          = 0,
1075                        .cra_module             = THIS_MODULE,
1076                        .cra_init               = atmel_sha_cra_init,
1077                        .cra_exit               = atmel_sha_cra_exit,
1078                }
1079        }
1080};
1081
1082static struct ahash_alg sha_384_512_algs[] = {
1083{
1084        .init           = atmel_sha_init,
1085        .update         = atmel_sha_update,
1086        .final          = atmel_sha_final,
1087        .finup          = atmel_sha_finup,
1088        .digest         = atmel_sha_digest,
1089        .halg = {
1090                .digestsize     = SHA384_DIGEST_SIZE,
1091                .base   = {
1092                        .cra_name               = "sha384",
1093                        .cra_driver_name        = "atmel-sha384",
1094                        .cra_priority           = 100,
1095                        .cra_flags              = CRYPTO_ALG_ASYNC |
1096                                                CRYPTO_ALG_NEED_FALLBACK,
1097                        .cra_blocksize          = SHA384_BLOCK_SIZE,
1098                        .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1099                        .cra_alignmask          = 0x3,
1100                        .cra_module             = THIS_MODULE,
1101                        .cra_init               = atmel_sha_cra_init,
1102                        .cra_exit               = atmel_sha_cra_exit,
1103                }
1104        }
1105},
1106{
1107        .init           = atmel_sha_init,
1108        .update         = atmel_sha_update,
1109        .final          = atmel_sha_final,
1110        .finup          = atmel_sha_finup,
1111        .digest         = atmel_sha_digest,
1112        .halg = {
1113                .digestsize     = SHA512_DIGEST_SIZE,
1114                .base   = {
1115                        .cra_name               = "sha512",
1116                        .cra_driver_name        = "atmel-sha512",
1117                        .cra_priority           = 100,
1118                        .cra_flags              = CRYPTO_ALG_ASYNC |
1119                                                CRYPTO_ALG_NEED_FALLBACK,
1120                        .cra_blocksize          = SHA512_BLOCK_SIZE,
1121                        .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1122                        .cra_alignmask          = 0x3,
1123                        .cra_module             = THIS_MODULE,
1124                        .cra_init               = atmel_sha_cra_init,
1125                        .cra_exit               = atmel_sha_cra_exit,
1126                }
1127        }
1128},
1129};
1130
1131static void atmel_sha_done_task(unsigned long data)
1132{
1133        struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;
1134        int err = 0;
1135
1136        if (!(SHA_FLAGS_BUSY & dd->flags)) {
1137                atmel_sha_handle_queue(dd, NULL);
1138                return;
1139        }
1140
1141        if (SHA_FLAGS_CPU & dd->flags) {
1142                if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
1143                        dd->flags &= ~SHA_FLAGS_OUTPUT_READY;
1144                        goto finish;
1145                }
1146        } else if (SHA_FLAGS_DMA_READY & dd->flags) {
1147                if (SHA_FLAGS_DMA_ACTIVE & dd->flags) {
1148                        dd->flags &= ~SHA_FLAGS_DMA_ACTIVE;
1149                        atmel_sha_update_dma_stop(dd);
1150                        if (dd->err) {
1151                                err = dd->err;
1152                                goto finish;
1153                        }
1154                }
1155                if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
1156                        /* hash or semi-hash ready */
1157                        dd->flags &= ~(SHA_FLAGS_DMA_READY |
1158                                                SHA_FLAGS_OUTPUT_READY);
1159                        err = atmel_sha_update_dma_start(dd);
1160                        if (err != -EINPROGRESS)
1161                                goto finish;
1162                }
1163        }
1164        return;
1165
1166finish:
1167        /* finish curent request */
1168        atmel_sha_finish_req(dd->req, err);
1169}
1170
1171static irqreturn_t atmel_sha_irq(int irq, void *dev_id)
1172{
1173        struct atmel_sha_dev *sha_dd = dev_id;
1174        u32 reg;
1175
1176        reg = atmel_sha_read(sha_dd, SHA_ISR);
1177        if (reg & atmel_sha_read(sha_dd, SHA_IMR)) {
1178                atmel_sha_write(sha_dd, SHA_IDR, reg);
1179                if (SHA_FLAGS_BUSY & sha_dd->flags) {
1180                        sha_dd->flags |= SHA_FLAGS_OUTPUT_READY;
1181                        if (!(SHA_FLAGS_CPU & sha_dd->flags))
1182                                sha_dd->flags |= SHA_FLAGS_DMA_READY;
1183                        tasklet_schedule(&sha_dd->done_task);
1184                } else {
1185                        dev_warn(sha_dd->dev, "SHA interrupt when no active requests.\n");
1186                }
1187                return IRQ_HANDLED;
1188        }
1189
1190        return IRQ_NONE;
1191}
1192
1193static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd)
1194{
1195        int i;
1196
1197        for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++)
1198                crypto_unregister_ahash(&sha_1_256_algs[i]);
1199
1200        if (dd->caps.has_sha224)
1201                crypto_unregister_ahash(&sha_224_alg);
1202
1203        if (dd->caps.has_sha_384_512) {
1204                for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++)
1205                        crypto_unregister_ahash(&sha_384_512_algs[i]);
1206        }
1207}
1208
1209static int atmel_sha_register_algs(struct atmel_sha_dev *dd)
1210{
1211        int err, i, j;
1212
1213        for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) {
1214                err = crypto_register_ahash(&sha_1_256_algs[i]);
1215                if (err)
1216                        goto err_sha_1_256_algs;
1217        }
1218
1219        if (dd->caps.has_sha224) {
1220                err = crypto_register_ahash(&sha_224_alg);
1221                if (err)
1222                        goto err_sha_224_algs;
1223        }
1224
1225        if (dd->caps.has_sha_384_512) {
1226                for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) {
1227                        err = crypto_register_ahash(&sha_384_512_algs[i]);
1228                        if (err)
1229                                goto err_sha_384_512_algs;
1230                }
1231        }
1232
1233        return 0;
1234
1235err_sha_384_512_algs:
1236        for (j = 0; j < i; j++)
1237                crypto_unregister_ahash(&sha_384_512_algs[j]);
1238        crypto_unregister_ahash(&sha_224_alg);
1239err_sha_224_algs:
1240        i = ARRAY_SIZE(sha_1_256_algs);
1241err_sha_1_256_algs:
1242        for (j = 0; j < i; j++)
1243                crypto_unregister_ahash(&sha_1_256_algs[j]);
1244
1245        return err;
1246}
1247
1248static bool atmel_sha_filter(struct dma_chan *chan, void *slave)
1249{
1250        struct at_dma_slave     *sl = slave;
1251
1252        if (sl && sl->dma_dev == chan->device->dev) {
1253                chan->private = sl;
1254                return true;
1255        } else {
1256                return false;
1257        }
1258}
1259
1260static int atmel_sha_dma_init(struct atmel_sha_dev *dd,
1261                                struct crypto_platform_data *pdata)
1262{
1263        int err = -ENOMEM;
1264        dma_cap_mask_t mask_in;
1265
1266        if (pdata && pdata->dma_slave->rxdata.dma_dev) {
1267                /* Try to grab DMA channel */
1268                dma_cap_zero(mask_in);
1269                dma_cap_set(DMA_SLAVE, mask_in);
1270
1271                dd->dma_lch_in.chan = dma_request_channel(mask_in,
1272                                atmel_sha_filter, &pdata->dma_slave->rxdata);
1273
1274                if (!dd->dma_lch_in.chan)
1275                        return err;
1276
1277                dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV;
1278                dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base +
1279                        SHA_REG_DIN(0);
1280                dd->dma_lch_in.dma_conf.src_maxburst = 1;
1281                dd->dma_lch_in.dma_conf.src_addr_width =
1282                        DMA_SLAVE_BUSWIDTH_4_BYTES;
1283                dd->dma_lch_in.dma_conf.dst_maxburst = 1;
1284                dd->dma_lch_in.dma_conf.dst_addr_width =
1285                        DMA_SLAVE_BUSWIDTH_4_BYTES;
1286                dd->dma_lch_in.dma_conf.device_fc = false;
1287
1288                return 0;
1289        }
1290
1291        return -ENODEV;
1292}
1293
1294static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd)
1295{
1296        dma_release_channel(dd->dma_lch_in.chan);
1297}
1298
1299static void atmel_sha_get_cap(struct atmel_sha_dev *dd)
1300{
1301
1302        dd->caps.has_dma = 0;
1303        dd->caps.has_dualbuff = 0;
1304        dd->caps.has_sha224 = 0;
1305        dd->caps.has_sha_384_512 = 0;
1306
1307        /* keep only major version number */
1308        switch (dd->hw_version & 0xff0) {
1309        case 0x410:
1310                dd->caps.has_dma = 1;
1311                dd->caps.has_dualbuff = 1;
1312                dd->caps.has_sha224 = 1;
1313                dd->caps.has_sha_384_512 = 1;
1314                break;
1315        case 0x400:
1316                dd->caps.has_dma = 1;
1317                dd->caps.has_dualbuff = 1;
1318                dd->caps.has_sha224 = 1;
1319                break;
1320        case 0x320:
1321                break;
1322        default:
1323                dev_warn(dd->dev,
1324                                "Unmanaged sha version, set minimum capabilities\n");
1325                break;
1326        }
1327}
1328
1329static int atmel_sha_probe(struct platform_device *pdev)
1330{
1331        struct atmel_sha_dev *sha_dd;
1332        struct crypto_platform_data     *pdata;
1333        struct device *dev = &pdev->dev;
1334        struct resource *sha_res;
1335        unsigned long sha_phys_size;
1336        int err;
1337
1338        sha_dd = kzalloc(sizeof(struct atmel_sha_dev), GFP_KERNEL);
1339        if (sha_dd == NULL) {
1340                dev_err(dev, "unable to alloc data struct.\n");
1341                err = -ENOMEM;
1342                goto sha_dd_err;
1343        }
1344
1345        sha_dd->dev = dev;
1346
1347        platform_set_drvdata(pdev, sha_dd);
1348
1349        INIT_LIST_HEAD(&sha_dd->list);
1350
1351        tasklet_init(&sha_dd->done_task, atmel_sha_done_task,
1352                                        (unsigned long)sha_dd);
1353
1354        crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH);
1355
1356        sha_dd->irq = -1;
1357
1358        /* Get the base address */
1359        sha_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1360        if (!sha_res) {
1361                dev_err(dev, "no MEM resource info\n");
1362                err = -ENODEV;
1363                goto res_err;
1364        }
1365        sha_dd->phys_base = sha_res->start;
1366        sha_phys_size = resource_size(sha_res);
1367
1368        /* Get the IRQ */
1369        sha_dd->irq = platform_get_irq(pdev,  0);
1370        if (sha_dd->irq < 0) {
1371                dev_err(dev, "no IRQ resource info\n");
1372                err = sha_dd->irq;
1373                goto res_err;
1374        }
1375
1376        err = request_irq(sha_dd->irq, atmel_sha_irq, IRQF_SHARED, "atmel-sha",
1377                                                sha_dd);
1378        if (err) {
1379                dev_err(dev, "unable to request sha irq.\n");
1380                goto res_err;
1381        }
1382
1383        /* Initializing the clock */
1384        sha_dd->iclk = clk_get(&pdev->dev, "sha_clk");
1385        if (IS_ERR(sha_dd->iclk)) {
1386                dev_err(dev, "clock intialization failed.\n");
1387                err = PTR_ERR(sha_dd->iclk);
1388                goto clk_err;
1389        }
1390
1391        sha_dd->io_base = ioremap(sha_dd->phys_base, sha_phys_size);
1392        if (!sha_dd->io_base) {
1393                dev_err(dev, "can't ioremap\n");
1394                err = -ENOMEM;
1395                goto sha_io_err;
1396        }
1397
1398        atmel_sha_hw_version_init(sha_dd);
1399
1400        atmel_sha_get_cap(sha_dd);
1401
1402        if (sha_dd->caps.has_dma) {
1403                pdata = pdev->dev.platform_data;
1404                if (!pdata) {
1405                        dev_err(&pdev->dev, "platform data not available\n");
1406                        err = -ENXIO;
1407                        goto err_pdata;
1408                }
1409                err = atmel_sha_dma_init(sha_dd, pdata);
1410                if (err)
1411                        goto err_sha_dma;
1412        }
1413
1414        spin_lock(&atmel_sha.lock);
1415        list_add_tail(&sha_dd->list, &atmel_sha.dev_list);
1416        spin_unlock(&atmel_sha.lock);
1417
1418        err = atmel_sha_register_algs(sha_dd);
1419        if (err)
1420                goto err_algs;
1421
1422        dev_info(dev, "Atmel SHA1/SHA256\n");
1423
1424        return 0;
1425
1426err_algs:
1427        spin_lock(&atmel_sha.lock);
1428        list_del(&sha_dd->list);
1429        spin_unlock(&atmel_sha.lock);
1430        if (sha_dd->caps.has_dma)
1431                atmel_sha_dma_cleanup(sha_dd);
1432err_sha_dma:
1433err_pdata:
1434        iounmap(sha_dd->io_base);
1435sha_io_err:
1436        clk_put(sha_dd->iclk);
1437clk_err:
1438        free_irq(sha_dd->irq, sha_dd);
1439res_err:
1440        tasklet_kill(&sha_dd->done_task);
1441        kfree(sha_dd);
1442        sha_dd = NULL;
1443sha_dd_err:
1444        dev_err(dev, "initialization failed.\n");
1445
1446        return err;
1447}
1448
1449static int atmel_sha_remove(struct platform_device *pdev)
1450{
1451        static struct atmel_sha_dev *sha_dd;
1452
1453        sha_dd = platform_get_drvdata(pdev);
1454        if (!sha_dd)
1455                return -ENODEV;
1456        spin_lock(&atmel_sha.lock);
1457        list_del(&sha_dd->list);
1458        spin_unlock(&atmel_sha.lock);
1459
1460        atmel_sha_unregister_algs(sha_dd);
1461
1462        tasklet_kill(&sha_dd->done_task);
1463
1464        if (sha_dd->caps.has_dma)
1465                atmel_sha_dma_cleanup(sha_dd);
1466
1467        iounmap(sha_dd->io_base);
1468
1469        clk_put(sha_dd->iclk);
1470
1471        if (sha_dd->irq >= 0)
1472                free_irq(sha_dd->irq, sha_dd);
1473
1474        kfree(sha_dd);
1475        sha_dd = NULL;
1476
1477        return 0;
1478}
1479
1480static struct platform_driver atmel_sha_driver = {
1481        .probe          = atmel_sha_probe,
1482        .remove         = atmel_sha_remove,
1483        .driver         = {
1484                .name   = "atmel_sha",
1485                .owner  = THIS_MODULE,
1486        },
1487};
1488
1489module_platform_driver(atmel_sha_driver);
1490
1491MODULE_DESCRIPTION("Atmel SHA (1/256/224/384/512) hw acceleration support.");
1492MODULE_LICENSE("GPL v2");
1493MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique");
1494
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.