linux/drivers/crypto/omap-sham.c
<<
>>
Prefs
   1/*
   2 * Cryptographic API.
   3 *
   4 * Support for OMAP SHA1/MD5 HW acceleration.
   5 *
   6 * Copyright (c) 2010 Nokia Corporation
   7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
   8 *
   9 * This program is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License version 2 as published
  11 * by the Free Software Foundation.
  12 *
  13 * Some ideas are from old omap-sha1-md5.c driver.
  14 */
  15
  16#define pr_fmt(fmt) "%s: " fmt, __func__
  17
  18#include <linux/err.h>
  19#include <linux/device.h>
  20#include <linux/module.h>
  21#include <linux/init.h>
  22#include <linux/errno.h>
  23#include <linux/interrupt.h>
  24#include <linux/kernel.h>
  25#include <linux/clk.h>
  26#include <linux/irq.h>
  27#include <linux/io.h>
  28#include <linux/platform_device.h>
  29#include <linux/scatterlist.h>
  30#include <linux/dma-mapping.h>
  31#include <linux/delay.h>
  32#include <linux/crypto.h>
  33#include <linux/cryptohash.h>
  34#include <crypto/scatterwalk.h>
  35#include <crypto/algapi.h>
  36#include <crypto/sha.h>
  37#include <crypto/hash.h>
  38#include <crypto/internal/hash.h>
  39
  40#include <plat/cpu.h>
  41#include <plat/dma.h>
  42#include <mach/irqs.h>
  43
  44#define SHA_REG_DIGEST(x)               (0x00 + ((x) * 0x04))
  45#define SHA_REG_DIN(x)                  (0x1C + ((x) * 0x04))
  46
  47#define SHA1_MD5_BLOCK_SIZE             SHA1_BLOCK_SIZE
  48#define MD5_DIGEST_SIZE                 16
  49
  50#define SHA_REG_DIGCNT                  0x14
  51
  52#define SHA_REG_CTRL                    0x18
  53#define SHA_REG_CTRL_LENGTH             (0xFFFFFFFF << 5)
  54#define SHA_REG_CTRL_CLOSE_HASH         (1 << 4)
  55#define SHA_REG_CTRL_ALGO_CONST         (1 << 3)
  56#define SHA_REG_CTRL_ALGO               (1 << 2)
  57#define SHA_REG_CTRL_INPUT_READY        (1 << 1)
  58#define SHA_REG_CTRL_OUTPUT_READY       (1 << 0)
  59
  60#define SHA_REG_REV                     0x5C
  61#define SHA_REG_REV_MAJOR               0xF0
  62#define SHA_REG_REV_MINOR               0x0F
  63
  64#define SHA_REG_MASK                    0x60
  65#define SHA_REG_MASK_DMA_EN             (1 << 3)
  66#define SHA_REG_MASK_IT_EN              (1 << 2)
  67#define SHA_REG_MASK_SOFTRESET          (1 << 1)
  68#define SHA_REG_AUTOIDLE                (1 << 0)
  69
  70#define SHA_REG_SYSSTATUS               0x64
  71#define SHA_REG_SYSSTATUS_RESETDONE     (1 << 0)
  72
  73#define DEFAULT_TIMEOUT_INTERVAL        HZ
  74
  75#define FLAGS_FIRST             0x0001
  76#define FLAGS_FINUP             0x0002
  77#define FLAGS_FINAL             0x0004
  78#define FLAGS_FAST              0x0008
  79#define FLAGS_SHA1              0x0010
  80#define FLAGS_DMA_ACTIVE        0x0020
  81#define FLAGS_OUTPUT_READY      0x0040
  82#define FLAGS_CLEAN             0x0080
  83#define FLAGS_INIT              0x0100
  84#define FLAGS_CPU               0x0200
  85#define FLAGS_HMAC              0x0400
  86
  87/* 3rd byte */
  88#define FLAGS_BUSY              16
  89
  90#define OP_UPDATE       1
  91#define OP_FINAL        2
  92
  93struct omap_sham_dev;
  94
  95struct omap_sham_reqctx {
  96        struct omap_sham_dev    *dd;
  97        unsigned long           flags;
  98        unsigned long           op;
  99
 100        size_t                  digcnt;
 101        u8                      *buffer;
 102        size_t                  bufcnt;
 103        size_t                  buflen;
 104        dma_addr_t              dma_addr;
 105
 106        /* walk state */
 107        struct scatterlist      *sg;
 108        unsigned int            offset; /* offset in current sg */
 109        unsigned int            total;  /* total request */
 110};
 111
 112struct omap_sham_hmac_ctx {
 113        struct crypto_shash     *shash;
 114        u8                      ipad[SHA1_MD5_BLOCK_SIZE];
 115        u8                      opad[SHA1_MD5_BLOCK_SIZE];
 116};
 117
 118struct omap_sham_ctx {
 119        struct omap_sham_dev    *dd;
 120
 121        unsigned long           flags;
 122
 123        /* fallback stuff */
 124        struct crypto_shash     *fallback;
 125
 126        struct omap_sham_hmac_ctx base[0];
 127};
 128
 129#define OMAP_SHAM_QUEUE_LENGTH  1
 130
 131struct omap_sham_dev {
 132        struct list_head        list;
 133        unsigned long           phys_base;
 134        struct device           *dev;
 135        void __iomem            *io_base;
 136        int                     irq;
 137        struct clk              *iclk;
 138        spinlock_t              lock;
 139        int                     dma;
 140        int                     dma_lch;
 141        struct tasklet_struct   done_task;
 142        struct tasklet_struct   queue_task;
 143
 144        unsigned long           flags;
 145        struct crypto_queue     queue;
 146        struct ahash_request    *req;
 147};
 148
 149struct omap_sham_drv {
 150        struct list_head        dev_list;
 151        spinlock_t              lock;
 152        unsigned long           flags;
 153};
 154
 155static struct omap_sham_drv sham = {
 156        .dev_list = LIST_HEAD_INIT(sham.dev_list),
 157        .lock = __SPIN_LOCK_UNLOCKED(sham.lock),
 158};
 159
 160static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset)
 161{
 162        return __raw_readl(dd->io_base + offset);
 163}
 164
 165static inline void omap_sham_write(struct omap_sham_dev *dd,
 166                                        u32 offset, u32 value)
 167{
 168        __raw_writel(value, dd->io_base + offset);
 169}
 170
 171static inline void omap_sham_write_mask(struct omap_sham_dev *dd, u32 address,
 172                                        u32 value, u32 mask)
 173{
 174        u32 val;
 175
 176        val = omap_sham_read(dd, address);
 177        val &= ~mask;
 178        val |= value;
 179        omap_sham_write(dd, address, val);
 180}
 181
 182static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit)
 183{
 184        unsigned long timeout = jiffies + DEFAULT_TIMEOUT_INTERVAL;
 185
 186        while (!(omap_sham_read(dd, offset) & bit)) {
 187                if (time_is_before_jiffies(timeout))
 188                        return -ETIMEDOUT;
 189        }
 190
 191        return 0;
 192}
 193
 194static void omap_sham_copy_hash(struct ahash_request *req, int out)
 195{
 196        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 197        u32 *hash = (u32 *)req->result;
 198        int i;
 199
 200        if (likely(ctx->flags & FLAGS_SHA1)) {
 201                /* SHA1 results are in big endian */
 202                for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++)
 203                        if (out)
 204                                hash[i] = be32_to_cpu(omap_sham_read(ctx->dd,
 205                                                        SHA_REG_DIGEST(i)));
 206                        else
 207                                omap_sham_write(ctx->dd, SHA_REG_DIGEST(i),
 208                                                        cpu_to_be32(hash[i]));
 209        } else {
 210                /* MD5 results are in little endian */
 211                for (i = 0; i < MD5_DIGEST_SIZE / sizeof(u32); i++)
 212                        if (out)
 213                                hash[i] = le32_to_cpu(omap_sham_read(ctx->dd,
 214                                                        SHA_REG_DIGEST(i)));
 215                        else
 216                                omap_sham_write(ctx->dd, SHA_REG_DIGEST(i),
 217                                                        cpu_to_le32(hash[i]));
 218        }
 219}
 220
 221static int omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
 222                                 int final, int dma)
 223{
 224        struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
 225        u32 val = length << 5, mask;
 226
 227        if (unlikely(!ctx->digcnt)) {
 228
 229                clk_enable(dd->iclk);
 230
 231                if (!(dd->flags & FLAGS_INIT)) {
 232                        omap_sham_write_mask(dd, SHA_REG_MASK,
 233                                SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET);
 234
 235                        if (omap_sham_wait(dd, SHA_REG_SYSSTATUS,
 236                                                SHA_REG_SYSSTATUS_RESETDONE))
 237                                return -ETIMEDOUT;
 238
 239                        dd->flags |= FLAGS_INIT;
 240                }
 241        } else {
 242                omap_sham_write(dd, SHA_REG_DIGCNT, ctx->digcnt);
 243        }
 244
 245        omap_sham_write_mask(dd, SHA_REG_MASK,
 246                SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0),
 247                SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
 248        /*
 249         * Setting ALGO_CONST only for the first iteration
 250         * and CLOSE_HASH only for the last one.
 251         */
 252        if (ctx->flags & FLAGS_SHA1)
 253                val |= SHA_REG_CTRL_ALGO;
 254        if (!ctx->digcnt)
 255                val |= SHA_REG_CTRL_ALGO_CONST;
 256        if (final)
 257                val |= SHA_REG_CTRL_CLOSE_HASH;
 258
 259        mask = SHA_REG_CTRL_ALGO_CONST | SHA_REG_CTRL_CLOSE_HASH |
 260                        SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH;
 261
 262        omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask);
 263
 264        return 0;
 265}
 266
 267static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
 268                              size_t length, int final)
 269{
 270        struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
 271        int err, count, len32;
 272        const u32 *buffer = (const u32 *)buf;
 273
 274        dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
 275                                                ctx->digcnt, length, final);
 276
 277        err = omap_sham_write_ctrl(dd, length, final, 0);
 278        if (err)
 279                return err;
 280
 281        if (omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY))
 282                return -ETIMEDOUT;
 283
 284        ctx->digcnt += length;
 285
 286        if (final)
 287                ctx->flags |= FLAGS_FINAL; /* catch last interrupt */
 288
 289        len32 = DIV_ROUND_UP(length, sizeof(u32));
 290
 291        for (count = 0; count < len32; count++)
 292                omap_sham_write(dd, SHA_REG_DIN(count), buffer[count]);
 293
 294        return -EINPROGRESS;
 295}
 296
 297static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
 298                              size_t length, int final)
 299{
 300        struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
 301        int err, len32;
 302
 303        dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
 304                                                ctx->digcnt, length, final);
 305
 306        /* flush cache entries related to our page */
 307        if (dma_addr == ctx->dma_addr)
 308                dma_sync_single_for_device(dd->dev, dma_addr, length,
 309                                           DMA_TO_DEVICE);
 310
 311        len32 = DIV_ROUND_UP(length, sizeof(u32));
 312
 313        omap_set_dma_transfer_params(dd->dma_lch, OMAP_DMA_DATA_TYPE_S32, len32,
 314                        1, OMAP_DMA_SYNC_PACKET, dd->dma,
 315                                OMAP_DMA_DST_SYNC_PREFETCH);
 316
 317        omap_set_dma_src_params(dd->dma_lch, 0, OMAP_DMA_AMODE_POST_INC,
 318                                dma_addr, 0, 0);
 319
 320        err = omap_sham_write_ctrl(dd, length, final, 1);
 321        if (err)
 322                return err;
 323
 324        ctx->digcnt += length;
 325
 326        if (final)
 327                ctx->flags |= FLAGS_FINAL; /* catch last interrupt */
 328
 329        dd->flags |= FLAGS_DMA_ACTIVE;
 330
 331        omap_start_dma(dd->dma_lch);
 332
 333        return -EINPROGRESS;
 334}
 335
 336static size_t omap_sham_append_buffer(struct omap_sham_reqctx *ctx,
 337                                const u8 *data, size_t length)
 338{
 339        size_t count = min(length, ctx->buflen - ctx->bufcnt);
 340
 341        count = min(count, ctx->total);
 342        if (count <= 0)
 343                return 0;
 344        memcpy(ctx->buffer + ctx->bufcnt, data, count);
 345        ctx->bufcnt += count;
 346
 347        return count;
 348}
 349
 350static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx)
 351{
 352        size_t count;
 353
 354        while (ctx->sg) {
 355                count = omap_sham_append_buffer(ctx,
 356                                sg_virt(ctx->sg) + ctx->offset,
 357                                ctx->sg->length - ctx->offset);
 358                if (!count)
 359                        break;
 360                ctx->offset += count;
 361                ctx->total -= count;
 362                if (ctx->offset == ctx->sg->length) {
 363                        ctx->sg = sg_next(ctx->sg);
 364                        if (ctx->sg)
 365                                ctx->offset = 0;
 366                        else
 367                                ctx->total = 0;
 368                }
 369        }
 370
 371        return 0;
 372}
 373
 374static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
 375{
 376        struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
 377        unsigned int final;
 378        size_t count;
 379
 380        if (!ctx->total)
 381                return 0;
 382
 383        omap_sham_append_sg(ctx);
 384
 385        final = (ctx->flags & FLAGS_FINUP) && !ctx->total;
 386
 387        dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
 388                                         ctx->bufcnt, ctx->digcnt, final);
 389
 390        if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {
 391                count = ctx->bufcnt;
 392                ctx->bufcnt = 0;
 393                return omap_sham_xmit_dma(dd, ctx->dma_addr, count, final);
 394        }
 395
 396        return 0;
 397}
 398
 399static int omap_sham_update_dma_fast(struct omap_sham_dev *dd)
 400{
 401        struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
 402        unsigned int length;
 403
 404        ctx->flags |= FLAGS_FAST;
 405
 406        length = min(ctx->total, sg_dma_len(ctx->sg));
 407        ctx->total = length;
 408
 409        if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
 410                dev_err(dd->dev, "dma_map_sg  error\n");
 411                return -EINVAL;
 412        }
 413
 414        ctx->total -= length;
 415
 416        return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, 1);
 417}
 418
 419static int omap_sham_update_cpu(struct omap_sham_dev *dd)
 420{
 421        struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
 422        int bufcnt;
 423
 424        omap_sham_append_sg(ctx);
 425        bufcnt = ctx->bufcnt;
 426        ctx->bufcnt = 0;
 427
 428        return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, 1);
 429}
 430
 431static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
 432{
 433        struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
 434
 435        omap_stop_dma(dd->dma_lch);
 436        if (ctx->flags & FLAGS_FAST)
 437                dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
 438
 439        return 0;
 440}
 441
 442static void omap_sham_cleanup(struct ahash_request *req)
 443{
 444        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 445        struct omap_sham_dev *dd = ctx->dd;
 446        unsigned long flags;
 447
 448        spin_lock_irqsave(&dd->lock, flags);
 449        if (ctx->flags & FLAGS_CLEAN) {
 450                spin_unlock_irqrestore(&dd->lock, flags);
 451                return;
 452        }
 453        ctx->flags |= FLAGS_CLEAN;
 454        spin_unlock_irqrestore(&dd->lock, flags);
 455
 456        if (ctx->digcnt)
 457                clk_disable(dd->iclk);
 458
 459        if (ctx->dma_addr)
 460                dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
 461                                 DMA_TO_DEVICE);
 462
 463        if (ctx->buffer)
 464                free_page((unsigned long)ctx->buffer);
 465
 466        dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);
 467}
 468
 469static int omap_sham_init(struct ahash_request *req)
 470{
 471        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 472        struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
 473        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 474        struct omap_sham_dev *dd = NULL, *tmp;
 475
 476        spin_lock_bh(&sham.lock);
 477        if (!tctx->dd) {
 478                list_for_each_entry(tmp, &sham.dev_list, list) {
 479                        dd = tmp;
 480                        break;
 481                }
 482                tctx->dd = dd;
 483        } else {
 484                dd = tctx->dd;
 485        }
 486        spin_unlock_bh(&sham.lock);
 487
 488        ctx->dd = dd;
 489
 490        ctx->flags = 0;
 491
 492        ctx->flags |= FLAGS_FIRST;
 493
 494        dev_dbg(dd->dev, "init: digest size: %d\n",
 495                crypto_ahash_digestsize(tfm));
 496
 497        if (crypto_ahash_digestsize(tfm) == SHA1_DIGEST_SIZE)
 498                ctx->flags |= FLAGS_SHA1;
 499
 500        ctx->bufcnt = 0;
 501        ctx->digcnt = 0;
 502
 503        ctx->buflen = PAGE_SIZE;
 504        ctx->buffer = (void *)__get_free_page(
 505                                (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 506                                GFP_KERNEL : GFP_ATOMIC);
 507        if (!ctx->buffer)
 508                return -ENOMEM;
 509
 510        ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
 511                                        DMA_TO_DEVICE);
 512        if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
 513                dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);
 514                free_page((unsigned long)ctx->buffer);
 515                return -EINVAL;
 516        }
 517
 518        if (tctx->flags & FLAGS_HMAC) {
 519                struct omap_sham_hmac_ctx *bctx = tctx->base;
 520
 521                memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE);
 522                ctx->bufcnt = SHA1_MD5_BLOCK_SIZE;
 523                ctx->flags |= FLAGS_HMAC;
 524        }
 525
 526        return 0;
 527
 528}
 529
 530static int omap_sham_update_req(struct omap_sham_dev *dd)
 531{
 532        struct ahash_request *req = dd->req;
 533        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 534        int err;
 535
 536        dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n",
 537                 ctx->total, ctx->digcnt, (ctx->flags & FLAGS_FINUP) != 0);
 538
 539        if (ctx->flags & FLAGS_CPU)
 540                err = omap_sham_update_cpu(dd);
 541        else if (ctx->flags & FLAGS_FAST)
 542                err = omap_sham_update_dma_fast(dd);
 543        else
 544                err = omap_sham_update_dma_slow(dd);
 545
 546        /* wait for dma completion before can take more data */
 547        dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);
 548
 549        return err;
 550}
 551
 552static int omap_sham_final_req(struct omap_sham_dev *dd)
 553{
 554        struct ahash_request *req = dd->req;
 555        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 556        int err = 0, use_dma = 1;
 557
 558        if (ctx->bufcnt <= 64)
 559                /* faster to handle last block with cpu */
 560                use_dma = 0;
 561
 562        if (use_dma)
 563                err = omap_sham_xmit_dma(dd, ctx->dma_addr, ctx->bufcnt, 1);
 564        else
 565                err = omap_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1);
 566
 567        ctx->bufcnt = 0;
 568
 569        if (err != -EINPROGRESS)
 570                omap_sham_cleanup(req);
 571
 572        dev_dbg(dd->dev, "final_req: err: %d\n", err);
 573
 574        return err;
 575}
 576
 577static int omap_sham_finish_req_hmac(struct ahash_request *req)
 578{
 579        struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
 580        struct omap_sham_hmac_ctx *bctx = tctx->base;
 581        int bs = crypto_shash_blocksize(bctx->shash);
 582        int ds = crypto_shash_digestsize(bctx->shash);
 583        struct {
 584                struct shash_desc shash;
 585                char ctx[crypto_shash_descsize(bctx->shash)];
 586        } desc;
 587
 588        desc.shash.tfm = bctx->shash;
 589        desc.shash.flags = 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */
 590
 591        return crypto_shash_init(&desc.shash) ?:
 592               crypto_shash_update(&desc.shash, bctx->opad, bs) ?:
 593               crypto_shash_finup(&desc.shash, req->result, ds, req->result);
 594}
 595
 596static void omap_sham_finish_req(struct ahash_request *req, int err)
 597{
 598        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 599
 600        if (!err) {
 601                omap_sham_copy_hash(ctx->dd->req, 1);
 602                if (ctx->flags & FLAGS_HMAC)
 603                        err = omap_sham_finish_req_hmac(req);
 604        }
 605
 606        if (ctx->flags & FLAGS_FINAL)
 607                omap_sham_cleanup(req);
 608
 609        clear_bit(FLAGS_BUSY, &ctx->dd->flags);
 610
 611        if (req->base.complete)
 612                req->base.complete(&req->base, err);
 613}
 614
 615static int omap_sham_handle_queue(struct omap_sham_dev *dd)
 616{
 617        struct crypto_async_request *async_req, *backlog;
 618        struct omap_sham_reqctx *ctx;
 619        struct ahash_request *req, *prev_req;
 620        unsigned long flags;
 621        int err = 0;
 622
 623        if (test_and_set_bit(FLAGS_BUSY, &dd->flags))
 624                return 0;
 625
 626        spin_lock_irqsave(&dd->lock, flags);
 627        backlog = crypto_get_backlog(&dd->queue);
 628        async_req = crypto_dequeue_request(&dd->queue);
 629        if (!async_req)
 630                clear_bit(FLAGS_BUSY, &dd->flags);
 631        spin_unlock_irqrestore(&dd->lock, flags);
 632
 633        if (!async_req)
 634                return 0;
 635
 636        if (backlog)
 637                backlog->complete(backlog, -EINPROGRESS);
 638
 639        req = ahash_request_cast(async_req);
 640
 641        prev_req = dd->req;
 642        dd->req = req;
 643
 644        ctx = ahash_request_ctx(req);
 645
 646        dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
 647                                                ctx->op, req->nbytes);
 648
 649        if (req != prev_req && ctx->digcnt)
 650                /* request has changed - restore hash */
 651                omap_sham_copy_hash(req, 0);
 652
 653        if (ctx->op == OP_UPDATE) {
 654                err = omap_sham_update_req(dd);
 655                if (err != -EINPROGRESS && (ctx->flags & FLAGS_FINUP))
 656                        /* no final() after finup() */
 657                        err = omap_sham_final_req(dd);
 658        } else if (ctx->op == OP_FINAL) {
 659                err = omap_sham_final_req(dd);
 660        }
 661
 662        if (err != -EINPROGRESS) {
 663                /* done_task will not finish it, so do it here */
 664                omap_sham_finish_req(req, err);
 665                tasklet_schedule(&dd->queue_task);
 666        }
 667
 668        dev_dbg(dd->dev, "exit, err: %d\n", err);
 669
 670        return err;
 671}
 672
 673static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
 674{
 675        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 676        struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
 677        struct omap_sham_dev *dd = tctx->dd;
 678        unsigned long flags;
 679        int err;
 680
 681        ctx->op = op;
 682
 683        spin_lock_irqsave(&dd->lock, flags);
 684        err = ahash_enqueue_request(&dd->queue, req);
 685        spin_unlock_irqrestore(&dd->lock, flags);
 686
 687        omap_sham_handle_queue(dd);
 688
 689        return err;
 690}
 691
 692static int omap_sham_update(struct ahash_request *req)
 693{
 694        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 695
 696        if (!req->nbytes)
 697                return 0;
 698
 699        ctx->total = req->nbytes;
 700        ctx->sg = req->src;
 701        ctx->offset = 0;
 702
 703        if (ctx->flags & FLAGS_FINUP) {
 704                if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) {
 705                        /*
 706                        * OMAP HW accel works only with buffers >= 9
 707                        * will switch to bypass in final()
 708                        * final has the same request and data
 709                        */
 710                        omap_sham_append_sg(ctx);
 711                        return 0;
 712                } else if (ctx->bufcnt + ctx->total <= 64) {
 713                        ctx->flags |= FLAGS_CPU;
 714                } else if (!ctx->bufcnt && sg_is_last(ctx->sg)) {
 715                        /* may be can use faster functions */
 716                        int aligned = IS_ALIGNED((u32)ctx->sg->offset,
 717                                                                sizeof(u32));
 718
 719                        if (aligned && (ctx->flags & FLAGS_FIRST))
 720                                /* digest: first and final */
 721                                ctx->flags |= FLAGS_FAST;
 722
 723                        ctx->flags &= ~FLAGS_FIRST;
 724                }
 725        } else if (ctx->bufcnt + ctx->total <= ctx->buflen) {
 726                /* if not finaup -> not fast */
 727                omap_sham_append_sg(ctx);
 728                return 0;
 729        }
 730
 731        return omap_sham_enqueue(req, OP_UPDATE);
 732}
 733
 734static int omap_sham_shash_digest(struct crypto_shash *shash, u32 flags,
 735                                  const u8 *data, unsigned int len, u8 *out)
 736{
 737        struct {
 738                struct shash_desc shash;
 739                char ctx[crypto_shash_descsize(shash)];
 740        } desc;
 741
 742        desc.shash.tfm = shash;
 743        desc.shash.flags = flags & CRYPTO_TFM_REQ_MAY_SLEEP;
 744
 745        return crypto_shash_digest(&desc.shash, data, len, out);
 746}
 747
 748static int omap_sham_final_shash(struct ahash_request *req)
 749{
 750        struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
 751        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 752
 753        return omap_sham_shash_digest(tctx->fallback, req->base.flags,
 754                                      ctx->buffer, ctx->bufcnt, req->result);
 755}
 756
 757static int omap_sham_final(struct ahash_request *req)
 758{
 759        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 760        int err = 0;
 761
 762        ctx->flags |= FLAGS_FINUP;
 763
 764        /* OMAP HW accel works only with buffers >= 9 */
 765        /* HMAC is always >= 9 because of ipad */
 766        if ((ctx->digcnt + ctx->bufcnt) < 9)
 767                err = omap_sham_final_shash(req);
 768        else if (ctx->bufcnt)
 769                return omap_sham_enqueue(req, OP_FINAL);
 770
 771        omap_sham_cleanup(req);
 772
 773        return err;
 774}
 775
 776static int omap_sham_finup(struct ahash_request *req)
 777{
 778        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 779        int err1, err2;
 780
 781        ctx->flags |= FLAGS_FINUP;
 782
 783        err1 = omap_sham_update(req);
 784        if (err1 == -EINPROGRESS)
 785                return err1;
 786        /*
 787         * final() has to be always called to cleanup resources
 788         * even if udpate() failed, except EINPROGRESS
 789         */
 790        err2 = omap_sham_final(req);
 791
 792        return err1 ?: err2;
 793}
 794
 795static int omap_sham_digest(struct ahash_request *req)
 796{
 797        return omap_sham_init(req) ?: omap_sham_finup(req);
 798}
 799
 800static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
 801                      unsigned int keylen)
 802{
 803        struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
 804        struct omap_sham_hmac_ctx *bctx = tctx->base;
 805        int bs = crypto_shash_blocksize(bctx->shash);
 806        int ds = crypto_shash_digestsize(bctx->shash);
 807        int err, i;
 808        err = crypto_shash_setkey(tctx->fallback, key, keylen);
 809        if (err)
 810                return err;
 811
 812        if (keylen > bs) {
 813                err = omap_sham_shash_digest(bctx->shash,
 814                                crypto_shash_get_flags(bctx->shash),
 815                                key, keylen, bctx->ipad);
 816                if (err)
 817                        return err;
 818                keylen = ds;
 819        } else {
 820                memcpy(bctx->ipad, key, keylen);
 821        }
 822
 823        memset(bctx->ipad + keylen, 0, bs - keylen);
 824        memcpy(bctx->opad, bctx->ipad, bs);
 825
 826        for (i = 0; i < bs; i++) {
 827                bctx->ipad[i] ^= 0x36;
 828                bctx->opad[i] ^= 0x5c;
 829        }
 830
 831        return err;
 832}
 833
 834static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
 835{
 836        struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
 837        const char *alg_name = crypto_tfm_alg_name(tfm);
 838
 839        /* Allocate a fallback and abort if it failed. */
 840        tctx->fallback = crypto_alloc_shash(alg_name, 0,
 841                                            CRYPTO_ALG_NEED_FALLBACK);
 842        if (IS_ERR(tctx->fallback)) {
 843                pr_err("omap-sham: fallback driver '%s' "
 844                                "could not be loaded.\n", alg_name);
 845                return PTR_ERR(tctx->fallback);
 846        }
 847
 848        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 849                                 sizeof(struct omap_sham_reqctx));
 850
 851        if (alg_base) {
 852                struct omap_sham_hmac_ctx *bctx = tctx->base;
 853                tctx->flags |= FLAGS_HMAC;
 854                bctx->shash = crypto_alloc_shash(alg_base, 0,
 855                                                CRYPTO_ALG_NEED_FALLBACK);
 856                if (IS_ERR(bctx->shash)) {
 857                        pr_err("omap-sham: base driver '%s' "
 858                                        "could not be loaded.\n", alg_base);
 859                        crypto_free_shash(tctx->fallback);
 860                        return PTR_ERR(bctx->shash);
 861                }
 862
 863        }
 864
 865        return 0;
 866}
 867
 868static int omap_sham_cra_init(struct crypto_tfm *tfm)
 869{
 870        return omap_sham_cra_init_alg(tfm, NULL);
 871}
 872
 873static int omap_sham_cra_sha1_init(struct crypto_tfm *tfm)
 874{
 875        return omap_sham_cra_init_alg(tfm, "sha1");
 876}
 877
 878static int omap_sham_cra_md5_init(struct crypto_tfm *tfm)
 879{
 880        return omap_sham_cra_init_alg(tfm, "md5");
 881}
 882
 883static void omap_sham_cra_exit(struct crypto_tfm *tfm)
 884{
 885        struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
 886
 887        crypto_free_shash(tctx->fallback);
 888        tctx->fallback = NULL;
 889
 890        if (tctx->flags & FLAGS_HMAC) {
 891                struct omap_sham_hmac_ctx *bctx = tctx->base;
 892                crypto_free_shash(bctx->shash);
 893        }
 894}
 895
 896static struct ahash_alg algs[] = {
 897{
 898        .init           = omap_sham_init,
 899        .update         = omap_sham_update,
 900        .final          = omap_sham_final,
 901        .finup          = omap_sham_finup,
 902        .digest         = omap_sham_digest,
 903        .halg.digestsize        = SHA1_DIGEST_SIZE,
 904        .halg.base      = {
 905                .cra_name               = "sha1",
 906                .cra_driver_name        = "omap-sha1",
 907                .cra_priority           = 100,
 908                .cra_flags              = CRYPTO_ALG_TYPE_AHASH |
 909                                                CRYPTO_ALG_ASYNC |
 910                                                CRYPTO_ALG_NEED_FALLBACK,
 911                .cra_blocksize          = SHA1_BLOCK_SIZE,
 912                .cra_ctxsize            = sizeof(struct omap_sham_ctx),
 913                .cra_alignmask          = 0,
 914                .cra_module             = THIS_MODULE,
 915                .cra_init               = omap_sham_cra_init,
 916                .cra_exit               = omap_sham_cra_exit,
 917        }
 918},
 919{
 920        .init           = omap_sham_init,
 921        .update         = omap_sham_update,
 922        .final          = omap_sham_final,
 923        .finup          = omap_sham_finup,
 924        .digest         = omap_sham_digest,
 925        .halg.digestsize        = MD5_DIGEST_SIZE,
 926        .halg.base      = {
 927                .cra_name               = "md5",
 928                .cra_driver_name        = "omap-md5",
 929                .cra_priority           = 100,
 930                .cra_flags              = CRYPTO_ALG_TYPE_AHASH |
 931                                                CRYPTO_ALG_ASYNC |
 932                                                CRYPTO_ALG_NEED_FALLBACK,
 933                .cra_blocksize          = SHA1_BLOCK_SIZE,
 934                .cra_ctxsize            = sizeof(struct omap_sham_ctx),
 935                .cra_alignmask          = 0,
 936                .cra_module             = THIS_MODULE,
 937                .cra_init               = omap_sham_cra_init,
 938                .cra_exit               = omap_sham_cra_exit,
 939        }
 940},
 941{
 942        .init           = omap_sham_init,
 943        .update         = omap_sham_update,
 944        .final          = omap_sham_final,
 945        .finup          = omap_sham_finup,
 946        .digest         = omap_sham_digest,
 947        .setkey         = omap_sham_setkey,
 948        .halg.digestsize        = SHA1_DIGEST_SIZE,
 949        .halg.base      = {
 950                .cra_name               = "hmac(sha1)",
 951                .cra_driver_name        = "omap-hmac-sha1",
 952                .cra_priority           = 100,
 953                .cra_flags              = CRYPTO_ALG_TYPE_AHASH |
 954                                                CRYPTO_ALG_ASYNC |
 955                                                CRYPTO_ALG_NEED_FALLBACK,
 956                .cra_blocksize          = SHA1_BLOCK_SIZE,
 957                .cra_ctxsize            = sizeof(struct omap_sham_ctx) +
 958                                        sizeof(struct omap_sham_hmac_ctx),
 959                .cra_alignmask          = 0,
 960                .cra_module             = THIS_MODULE,
 961                .cra_init               = omap_sham_cra_sha1_init,
 962                .cra_exit               = omap_sham_cra_exit,
 963        }
 964},
 965{
 966        .init           = omap_sham_init,
 967        .update         = omap_sham_update,
 968        .final          = omap_sham_final,
 969        .finup          = omap_sham_finup,
 970        .digest         = omap_sham_digest,
 971        .setkey         = omap_sham_setkey,
 972        .halg.digestsize        = MD5_DIGEST_SIZE,
 973        .halg.base      = {
 974                .cra_name               = "hmac(md5)",
 975                .cra_driver_name        = "omap-hmac-md5",
 976                .cra_priority           = 100,
 977                .cra_flags              = CRYPTO_ALG_TYPE_AHASH |
 978                                                CRYPTO_ALG_ASYNC |
 979                                                CRYPTO_ALG_NEED_FALLBACK,
 980                .cra_blocksize          = SHA1_BLOCK_SIZE,
 981                .cra_ctxsize            = sizeof(struct omap_sham_ctx) +
 982                                        sizeof(struct omap_sham_hmac_ctx),
 983                .cra_alignmask          = 0,
 984                .cra_module             = THIS_MODULE,
 985                .cra_init               = omap_sham_cra_md5_init,
 986                .cra_exit               = omap_sham_cra_exit,
 987        }
 988}
 989};
 990
 991static void omap_sham_done_task(unsigned long data)
 992{
 993        struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
 994        struct ahash_request *req = dd->req;
 995        struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
 996        int ready = 1;
 997
 998        if (ctx->flags & FLAGS_OUTPUT_READY) {
 999                ctx->flags &= ~FLAGS_OUTPUT_READY;
1000                ready = 1;
1001        }
1002
1003        if (dd->flags & FLAGS_DMA_ACTIVE) {
1004                dd->flags &= ~FLAGS_DMA_ACTIVE;
1005                omap_sham_update_dma_stop(dd);
1006                omap_sham_update_dma_slow(dd);
1007        }
1008
1009        if (ready && !(dd->flags & FLAGS_DMA_ACTIVE)) {
1010                dev_dbg(dd->dev, "update done\n");
1011                /* finish curent request */
1012                omap_sham_finish_req(req, 0);
1013                /* start new request */
1014                omap_sham_handle_queue(dd);
1015        }
1016}
1017
1018static void omap_sham_queue_task(unsigned long data)
1019{
1020        struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
1021
1022        omap_sham_handle_queue(dd);
1023}
1024
1025static irqreturn_t omap_sham_irq(int irq, void *dev_id)
1026{
1027        struct omap_sham_dev *dd = dev_id;
1028        struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
1029
1030        if (!ctx) {
1031                dev_err(dd->dev, "unknown interrupt.\n");
1032                return IRQ_HANDLED;
1033        }
1034
1035        if (unlikely(ctx->flags & FLAGS_FINAL))
1036                /* final -> allow device to go to power-saving mode */
1037                omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH);
1038
1039        omap_sham_write_mask(dd, SHA_REG_CTRL, SHA_REG_CTRL_OUTPUT_READY,
1040                                 SHA_REG_CTRL_OUTPUT_READY);
1041        omap_sham_read(dd, SHA_REG_CTRL);
1042
1043        ctx->flags |= FLAGS_OUTPUT_READY;
1044        tasklet_schedule(&dd->done_task);
1045
1046        return IRQ_HANDLED;
1047}
1048
1049static void omap_sham_dma_callback(int lch, u16 ch_status, void *data)
1050{
1051        struct omap_sham_dev *dd = data;
1052
1053        if (likely(lch == dd->dma_lch))
1054                tasklet_schedule(&dd->done_task);
1055}
1056
1057static int omap_sham_dma_init(struct omap_sham_dev *dd)
1058{
1059        int err;
1060
1061        dd->dma_lch = -1;
1062
1063        err = omap_request_dma(dd->dma, dev_name(dd->dev),
1064                        omap_sham_dma_callback, dd, &dd->dma_lch);
1065        if (err) {
1066                dev_err(dd->dev, "Unable to request DMA channel\n");
1067                return err;
1068        }
1069        omap_set_dma_dest_params(dd->dma_lch, 0,
1070                        OMAP_DMA_AMODE_CONSTANT,
1071                        dd->phys_base + SHA_REG_DIN(0), 0, 16);
1072
1073        omap_set_dma_dest_burst_mode(dd->dma_lch,
1074                        OMAP_DMA_DATA_BURST_16);
1075
1076        omap_set_dma_src_burst_mode(dd->dma_lch,
1077                        OMAP_DMA_DATA_BURST_4);
1078
1079        return 0;
1080}
1081
1082static void omap_sham_dma_cleanup(struct omap_sham_dev *dd)
1083{
1084        if (dd->dma_lch >= 0) {
1085                omap_free_dma(dd->dma_lch);
1086                dd->dma_lch = -1;
1087        }
1088}
1089
1090static int __devinit omap_sham_probe(struct platform_device *pdev)
1091{
1092        struct omap_sham_dev *dd;
1093        struct device *dev = &pdev->dev;
1094        struct resource *res;
1095        int err, i, j;
1096
1097        dd = kzalloc(sizeof(struct omap_sham_dev), GFP_KERNEL);
1098        if (dd == NULL) {
1099                dev_err(dev, "unable to alloc data struct.\n");
1100                err = -ENOMEM;
1101                goto data_err;
1102        }
1103        dd->dev = dev;
1104        platform_set_drvdata(pdev, dd);
1105
1106        INIT_LIST_HEAD(&dd->list);
1107        spin_lock_init(&dd->lock);
1108        tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
1109        tasklet_init(&dd->queue_task, omap_sham_queue_task, (unsigned long)dd);
1110        crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
1111
1112        dd->irq = -1;
1113
1114        /* Get the base address */
1115        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1116        if (!res) {
1117                dev_err(dev, "no MEM resource info\n");
1118                err = -ENODEV;
1119                goto res_err;
1120        }
1121        dd->phys_base = res->start;
1122
1123        /* Get the DMA */
1124        res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
1125        if (!res) {
1126                dev_err(dev, "no DMA resource info\n");
1127                err = -ENODEV;
1128                goto res_err;
1129        }
1130        dd->dma = res->start;
1131
1132        /* Get the IRQ */
1133        dd->irq = platform_get_irq(pdev,  0);
1134        if (dd->irq < 0) {
1135                dev_err(dev, "no IRQ resource info\n");
1136                err = dd->irq;
1137                goto res_err;
1138        }
1139
1140        err = request_irq(dd->irq, omap_sham_irq,
1141                        IRQF_TRIGGER_LOW, dev_name(dev), dd);
1142        if (err) {
1143                dev_err(dev, "unable to request irq.\n");
1144                goto res_err;
1145        }
1146
1147        err = omap_sham_dma_init(dd);
1148        if (err)
1149                goto dma_err;
1150
1151        /* Initializing the clock */
1152        dd->iclk = clk_get(dev, "ick");
1153        if (!dd->iclk) {
1154                dev_err(dev, "clock intialization failed.\n");
1155                err = -ENODEV;
1156                goto clk_err;
1157        }
1158
1159        dd->io_base = ioremap(dd->phys_base, SZ_4K);
1160        if (!dd->io_base) {
1161                dev_err(dev, "can't ioremap\n");
1162                err = -ENOMEM;
1163                goto io_err;
1164        }
1165
1166        clk_enable(dd->iclk);
1167        dev_info(dev, "hw accel on OMAP rev %u.%u\n",
1168                (omap_sham_read(dd, SHA_REG_REV) & SHA_REG_REV_MAJOR) >> 4,
1169                omap_sham_read(dd, SHA_REG_REV) & SHA_REG_REV_MINOR);
1170        clk_disable(dd->iclk);
1171
1172        spin_lock(&sham.lock);
1173        list_add_tail(&dd->list, &sham.dev_list);
1174        spin_unlock(&sham.lock);
1175
1176        for (i = 0; i < ARRAY_SIZE(algs); i++) {
1177                err = crypto_register_ahash(&algs[i]);
1178                if (err)
1179                        goto err_algs;
1180        }
1181
1182        return 0;
1183
1184err_algs:
1185        for (j = 0; j < i; j++)
1186                crypto_unregister_ahash(&algs[j]);
1187        iounmap(dd->io_base);
1188io_err:
1189        clk_put(dd->iclk);
1190clk_err:
1191        omap_sham_dma_cleanup(dd);
1192dma_err:
1193        if (dd->irq >= 0)
1194                free_irq(dd->irq, dd);
1195res_err:
1196        kfree(dd);
1197        dd = NULL;
1198data_err:
1199        dev_err(dev, "initialization failed.\n");
1200
1201        return err;
1202}
1203
1204static int __devexit omap_sham_remove(struct platform_device *pdev)
1205{
1206        static struct omap_sham_dev *dd;
1207        int i;
1208
1209        dd = platform_get_drvdata(pdev);
1210        if (!dd)
1211                return -ENODEV;
1212        spin_lock(&sham.lock);
1213        list_del(&dd->list);
1214        spin_unlock(&sham.lock);
1215        for (i = 0; i < ARRAY_SIZE(algs); i++)
1216                crypto_unregister_ahash(&algs[i]);
1217        tasklet_kill(&dd->done_task);
1218        tasklet_kill(&dd->queue_task);
1219        iounmap(dd->io_base);
1220        clk_put(dd->iclk);
1221        omap_sham_dma_cleanup(dd);
1222        if (dd->irq >= 0)
1223                free_irq(dd->irq, dd);
1224        kfree(dd);
1225        dd = NULL;
1226
1227        return 0;
1228}
1229
1230static struct platform_driver omap_sham_driver = {
1231        .probe  = omap_sham_probe,
1232        .remove = omap_sham_remove,
1233        .driver = {
1234                .name   = "omap-sham",
1235                .owner  = THIS_MODULE,
1236        },
1237};
1238
1239static int __init omap_sham_mod_init(void)
1240{
1241        pr_info("loading %s driver\n", "omap-sham");
1242
1243        if (!cpu_class_is_omap2() ||
1244                omap_type() != OMAP2_DEVICE_TYPE_SEC) {
1245                pr_err("Unsupported cpu\n");
1246                return -ENODEV;
1247        }
1248
1249        return platform_driver_register(&omap_sham_driver);
1250}
1251
1252static void __exit omap_sham_mod_exit(void)
1253{
1254        platform_driver_unregister(&omap_sham_driver);
1255}
1256
1257module_init(omap_sham_mod_init);
1258module_exit(omap_sham_mod_exit);
1259
1260MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
1261MODULE_LICENSE("GPL v2");
1262MODULE_AUTHOR("Dmitry Kasatkin");
1263