linux/drivers/crypto/omap-aes.c
<<
>>
Prefs
   1/*
   2 * Cryptographic API.
   3 *
   4 * Support for OMAP AES HW acceleration.
   5 *
   6 * Copyright (c) 2010 Nokia Corporation
   7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
   8 *
   9 * This program is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License version 2 as published
  11 * by the Free Software Foundation.
  12 *
  13 */
  14
  15#define pr_fmt(fmt) "%s: " fmt, __func__
  16
  17#include <linux/err.h>
  18#include <linux/module.h>
  19#include <linux/init.h>
  20#include <linux/errno.h>
  21#include <linux/kernel.h>
  22#include <linux/clk.h>
  23#include <linux/platform_device.h>
  24#include <linux/scatterlist.h>
  25#include <linux/dma-mapping.h>
  26#include <linux/io.h>
  27#include <linux/crypto.h>
  28#include <linux/interrupt.h>
  29#include <crypto/scatterwalk.h>
  30#include <crypto/aes.h>
  31
  32#include <linux/omap-dma.h>
  33
  34/* OMAP TRM gives bitfields as start:end, where start is the higher bit
  35   number. For example 7:0 */
  36#define FLD_MASK(start, end)    (((1 << ((start) - (end) + 1)) - 1) << (end))
  37#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
  38
  39#define AES_REG_KEY(x)                  (0x1C - ((x ^ 0x01) * 0x04))
  40#define AES_REG_IV(x)                   (0x20 + ((x) * 0x04))
  41
  42#define AES_REG_CTRL                    0x30
  43#define AES_REG_CTRL_CTR_WIDTH          (1 << 7)
  44#define AES_REG_CTRL_CTR                (1 << 6)
  45#define AES_REG_CTRL_CBC                (1 << 5)
  46#define AES_REG_CTRL_KEY_SIZE           (3 << 3)
  47#define AES_REG_CTRL_DIRECTION          (1 << 2)
  48#define AES_REG_CTRL_INPUT_READY        (1 << 1)
  49#define AES_REG_CTRL_OUTPUT_READY       (1 << 0)
  50
  51#define AES_REG_DATA                    0x34
  52#define AES_REG_DATA_N(x)               (0x34 + ((x) * 0x04))
  53
  54#define AES_REG_REV                     0x44
  55#define AES_REG_REV_MAJOR               0xF0
  56#define AES_REG_REV_MINOR               0x0F
  57
  58#define AES_REG_MASK                    0x48
  59#define AES_REG_MASK_SIDLE              (1 << 6)
  60#define AES_REG_MASK_START              (1 << 5)
  61#define AES_REG_MASK_DMA_OUT_EN         (1 << 3)
  62#define AES_REG_MASK_DMA_IN_EN          (1 << 2)
  63#define AES_REG_MASK_SOFTRESET          (1 << 1)
  64#define AES_REG_AUTOIDLE                (1 << 0)
  65
  66#define AES_REG_SYSSTATUS               0x4C
  67#define AES_REG_SYSSTATUS_RESETDONE     (1 << 0)
  68
  69#define DEFAULT_TIMEOUT         (5*HZ)
  70
  71#define FLAGS_MODE_MASK         0x000f
  72#define FLAGS_ENCRYPT           BIT(0)
  73#define FLAGS_CBC               BIT(1)
  74#define FLAGS_GIV               BIT(2)
  75
  76#define FLAGS_INIT              BIT(4)
  77#define FLAGS_FAST              BIT(5)
  78#define FLAGS_BUSY              BIT(6)
  79
  80struct omap_aes_ctx {
  81        struct omap_aes_dev *dd;
  82
  83        int             keylen;
  84        u32             key[AES_KEYSIZE_256 / sizeof(u32)];
  85        unsigned long   flags;
  86};
  87
  88struct omap_aes_reqctx {
  89        unsigned long mode;
  90};
  91
  92#define OMAP_AES_QUEUE_LENGTH   1
  93#define OMAP_AES_CACHE_SIZE     0
  94
  95struct omap_aes_dev {
  96        struct list_head        list;
  97        unsigned long           phys_base;
  98        void __iomem            *io_base;
  99        struct clk              *iclk;
 100        struct omap_aes_ctx     *ctx;
 101        struct device           *dev;
 102        unsigned long           flags;
 103        int                     err;
 104
 105        spinlock_t              lock;
 106        struct crypto_queue     queue;
 107
 108        struct tasklet_struct   done_task;
 109        struct tasklet_struct   queue_task;
 110
 111        struct ablkcipher_request       *req;
 112        size_t                          total;
 113        struct scatterlist              *in_sg;
 114        size_t                          in_offset;
 115        struct scatterlist              *out_sg;
 116        size_t                          out_offset;
 117
 118        size_t                  buflen;
 119        void                    *buf_in;
 120        size_t                  dma_size;
 121        int                     dma_in;
 122        int                     dma_lch_in;
 123        dma_addr_t              dma_addr_in;
 124        void                    *buf_out;
 125        int                     dma_out;
 126        int                     dma_lch_out;
 127        dma_addr_t              dma_addr_out;
 128};
 129
 130/* keep registered devices data here */
 131static LIST_HEAD(dev_list);
 132static DEFINE_SPINLOCK(list_lock);
 133
 134static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
 135{
 136        return __raw_readl(dd->io_base + offset);
 137}
 138
 139static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
 140                                  u32 value)
 141{
 142        __raw_writel(value, dd->io_base + offset);
 143}
 144
 145static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
 146                                        u32 value, u32 mask)
 147{
 148        u32 val;
 149
 150        val = omap_aes_read(dd, offset);
 151        val &= ~mask;
 152        val |= value;
 153        omap_aes_write(dd, offset, val);
 154}
 155
 156static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
 157                                        u32 *value, int count)
 158{
 159        for (; count--; value++, offset += 4)
 160                omap_aes_write(dd, offset, *value);
 161}
 162
 163static int omap_aes_wait(struct omap_aes_dev *dd, u32 offset, u32 bit)
 164{
 165        unsigned long timeout = jiffies + DEFAULT_TIMEOUT;
 166
 167        while (!(omap_aes_read(dd, offset) & bit)) {
 168                if (time_is_before_jiffies(timeout)) {
 169                        dev_err(dd->dev, "omap-aes timeout\n");
 170                        return -ETIMEDOUT;
 171                }
 172        }
 173        return 0;
 174}
 175
 176static int omap_aes_hw_init(struct omap_aes_dev *dd)
 177{
 178        /*
 179         * clocks are enabled when request starts and disabled when finished.
 180         * It may be long delays between requests.
 181         * Device might go to off mode to save power.
 182         */
 183        clk_enable(dd->iclk);
 184
 185        if (!(dd->flags & FLAGS_INIT)) {
 186                /* is it necessary to reset before every operation? */
 187                omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_SOFTRESET,
 188                                        AES_REG_MASK_SOFTRESET);
 189                /*
 190                 * prevent OCP bus error (SRESP) in case an access to the module
 191                 * is performed while the module is coming out of soft reset
 192                 */
 193                __asm__ __volatile__("nop");
 194                __asm__ __volatile__("nop");
 195
 196                if (omap_aes_wait(dd, AES_REG_SYSSTATUS,
 197                                AES_REG_SYSSTATUS_RESETDONE))
 198                        return -ETIMEDOUT;
 199
 200                dd->flags |= FLAGS_INIT;
 201                dd->err = 0;
 202        }
 203
 204        return 0;
 205}
 206
 207static int omap_aes_write_ctrl(struct omap_aes_dev *dd)
 208{
 209        unsigned int key32;
 210        int i, err;
 211        u32 val, mask;
 212
 213        err = omap_aes_hw_init(dd);
 214        if (err)
 215                return err;
 216
 217        val = 0;
 218        if (dd->dma_lch_out >= 0)
 219                val |= AES_REG_MASK_DMA_OUT_EN;
 220        if (dd->dma_lch_in >= 0)
 221                val |= AES_REG_MASK_DMA_IN_EN;
 222
 223        mask = AES_REG_MASK_DMA_IN_EN | AES_REG_MASK_DMA_OUT_EN;
 224
 225        omap_aes_write_mask(dd, AES_REG_MASK, val, mask);
 226
 227        key32 = dd->ctx->keylen / sizeof(u32);
 228
 229        /* it seems a key should always be set even if it has not changed */
 230        for (i = 0; i < key32; i++) {
 231                omap_aes_write(dd, AES_REG_KEY(i),
 232                        __le32_to_cpu(dd->ctx->key[i]));
 233        }
 234
 235        if ((dd->flags & FLAGS_CBC) && dd->req->info)
 236                omap_aes_write_n(dd, AES_REG_IV(0), dd->req->info, 4);
 237
 238        val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
 239        if (dd->flags & FLAGS_CBC)
 240                val |= AES_REG_CTRL_CBC;
 241        if (dd->flags & FLAGS_ENCRYPT)
 242                val |= AES_REG_CTRL_DIRECTION;
 243
 244        mask = AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION |
 245                        AES_REG_CTRL_KEY_SIZE;
 246
 247        omap_aes_write_mask(dd, AES_REG_CTRL, val, mask);
 248
 249        /* IN */
 250        omap_set_dma_dest_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_CONSTANT,
 251                                 dd->phys_base + AES_REG_DATA, 0, 4);
 252
 253        omap_set_dma_dest_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
 254        omap_set_dma_src_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
 255
 256        /* OUT */
 257        omap_set_dma_src_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_CONSTANT,
 258                                dd->phys_base + AES_REG_DATA, 0, 4);
 259
 260        omap_set_dma_src_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
 261        omap_set_dma_dest_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
 262
 263        return 0;
 264}
 265
 266static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx)
 267{
 268        struct omap_aes_dev *dd = NULL, *tmp;
 269
 270        spin_lock_bh(&list_lock);
 271        if (!ctx->dd) {
 272                list_for_each_entry(tmp, &dev_list, list) {
 273                        /* FIXME: take fist available aes core */
 274                        dd = tmp;
 275                        break;
 276                }
 277                ctx->dd = dd;
 278        } else {
 279                /* already found before */
 280                dd = ctx->dd;
 281        }
 282        spin_unlock_bh(&list_lock);
 283
 284        return dd;
 285}
 286
 287static void omap_aes_dma_callback(int lch, u16 ch_status, void *data)
 288{
 289        struct omap_aes_dev *dd = data;
 290
 291        if (ch_status != OMAP_DMA_BLOCK_IRQ) {
 292                pr_err("omap-aes DMA error status: 0x%hx\n", ch_status);
 293                dd->err = -EIO;
 294                dd->flags &= ~FLAGS_INIT; /* request to re-initialize */
 295        } else if (lch == dd->dma_lch_in) {
 296                return;
 297        }
 298
 299        /* dma_lch_out - completed */
 300        tasklet_schedule(&dd->done_task);
 301}
 302
 303static int omap_aes_dma_init(struct omap_aes_dev *dd)
 304{
 305        int err = -ENOMEM;
 306
 307        dd->dma_lch_out = -1;
 308        dd->dma_lch_in = -1;
 309
 310        dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
 311        dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
 312        dd->buflen = PAGE_SIZE << OMAP_AES_CACHE_SIZE;
 313        dd->buflen &= ~(AES_BLOCK_SIZE - 1);
 314
 315        if (!dd->buf_in || !dd->buf_out) {
 316                dev_err(dd->dev, "unable to alloc pages.\n");
 317                goto err_alloc;
 318        }
 319
 320        /* MAP here */
 321        dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen,
 322                                         DMA_TO_DEVICE);
 323        if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
 324                dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 325                err = -EINVAL;
 326                goto err_map_in;
 327        }
 328
 329        dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen,
 330                                          DMA_FROM_DEVICE);
 331        if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
 332                dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 333                err = -EINVAL;
 334                goto err_map_out;
 335        }
 336
 337        err = omap_request_dma(dd->dma_in, "omap-aes-rx",
 338                               omap_aes_dma_callback, dd, &dd->dma_lch_in);
 339        if (err) {
 340                dev_err(dd->dev, "Unable to request DMA channel\n");
 341                goto err_dma_in;
 342        }
 343        err = omap_request_dma(dd->dma_out, "omap-aes-tx",
 344                               omap_aes_dma_callback, dd, &dd->dma_lch_out);
 345        if (err) {
 346                dev_err(dd->dev, "Unable to request DMA channel\n");
 347                goto err_dma_out;
 348        }
 349
 350        return 0;
 351
 352err_dma_out:
 353        omap_free_dma(dd->dma_lch_in);
 354err_dma_in:
 355        dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
 356                         DMA_FROM_DEVICE);
 357err_map_out:
 358        dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
 359err_map_in:
 360        free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
 361        free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
 362err_alloc:
 363        if (err)
 364                pr_err("error: %d\n", err);
 365        return err;
 366}
 367
 368static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
 369{
 370        omap_free_dma(dd->dma_lch_out);
 371        omap_free_dma(dd->dma_lch_in);
 372        dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
 373                         DMA_FROM_DEVICE);
 374        dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
 375        free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
 376        free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
 377}
 378
 379static void sg_copy_buf(void *buf, struct scatterlist *sg,
 380                              unsigned int start, unsigned int nbytes, int out)
 381{
 382        struct scatter_walk walk;
 383
 384        if (!nbytes)
 385                return;
 386
 387        scatterwalk_start(&walk, sg);
 388        scatterwalk_advance(&walk, start);
 389        scatterwalk_copychunks(buf, &walk, nbytes, out);
 390        scatterwalk_done(&walk, out, 0);
 391}
 392
 393static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf,
 394                   size_t buflen, size_t total, int out)
 395{
 396        unsigned int count, off = 0;
 397
 398        while (buflen && total) {
 399                count = min((*sg)->length - *offset, total);
 400                count = min(count, buflen);
 401
 402                if (!count)
 403                        return off;
 404
 405                /*
 406                 * buflen and total are AES_BLOCK_SIZE size aligned,
 407                 * so count should be also aligned
 408                 */
 409
 410                sg_copy_buf(buf + off, *sg, *offset, count, out);
 411
 412                off += count;
 413                buflen -= count;
 414                *offset += count;
 415                total -= count;
 416
 417                if (*offset == (*sg)->length) {
 418                        *sg = sg_next(*sg);
 419                        if (*sg)
 420                                *offset = 0;
 421                        else
 422                                total = 0;
 423                }
 424        }
 425
 426        return off;
 427}
 428
 429static int omap_aes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
 430                               dma_addr_t dma_addr_out, int length)
 431{
 432        struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 433        struct omap_aes_dev *dd = ctx->dd;
 434        int len32;
 435
 436        pr_debug("len: %d\n", length);
 437
 438        dd->dma_size = length;
 439
 440        if (!(dd->flags & FLAGS_FAST))
 441                dma_sync_single_for_device(dd->dev, dma_addr_in, length,
 442                                           DMA_TO_DEVICE);
 443
 444        len32 = DIV_ROUND_UP(length, sizeof(u32));
 445
 446        /* IN */
 447        omap_set_dma_transfer_params(dd->dma_lch_in, OMAP_DMA_DATA_TYPE_S32,
 448                                     len32, 1, OMAP_DMA_SYNC_PACKET, dd->dma_in,
 449                                        OMAP_DMA_DST_SYNC);
 450
 451        omap_set_dma_src_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_POST_INC,
 452                                dma_addr_in, 0, 0);
 453
 454        /* OUT */
 455        omap_set_dma_transfer_params(dd->dma_lch_out, OMAP_DMA_DATA_TYPE_S32,
 456                                     len32, 1, OMAP_DMA_SYNC_PACKET,
 457                                        dd->dma_out, OMAP_DMA_SRC_SYNC);
 458
 459        omap_set_dma_dest_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_POST_INC,
 460                                 dma_addr_out, 0, 0);
 461
 462        omap_start_dma(dd->dma_lch_in);
 463        omap_start_dma(dd->dma_lch_out);
 464
 465        /* start DMA or disable idle mode */
 466        omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_START,
 467                            AES_REG_MASK_START);
 468
 469        return 0;
 470}
 471
 472static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
 473{
 474        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
 475                                        crypto_ablkcipher_reqtfm(dd->req));
 476        int err, fast = 0, in, out;
 477        size_t count;
 478        dma_addr_t addr_in, addr_out;
 479
 480        pr_debug("total: %d\n", dd->total);
 481
 482        if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) {
 483                /* check for alignment */
 484                in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32));
 485                out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32));
 486
 487                fast = in && out;
 488        }
 489
 490        if (fast)  {
 491                count = min(dd->total, sg_dma_len(dd->in_sg));
 492                count = min(count, sg_dma_len(dd->out_sg));
 493
 494                if (count != dd->total) {
 495                        pr_err("request length != buffer length\n");
 496                        return -EINVAL;
 497                }
 498
 499                pr_debug("fast\n");
 500
 501                err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 502                if (!err) {
 503                        dev_err(dd->dev, "dma_map_sg() error\n");
 504                        return -EINVAL;
 505                }
 506
 507                err = dma_map_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 508                if (!err) {
 509                        dev_err(dd->dev, "dma_map_sg() error\n");
 510                        dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 511                        return -EINVAL;
 512                }
 513
 514                addr_in = sg_dma_address(dd->in_sg);
 515                addr_out = sg_dma_address(dd->out_sg);
 516
 517                dd->flags |= FLAGS_FAST;
 518
 519        } else {
 520                /* use cache buffers */
 521                count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in,
 522                                 dd->buflen, dd->total, 0);
 523
 524                addr_in = dd->dma_addr_in;
 525                addr_out = dd->dma_addr_out;
 526
 527                dd->flags &= ~FLAGS_FAST;
 528
 529        }
 530
 531        dd->total -= count;
 532
 533        err = omap_aes_crypt_dma(tfm, addr_in, addr_out, count);
 534        if (err) {
 535                dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 536                dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
 537        }
 538
 539        return err;
 540}
 541
 542static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
 543{
 544        struct ablkcipher_request *req = dd->req;
 545
 546        pr_debug("err: %d\n", err);
 547
 548        clk_disable(dd->iclk);
 549        dd->flags &= ~FLAGS_BUSY;
 550
 551        req->base.complete(&req->base, err);
 552}
 553
 554static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
 555{
 556        int err = 0;
 557        size_t count;
 558
 559        pr_debug("total: %d\n", dd->total);
 560
 561        omap_aes_write_mask(dd, AES_REG_MASK, 0, AES_REG_MASK_START);
 562
 563        omap_stop_dma(dd->dma_lch_in);
 564        omap_stop_dma(dd->dma_lch_out);
 565
 566        if (dd->flags & FLAGS_FAST) {
 567                dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 568                dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 569        } else {
 570                dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
 571                                           dd->dma_size, DMA_FROM_DEVICE);
 572
 573                /* copy data */
 574                count = sg_copy(&dd->out_sg, &dd->out_offset, dd->buf_out,
 575                                 dd->buflen, dd->dma_size, 1);
 576                if (count != dd->dma_size) {
 577                        err = -EINVAL;
 578                        pr_err("not all data converted: %u\n", count);
 579                }
 580        }
 581
 582        return err;
 583}
 584
 585static int omap_aes_handle_queue(struct omap_aes_dev *dd,
 586                               struct ablkcipher_request *req)
 587{
 588        struct crypto_async_request *async_req, *backlog;
 589        struct omap_aes_ctx *ctx;
 590        struct omap_aes_reqctx *rctx;
 591        unsigned long flags;
 592        int err, ret = 0;
 593
 594        spin_lock_irqsave(&dd->lock, flags);
 595        if (req)
 596                ret = ablkcipher_enqueue_request(&dd->queue, req);
 597        if (dd->flags & FLAGS_BUSY) {
 598                spin_unlock_irqrestore(&dd->lock, flags);
 599                return ret;
 600        }
 601        backlog = crypto_get_backlog(&dd->queue);
 602        async_req = crypto_dequeue_request(&dd->queue);
 603        if (async_req)
 604                dd->flags |= FLAGS_BUSY;
 605        spin_unlock_irqrestore(&dd->lock, flags);
 606
 607        if (!async_req)
 608                return ret;
 609
 610        if (backlog)
 611                backlog->complete(backlog, -EINPROGRESS);
 612
 613        req = ablkcipher_request_cast(async_req);
 614
 615        /* assign new request to device */
 616        dd->req = req;
 617        dd->total = req->nbytes;
 618        dd->in_offset = 0;
 619        dd->in_sg = req->src;
 620        dd->out_offset = 0;
 621        dd->out_sg = req->dst;
 622
 623        rctx = ablkcipher_request_ctx(req);
 624        ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
 625        rctx->mode &= FLAGS_MODE_MASK;
 626        dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 627
 628        dd->ctx = ctx;
 629        ctx->dd = dd;
 630
 631        err = omap_aes_write_ctrl(dd);
 632        if (!err)
 633                err = omap_aes_crypt_dma_start(dd);
 634        if (err) {
 635                /* aes_task will not finish it, so do it here */
 636                omap_aes_finish_req(dd, err);
 637                tasklet_schedule(&dd->queue_task);
 638        }
 639
 640        return ret; /* return ret, which is enqueue return value */
 641}
 642
 643static void omap_aes_done_task(unsigned long data)
 644{
 645        struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
 646        int err;
 647
 648        pr_debug("enter\n");
 649
 650        err = omap_aes_crypt_dma_stop(dd);
 651
 652        err = dd->err ? : err;
 653
 654        if (dd->total && !err) {
 655                err = omap_aes_crypt_dma_start(dd);
 656                if (!err)
 657                        return; /* DMA started. Not fininishing. */
 658        }
 659
 660        omap_aes_finish_req(dd, err);
 661        omap_aes_handle_queue(dd, NULL);
 662
 663        pr_debug("exit\n");
 664}
 665
 666static void omap_aes_queue_task(unsigned long data)
 667{
 668        struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
 669
 670        omap_aes_handle_queue(dd, NULL);
 671}
 672
 673static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
 674{
 675        struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
 676                        crypto_ablkcipher_reqtfm(req));
 677        struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
 678        struct omap_aes_dev *dd;
 679
 680        pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
 681                  !!(mode & FLAGS_ENCRYPT),
 682                  !!(mode & FLAGS_CBC));
 683
 684        if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
 685                pr_err("request size is not exact amount of AES blocks\n");
 686                return -EINVAL;
 687        }
 688
 689        dd = omap_aes_find_dev(ctx);
 690        if (!dd)
 691                return -ENODEV;
 692
 693        rctx->mode = mode;
 694
 695        return omap_aes_handle_queue(dd, req);
 696}
 697
 698/* ********************** ALG API ************************************ */
 699
 700static int omap_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 701                           unsigned int keylen)
 702{
 703        struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 704
 705        if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 706                   keylen != AES_KEYSIZE_256)
 707                return -EINVAL;
 708
 709        pr_debug("enter, keylen: %d\n", keylen);
 710
 711        memcpy(ctx->key, key, keylen);
 712        ctx->keylen = keylen;
 713
 714        return 0;
 715}
 716
 717static int omap_aes_ecb_encrypt(struct ablkcipher_request *req)
 718{
 719        return omap_aes_crypt(req, FLAGS_ENCRYPT);
 720}
 721
 722static int omap_aes_ecb_decrypt(struct ablkcipher_request *req)
 723{
 724        return omap_aes_crypt(req, 0);
 725}
 726
 727static int omap_aes_cbc_encrypt(struct ablkcipher_request *req)
 728{
 729        return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
 730}
 731
 732static int omap_aes_cbc_decrypt(struct ablkcipher_request *req)
 733{
 734        return omap_aes_crypt(req, FLAGS_CBC);
 735}
 736
 737static int omap_aes_cra_init(struct crypto_tfm *tfm)
 738{
 739        pr_debug("enter\n");
 740
 741        tfm->crt_ablkcipher.reqsize = sizeof(struct omap_aes_reqctx);
 742
 743        return 0;
 744}
 745
 746static void omap_aes_cra_exit(struct crypto_tfm *tfm)
 747{
 748        pr_debug("enter\n");
 749}
 750
 751/* ********************** ALGS ************************************ */
 752
 753static struct crypto_alg algs[] = {
 754{
 755        .cra_name               = "ecb(aes)",
 756        .cra_driver_name        = "ecb-aes-omap",
 757        .cra_priority           = 100,
 758        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 759                                  CRYPTO_ALG_KERN_DRIVER_ONLY |
 760                                  CRYPTO_ALG_ASYNC,
 761        .cra_blocksize          = AES_BLOCK_SIZE,
 762        .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 763        .cra_alignmask          = 0,
 764        .cra_type               = &crypto_ablkcipher_type,
 765        .cra_module             = THIS_MODULE,
 766        .cra_init               = omap_aes_cra_init,
 767        .cra_exit               = omap_aes_cra_exit,
 768        .cra_u.ablkcipher = {
 769                .min_keysize    = AES_MIN_KEY_SIZE,
 770                .max_keysize    = AES_MAX_KEY_SIZE,
 771                .setkey         = omap_aes_setkey,
 772                .encrypt        = omap_aes_ecb_encrypt,
 773                .decrypt        = omap_aes_ecb_decrypt,
 774        }
 775},
 776{
 777        .cra_name               = "cbc(aes)",
 778        .cra_driver_name        = "cbc-aes-omap",
 779        .cra_priority           = 100,
 780        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
 781                                  CRYPTO_ALG_KERN_DRIVER_ONLY |
 782                                  CRYPTO_ALG_ASYNC,
 783        .cra_blocksize          = AES_BLOCK_SIZE,
 784        .cra_ctxsize            = sizeof(struct omap_aes_ctx),
 785        .cra_alignmask          = 0,
 786        .cra_type               = &crypto_ablkcipher_type,
 787        .cra_module             = THIS_MODULE,
 788        .cra_init               = omap_aes_cra_init,
 789        .cra_exit               = omap_aes_cra_exit,
 790        .cra_u.ablkcipher = {
 791                .min_keysize    = AES_MIN_KEY_SIZE,
 792                .max_keysize    = AES_MAX_KEY_SIZE,
 793                .ivsize         = AES_BLOCK_SIZE,
 794                .setkey         = omap_aes_setkey,
 795                .encrypt        = omap_aes_cbc_encrypt,
 796                .decrypt        = omap_aes_cbc_decrypt,
 797        }
 798}
 799};
 800
 801static int omap_aes_probe(struct platform_device *pdev)
 802{
 803        struct device *dev = &pdev->dev;
 804        struct omap_aes_dev *dd;
 805        struct resource *res;
 806        int err = -ENOMEM, i, j;
 807        u32 reg;
 808
 809        dd = kzalloc(sizeof(struct omap_aes_dev), GFP_KERNEL);
 810        if (dd == NULL) {
 811                dev_err(dev, "unable to alloc data struct.\n");
 812                goto err_data;
 813        }
 814        dd->dev = dev;
 815        platform_set_drvdata(pdev, dd);
 816
 817        spin_lock_init(&dd->lock);
 818        crypto_init_queue(&dd->queue, OMAP_AES_QUEUE_LENGTH);
 819
 820        /* Get the base address */
 821        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 822        if (!res) {
 823                dev_err(dev, "invalid resource type\n");
 824                err = -ENODEV;
 825                goto err_res;
 826        }
 827        dd->phys_base = res->start;
 828
 829        /* Get the DMA */
 830        res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
 831        if (!res)
 832                dev_info(dev, "no DMA info\n");
 833        else
 834                dd->dma_out = res->start;
 835
 836        /* Get the DMA */
 837        res = platform_get_resource(pdev, IORESOURCE_DMA, 1);
 838        if (!res)
 839                dev_info(dev, "no DMA info\n");
 840        else
 841                dd->dma_in = res->start;
 842
 843        /* Initializing the clock */
 844        dd->iclk = clk_get(dev, "ick");
 845        if (IS_ERR(dd->iclk)) {
 846                dev_err(dev, "clock intialization failed.\n");
 847                err = PTR_ERR(dd->iclk);
 848                goto err_res;
 849        }
 850
 851        dd->io_base = ioremap(dd->phys_base, SZ_4K);
 852        if (!dd->io_base) {
 853                dev_err(dev, "can't ioremap\n");
 854                err = -ENOMEM;
 855                goto err_io;
 856        }
 857
 858        clk_enable(dd->iclk);
 859        reg = omap_aes_read(dd, AES_REG_REV);
 860        dev_info(dev, "OMAP AES hw accel rev: %u.%u\n",
 861                 (reg & AES_REG_REV_MAJOR) >> 4, reg & AES_REG_REV_MINOR);
 862        clk_disable(dd->iclk);
 863
 864        tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd);
 865        tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd);
 866
 867        err = omap_aes_dma_init(dd);
 868        if (err)
 869                goto err_dma;
 870
 871        INIT_LIST_HEAD(&dd->list);
 872        spin_lock(&list_lock);
 873        list_add_tail(&dd->list, &dev_list);
 874        spin_unlock(&list_lock);
 875
 876        for (i = 0; i < ARRAY_SIZE(algs); i++) {
 877                pr_debug("i: %d\n", i);
 878                err = crypto_register_alg(&algs[i]);
 879                if (err)
 880                        goto err_algs;
 881        }
 882
 883        pr_info("probe() done\n");
 884
 885        return 0;
 886err_algs:
 887        for (j = 0; j < i; j++)
 888                crypto_unregister_alg(&algs[j]);
 889        omap_aes_dma_cleanup(dd);
 890err_dma:
 891        tasklet_kill(&dd->done_task);
 892        tasklet_kill(&dd->queue_task);
 893        iounmap(dd->io_base);
 894err_io:
 895        clk_put(dd->iclk);
 896err_res:
 897        kfree(dd);
 898        dd = NULL;
 899err_data:
 900        dev_err(dev, "initialization failed.\n");
 901        return err;
 902}
 903
 904static int omap_aes_remove(struct platform_device *pdev)
 905{
 906        struct omap_aes_dev *dd = platform_get_drvdata(pdev);
 907        int i;
 908
 909        if (!dd)
 910                return -ENODEV;
 911
 912        spin_lock(&list_lock);
 913        list_del(&dd->list);
 914        spin_unlock(&list_lock);
 915
 916        for (i = 0; i < ARRAY_SIZE(algs); i++)
 917                crypto_unregister_alg(&algs[i]);
 918
 919        tasklet_kill(&dd->done_task);
 920        tasklet_kill(&dd->queue_task);
 921        omap_aes_dma_cleanup(dd);
 922        iounmap(dd->io_base);
 923        clk_put(dd->iclk);
 924        kfree(dd);
 925        dd = NULL;
 926
 927        return 0;
 928}
 929
 930static struct platform_driver omap_aes_driver = {
 931        .probe  = omap_aes_probe,
 932        .remove = omap_aes_remove,
 933        .driver = {
 934                .name   = "omap-aes",
 935                .owner  = THIS_MODULE,
 936        },
 937};
 938
 939static int __init omap_aes_mod_init(void)
 940{
 941        pr_info("loading %s driver\n", "omap-aes");
 942
 943        return  platform_driver_register(&omap_aes_driver);
 944}
 945
 946static void __exit omap_aes_mod_exit(void)
 947{
 948        platform_driver_unregister(&omap_aes_driver);
 949}
 950
 951module_init(omap_aes_mod_init);
 952module_exit(omap_aes_mod_exit);
 953
 954MODULE_DESCRIPTION("OMAP AES hw acceleration support.");
 955MODULE_LICENSE("GPL v2");
 956MODULE_AUTHOR("Dmitry Kasatkin");
 957
 958
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.