linux/drivers/crypto/atmel-tdes.c
<<
>>
Prefs
   1/*
   2 * Cryptographic API.
   3 *
   4 * Support for ATMEL DES/TDES HW acceleration.
   5 *
   6 * Copyright (c) 2012 Eukr\xC3\xA9a Electromatique - ATMEL
   7 * Author: Nicolas Royer <nicolas@eukrea.com>
   8 *
   9 * This program is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License version 2 as published
  11 * by the Free Software Foundation.
  12 *
  13 * Some ideas are from omap-aes.c drivers.
  14 */
  15
  16
  17#include <linux/kernel.h>
  18#include <linux/module.h>
  19#include <linux/slab.h>
  20#include <linux/err.h>
  21#include <linux/clk.h>
  22#include <linux/io.h>
  23#include <linux/hw_random.h>
  24#include <linux/platform_device.h>
  25
  26#include <linux/device.h>
  27#include <linux/init.h>
  28#include <linux/errno.h>
  29#include <linux/interrupt.h>
  30#include <linux/irq.h>
  31#include <linux/scatterlist.h>
  32#include <linux/dma-mapping.h>
  33#include <linux/of_device.h>
  34#include <linux/delay.h>
  35#include <linux/crypto.h>
  36#include <linux/cryptohash.h>
  37#include <crypto/scatterwalk.h>
  38#include <crypto/algapi.h>
  39#include <crypto/des.h>
  40#include <crypto/hash.h>
  41#include <crypto/internal/hash.h>
  42#include <linux/platform_data/crypto-atmel.h>
  43#include "atmel-tdes-regs.h"
  44
  45/* TDES flags  */
  46#define TDES_FLAGS_MODE_MASK            0x00ff
  47#define TDES_FLAGS_ENCRYPT      BIT(0)
  48#define TDES_FLAGS_CBC          BIT(1)
  49#define TDES_FLAGS_CFB          BIT(2)
  50#define TDES_FLAGS_CFB8         BIT(3)
  51#define TDES_FLAGS_CFB16        BIT(4)
  52#define TDES_FLAGS_CFB32        BIT(5)
  53#define TDES_FLAGS_CFB64        BIT(6)
  54#define TDES_FLAGS_OFB          BIT(7)
  55
  56#define TDES_FLAGS_INIT         BIT(16)
  57#define TDES_FLAGS_FAST         BIT(17)
  58#define TDES_FLAGS_BUSY         BIT(18)
  59#define TDES_FLAGS_DMA          BIT(19)
  60
  61#define ATMEL_TDES_QUEUE_LENGTH 50
  62
  63#define CFB8_BLOCK_SIZE         1
  64#define CFB16_BLOCK_SIZE        2
  65#define CFB32_BLOCK_SIZE        4
  66
  67struct atmel_tdes_caps {
  68        bool    has_dma;
  69        u32             has_cfb_3keys;
  70};
  71
  72struct atmel_tdes_dev;
  73
  74struct atmel_tdes_ctx {
  75        struct atmel_tdes_dev *dd;
  76
  77        int             keylen;
  78        u32             key[3*DES_KEY_SIZE / sizeof(u32)];
  79        unsigned long   flags;
  80
  81        u16             block_size;
  82};
  83
  84struct atmel_tdes_reqctx {
  85        unsigned long mode;
  86};
  87
  88struct atmel_tdes_dma {
  89        struct dma_chan                 *chan;
  90        struct dma_slave_config dma_conf;
  91};
  92
  93struct atmel_tdes_dev {
  94        struct list_head        list;
  95        unsigned long           phys_base;
  96        void __iomem            *io_base;
  97
  98        struct atmel_tdes_ctx   *ctx;
  99        struct device           *dev;
 100        struct clk                      *iclk;
 101        int                                     irq;
 102
 103        unsigned long           flags;
 104        int                     err;
 105
 106        spinlock_t              lock;
 107        struct crypto_queue     queue;
 108
 109        struct tasklet_struct   done_task;
 110        struct tasklet_struct   queue_task;
 111
 112        struct ablkcipher_request       *req;
 113        size_t                          total;
 114
 115        struct scatterlist      *in_sg;
 116        unsigned int            nb_in_sg;
 117        size_t                          in_offset;
 118        struct scatterlist      *out_sg;
 119        unsigned int            nb_out_sg;
 120        size_t                          out_offset;
 121
 122        size_t  buflen;
 123        size_t  dma_size;
 124
 125        void    *buf_in;
 126        int             dma_in;
 127        dma_addr_t      dma_addr_in;
 128        struct atmel_tdes_dma   dma_lch_in;
 129
 130        void    *buf_out;
 131        int             dma_out;
 132        dma_addr_t      dma_addr_out;
 133        struct atmel_tdes_dma   dma_lch_out;
 134
 135        struct atmel_tdes_caps  caps;
 136
 137        u32     hw_version;
 138};
 139
 140struct atmel_tdes_drv {
 141        struct list_head        dev_list;
 142        spinlock_t              lock;
 143};
 144
 145static struct atmel_tdes_drv atmel_tdes = {
 146        .dev_list = LIST_HEAD_INIT(atmel_tdes.dev_list),
 147        .lock = __SPIN_LOCK_UNLOCKED(atmel_tdes.lock),
 148};
 149
 150static int atmel_tdes_sg_copy(struct scatterlist **sg, size_t *offset,
 151                        void *buf, size_t buflen, size_t total, int out)
 152{
 153        unsigned int count, off = 0;
 154
 155        while (buflen && total) {
 156                count = min((*sg)->length - *offset, total);
 157                count = min(count, buflen);
 158
 159                if (!count)
 160                        return off;
 161
 162                scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out);
 163
 164                off += count;
 165                buflen -= count;
 166                *offset += count;
 167                total -= count;
 168
 169                if (*offset == (*sg)->length) {
 170                        *sg = sg_next(*sg);
 171                        if (*sg)
 172                                *offset = 0;
 173                        else
 174                                total = 0;
 175                }
 176        }
 177
 178        return off;
 179}
 180
 181static inline u32 atmel_tdes_read(struct atmel_tdes_dev *dd, u32 offset)
 182{
 183        return readl_relaxed(dd->io_base + offset);
 184}
 185
 186static inline void atmel_tdes_write(struct atmel_tdes_dev *dd,
 187                                        u32 offset, u32 value)
 188{
 189        writel_relaxed(value, dd->io_base + offset);
 190}
 191
 192static void atmel_tdes_write_n(struct atmel_tdes_dev *dd, u32 offset,
 193                                        u32 *value, int count)
 194{
 195        for (; count--; value++, offset += 4)
 196                atmel_tdes_write(dd, offset, *value);
 197}
 198
 199static struct atmel_tdes_dev *atmel_tdes_find_dev(struct atmel_tdes_ctx *ctx)
 200{
 201        struct atmel_tdes_dev *tdes_dd = NULL;
 202        struct atmel_tdes_dev *tmp;
 203
 204        spin_lock_bh(&atmel_tdes.lock);
 205        if (!ctx->dd) {
 206                list_for_each_entry(tmp, &atmel_tdes.dev_list, list) {
 207                        tdes_dd = tmp;
 208                        break;
 209                }
 210                ctx->dd = tdes_dd;
 211        } else {
 212                tdes_dd = ctx->dd;
 213        }
 214        spin_unlock_bh(&atmel_tdes.lock);
 215
 216        return tdes_dd;
 217}
 218
 219static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd)
 220{
 221        int err;
 222
 223        err = clk_prepare_enable(dd->iclk);
 224        if (err)
 225                return err;
 226
 227        if (!(dd->flags & TDES_FLAGS_INIT)) {
 228                atmel_tdes_write(dd, TDES_CR, TDES_CR_SWRST);
 229                dd->flags |= TDES_FLAGS_INIT;
 230                dd->err = 0;
 231        }
 232
 233        return 0;
 234}
 235
 236static inline unsigned int atmel_tdes_get_version(struct atmel_tdes_dev *dd)
 237{
 238        return atmel_tdes_read(dd, TDES_HW_VERSION) & 0x00000fff;
 239}
 240
 241static void atmel_tdes_hw_version_init(struct atmel_tdes_dev *dd)
 242{
 243        atmel_tdes_hw_init(dd);
 244
 245        dd->hw_version = atmel_tdes_get_version(dd);
 246
 247        dev_info(dd->dev,
 248                        "version: 0x%x\n", dd->hw_version);
 249
 250        clk_disable_unprepare(dd->iclk);
 251}
 252
 253static void atmel_tdes_dma_callback(void *data)
 254{
 255        struct atmel_tdes_dev *dd = data;
 256
 257        /* dma_lch_out - completed */
 258        tasklet_schedule(&dd->done_task);
 259}
 260
 261static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd)
 262{
 263        int err;
 264        u32 valcr = 0, valmr = TDES_MR_SMOD_PDC;
 265
 266        err = atmel_tdes_hw_init(dd);
 267
 268        if (err)
 269                return err;
 270
 271        if (!dd->caps.has_dma)
 272                atmel_tdes_write(dd, TDES_PTCR,
 273                        TDES_PTCR_TXTDIS | TDES_PTCR_RXTDIS);
 274
 275        /* MR register must be set before IV registers */
 276        if (dd->ctx->keylen > (DES_KEY_SIZE << 1)) {
 277                valmr |= TDES_MR_KEYMOD_3KEY;
 278                valmr |= TDES_MR_TDESMOD_TDES;
 279        } else if (dd->ctx->keylen > DES_KEY_SIZE) {
 280                valmr |= TDES_MR_KEYMOD_2KEY;
 281                valmr |= TDES_MR_TDESMOD_TDES;
 282        } else {
 283                valmr |= TDES_MR_TDESMOD_DES;
 284        }
 285
 286        if (dd->flags & TDES_FLAGS_CBC) {
 287                valmr |= TDES_MR_OPMOD_CBC;
 288        } else if (dd->flags & TDES_FLAGS_CFB) {
 289                valmr |= TDES_MR_OPMOD_CFB;
 290
 291                if (dd->flags & TDES_FLAGS_CFB8)
 292                        valmr |= TDES_MR_CFBS_8b;
 293                else if (dd->flags & TDES_FLAGS_CFB16)
 294                        valmr |= TDES_MR_CFBS_16b;
 295                else if (dd->flags & TDES_FLAGS_CFB32)
 296                        valmr |= TDES_MR_CFBS_32b;
 297                else if (dd->flags & TDES_FLAGS_CFB64)
 298                        valmr |= TDES_MR_CFBS_64b;
 299        } else if (dd->flags & TDES_FLAGS_OFB) {
 300                valmr |= TDES_MR_OPMOD_OFB;
 301        }
 302
 303        if ((dd->flags & TDES_FLAGS_ENCRYPT) || (dd->flags & TDES_FLAGS_OFB))
 304                valmr |= TDES_MR_CYPHER_ENC;
 305
 306        atmel_tdes_write(dd, TDES_CR, valcr);
 307        atmel_tdes_write(dd, TDES_MR, valmr);
 308
 309        atmel_tdes_write_n(dd, TDES_KEY1W1R, dd->ctx->key,
 310                                                dd->ctx->keylen >> 2);
 311
 312        if (((dd->flags & TDES_FLAGS_CBC) || (dd->flags & TDES_FLAGS_CFB) ||
 313                (dd->flags & TDES_FLAGS_OFB)) && dd->req->info) {
 314                atmel_tdes_write_n(dd, TDES_IV1R, dd->req->info, 2);
 315        }
 316
 317        return 0;
 318}
 319
 320static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd)
 321{
 322        int err = 0;
 323        size_t count;
 324
 325        atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);
 326
 327        if (dd->flags & TDES_FLAGS_FAST) {
 328                dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 329                dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 330        } else {
 331                dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
 332                                           dd->dma_size, DMA_FROM_DEVICE);
 333
 334                /* copy data */
 335                count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
 336                                dd->buf_out, dd->buflen, dd->dma_size, 1);
 337                if (count != dd->dma_size) {
 338                        err = -EINVAL;
 339                        pr_err("not all data converted: %u\n", count);
 340                }
 341        }
 342
 343        return err;
 344}
 345
 346static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd)
 347{
 348        int err = -ENOMEM;
 349
 350        dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0);
 351        dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0);
 352        dd->buflen = PAGE_SIZE;
 353        dd->buflen &= ~(DES_BLOCK_SIZE - 1);
 354
 355        if (!dd->buf_in || !dd->buf_out) {
 356                dev_err(dd->dev, "unable to alloc pages.\n");
 357                goto err_alloc;
 358        }
 359
 360        /* MAP here */
 361        dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in,
 362                                        dd->buflen, DMA_TO_DEVICE);
 363        if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
 364                dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 365                err = -EINVAL;
 366                goto err_map_in;
 367        }
 368
 369        dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out,
 370                                        dd->buflen, DMA_FROM_DEVICE);
 371        if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
 372                dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 373                err = -EINVAL;
 374                goto err_map_out;
 375        }
 376
 377        return 0;
 378
 379err_map_out:
 380        dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
 381                DMA_TO_DEVICE);
 382err_map_in:
 383err_alloc:
 384        free_page((unsigned long)dd->buf_out);
 385        free_page((unsigned long)dd->buf_in);
 386        if (err)
 387                pr_err("error: %d\n", err);
 388        return err;
 389}
 390
 391static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd)
 392{
 393        dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
 394                         DMA_FROM_DEVICE);
 395        dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
 396                DMA_TO_DEVICE);
 397        free_page((unsigned long)dd->buf_out);
 398        free_page((unsigned long)dd->buf_in);
 399}
 400
 401static int atmel_tdes_crypt_pdc(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
 402                               dma_addr_t dma_addr_out, int length)
 403{
 404        struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm);
 405        struct atmel_tdes_dev *dd = ctx->dd;
 406        int len32;
 407
 408        dd->dma_size = length;
 409
 410        if (!(dd->flags & TDES_FLAGS_FAST)) {
 411                dma_sync_single_for_device(dd->dev, dma_addr_in, length,
 412                                           DMA_TO_DEVICE);
 413        }
 414
 415        if ((dd->flags & TDES_FLAGS_CFB) && (dd->flags & TDES_FLAGS_CFB8))
 416                len32 = DIV_ROUND_UP(length, sizeof(u8));
 417        else if ((dd->flags & TDES_FLAGS_CFB) && (dd->flags & TDES_FLAGS_CFB16))
 418                len32 = DIV_ROUND_UP(length, sizeof(u16));
 419        else
 420                len32 = DIV_ROUND_UP(length, sizeof(u32));
 421
 422        atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);
 423        atmel_tdes_write(dd, TDES_TPR, dma_addr_in);
 424        atmel_tdes_write(dd, TDES_TCR, len32);
 425        atmel_tdes_write(dd, TDES_RPR, dma_addr_out);
 426        atmel_tdes_write(dd, TDES_RCR, len32);
 427
 428        /* Enable Interrupt */
 429        atmel_tdes_write(dd, TDES_IER, TDES_INT_ENDRX);
 430
 431        /* Start DMA transfer */
 432        atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTEN | TDES_PTCR_RXTEN);
 433
 434        return 0;
 435}
 436
 437static int atmel_tdes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
 438                               dma_addr_t dma_addr_out, int length)
 439{
 440        struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm);
 441        struct atmel_tdes_dev *dd = ctx->dd;
 442        struct scatterlist sg[2];
 443        struct dma_async_tx_descriptor  *in_desc, *out_desc;
 444
 445        dd->dma_size = length;
 446
 447        if (!(dd->flags & TDES_FLAGS_FAST)) {
 448                dma_sync_single_for_device(dd->dev, dma_addr_in, length,
 449                                           DMA_TO_DEVICE);
 450        }
 451
 452        if (dd->flags & TDES_FLAGS_CFB8) {
 453                dd->dma_lch_in.dma_conf.dst_addr_width =
 454                        DMA_SLAVE_BUSWIDTH_1_BYTE;
 455                dd->dma_lch_out.dma_conf.src_addr_width =
 456                        DMA_SLAVE_BUSWIDTH_1_BYTE;
 457        } else if (dd->flags & TDES_FLAGS_CFB16) {
 458                dd->dma_lch_in.dma_conf.dst_addr_width =
 459                        DMA_SLAVE_BUSWIDTH_2_BYTES;
 460                dd->dma_lch_out.dma_conf.src_addr_width =
 461                        DMA_SLAVE_BUSWIDTH_2_BYTES;
 462        } else {
 463                dd->dma_lch_in.dma_conf.dst_addr_width =
 464                        DMA_SLAVE_BUSWIDTH_4_BYTES;
 465                dd->dma_lch_out.dma_conf.src_addr_width =
 466                        DMA_SLAVE_BUSWIDTH_4_BYTES;
 467        }
 468
 469        dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);
 470        dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf);
 471
 472        dd->flags |= TDES_FLAGS_DMA;
 473
 474        sg_init_table(&sg[0], 1);
 475        sg_dma_address(&sg[0]) = dma_addr_in;
 476        sg_dma_len(&sg[0]) = length;
 477
 478        sg_init_table(&sg[1], 1);
 479        sg_dma_address(&sg[1]) = dma_addr_out;
 480        sg_dma_len(&sg[1]) = length;
 481
 482        in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0],
 483                                1, DMA_MEM_TO_DEV,
 484                                DMA_PREP_INTERRUPT  |  DMA_CTRL_ACK);
 485        if (!in_desc)
 486                return -EINVAL;
 487
 488        out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1],
 489                                1, DMA_DEV_TO_MEM,
 490                                DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 491        if (!out_desc)
 492                return -EINVAL;
 493
 494        out_desc->callback = atmel_tdes_dma_callback;
 495        out_desc->callback_param = dd;
 496
 497        dmaengine_submit(out_desc);
 498        dma_async_issue_pending(dd->dma_lch_out.chan);
 499
 500        dmaengine_submit(in_desc);
 501        dma_async_issue_pending(dd->dma_lch_in.chan);
 502
 503        return 0;
 504}
 505
 506static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd)
 507{
 508        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
 509                                        crypto_ablkcipher_reqtfm(dd->req));
 510        int err, fast = 0, in, out;
 511        size_t count;
 512        dma_addr_t addr_in, addr_out;
 513
 514        if ((!dd->in_offset) && (!dd->out_offset)) {
 515                /* check for alignment */
 516                in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) &&
 517                        IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size);
 518                out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) &&
 519                        IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size);
 520                fast = in && out;
 521
 522                if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg))
 523                        fast = 0;
 524        }
 525
 526
 527        if (fast)  {
 528                count = min(dd->total, sg_dma_len(dd->in_sg));
 529                count = min(count, sg_dma_len(dd->out_sg));
 530
 531                err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 532                if (!err) {
 533                        dev_err(dd->dev, "dma_map_sg() error\n");
 534                        return -EINVAL;
 535                }
 536
 537                err = dma_map_sg(dd->dev, dd->out_sg, 1,
 538                                DMA_FROM_DEVICE);
 539                if (!err) {
 540                        dev_err(dd->dev, "dma_map_sg() error\n");
 541                        dma_unmap_sg(dd->dev, dd->in_sg, 1,
 542                                DMA_TO_DEVICE);
 543                        return -EINVAL;
 544                }
 545
 546                addr_in = sg_dma_address(dd->in_sg);
 547                addr_out = sg_dma_address(dd->out_sg);
 548
 549                dd->flags |= TDES_FLAGS_FAST;
 550
 551        } else {
 552                /* use cache buffers */
 553                count = atmel_tdes_sg_copy(&dd->in_sg, &dd->in_offset,
 554                                dd->buf_in, dd->buflen, dd->total, 0);
 555
 556                addr_in = dd->dma_addr_in;
 557                addr_out = dd->dma_addr_out;
 558
 559                dd->flags &= ~TDES_FLAGS_FAST;
 560        }
 561
 562        dd->total -= count;
 563
 564        if (dd->caps.has_dma)
 565                err = atmel_tdes_crypt_dma(tfm, addr_in, addr_out, count);
 566        else
 567                err = atmel_tdes_crypt_pdc(tfm, addr_in, addr_out, count);
 568
 569        if (err && (dd->flags & TDES_FLAGS_FAST)) {
 570                dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 571                dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
 572        }
 573
 574        return err;
 575}
 576
 577static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err)
 578{
 579        struct ablkcipher_request *req = dd->req;
 580
 581        clk_disable_unprepare(dd->iclk);
 582
 583        dd->flags &= ~TDES_FLAGS_BUSY;
 584
 585        req->base.complete(&req->base, err);
 586}
 587
 588static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd,
 589                               struct ablkcipher_request *req)
 590{
 591        struct crypto_async_request *async_req, *backlog;
 592        struct atmel_tdes_ctx *ctx;
 593        struct atmel_tdes_reqctx *rctx;
 594        unsigned long flags;
 595        int err, ret = 0;
 596
 597        spin_lock_irqsave(&dd->lock, flags);
 598        if (req)
 599                ret = ablkcipher_enqueue_request(&dd->queue, req);
 600        if (dd->flags & TDES_FLAGS_BUSY) {
 601                spin_unlock_irqrestore(&dd->lock, flags);
 602                return ret;
 603        }
 604        backlog = crypto_get_backlog(&dd->queue);
 605        async_req = crypto_dequeue_request(&dd->queue);
 606        if (async_req)
 607                dd->flags |= TDES_FLAGS_BUSY;
 608        spin_unlock_irqrestore(&dd->lock, flags);
 609
 610        if (!async_req)
 611                return ret;
 612
 613        if (backlog)
 614                backlog->complete(backlog, -EINPROGRESS);
 615
 616        req = ablkcipher_request_cast(async_req);
 617
 618        /* assign new request to device */
 619        dd->req = req;
 620        dd->total = req->nbytes;
 621        dd->in_offset = 0;
 622        dd->in_sg = req->src;
 623        dd->out_offset = 0;
 624        dd->out_sg = req->dst;
 625
 626        rctx = ablkcipher_request_ctx(req);
 627        ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
 628        rctx->mode &= TDES_FLAGS_MODE_MASK;
 629        dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode;
 630        dd->ctx = ctx;
 631        ctx->dd = dd;
 632
 633        err = atmel_tdes_write_ctrl(dd);
 634        if (!err)
 635                err = atmel_tdes_crypt_start(dd);
 636        if (err) {
 637                /* des_task will not finish it, so do it here */
 638                atmel_tdes_finish_req(dd, err);
 639                tasklet_schedule(&dd->queue_task);
 640        }
 641
 642        return ret;
 643}
 644
 645static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd)
 646{
 647        int err = -EINVAL;
 648        size_t count;
 649
 650        if (dd->flags & TDES_FLAGS_DMA) {
 651                err = 0;
 652                if  (dd->flags & TDES_FLAGS_FAST) {
 653                        dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 654                        dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 655                } else {
 656                        dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
 657                                dd->dma_size, DMA_FROM_DEVICE);
 658
 659                        /* copy data */
 660                        count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
 661                                dd->buf_out, dd->buflen, dd->dma_size, 1);
 662                        if (count != dd->dma_size) {
 663                                err = -EINVAL;
 664                                pr_err("not all data converted: %u\n", count);
 665                        }
 666                }
 667        }
 668        return err;
 669}
 670
 671static int atmel_tdes_crypt(struct ablkcipher_request *req, unsigned long mode)
 672{
 673        struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(
 674                        crypto_ablkcipher_reqtfm(req));
 675        struct atmel_tdes_reqctx *rctx = ablkcipher_request_ctx(req);
 676
 677        if (mode & TDES_FLAGS_CFB8) {
 678                if (!IS_ALIGNED(req->nbytes, CFB8_BLOCK_SIZE)) {
 679                        pr_err("request size is not exact amount of CFB8 blocks\n");
 680                        return -EINVAL;
 681                }
 682                ctx->block_size = CFB8_BLOCK_SIZE;
 683        } else if (mode & TDES_FLAGS_CFB16) {
 684                if (!IS_ALIGNED(req->nbytes, CFB16_BLOCK_SIZE)) {
 685                        pr_err("request size is not exact amount of CFB16 blocks\n");
 686                        return -EINVAL;
 687                }
 688                ctx->block_size = CFB16_BLOCK_SIZE;
 689        } else if (mode & TDES_FLAGS_CFB32) {
 690                if (!IS_ALIGNED(req->nbytes, CFB32_BLOCK_SIZE)) {
 691                        pr_err("request size is not exact amount of CFB32 blocks\n");
 692                        return -EINVAL;
 693                }
 694                ctx->block_size = CFB32_BLOCK_SIZE;
 695        } else {
 696                if (!IS_ALIGNED(req->nbytes, DES_BLOCK_SIZE)) {
 697                        pr_err("request size is not exact amount of DES blocks\n");
 698                        return -EINVAL;
 699                }
 700                ctx->block_size = DES_BLOCK_SIZE;
 701        }
 702
 703        rctx->mode = mode;
 704
 705        return atmel_tdes_handle_queue(ctx->dd, req);
 706}
 707
 708static bool atmel_tdes_filter(struct dma_chan *chan, void *slave)
 709{
 710        struct at_dma_slave     *sl = slave;
 711
 712        if (sl && sl->dma_dev == chan->device->dev) {
 713                chan->private = sl;
 714                return true;
 715        } else {
 716                return false;
 717        }
 718}
 719
 720static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd,
 721                        struct crypto_platform_data *pdata)
 722{
 723        int err = -ENOMEM;
 724        dma_cap_mask_t mask;
 725
 726        dma_cap_zero(mask);
 727        dma_cap_set(DMA_SLAVE, mask);
 728
 729        /* Try to grab 2 DMA channels */
 730        dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask,
 731                        atmel_tdes_filter, &pdata->dma_slave->rxdata, dd->dev, "tx");
 732        if (!dd->dma_lch_in.chan)
 733                goto err_dma_in;
 734
 735        dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV;
 736        dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base +
 737                TDES_IDATA1R;
 738        dd->dma_lch_in.dma_conf.src_maxburst = 1;
 739        dd->dma_lch_in.dma_conf.src_addr_width =
 740                DMA_SLAVE_BUSWIDTH_4_BYTES;
 741        dd->dma_lch_in.dma_conf.dst_maxburst = 1;
 742        dd->dma_lch_in.dma_conf.dst_addr_width =
 743                DMA_SLAVE_BUSWIDTH_4_BYTES;
 744        dd->dma_lch_in.dma_conf.device_fc = false;
 745
 746        dd->dma_lch_out.chan = dma_request_slave_channel_compat(mask,
 747                        atmel_tdes_filter, &pdata->dma_slave->txdata, dd->dev, "rx");
 748        if (!dd->dma_lch_out.chan)
 749                goto err_dma_out;
 750
 751        dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM;
 752        dd->dma_lch_out.dma_conf.src_addr = dd->phys_base +
 753                TDES_ODATA1R;
 754        dd->dma_lch_out.dma_conf.src_maxburst = 1;
 755        dd->dma_lch_out.dma_conf.src_addr_width =
 756                DMA_SLAVE_BUSWIDTH_4_BYTES;
 757        dd->dma_lch_out.dma_conf.dst_maxburst = 1;
 758        dd->dma_lch_out.dma_conf.dst_addr_width =
 759                DMA_SLAVE_BUSWIDTH_4_BYTES;
 760        dd->dma_lch_out.dma_conf.device_fc = false;
 761
 762        return 0;
 763
 764err_dma_out:
 765        dma_release_channel(dd->dma_lch_in.chan);
 766err_dma_in:
 767        dev_warn(dd->dev, "no DMA channel available\n");
 768        return err;
 769}
 770
 771static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd)
 772{
 773        dma_release_channel(dd->dma_lch_in.chan);
 774        dma_release_channel(dd->dma_lch_out.chan);
 775}
 776
 777static int atmel_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 778                           unsigned int keylen)
 779{
 780        u32 tmp[DES_EXPKEY_WORDS];
 781        int err;
 782        struct crypto_tfm *ctfm = crypto_ablkcipher_tfm(tfm);
 783
 784        struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 785
 786        if (keylen != DES_KEY_SIZE) {
 787                crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 788                return -EINVAL;
 789        }
 790
 791        err = des_ekey(tmp, key);
 792        if (err == 0 && (ctfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 793                ctfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
 794                return -EINVAL;
 795        }
 796
 797        memcpy(ctx->key, key, keylen);
 798        ctx->keylen = keylen;
 799
 800        return 0;
 801}
 802
 803static int atmel_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 804                           unsigned int keylen)
 805{
 806        struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 807        const char *alg_name;
 808
 809        alg_name = crypto_tfm_alg_name(crypto_ablkcipher_tfm(tfm));
 810
 811        /*
 812         * HW bug in cfb 3-keys mode.
 813         */
 814        if (!ctx->dd->caps.has_cfb_3keys && strstr(alg_name, "cfb")
 815                        && (keylen != 2*DES_KEY_SIZE)) {
 816                crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 817                return -EINVAL;
 818        } else if ((keylen != 2*DES_KEY_SIZE) && (keylen != 3*DES_KEY_SIZE)) {
 819                crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 820                return -EINVAL;
 821        }
 822
 823        memcpy(ctx->key, key, keylen);
 824        ctx->keylen = keylen;
 825
 826        return 0;
 827}
 828
 829static int atmel_tdes_ecb_encrypt(struct ablkcipher_request *req)
 830{
 831        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT);
 832}
 833
 834static int atmel_tdes_ecb_decrypt(struct ablkcipher_request *req)
 835{
 836        return atmel_tdes_crypt(req, 0);
 837}
 838
 839static int atmel_tdes_cbc_encrypt(struct ablkcipher_request *req)
 840{
 841        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CBC);
 842}
 843
 844static int atmel_tdes_cbc_decrypt(struct ablkcipher_request *req)
 845{
 846        return atmel_tdes_crypt(req, TDES_FLAGS_CBC);
 847}
 848static int atmel_tdes_cfb_encrypt(struct ablkcipher_request *req)
 849{
 850        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB);
 851}
 852
 853static int atmel_tdes_cfb_decrypt(struct ablkcipher_request *req)
 854{
 855        return atmel_tdes_crypt(req, TDES_FLAGS_CFB);
 856}
 857
 858static int atmel_tdes_cfb8_encrypt(struct ablkcipher_request *req)
 859{
 860        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB |
 861                                                TDES_FLAGS_CFB8);
 862}
 863
 864static int atmel_tdes_cfb8_decrypt(struct ablkcipher_request *req)
 865{
 866        return atmel_tdes_crypt(req, TDES_FLAGS_CFB | TDES_FLAGS_CFB8);
 867}
 868
 869static int atmel_tdes_cfb16_encrypt(struct ablkcipher_request *req)
 870{
 871        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB |
 872                                                TDES_FLAGS_CFB16);
 873}
 874
 875static int atmel_tdes_cfb16_decrypt(struct ablkcipher_request *req)
 876{
 877        return atmel_tdes_crypt(req, TDES_FLAGS_CFB | TDES_FLAGS_CFB16);
 878}
 879
 880static int atmel_tdes_cfb32_encrypt(struct ablkcipher_request *req)
 881{
 882        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB |
 883                                                TDES_FLAGS_CFB32);
 884}
 885
 886static int atmel_tdes_cfb32_decrypt(struct ablkcipher_request *req)
 887{
 888        return atmel_tdes_crypt(req, TDES_FLAGS_CFB | TDES_FLAGS_CFB32);
 889}
 890
 891static int atmel_tdes_ofb_encrypt(struct ablkcipher_request *req)
 892{
 893        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_OFB);
 894}
 895
 896static int atmel_tdes_ofb_decrypt(struct ablkcipher_request *req)
 897{
 898        return atmel_tdes_crypt(req, TDES_FLAGS_OFB);
 899}
 900
 901static int atmel_tdes_cra_init(struct crypto_tfm *tfm)
 902{
 903        struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm);
 904        struct atmel_tdes_dev *dd;
 905
 906        tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_tdes_reqctx);
 907
 908        dd = atmel_tdes_find_dev(ctx);
 909        if (!dd)
 910                return -ENODEV;
 911
 912        return 0;
 913}
 914
 915static void atmel_tdes_cra_exit(struct crypto_tfm *tfm)
 916{
 917}
 918
 919static struct crypto_alg tdes_algs[] = {
 920{
 921        .cra_name               = "ecb(des)",
 922        .cra_driver_name        = "atmel-ecb-des",
 923        .cra_priority           = 100,
 924        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 925        .cra_blocksize          = DES_BLOCK_SIZE,
 926        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
 927        .cra_alignmask          = 0x7,
 928        .cra_type               = &crypto_ablkcipher_type,
 929        .cra_module             = THIS_MODULE,
 930        .cra_init               = atmel_tdes_cra_init,
 931        .cra_exit               = atmel_tdes_cra_exit,
 932        .cra_u.ablkcipher = {
 933                .min_keysize    = DES_KEY_SIZE,
 934                .max_keysize    = DES_KEY_SIZE,
 935                .setkey         = atmel_des_setkey,
 936                .encrypt        = atmel_tdes_ecb_encrypt,
 937                .decrypt        = atmel_tdes_ecb_decrypt,
 938        }
 939},
 940{
 941        .cra_name               = "cbc(des)",
 942        .cra_driver_name        = "atmel-cbc-des",
 943        .cra_priority           = 100,
 944        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 945        .cra_blocksize          = DES_BLOCK_SIZE,
 946        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
 947        .cra_alignmask          = 0x7,
 948        .cra_type               = &crypto_ablkcipher_type,
 949        .cra_module             = THIS_MODULE,
 950        .cra_init               = atmel_tdes_cra_init,
 951        .cra_exit               = atmel_tdes_cra_exit,
 952        .cra_u.ablkcipher = {
 953                .min_keysize    = DES_KEY_SIZE,
 954                .max_keysize    = DES_KEY_SIZE,
 955                .ivsize         = DES_BLOCK_SIZE,
 956                .setkey         = atmel_des_setkey,
 957                .encrypt        = atmel_tdes_cbc_encrypt,
 958                .decrypt        = atmel_tdes_cbc_decrypt,
 959        }
 960},
 961{
 962        .cra_name               = "cfb(des)",
 963        .cra_driver_name        = "atmel-cfb-des",
 964        .cra_priority           = 100,
 965        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 966        .cra_blocksize          = DES_BLOCK_SIZE,
 967        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
 968        .cra_alignmask          = 0x7,
 969        .cra_type               = &crypto_ablkcipher_type,
 970        .cra_module             = THIS_MODULE,
 971        .cra_init               = atmel_tdes_cra_init,
 972        .cra_exit               = atmel_tdes_cra_exit,
 973        .cra_u.ablkcipher = {
 974                .min_keysize    = DES_KEY_SIZE,
 975                .max_keysize    = DES_KEY_SIZE,
 976                .ivsize         = DES_BLOCK_SIZE,
 977                .setkey         = atmel_des_setkey,
 978                .encrypt        = atmel_tdes_cfb_encrypt,
 979                .decrypt        = atmel_tdes_cfb_decrypt,
 980        }
 981},
 982{
 983        .cra_name               = "cfb8(des)",
 984        .cra_driver_name        = "atmel-cfb8-des",
 985        .cra_priority           = 100,
 986        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 987        .cra_blocksize          = CFB8_BLOCK_SIZE,
 988        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
 989        .cra_alignmask          = 0,
 990        .cra_type               = &crypto_ablkcipher_type,
 991        .cra_module             = THIS_MODULE,
 992        .cra_init               = atmel_tdes_cra_init,
 993        .cra_exit               = atmel_tdes_cra_exit,
 994        .cra_u.ablkcipher = {
 995                .min_keysize    = DES_KEY_SIZE,
 996                .max_keysize    = DES_KEY_SIZE,
 997                .ivsize         = DES_BLOCK_SIZE,
 998                .setkey         = atmel_des_setkey,
 999                .encrypt        = atmel_tdes_cfb8_encrypt,
1000                .decrypt        = atmel_tdes_cfb8_decrypt,
1001        }
1002},
1003{
1004        .cra_name               = "cfb16(des)",
1005        .cra_driver_name        = "atmel-cfb16-des",
1006        .cra_priority           = 100,
1007        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1008        .cra_blocksize          = CFB16_BLOCK_SIZE,
1009        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1010        .cra_alignmask          = 0x1,
1011        .cra_type               = &crypto_ablkcipher_type,
1012        .cra_module             = THIS_MODULE,
1013        .cra_init               = atmel_tdes_cra_init,
1014        .cra_exit               = atmel_tdes_cra_exit,
1015        .cra_u.ablkcipher = {
1016                .min_keysize    = DES_KEY_SIZE,
1017                .max_keysize    = DES_KEY_SIZE,
1018                .ivsize         = DES_BLOCK_SIZE,
1019                .setkey         = atmel_des_setkey,
1020                .encrypt        = atmel_tdes_cfb16_encrypt,
1021                .decrypt        = atmel_tdes_cfb16_decrypt,
1022        }
1023},
1024{
1025        .cra_name               = "cfb32(des)",
1026        .cra_driver_name        = "atmel-cfb32-des",
1027        .cra_priority           = 100,
1028        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1029        .cra_blocksize          = CFB32_BLOCK_SIZE,
1030        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1031        .cra_alignmask          = 0x3,
1032        .cra_type               = &crypto_ablkcipher_type,
1033        .cra_module             = THIS_MODULE,
1034        .cra_init               = atmel_tdes_cra_init,
1035        .cra_exit               = atmel_tdes_cra_exit,
1036        .cra_u.ablkcipher = {
1037                .min_keysize    = DES_KEY_SIZE,
1038                .max_keysize    = DES_KEY_SIZE,
1039                .ivsize         = DES_BLOCK_SIZE,
1040                .setkey         = atmel_des_setkey,
1041                .encrypt        = atmel_tdes_cfb32_encrypt,
1042                .decrypt        = atmel_tdes_cfb32_decrypt,
1043        }
1044},
1045{
1046        .cra_name               = "ofb(des)",
1047        .cra_driver_name        = "atmel-ofb-des",
1048        .cra_priority           = 100,
1049        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1050        .cra_blocksize          = DES_BLOCK_SIZE,
1051        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1052        .cra_alignmask          = 0x7,
1053        .cra_type               = &crypto_ablkcipher_type,
1054        .cra_module             = THIS_MODULE,
1055        .cra_init               = atmel_tdes_cra_init,
1056        .cra_exit               = atmel_tdes_cra_exit,
1057        .cra_u.ablkcipher = {
1058                .min_keysize    = DES_KEY_SIZE,
1059                .max_keysize    = DES_KEY_SIZE,
1060                .ivsize         = DES_BLOCK_SIZE,
1061                .setkey         = atmel_des_setkey,
1062                .encrypt        = atmel_tdes_ofb_encrypt,
1063                .decrypt        = atmel_tdes_ofb_decrypt,
1064        }
1065},
1066{
1067        .cra_name               = "ecb(des3_ede)",
1068        .cra_driver_name        = "atmel-ecb-tdes",
1069        .cra_priority           = 100,
1070        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1071        .cra_blocksize          = DES_BLOCK_SIZE,
1072        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1073        .cra_alignmask          = 0x7,
1074        .cra_type               = &crypto_ablkcipher_type,
1075        .cra_module             = THIS_MODULE,
1076        .cra_init               = atmel_tdes_cra_init,
1077        .cra_exit               = atmel_tdes_cra_exit,
1078        .cra_u.ablkcipher = {
1079                .min_keysize    = 2 * DES_KEY_SIZE,
1080                .max_keysize    = 3 * DES_KEY_SIZE,
1081                .setkey         = atmel_tdes_setkey,
1082                .encrypt        = atmel_tdes_ecb_encrypt,
1083                .decrypt        = atmel_tdes_ecb_decrypt,
1084        }
1085},
1086{
1087        .cra_name               = "cbc(des3_ede)",
1088        .cra_driver_name        = "atmel-cbc-tdes",
1089        .cra_priority           = 100,
1090        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1091        .cra_blocksize          = DES_BLOCK_SIZE,
1092        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1093        .cra_alignmask          = 0x7,
1094        .cra_type               = &crypto_ablkcipher_type,
1095        .cra_module             = THIS_MODULE,
1096        .cra_init               = atmel_tdes_cra_init,
1097        .cra_exit               = atmel_tdes_cra_exit,
1098        .cra_u.ablkcipher = {
1099                .min_keysize    = 2*DES_KEY_SIZE,
1100                .max_keysize    = 3*DES_KEY_SIZE,
1101                .ivsize         = DES_BLOCK_SIZE,
1102                .setkey         = atmel_tdes_setkey,
1103                .encrypt        = atmel_tdes_cbc_encrypt,
1104                .decrypt        = atmel_tdes_cbc_decrypt,
1105        }
1106},
1107{
1108        .cra_name               = "cfb(des3_ede)",
1109        .cra_driver_name        = "atmel-cfb-tdes",
1110        .cra_priority           = 100,
1111        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1112        .cra_blocksize          = DES_BLOCK_SIZE,
1113        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1114        .cra_alignmask          = 0x7,
1115        .cra_type               = &crypto_ablkcipher_type,
1116        .cra_module             = THIS_MODULE,
1117        .cra_init               = atmel_tdes_cra_init,
1118        .cra_exit               = atmel_tdes_cra_exit,
1119        .cra_u.ablkcipher = {
1120                .min_keysize    = 2*DES_KEY_SIZE,
1121                .max_keysize    = 2*DES_KEY_SIZE,
1122                .ivsize         = DES_BLOCK_SIZE,
1123                .setkey         = atmel_tdes_setkey,
1124                .encrypt        = atmel_tdes_cfb_encrypt,
1125                .decrypt        = atmel_tdes_cfb_decrypt,
1126        }
1127},
1128{
1129        .cra_name               = "cfb8(des3_ede)",
1130        .cra_driver_name        = "atmel-cfb8-tdes",
1131        .cra_priority           = 100,
1132        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1133        .cra_blocksize          = CFB8_BLOCK_SIZE,
1134        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1135        .cra_alignmask          = 0,
1136        .cra_type               = &crypto_ablkcipher_type,
1137        .cra_module             = THIS_MODULE,
1138        .cra_init               = atmel_tdes_cra_init,
1139        .cra_exit               = atmel_tdes_cra_exit,
1140        .cra_u.ablkcipher = {
1141                .min_keysize    = 2*DES_KEY_SIZE,
1142                .max_keysize    = 2*DES_KEY_SIZE,
1143                .ivsize         = DES_BLOCK_SIZE,
1144                .setkey         = atmel_tdes_setkey,
1145                .encrypt        = atmel_tdes_cfb8_encrypt,
1146                .decrypt        = atmel_tdes_cfb8_decrypt,
1147        }
1148},
1149{
1150        .cra_name               = "cfb16(des3_ede)",
1151        .cra_driver_name        = "atmel-cfb16-tdes",
1152        .cra_priority           = 100,
1153        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1154        .cra_blocksize          = CFB16_BLOCK_SIZE,
1155        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1156        .cra_alignmask          = 0x1,
1157        .cra_type               = &crypto_ablkcipher_type,
1158        .cra_module             = THIS_MODULE,
1159        .cra_init               = atmel_tdes_cra_init,
1160        .cra_exit               = atmel_tdes_cra_exit,
1161        .cra_u.ablkcipher = {
1162                .min_keysize    = 2*DES_KEY_SIZE,
1163                .max_keysize    = 2*DES_KEY_SIZE,
1164                .ivsize         = DES_BLOCK_SIZE,
1165                .setkey         = atmel_tdes_setkey,
1166                .encrypt        = atmel_tdes_cfb16_encrypt,
1167                .decrypt        = atmel_tdes_cfb16_decrypt,
1168        }
1169},
1170{
1171        .cra_name               = "cfb32(des3_ede)",
1172        .cra_driver_name        = "atmel-cfb32-tdes",
1173        .cra_priority           = 100,
1174        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1175        .cra_blocksize          = CFB32_BLOCK_SIZE,
1176        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1177        .cra_alignmask          = 0x3,
1178        .cra_type               = &crypto_ablkcipher_type,
1179        .cra_module             = THIS_MODULE,
1180        .cra_init               = atmel_tdes_cra_init,
1181        .cra_exit               = atmel_tdes_cra_exit,
1182        .cra_u.ablkcipher = {
1183                .min_keysize    = 2*DES_KEY_SIZE,
1184                .max_keysize    = 2*DES_KEY_SIZE,
1185                .ivsize         = DES_BLOCK_SIZE,
1186                .setkey         = atmel_tdes_setkey,
1187                .encrypt        = atmel_tdes_cfb32_encrypt,
1188                .decrypt        = atmel_tdes_cfb32_decrypt,
1189        }
1190},
1191{
1192        .cra_name               = "ofb(des3_ede)",
1193        .cra_driver_name        = "atmel-ofb-tdes",
1194        .cra_priority           = 100,
1195        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1196        .cra_blocksize          = DES_BLOCK_SIZE,
1197        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1198        .cra_alignmask          = 0x7,
1199        .cra_type               = &crypto_ablkcipher_type,
1200        .cra_module             = THIS_MODULE,
1201        .cra_init               = atmel_tdes_cra_init,
1202        .cra_exit               = atmel_tdes_cra_exit,
1203        .cra_u.ablkcipher = {
1204                .min_keysize    = 2*DES_KEY_SIZE,
1205                .max_keysize    = 3*DES_KEY_SIZE,
1206                .ivsize         = DES_BLOCK_SIZE,
1207                .setkey         = atmel_tdes_setkey,
1208                .encrypt        = atmel_tdes_ofb_encrypt,
1209                .decrypt        = atmel_tdes_ofb_decrypt,
1210        }
1211},
1212};
1213
1214static void atmel_tdes_queue_task(unsigned long data)
1215{
1216        struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *)data;
1217
1218        atmel_tdes_handle_queue(dd, NULL);
1219}
1220
1221static void atmel_tdes_done_task(unsigned long data)
1222{
1223        struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *) data;
1224        int err;
1225
1226        if (!(dd->flags & TDES_FLAGS_DMA))
1227                err = atmel_tdes_crypt_pdc_stop(dd);
1228        else
1229                err = atmel_tdes_crypt_dma_stop(dd);
1230
1231        err = dd->err ? : err;
1232
1233        if (dd->total && !err) {
1234                if (dd->flags & TDES_FLAGS_FAST) {
1235                        dd->in_sg = sg_next(dd->in_sg);
1236                        dd->out_sg = sg_next(dd->out_sg);
1237                        if (!dd->in_sg || !dd->out_sg)
1238                                err = -EINVAL;
1239                }
1240                if (!err)
1241                        err = atmel_tdes_crypt_start(dd);
1242                if (!err)
1243                        return; /* DMA started. Not fininishing. */
1244        }
1245
1246        atmel_tdes_finish_req(dd, err);
1247        atmel_tdes_handle_queue(dd, NULL);
1248}
1249
1250static irqreturn_t atmel_tdes_irq(int irq, void *dev_id)
1251{
1252        struct atmel_tdes_dev *tdes_dd = dev_id;
1253        u32 reg;
1254
1255        reg = atmel_tdes_read(tdes_dd, TDES_ISR);
1256        if (reg & atmel_tdes_read(tdes_dd, TDES_IMR)) {
1257                atmel_tdes_write(tdes_dd, TDES_IDR, reg);
1258                if (TDES_FLAGS_BUSY & tdes_dd->flags)
1259                        tasklet_schedule(&tdes_dd->done_task);
1260                else
1261                        dev_warn(tdes_dd->dev, "TDES interrupt when no active requests.\n");
1262                return IRQ_HANDLED;
1263        }
1264
1265        return IRQ_NONE;
1266}
1267
1268static void atmel_tdes_unregister_algs(struct atmel_tdes_dev *dd)
1269{
1270        int i;
1271
1272        for (i = 0; i < ARRAY_SIZE(tdes_algs); i++)
1273                crypto_unregister_alg(&tdes_algs[i]);
1274}
1275
1276static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd)
1277{
1278        int err, i, j;
1279
1280        for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) {
1281                err = crypto_register_alg(&tdes_algs[i]);
1282                if (err)
1283                        goto err_tdes_algs;
1284        }
1285
1286        return 0;
1287
1288err_tdes_algs:
1289        for (j = 0; j < i; j++)
1290                crypto_unregister_alg(&tdes_algs[j]);
1291
1292        return err;
1293}
1294
1295static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd)
1296{
1297
1298        dd->caps.has_dma = 0;
1299        dd->caps.has_cfb_3keys = 0;
1300
1301        /* keep only major version number */
1302        switch (dd->hw_version & 0xf00) {
1303        case 0x700:
1304                dd->caps.has_dma = 1;
1305                dd->caps.has_cfb_3keys = 1;
1306                break;
1307        case 0x600:
1308                break;
1309        default:
1310                dev_warn(dd->dev,
1311                                "Unmanaged tdes version, set minimum capabilities\n");
1312                break;
1313        }
1314}
1315
1316#if defined(CONFIG_OF)
1317static const struct of_device_id atmel_tdes_dt_ids[] = {
1318        { .compatible = "atmel,at91sam9g46-tdes" },
1319        { /* sentinel */ }
1320};
1321MODULE_DEVICE_TABLE(of, atmel_tdes_dt_ids);
1322
1323static struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *pdev)
1324{
1325        struct device_node *np = pdev->dev.of_node;
1326        struct crypto_platform_data *pdata;
1327
1328        if (!np) {
1329                dev_err(&pdev->dev, "device node not found\n");
1330                return ERR_PTR(-EINVAL);
1331        }
1332
1333        pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1334        if (!pdata) {
1335                dev_err(&pdev->dev, "could not allocate memory for pdata\n");
1336                return ERR_PTR(-ENOMEM);
1337        }
1338
1339        pdata->dma_slave = devm_kzalloc(&pdev->dev,
1340                                        sizeof(*(pdata->dma_slave)),
1341                                        GFP_KERNEL);
1342        if (!pdata->dma_slave) {
1343                dev_err(&pdev->dev, "could not allocate memory for dma_slave\n");
1344                return ERR_PTR(-ENOMEM);
1345        }
1346
1347        return pdata;
1348}
1349#else /* CONFIG_OF */
1350static inline struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *pdev)
1351{
1352        return ERR_PTR(-EINVAL);
1353}
1354#endif
1355
1356static int atmel_tdes_probe(struct platform_device *pdev)
1357{
1358        struct atmel_tdes_dev *tdes_dd;
1359        struct crypto_platform_data     *pdata;
1360        struct device *dev = &pdev->dev;
1361        struct resource *tdes_res;
1362        int err;
1363
1364        tdes_dd = devm_kmalloc(&pdev->dev, sizeof(*tdes_dd), GFP_KERNEL);
1365        if (tdes_dd == NULL) {
1366                dev_err(dev, "unable to alloc data struct.\n");
1367                err = -ENOMEM;
1368                goto tdes_dd_err;
1369        }
1370
1371        tdes_dd->dev = dev;
1372
1373        platform_set_drvdata(pdev, tdes_dd);
1374
1375        INIT_LIST_HEAD(&tdes_dd->list);
1376        spin_lock_init(&tdes_dd->lock);
1377
1378        tasklet_init(&tdes_dd->done_task, atmel_tdes_done_task,
1379                                        (unsigned long)tdes_dd);
1380        tasklet_init(&tdes_dd->queue_task, atmel_tdes_queue_task,
1381                                        (unsigned long)tdes_dd);
1382
1383        crypto_init_queue(&tdes_dd->queue, ATMEL_TDES_QUEUE_LENGTH);
1384
1385        tdes_dd->irq = -1;
1386
1387        /* Get the base address */
1388        tdes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1389        if (!tdes_res) {
1390                dev_err(dev, "no MEM resource info\n");
1391                err = -ENODEV;
1392                goto res_err;
1393        }
1394        tdes_dd->phys_base = tdes_res->start;
1395
1396        /* Get the IRQ */
1397        tdes_dd->irq = platform_get_irq(pdev,  0);
1398        if (tdes_dd->irq < 0) {
1399                dev_err(dev, "no IRQ resource info\n");
1400                err = tdes_dd->irq;
1401                goto res_err;
1402        }
1403
1404        err = devm_request_irq(&pdev->dev, tdes_dd->irq, atmel_tdes_irq,
1405                               IRQF_SHARED, "atmel-tdes", tdes_dd);
1406        if (err) {
1407                dev_err(dev, "unable to request tdes irq.\n");
1408                goto res_err;
1409        }
1410
1411        /* Initializing the clock */
1412        tdes_dd->iclk = devm_clk_get(&pdev->dev, "tdes_clk");
1413        if (IS_ERR(tdes_dd->iclk)) {
1414                dev_err(dev, "clock initialization failed.\n");
1415                err = PTR_ERR(tdes_dd->iclk);
1416                goto res_err;
1417        }
1418
1419        tdes_dd->io_base = devm_ioremap_resource(&pdev->dev, tdes_res);
1420        if (IS_ERR(tdes_dd->io_base)) {
1421                dev_err(dev, "can't ioremap\n");
1422                err = PTR_ERR(tdes_dd->io_base);
1423                goto res_err;
1424        }
1425
1426        atmel_tdes_hw_version_init(tdes_dd);
1427
1428        atmel_tdes_get_cap(tdes_dd);
1429
1430        err = atmel_tdes_buff_init(tdes_dd);
1431        if (err)
1432                goto err_tdes_buff;
1433
1434        if (tdes_dd->caps.has_dma) {
1435                pdata = pdev->dev.platform_data;
1436                if (!pdata) {
1437                        pdata = atmel_tdes_of_init(pdev);
1438                        if (IS_ERR(pdata)) {
1439                                dev_err(&pdev->dev, "platform data not available\n");
1440                                err = PTR_ERR(pdata);
1441                                goto err_pdata;
1442                        }
1443                }
1444                if (!pdata->dma_slave) {
1445                        err = -ENXIO;
1446                        goto err_pdata;
1447                }
1448                err = atmel_tdes_dma_init(tdes_dd, pdata);
1449                if (err)
1450                        goto err_tdes_dma;
1451
1452                dev_info(dev, "using %s, %s for DMA transfers\n",
1453                                dma_chan_name(tdes_dd->dma_lch_in.chan),
1454                                dma_chan_name(tdes_dd->dma_lch_out.chan));
1455        }
1456
1457        spin_lock(&atmel_tdes.lock);
1458        list_add_tail(&tdes_dd->list, &atmel_tdes.dev_list);
1459        spin_unlock(&atmel_tdes.lock);
1460
1461        err = atmel_tdes_register_algs(tdes_dd);
1462        if (err)
1463                goto err_algs;
1464
1465        dev_info(dev, "Atmel DES/TDES\n");
1466
1467        return 0;
1468
1469err_algs:
1470        spin_lock(&atmel_tdes.lock);
1471        list_del(&tdes_dd->list);
1472        spin_unlock(&atmel_tdes.lock);
1473        if (tdes_dd->caps.has_dma)
1474                atmel_tdes_dma_cleanup(tdes_dd);
1475err_tdes_dma:
1476err_pdata:
1477        atmel_tdes_buff_cleanup(tdes_dd);
1478err_tdes_buff:
1479res_err:
1480        tasklet_kill(&tdes_dd->done_task);
1481        tasklet_kill(&tdes_dd->queue_task);
1482tdes_dd_err:
1483        dev_err(dev, "initialization failed.\n");
1484
1485        return err;
1486}
1487
1488static int atmel_tdes_remove(struct platform_device *pdev)
1489{
1490        static struct atmel_tdes_dev *tdes_dd;
1491
1492        tdes_dd = platform_get_drvdata(pdev);
1493        if (!tdes_dd)
1494                return -ENODEV;
1495        spin_lock(&atmel_tdes.lock);
1496        list_del(&tdes_dd->list);
1497        spin_unlock(&atmel_tdes.lock);
1498
1499        atmel_tdes_unregister_algs(tdes_dd);
1500
1501        tasklet_kill(&tdes_dd->done_task);
1502        tasklet_kill(&tdes_dd->queue_task);
1503
1504        if (tdes_dd->caps.has_dma)
1505                atmel_tdes_dma_cleanup(tdes_dd);
1506
1507        atmel_tdes_buff_cleanup(tdes_dd);
1508
1509        return 0;
1510}
1511
1512static struct platform_driver atmel_tdes_driver = {
1513        .probe          = atmel_tdes_probe,
1514        .remove         = atmel_tdes_remove,
1515        .driver         = {
1516                .name   = "atmel_tdes",
1517                .of_match_table = of_match_ptr(atmel_tdes_dt_ids),
1518        },
1519};
1520
1521module_platform_driver(atmel_tdes_driver);
1522
1523MODULE_DESCRIPTION("Atmel DES/TDES hw acceleration support.");
1524MODULE_LICENSE("GPL v2");
1525MODULE_AUTHOR("Nicolas Royer - Eukr\xC3\xA9a Electromatique");
1526
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.