linux/drivers/crypto/atmel-tdes.c
<<
>>
Prefs
   1/*
   2 * Cryptographic API.
   3 *
   4 * Support for ATMEL DES/TDES HW acceleration.
   5 *
   6 * Copyright (c) 2012 Eukréa Electromatique - ATMEL
   7 * Author: Nicolas Royer <nicolas@eukrea.com>
   8 *
   9 * This program is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License version 2 as published
  11 * by the Free Software Foundation.
  12 *
  13 * Some ideas are from omap-aes.c drivers.
  14 */
  15
  16
  17#include <linux/kernel.h>
  18#include <linux/module.h>
  19#include <linux/slab.h>
  20#include <linux/err.h>
  21#include <linux/clk.h>
  22#include <linux/io.h>
  23#include <linux/hw_random.h>
  24#include <linux/platform_device.h>
  25
  26#include <linux/device.h>
  27#include <linux/init.h>
  28#include <linux/errno.h>
  29#include <linux/interrupt.h>
  30#include <linux/irq.h>
  31#include <linux/scatterlist.h>
  32#include <linux/dma-mapping.h>
  33#include <linux/of_device.h>
  34#include <linux/delay.h>
  35#include <linux/crypto.h>
  36#include <linux/cryptohash.h>
  37#include <crypto/scatterwalk.h>
  38#include <crypto/algapi.h>
  39#include <crypto/des.h>
  40#include <crypto/hash.h>
  41#include <crypto/internal/hash.h>
  42#include <linux/platform_data/crypto-atmel.h>
  43#include "atmel-tdes-regs.h"
  44
  45/* TDES flags  */
  46#define TDES_FLAGS_MODE_MASK            0x00ff
  47#define TDES_FLAGS_ENCRYPT      BIT(0)
  48#define TDES_FLAGS_CBC          BIT(1)
  49#define TDES_FLAGS_CFB          BIT(2)
  50#define TDES_FLAGS_CFB8         BIT(3)
  51#define TDES_FLAGS_CFB16        BIT(4)
  52#define TDES_FLAGS_CFB32        BIT(5)
  53#define TDES_FLAGS_CFB64        BIT(6)
  54#define TDES_FLAGS_OFB          BIT(7)
  55
  56#define TDES_FLAGS_INIT         BIT(16)
  57#define TDES_FLAGS_FAST         BIT(17)
  58#define TDES_FLAGS_BUSY         BIT(18)
  59#define TDES_FLAGS_DMA          BIT(19)
  60
  61#define ATMEL_TDES_QUEUE_LENGTH 50
  62
  63#define CFB8_BLOCK_SIZE         1
  64#define CFB16_BLOCK_SIZE        2
  65#define CFB32_BLOCK_SIZE        4
  66
  67struct atmel_tdes_caps {
  68        bool    has_dma;
  69        u32             has_cfb_3keys;
  70};
  71
  72struct atmel_tdes_dev;
  73
  74struct atmel_tdes_ctx {
  75        struct atmel_tdes_dev *dd;
  76
  77        int             keylen;
  78        u32             key[3*DES_KEY_SIZE / sizeof(u32)];
  79        unsigned long   flags;
  80
  81        u16             block_size;
  82};
  83
  84struct atmel_tdes_reqctx {
  85        unsigned long mode;
  86};
  87
  88struct atmel_tdes_dma {
  89        struct dma_chan                 *chan;
  90        struct dma_slave_config dma_conf;
  91};
  92
  93struct atmel_tdes_dev {
  94        struct list_head        list;
  95        unsigned long           phys_base;
  96        void __iomem            *io_base;
  97
  98        struct atmel_tdes_ctx   *ctx;
  99        struct device           *dev;
 100        struct clk                      *iclk;
 101        int                                     irq;
 102
 103        unsigned long           flags;
 104        int                     err;
 105
 106        spinlock_t              lock;
 107        struct crypto_queue     queue;
 108
 109        struct tasklet_struct   done_task;
 110        struct tasklet_struct   queue_task;
 111
 112        struct ablkcipher_request       *req;
 113        size_t                          total;
 114
 115        struct scatterlist      *in_sg;
 116        unsigned int            nb_in_sg;
 117        size_t                          in_offset;
 118        struct scatterlist      *out_sg;
 119        unsigned int            nb_out_sg;
 120        size_t                          out_offset;
 121
 122        size_t  buflen;
 123        size_t  dma_size;
 124
 125        void    *buf_in;
 126        int             dma_in;
 127        dma_addr_t      dma_addr_in;
 128        struct atmel_tdes_dma   dma_lch_in;
 129
 130        void    *buf_out;
 131        int             dma_out;
 132        dma_addr_t      dma_addr_out;
 133        struct atmel_tdes_dma   dma_lch_out;
 134
 135        struct atmel_tdes_caps  caps;
 136
 137        u32     hw_version;
 138};
 139
 140struct atmel_tdes_drv {
 141        struct list_head        dev_list;
 142        spinlock_t              lock;
 143};
 144
 145static struct atmel_tdes_drv atmel_tdes = {
 146        .dev_list = LIST_HEAD_INIT(atmel_tdes.dev_list),
 147        .lock = __SPIN_LOCK_UNLOCKED(atmel_tdes.lock),
 148};
 149
 150static int atmel_tdes_sg_copy(struct scatterlist **sg, size_t *offset,
 151                        void *buf, size_t buflen, size_t total, int out)
 152{
 153        unsigned int count, off = 0;
 154
 155        while (buflen && total) {
 156                count = min((*sg)->length - *offset, total);
 157                count = min(count, buflen);
 158
 159                if (!count)
 160                        return off;
 161
 162                scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out);
 163
 164                off += count;
 165                buflen -= count;
 166                *offset += count;
 167                total -= count;
 168
 169                if (*offset == (*sg)->length) {
 170                        *sg = sg_next(*sg);
 171                        if (*sg)
 172                                *offset = 0;
 173                        else
 174                                total = 0;
 175                }
 176        }
 177
 178        return off;
 179}
 180
 181static inline u32 atmel_tdes_read(struct atmel_tdes_dev *dd, u32 offset)
 182{
 183        return readl_relaxed(dd->io_base + offset);
 184}
 185
 186static inline void atmel_tdes_write(struct atmel_tdes_dev *dd,
 187                                        u32 offset, u32 value)
 188{
 189        writel_relaxed(value, dd->io_base + offset);
 190}
 191
 192static void atmel_tdes_write_n(struct atmel_tdes_dev *dd, u32 offset,
 193                                        u32 *value, int count)
 194{
 195        for (; count--; value++, offset += 4)
 196                atmel_tdes_write(dd, offset, *value);
 197}
 198
 199static struct atmel_tdes_dev *atmel_tdes_find_dev(struct atmel_tdes_ctx *ctx)
 200{
 201        struct atmel_tdes_dev *tdes_dd = NULL;
 202        struct atmel_tdes_dev *tmp;
 203
 204        spin_lock_bh(&atmel_tdes.lock);
 205        if (!ctx->dd) {
 206                list_for_each_entry(tmp, &atmel_tdes.dev_list, list) {
 207                        tdes_dd = tmp;
 208                        break;
 209                }
 210                ctx->dd = tdes_dd;
 211        } else {
 212                tdes_dd = ctx->dd;
 213        }
 214        spin_unlock_bh(&atmel_tdes.lock);
 215
 216        return tdes_dd;
 217}
 218
 219static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd)
 220{
 221        clk_prepare_enable(dd->iclk);
 222
 223        if (!(dd->flags & TDES_FLAGS_INIT)) {
 224                atmel_tdes_write(dd, TDES_CR, TDES_CR_SWRST);
 225                dd->flags |= TDES_FLAGS_INIT;
 226                dd->err = 0;
 227        }
 228
 229        return 0;
 230}
 231
 232static inline unsigned int atmel_tdes_get_version(struct atmel_tdes_dev *dd)
 233{
 234        return atmel_tdes_read(dd, TDES_HW_VERSION) & 0x00000fff;
 235}
 236
 237static void atmel_tdes_hw_version_init(struct atmel_tdes_dev *dd)
 238{
 239        atmel_tdes_hw_init(dd);
 240
 241        dd->hw_version = atmel_tdes_get_version(dd);
 242
 243        dev_info(dd->dev,
 244                        "version: 0x%x\n", dd->hw_version);
 245
 246        clk_disable_unprepare(dd->iclk);
 247}
 248
 249static void atmel_tdes_dma_callback(void *data)
 250{
 251        struct atmel_tdes_dev *dd = data;
 252
 253        /* dma_lch_out - completed */
 254        tasklet_schedule(&dd->done_task);
 255}
 256
 257static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd)
 258{
 259        int err;
 260        u32 valcr = 0, valmr = TDES_MR_SMOD_PDC;
 261
 262        err = atmel_tdes_hw_init(dd);
 263
 264        if (err)
 265                return err;
 266
 267        if (!dd->caps.has_dma)
 268                atmel_tdes_write(dd, TDES_PTCR,
 269                        TDES_PTCR_TXTDIS | TDES_PTCR_RXTDIS);
 270
 271        /* MR register must be set before IV registers */
 272        if (dd->ctx->keylen > (DES_KEY_SIZE << 1)) {
 273                valmr |= TDES_MR_KEYMOD_3KEY;
 274                valmr |= TDES_MR_TDESMOD_TDES;
 275        } else if (dd->ctx->keylen > DES_KEY_SIZE) {
 276                valmr |= TDES_MR_KEYMOD_2KEY;
 277                valmr |= TDES_MR_TDESMOD_TDES;
 278        } else {
 279                valmr |= TDES_MR_TDESMOD_DES;
 280        }
 281
 282        if (dd->flags & TDES_FLAGS_CBC) {
 283                valmr |= TDES_MR_OPMOD_CBC;
 284        } else if (dd->flags & TDES_FLAGS_CFB) {
 285                valmr |= TDES_MR_OPMOD_CFB;
 286
 287                if (dd->flags & TDES_FLAGS_CFB8)
 288                        valmr |= TDES_MR_CFBS_8b;
 289                else if (dd->flags & TDES_FLAGS_CFB16)
 290                        valmr |= TDES_MR_CFBS_16b;
 291                else if (dd->flags & TDES_FLAGS_CFB32)
 292                        valmr |= TDES_MR_CFBS_32b;
 293                else if (dd->flags & TDES_FLAGS_CFB64)
 294                        valmr |= TDES_MR_CFBS_64b;
 295        } else if (dd->flags & TDES_FLAGS_OFB) {
 296                valmr |= TDES_MR_OPMOD_OFB;
 297        }
 298
 299        if ((dd->flags & TDES_FLAGS_ENCRYPT) || (dd->flags & TDES_FLAGS_OFB))
 300                valmr |= TDES_MR_CYPHER_ENC;
 301
 302        atmel_tdes_write(dd, TDES_CR, valcr);
 303        atmel_tdes_write(dd, TDES_MR, valmr);
 304
 305        atmel_tdes_write_n(dd, TDES_KEY1W1R, dd->ctx->key,
 306                                                dd->ctx->keylen >> 2);
 307
 308        if (((dd->flags & TDES_FLAGS_CBC) || (dd->flags & TDES_FLAGS_CFB) ||
 309                (dd->flags & TDES_FLAGS_OFB)) && dd->req->info) {
 310                atmel_tdes_write_n(dd, TDES_IV1R, dd->req->info, 2);
 311        }
 312
 313        return 0;
 314}
 315
 316static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd)
 317{
 318        int err = 0;
 319        size_t count;
 320
 321        atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);
 322
 323        if (dd->flags & TDES_FLAGS_FAST) {
 324                dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 325                dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 326        } else {
 327                dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
 328                                           dd->dma_size, DMA_FROM_DEVICE);
 329
 330                /* copy data */
 331                count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
 332                                dd->buf_out, dd->buflen, dd->dma_size, 1);
 333                if (count != dd->dma_size) {
 334                        err = -EINVAL;
 335                        pr_err("not all data converted: %u\n", count);
 336                }
 337        }
 338
 339        return err;
 340}
 341
 342static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd)
 343{
 344        int err = -ENOMEM;
 345
 346        dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0);
 347        dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0);
 348        dd->buflen = PAGE_SIZE;
 349        dd->buflen &= ~(DES_BLOCK_SIZE - 1);
 350
 351        if (!dd->buf_in || !dd->buf_out) {
 352                dev_err(dd->dev, "unable to alloc pages.\n");
 353                goto err_alloc;
 354        }
 355
 356        /* MAP here */
 357        dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in,
 358                                        dd->buflen, DMA_TO_DEVICE);
 359        if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
 360                dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 361                err = -EINVAL;
 362                goto err_map_in;
 363        }
 364
 365        dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out,
 366                                        dd->buflen, DMA_FROM_DEVICE);
 367        if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
 368                dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
 369                err = -EINVAL;
 370                goto err_map_out;
 371        }
 372
 373        return 0;
 374
 375err_map_out:
 376        dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
 377                DMA_TO_DEVICE);
 378err_map_in:
 379        free_page((unsigned long)dd->buf_out);
 380        free_page((unsigned long)dd->buf_in);
 381err_alloc:
 382        if (err)
 383                pr_err("error: %d\n", err);
 384        return err;
 385}
 386
 387static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd)
 388{
 389        dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
 390                         DMA_FROM_DEVICE);
 391        dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,
 392                DMA_TO_DEVICE);
 393        free_page((unsigned long)dd->buf_out);
 394        free_page((unsigned long)dd->buf_in);
 395}
 396
 397static int atmel_tdes_crypt_pdc(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
 398                               dma_addr_t dma_addr_out, int length)
 399{
 400        struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm);
 401        struct atmel_tdes_dev *dd = ctx->dd;
 402        int len32;
 403
 404        dd->dma_size = length;
 405
 406        if (!(dd->flags & TDES_FLAGS_FAST)) {
 407                dma_sync_single_for_device(dd->dev, dma_addr_in, length,
 408                                           DMA_TO_DEVICE);
 409        }
 410
 411        if ((dd->flags & TDES_FLAGS_CFB) && (dd->flags & TDES_FLAGS_CFB8))
 412                len32 = DIV_ROUND_UP(length, sizeof(u8));
 413        else if ((dd->flags & TDES_FLAGS_CFB) && (dd->flags & TDES_FLAGS_CFB16))
 414                len32 = DIV_ROUND_UP(length, sizeof(u16));
 415        else
 416                len32 = DIV_ROUND_UP(length, sizeof(u32));
 417
 418        atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);
 419        atmel_tdes_write(dd, TDES_TPR, dma_addr_in);
 420        atmel_tdes_write(dd, TDES_TCR, len32);
 421        atmel_tdes_write(dd, TDES_RPR, dma_addr_out);
 422        atmel_tdes_write(dd, TDES_RCR, len32);
 423
 424        /* Enable Interrupt */
 425        atmel_tdes_write(dd, TDES_IER, TDES_INT_ENDRX);
 426
 427        /* Start DMA transfer */
 428        atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTEN | TDES_PTCR_RXTEN);
 429
 430        return 0;
 431}
 432
 433static int atmel_tdes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
 434                               dma_addr_t dma_addr_out, int length)
 435{
 436        struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm);
 437        struct atmel_tdes_dev *dd = ctx->dd;
 438        struct scatterlist sg[2];
 439        struct dma_async_tx_descriptor  *in_desc, *out_desc;
 440
 441        dd->dma_size = length;
 442
 443        if (!(dd->flags & TDES_FLAGS_FAST)) {
 444                dma_sync_single_for_device(dd->dev, dma_addr_in, length,
 445                                           DMA_TO_DEVICE);
 446        }
 447
 448        if (dd->flags & TDES_FLAGS_CFB8) {
 449                dd->dma_lch_in.dma_conf.dst_addr_width =
 450                        DMA_SLAVE_BUSWIDTH_1_BYTE;
 451                dd->dma_lch_out.dma_conf.src_addr_width =
 452                        DMA_SLAVE_BUSWIDTH_1_BYTE;
 453        } else if (dd->flags & TDES_FLAGS_CFB16) {
 454                dd->dma_lch_in.dma_conf.dst_addr_width =
 455                        DMA_SLAVE_BUSWIDTH_2_BYTES;
 456                dd->dma_lch_out.dma_conf.src_addr_width =
 457                        DMA_SLAVE_BUSWIDTH_2_BYTES;
 458        } else {
 459                dd->dma_lch_in.dma_conf.dst_addr_width =
 460                        DMA_SLAVE_BUSWIDTH_4_BYTES;
 461                dd->dma_lch_out.dma_conf.src_addr_width =
 462                        DMA_SLAVE_BUSWIDTH_4_BYTES;
 463        }
 464
 465        dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);
 466        dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf);
 467
 468        dd->flags |= TDES_FLAGS_DMA;
 469
 470        sg_init_table(&sg[0], 1);
 471        sg_dma_address(&sg[0]) = dma_addr_in;
 472        sg_dma_len(&sg[0]) = length;
 473
 474        sg_init_table(&sg[1], 1);
 475        sg_dma_address(&sg[1]) = dma_addr_out;
 476        sg_dma_len(&sg[1]) = length;
 477
 478        in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0],
 479                                1, DMA_MEM_TO_DEV,
 480                                DMA_PREP_INTERRUPT  |  DMA_CTRL_ACK);
 481        if (!in_desc)
 482                return -EINVAL;
 483
 484        out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1],
 485                                1, DMA_DEV_TO_MEM,
 486                                DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 487        if (!out_desc)
 488                return -EINVAL;
 489
 490        out_desc->callback = atmel_tdes_dma_callback;
 491        out_desc->callback_param = dd;
 492
 493        dmaengine_submit(out_desc);
 494        dma_async_issue_pending(dd->dma_lch_out.chan);
 495
 496        dmaengine_submit(in_desc);
 497        dma_async_issue_pending(dd->dma_lch_in.chan);
 498
 499        return 0;
 500}
 501
 502static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd)
 503{
 504        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
 505                                        crypto_ablkcipher_reqtfm(dd->req));
 506        int err, fast = 0, in, out;
 507        size_t count;
 508        dma_addr_t addr_in, addr_out;
 509
 510        if ((!dd->in_offset) && (!dd->out_offset)) {
 511                /* check for alignment */
 512                in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) &&
 513                        IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size);
 514                out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) &&
 515                        IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size);
 516                fast = in && out;
 517
 518                if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg))
 519                        fast = 0;
 520        }
 521
 522
 523        if (fast)  {
 524                count = min(dd->total, sg_dma_len(dd->in_sg));
 525                count = min(count, sg_dma_len(dd->out_sg));
 526
 527                err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 528                if (!err) {
 529                        dev_err(dd->dev, "dma_map_sg() error\n");
 530                        return -EINVAL;
 531                }
 532
 533                err = dma_map_sg(dd->dev, dd->out_sg, 1,
 534                                DMA_FROM_DEVICE);
 535                if (!err) {
 536                        dev_err(dd->dev, "dma_map_sg() error\n");
 537                        dma_unmap_sg(dd->dev, dd->in_sg, 1,
 538                                DMA_TO_DEVICE);
 539                        return -EINVAL;
 540                }
 541
 542                addr_in = sg_dma_address(dd->in_sg);
 543                addr_out = sg_dma_address(dd->out_sg);
 544
 545                dd->flags |= TDES_FLAGS_FAST;
 546
 547        } else {
 548                /* use cache buffers */
 549                count = atmel_tdes_sg_copy(&dd->in_sg, &dd->in_offset,
 550                                dd->buf_in, dd->buflen, dd->total, 0);
 551
 552                addr_in = dd->dma_addr_in;
 553                addr_out = dd->dma_addr_out;
 554
 555                dd->flags &= ~TDES_FLAGS_FAST;
 556        }
 557
 558        dd->total -= count;
 559
 560        if (dd->caps.has_dma)
 561                err = atmel_tdes_crypt_dma(tfm, addr_in, addr_out, count);
 562        else
 563                err = atmel_tdes_crypt_pdc(tfm, addr_in, addr_out, count);
 564
 565        if (err && (dd->flags & TDES_FLAGS_FAST)) {
 566                dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 567                dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
 568        }
 569
 570        return err;
 571}
 572
 573static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err)
 574{
 575        struct ablkcipher_request *req = dd->req;
 576
 577        clk_disable_unprepare(dd->iclk);
 578
 579        dd->flags &= ~TDES_FLAGS_BUSY;
 580
 581        req->base.complete(&req->base, err);
 582}
 583
 584static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd,
 585                               struct ablkcipher_request *req)
 586{
 587        struct crypto_async_request *async_req, *backlog;
 588        struct atmel_tdes_ctx *ctx;
 589        struct atmel_tdes_reqctx *rctx;
 590        unsigned long flags;
 591        int err, ret = 0;
 592
 593        spin_lock_irqsave(&dd->lock, flags);
 594        if (req)
 595                ret = ablkcipher_enqueue_request(&dd->queue, req);
 596        if (dd->flags & TDES_FLAGS_BUSY) {
 597                spin_unlock_irqrestore(&dd->lock, flags);
 598                return ret;
 599        }
 600        backlog = crypto_get_backlog(&dd->queue);
 601        async_req = crypto_dequeue_request(&dd->queue);
 602        if (async_req)
 603                dd->flags |= TDES_FLAGS_BUSY;
 604        spin_unlock_irqrestore(&dd->lock, flags);
 605
 606        if (!async_req)
 607                return ret;
 608
 609        if (backlog)
 610                backlog->complete(backlog, -EINPROGRESS);
 611
 612        req = ablkcipher_request_cast(async_req);
 613
 614        /* assign new request to device */
 615        dd->req = req;
 616        dd->total = req->nbytes;
 617        dd->in_offset = 0;
 618        dd->in_sg = req->src;
 619        dd->out_offset = 0;
 620        dd->out_sg = req->dst;
 621
 622        rctx = ablkcipher_request_ctx(req);
 623        ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
 624        rctx->mode &= TDES_FLAGS_MODE_MASK;
 625        dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode;
 626        dd->ctx = ctx;
 627        ctx->dd = dd;
 628
 629        err = atmel_tdes_write_ctrl(dd);
 630        if (!err)
 631                err = atmel_tdes_crypt_start(dd);
 632        if (err) {
 633                /* des_task will not finish it, so do it here */
 634                atmel_tdes_finish_req(dd, err);
 635                tasklet_schedule(&dd->queue_task);
 636        }
 637
 638        return ret;
 639}
 640
 641static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd)
 642{
 643        int err = -EINVAL;
 644        size_t count;
 645
 646        if (dd->flags & TDES_FLAGS_DMA) {
 647                err = 0;
 648                if  (dd->flags & TDES_FLAGS_FAST) {
 649                        dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
 650                        dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
 651                } else {
 652                        dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
 653                                dd->dma_size, DMA_FROM_DEVICE);
 654
 655                        /* copy data */
 656                        count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,
 657                                dd->buf_out, dd->buflen, dd->dma_size, 1);
 658                        if (count != dd->dma_size) {
 659                                err = -EINVAL;
 660                                pr_err("not all data converted: %u\n", count);
 661                        }
 662                }
 663        }
 664        return err;
 665}
 666
 667static int atmel_tdes_crypt(struct ablkcipher_request *req, unsigned long mode)
 668{
 669        struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(
 670                        crypto_ablkcipher_reqtfm(req));
 671        struct atmel_tdes_reqctx *rctx = ablkcipher_request_ctx(req);
 672
 673        if (mode & TDES_FLAGS_CFB8) {
 674                if (!IS_ALIGNED(req->nbytes, CFB8_BLOCK_SIZE)) {
 675                        pr_err("request size is not exact amount of CFB8 blocks\n");
 676                        return -EINVAL;
 677                }
 678                ctx->block_size = CFB8_BLOCK_SIZE;
 679        } else if (mode & TDES_FLAGS_CFB16) {
 680                if (!IS_ALIGNED(req->nbytes, CFB16_BLOCK_SIZE)) {
 681                        pr_err("request size is not exact amount of CFB16 blocks\n");
 682                        return -EINVAL;
 683                }
 684                ctx->block_size = CFB16_BLOCK_SIZE;
 685        } else if (mode & TDES_FLAGS_CFB32) {
 686                if (!IS_ALIGNED(req->nbytes, CFB32_BLOCK_SIZE)) {
 687                        pr_err("request size is not exact amount of CFB32 blocks\n");
 688                        return -EINVAL;
 689                }
 690                ctx->block_size = CFB32_BLOCK_SIZE;
 691        } else {
 692                if (!IS_ALIGNED(req->nbytes, DES_BLOCK_SIZE)) {
 693                        pr_err("request size is not exact amount of DES blocks\n");
 694                        return -EINVAL;
 695                }
 696                ctx->block_size = DES_BLOCK_SIZE;
 697        }
 698
 699        rctx->mode = mode;
 700
 701        return atmel_tdes_handle_queue(ctx->dd, req);
 702}
 703
 704static bool atmel_tdes_filter(struct dma_chan *chan, void *slave)
 705{
 706        struct at_dma_slave     *sl = slave;
 707
 708        if (sl && sl->dma_dev == chan->device->dev) {
 709                chan->private = sl;
 710                return true;
 711        } else {
 712                return false;
 713        }
 714}
 715
 716static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd,
 717                        struct crypto_platform_data *pdata)
 718{
 719        int err = -ENOMEM;
 720        dma_cap_mask_t mask;
 721
 722        dma_cap_zero(mask);
 723        dma_cap_set(DMA_SLAVE, mask);
 724
 725        /* Try to grab 2 DMA channels */
 726        dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask,
 727                        atmel_tdes_filter, &pdata->dma_slave->rxdata, dd->dev, "tx");
 728        if (!dd->dma_lch_in.chan)
 729                goto err_dma_in;
 730
 731        dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV;
 732        dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base +
 733                TDES_IDATA1R;
 734        dd->dma_lch_in.dma_conf.src_maxburst = 1;
 735        dd->dma_lch_in.dma_conf.src_addr_width =
 736                DMA_SLAVE_BUSWIDTH_4_BYTES;
 737        dd->dma_lch_in.dma_conf.dst_maxburst = 1;
 738        dd->dma_lch_in.dma_conf.dst_addr_width =
 739                DMA_SLAVE_BUSWIDTH_4_BYTES;
 740        dd->dma_lch_in.dma_conf.device_fc = false;
 741
 742        dd->dma_lch_out.chan = dma_request_slave_channel_compat(mask,
 743                        atmel_tdes_filter, &pdata->dma_slave->txdata, dd->dev, "rx");
 744        if (!dd->dma_lch_out.chan)
 745                goto err_dma_out;
 746
 747        dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM;
 748        dd->dma_lch_out.dma_conf.src_addr = dd->phys_base +
 749                TDES_ODATA1R;
 750        dd->dma_lch_out.dma_conf.src_maxburst = 1;
 751        dd->dma_lch_out.dma_conf.src_addr_width =
 752                DMA_SLAVE_BUSWIDTH_4_BYTES;
 753        dd->dma_lch_out.dma_conf.dst_maxburst = 1;
 754        dd->dma_lch_out.dma_conf.dst_addr_width =
 755                DMA_SLAVE_BUSWIDTH_4_BYTES;
 756        dd->dma_lch_out.dma_conf.device_fc = false;
 757
 758        return 0;
 759
 760err_dma_out:
 761        dma_release_channel(dd->dma_lch_in.chan);
 762err_dma_in:
 763        dev_warn(dd->dev, "no DMA channel available\n");
 764        return err;
 765}
 766
 767static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd)
 768{
 769        dma_release_channel(dd->dma_lch_in.chan);
 770        dma_release_channel(dd->dma_lch_out.chan);
 771}
 772
 773static int atmel_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 774                           unsigned int keylen)
 775{
 776        u32 tmp[DES_EXPKEY_WORDS];
 777        int err;
 778        struct crypto_tfm *ctfm = crypto_ablkcipher_tfm(tfm);
 779
 780        struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 781
 782        if (keylen != DES_KEY_SIZE) {
 783                crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 784                return -EINVAL;
 785        }
 786
 787        err = des_ekey(tmp, key);
 788        if (err == 0 && (ctfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 789                ctfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
 790                return -EINVAL;
 791        }
 792
 793        memcpy(ctx->key, key, keylen);
 794        ctx->keylen = keylen;
 795
 796        return 0;
 797}
 798
 799static int atmel_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 800                           unsigned int keylen)
 801{
 802        struct atmel_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 803        const char *alg_name;
 804
 805        alg_name = crypto_tfm_alg_name(crypto_ablkcipher_tfm(tfm));
 806
 807        /*
 808         * HW bug in cfb 3-keys mode.
 809         */
 810        if (!ctx->dd->caps.has_cfb_3keys && strstr(alg_name, "cfb")
 811                        && (keylen != 2*DES_KEY_SIZE)) {
 812                crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 813                return -EINVAL;
 814        } else if ((keylen != 2*DES_KEY_SIZE) && (keylen != 3*DES_KEY_SIZE)) {
 815                crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 816                return -EINVAL;
 817        }
 818
 819        memcpy(ctx->key, key, keylen);
 820        ctx->keylen = keylen;
 821
 822        return 0;
 823}
 824
 825static int atmel_tdes_ecb_encrypt(struct ablkcipher_request *req)
 826{
 827        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT);
 828}
 829
 830static int atmel_tdes_ecb_decrypt(struct ablkcipher_request *req)
 831{
 832        return atmel_tdes_crypt(req, 0);
 833}
 834
 835static int atmel_tdes_cbc_encrypt(struct ablkcipher_request *req)
 836{
 837        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CBC);
 838}
 839
 840static int atmel_tdes_cbc_decrypt(struct ablkcipher_request *req)
 841{
 842        return atmel_tdes_crypt(req, TDES_FLAGS_CBC);
 843}
 844static int atmel_tdes_cfb_encrypt(struct ablkcipher_request *req)
 845{
 846        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB);
 847}
 848
 849static int atmel_tdes_cfb_decrypt(struct ablkcipher_request *req)
 850{
 851        return atmel_tdes_crypt(req, TDES_FLAGS_CFB);
 852}
 853
 854static int atmel_tdes_cfb8_encrypt(struct ablkcipher_request *req)
 855{
 856        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB |
 857                                                TDES_FLAGS_CFB8);
 858}
 859
 860static int atmel_tdes_cfb8_decrypt(struct ablkcipher_request *req)
 861{
 862        return atmel_tdes_crypt(req, TDES_FLAGS_CFB | TDES_FLAGS_CFB8);
 863}
 864
 865static int atmel_tdes_cfb16_encrypt(struct ablkcipher_request *req)
 866{
 867        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB |
 868                                                TDES_FLAGS_CFB16);
 869}
 870
 871static int atmel_tdes_cfb16_decrypt(struct ablkcipher_request *req)
 872{
 873        return atmel_tdes_crypt(req, TDES_FLAGS_CFB | TDES_FLAGS_CFB16);
 874}
 875
 876static int atmel_tdes_cfb32_encrypt(struct ablkcipher_request *req)
 877{
 878        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_CFB |
 879                                                TDES_FLAGS_CFB32);
 880}
 881
 882static int atmel_tdes_cfb32_decrypt(struct ablkcipher_request *req)
 883{
 884        return atmel_tdes_crypt(req, TDES_FLAGS_CFB | TDES_FLAGS_CFB32);
 885}
 886
 887static int atmel_tdes_ofb_encrypt(struct ablkcipher_request *req)
 888{
 889        return atmel_tdes_crypt(req, TDES_FLAGS_ENCRYPT | TDES_FLAGS_OFB);
 890}
 891
 892static int atmel_tdes_ofb_decrypt(struct ablkcipher_request *req)
 893{
 894        return atmel_tdes_crypt(req, TDES_FLAGS_OFB);
 895}
 896
 897static int atmel_tdes_cra_init(struct crypto_tfm *tfm)
 898{
 899        struct atmel_tdes_ctx *ctx = crypto_tfm_ctx(tfm);
 900        struct atmel_tdes_dev *dd;
 901
 902        tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_tdes_reqctx);
 903
 904        dd = atmel_tdes_find_dev(ctx);
 905        if (!dd)
 906                return -ENODEV;
 907
 908        return 0;
 909}
 910
 911static void atmel_tdes_cra_exit(struct crypto_tfm *tfm)
 912{
 913}
 914
 915static struct crypto_alg tdes_algs[] = {
 916{
 917        .cra_name               = "ecb(des)",
 918        .cra_driver_name        = "atmel-ecb-des",
 919        .cra_priority           = 100,
 920        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 921        .cra_blocksize          = DES_BLOCK_SIZE,
 922        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
 923        .cra_alignmask          = 0x7,
 924        .cra_type               = &crypto_ablkcipher_type,
 925        .cra_module             = THIS_MODULE,
 926        .cra_init               = atmel_tdes_cra_init,
 927        .cra_exit               = atmel_tdes_cra_exit,
 928        .cra_u.ablkcipher = {
 929                .min_keysize    = DES_KEY_SIZE,
 930                .max_keysize    = DES_KEY_SIZE,
 931                .setkey         = atmel_des_setkey,
 932                .encrypt        = atmel_tdes_ecb_encrypt,
 933                .decrypt        = atmel_tdes_ecb_decrypt,
 934        }
 935},
 936{
 937        .cra_name               = "cbc(des)",
 938        .cra_driver_name        = "atmel-cbc-des",
 939        .cra_priority           = 100,
 940        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 941        .cra_blocksize          = DES_BLOCK_SIZE,
 942        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
 943        .cra_alignmask          = 0x7,
 944        .cra_type               = &crypto_ablkcipher_type,
 945        .cra_module             = THIS_MODULE,
 946        .cra_init               = atmel_tdes_cra_init,
 947        .cra_exit               = atmel_tdes_cra_exit,
 948        .cra_u.ablkcipher = {
 949                .min_keysize    = DES_KEY_SIZE,
 950                .max_keysize    = DES_KEY_SIZE,
 951                .ivsize         = DES_BLOCK_SIZE,
 952                .setkey         = atmel_des_setkey,
 953                .encrypt        = atmel_tdes_cbc_encrypt,
 954                .decrypt        = atmel_tdes_cbc_decrypt,
 955        }
 956},
 957{
 958        .cra_name               = "cfb(des)",
 959        .cra_driver_name        = "atmel-cfb-des",
 960        .cra_priority           = 100,
 961        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 962        .cra_blocksize          = DES_BLOCK_SIZE,
 963        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
 964        .cra_alignmask          = 0x7,
 965        .cra_type               = &crypto_ablkcipher_type,
 966        .cra_module             = THIS_MODULE,
 967        .cra_init               = atmel_tdes_cra_init,
 968        .cra_exit               = atmel_tdes_cra_exit,
 969        .cra_u.ablkcipher = {
 970                .min_keysize    = DES_KEY_SIZE,
 971                .max_keysize    = DES_KEY_SIZE,
 972                .ivsize         = DES_BLOCK_SIZE,
 973                .setkey         = atmel_des_setkey,
 974                .encrypt        = atmel_tdes_cfb_encrypt,
 975                .decrypt        = atmel_tdes_cfb_decrypt,
 976        }
 977},
 978{
 979        .cra_name               = "cfb8(des)",
 980        .cra_driver_name        = "atmel-cfb8-des",
 981        .cra_priority           = 100,
 982        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 983        .cra_blocksize          = CFB8_BLOCK_SIZE,
 984        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
 985        .cra_alignmask          = 0,
 986        .cra_type               = &crypto_ablkcipher_type,
 987        .cra_module             = THIS_MODULE,
 988        .cra_init               = atmel_tdes_cra_init,
 989        .cra_exit               = atmel_tdes_cra_exit,
 990        .cra_u.ablkcipher = {
 991                .min_keysize    = DES_KEY_SIZE,
 992                .max_keysize    = DES_KEY_SIZE,
 993                .ivsize         = DES_BLOCK_SIZE,
 994                .setkey         = atmel_des_setkey,
 995                .encrypt        = atmel_tdes_cfb8_encrypt,
 996                .decrypt        = atmel_tdes_cfb8_decrypt,
 997        }
 998},
 999{
1000        .cra_name               = "cfb16(des)",
1001        .cra_driver_name        = "atmel-cfb16-des",
1002        .cra_priority           = 100,
1003        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1004        .cra_blocksize          = CFB16_BLOCK_SIZE,
1005        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1006        .cra_alignmask          = 0x1,
1007        .cra_type               = &crypto_ablkcipher_type,
1008        .cra_module             = THIS_MODULE,
1009        .cra_init               = atmel_tdes_cra_init,
1010        .cra_exit               = atmel_tdes_cra_exit,
1011        .cra_u.ablkcipher = {
1012                .min_keysize    = DES_KEY_SIZE,
1013                .max_keysize    = DES_KEY_SIZE,
1014                .ivsize         = DES_BLOCK_SIZE,
1015                .setkey         = atmel_des_setkey,
1016                .encrypt        = atmel_tdes_cfb16_encrypt,
1017                .decrypt        = atmel_tdes_cfb16_decrypt,
1018        }
1019},
1020{
1021        .cra_name               = "cfb32(des)",
1022        .cra_driver_name        = "atmel-cfb32-des",
1023        .cra_priority           = 100,
1024        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1025        .cra_blocksize          = CFB32_BLOCK_SIZE,
1026        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1027        .cra_alignmask          = 0x3,
1028        .cra_type               = &crypto_ablkcipher_type,
1029        .cra_module             = THIS_MODULE,
1030        .cra_init               = atmel_tdes_cra_init,
1031        .cra_exit               = atmel_tdes_cra_exit,
1032        .cra_u.ablkcipher = {
1033                .min_keysize    = DES_KEY_SIZE,
1034                .max_keysize    = DES_KEY_SIZE,
1035                .ivsize         = DES_BLOCK_SIZE,
1036                .setkey         = atmel_des_setkey,
1037                .encrypt        = atmel_tdes_cfb32_encrypt,
1038                .decrypt        = atmel_tdes_cfb32_decrypt,
1039        }
1040},
1041{
1042        .cra_name               = "ofb(des)",
1043        .cra_driver_name        = "atmel-ofb-des",
1044        .cra_priority           = 100,
1045        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1046        .cra_blocksize          = DES_BLOCK_SIZE,
1047        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1048        .cra_alignmask          = 0x7,
1049        .cra_type               = &crypto_ablkcipher_type,
1050        .cra_module             = THIS_MODULE,
1051        .cra_init               = atmel_tdes_cra_init,
1052        .cra_exit               = atmel_tdes_cra_exit,
1053        .cra_u.ablkcipher = {
1054                .min_keysize    = DES_KEY_SIZE,
1055                .max_keysize    = DES_KEY_SIZE,
1056                .ivsize         = DES_BLOCK_SIZE,
1057                .setkey         = atmel_des_setkey,
1058                .encrypt        = atmel_tdes_ofb_encrypt,
1059                .decrypt        = atmel_tdes_ofb_decrypt,
1060        }
1061},
1062{
1063        .cra_name               = "ecb(des3_ede)",
1064        .cra_driver_name        = "atmel-ecb-tdes",
1065        .cra_priority           = 100,
1066        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1067        .cra_blocksize          = DES_BLOCK_SIZE,
1068        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1069        .cra_alignmask          = 0x7,
1070        .cra_type               = &crypto_ablkcipher_type,
1071        .cra_module             = THIS_MODULE,
1072        .cra_init               = atmel_tdes_cra_init,
1073        .cra_exit               = atmel_tdes_cra_exit,
1074        .cra_u.ablkcipher = {
1075                .min_keysize    = 2 * DES_KEY_SIZE,
1076                .max_keysize    = 3 * DES_KEY_SIZE,
1077                .setkey         = atmel_tdes_setkey,
1078                .encrypt        = atmel_tdes_ecb_encrypt,
1079                .decrypt        = atmel_tdes_ecb_decrypt,
1080        }
1081},
1082{
1083        .cra_name               = "cbc(des3_ede)",
1084        .cra_driver_name        = "atmel-cbc-tdes",
1085        .cra_priority           = 100,
1086        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1087        .cra_blocksize          = DES_BLOCK_SIZE,
1088        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1089        .cra_alignmask          = 0x7,
1090        .cra_type               = &crypto_ablkcipher_type,
1091        .cra_module             = THIS_MODULE,
1092        .cra_init               = atmel_tdes_cra_init,
1093        .cra_exit               = atmel_tdes_cra_exit,
1094        .cra_u.ablkcipher = {
1095                .min_keysize    = 2*DES_KEY_SIZE,
1096                .max_keysize    = 3*DES_KEY_SIZE,
1097                .ivsize         = DES_BLOCK_SIZE,
1098                .setkey         = atmel_tdes_setkey,
1099                .encrypt        = atmel_tdes_cbc_encrypt,
1100                .decrypt        = atmel_tdes_cbc_decrypt,
1101        }
1102},
1103{
1104        .cra_name               = "cfb(des3_ede)",
1105        .cra_driver_name        = "atmel-cfb-tdes",
1106        .cra_priority           = 100,
1107        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1108        .cra_blocksize          = DES_BLOCK_SIZE,
1109        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1110        .cra_alignmask          = 0x7,
1111        .cra_type               = &crypto_ablkcipher_type,
1112        .cra_module             = THIS_MODULE,
1113        .cra_init               = atmel_tdes_cra_init,
1114        .cra_exit               = atmel_tdes_cra_exit,
1115        .cra_u.ablkcipher = {
1116                .min_keysize    = 2*DES_KEY_SIZE,
1117                .max_keysize    = 2*DES_KEY_SIZE,
1118                .ivsize         = DES_BLOCK_SIZE,
1119                .setkey         = atmel_tdes_setkey,
1120                .encrypt        = atmel_tdes_cfb_encrypt,
1121                .decrypt        = atmel_tdes_cfb_decrypt,
1122        }
1123},
1124{
1125        .cra_name               = "cfb8(des3_ede)",
1126        .cra_driver_name        = "atmel-cfb8-tdes",
1127        .cra_priority           = 100,
1128        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1129        .cra_blocksize          = CFB8_BLOCK_SIZE,
1130        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1131        .cra_alignmask          = 0,
1132        .cra_type               = &crypto_ablkcipher_type,
1133        .cra_module             = THIS_MODULE,
1134        .cra_init               = atmel_tdes_cra_init,
1135        .cra_exit               = atmel_tdes_cra_exit,
1136        .cra_u.ablkcipher = {
1137                .min_keysize    = 2*DES_KEY_SIZE,
1138                .max_keysize    = 2*DES_KEY_SIZE,
1139                .ivsize         = DES_BLOCK_SIZE,
1140                .setkey         = atmel_tdes_setkey,
1141                .encrypt        = atmel_tdes_cfb8_encrypt,
1142                .decrypt        = atmel_tdes_cfb8_decrypt,
1143        }
1144},
1145{
1146        .cra_name               = "cfb16(des3_ede)",
1147        .cra_driver_name        = "atmel-cfb16-tdes",
1148        .cra_priority           = 100,
1149        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1150        .cra_blocksize          = CFB16_BLOCK_SIZE,
1151        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1152        .cra_alignmask          = 0x1,
1153        .cra_type               = &crypto_ablkcipher_type,
1154        .cra_module             = THIS_MODULE,
1155        .cra_init               = atmel_tdes_cra_init,
1156        .cra_exit               = atmel_tdes_cra_exit,
1157        .cra_u.ablkcipher = {
1158                .min_keysize    = 2*DES_KEY_SIZE,
1159                .max_keysize    = 2*DES_KEY_SIZE,
1160                .ivsize         = DES_BLOCK_SIZE,
1161                .setkey         = atmel_tdes_setkey,
1162                .encrypt        = atmel_tdes_cfb16_encrypt,
1163                .decrypt        = atmel_tdes_cfb16_decrypt,
1164        }
1165},
1166{
1167        .cra_name               = "cfb32(des3_ede)",
1168        .cra_driver_name        = "atmel-cfb32-tdes",
1169        .cra_priority           = 100,
1170        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1171        .cra_blocksize          = CFB32_BLOCK_SIZE,
1172        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1173        .cra_alignmask          = 0x3,
1174        .cra_type               = &crypto_ablkcipher_type,
1175        .cra_module             = THIS_MODULE,
1176        .cra_init               = atmel_tdes_cra_init,
1177        .cra_exit               = atmel_tdes_cra_exit,
1178        .cra_u.ablkcipher = {
1179                .min_keysize    = 2*DES_KEY_SIZE,
1180                .max_keysize    = 2*DES_KEY_SIZE,
1181                .ivsize         = DES_BLOCK_SIZE,
1182                .setkey         = atmel_tdes_setkey,
1183                .encrypt        = atmel_tdes_cfb32_encrypt,
1184                .decrypt        = atmel_tdes_cfb32_decrypt,
1185        }
1186},
1187{
1188        .cra_name               = "ofb(des3_ede)",
1189        .cra_driver_name        = "atmel-ofb-tdes",
1190        .cra_priority           = 100,
1191        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1192        .cra_blocksize          = DES_BLOCK_SIZE,
1193        .cra_ctxsize            = sizeof(struct atmel_tdes_ctx),
1194        .cra_alignmask          = 0x7,
1195        .cra_type               = &crypto_ablkcipher_type,
1196        .cra_module             = THIS_MODULE,
1197        .cra_init               = atmel_tdes_cra_init,
1198        .cra_exit               = atmel_tdes_cra_exit,
1199        .cra_u.ablkcipher = {
1200                .min_keysize    = 2*DES_KEY_SIZE,
1201                .max_keysize    = 3*DES_KEY_SIZE,
1202                .ivsize         = DES_BLOCK_SIZE,
1203                .setkey         = atmel_tdes_setkey,
1204                .encrypt        = atmel_tdes_ofb_encrypt,
1205                .decrypt        = atmel_tdes_ofb_decrypt,
1206        }
1207},
1208};
1209
1210static void atmel_tdes_queue_task(unsigned long data)
1211{
1212        struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *)data;
1213
1214        atmel_tdes_handle_queue(dd, NULL);
1215}
1216
1217static void atmel_tdes_done_task(unsigned long data)
1218{
1219        struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *) data;
1220        int err;
1221
1222        if (!(dd->flags & TDES_FLAGS_DMA))
1223                err = atmel_tdes_crypt_pdc_stop(dd);
1224        else
1225                err = atmel_tdes_crypt_dma_stop(dd);
1226
1227        err = dd->err ? : err;
1228
1229        if (dd->total && !err) {
1230                if (dd->flags & TDES_FLAGS_FAST) {
1231                        dd->in_sg = sg_next(dd->in_sg);
1232                        dd->out_sg = sg_next(dd->out_sg);
1233                        if (!dd->in_sg || !dd->out_sg)
1234                                err = -EINVAL;
1235                }
1236                if (!err)
1237                        err = atmel_tdes_crypt_start(dd);
1238                if (!err)
1239                        return; /* DMA started. Not fininishing. */
1240        }
1241
1242        atmel_tdes_finish_req(dd, err);
1243        atmel_tdes_handle_queue(dd, NULL);
1244}
1245
1246static irqreturn_t atmel_tdes_irq(int irq, void *dev_id)
1247{
1248        struct atmel_tdes_dev *tdes_dd = dev_id;
1249        u32 reg;
1250
1251        reg = atmel_tdes_read(tdes_dd, TDES_ISR);
1252        if (reg & atmel_tdes_read(tdes_dd, TDES_IMR)) {
1253                atmel_tdes_write(tdes_dd, TDES_IDR, reg);
1254                if (TDES_FLAGS_BUSY & tdes_dd->flags)
1255                        tasklet_schedule(&tdes_dd->done_task);
1256                else
1257                        dev_warn(tdes_dd->dev, "TDES interrupt when no active requests.\n");
1258                return IRQ_HANDLED;
1259        }
1260
1261        return IRQ_NONE;
1262}
1263
1264static void atmel_tdes_unregister_algs(struct atmel_tdes_dev *dd)
1265{
1266        int i;
1267
1268        for (i = 0; i < ARRAY_SIZE(tdes_algs); i++)
1269                crypto_unregister_alg(&tdes_algs[i]);
1270}
1271
1272static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd)
1273{
1274        int err, i, j;
1275
1276        for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) {
1277                err = crypto_register_alg(&tdes_algs[i]);
1278                if (err)
1279                        goto err_tdes_algs;
1280        }
1281
1282        return 0;
1283
1284err_tdes_algs:
1285        for (j = 0; j < i; j++)
1286                crypto_unregister_alg(&tdes_algs[j]);
1287
1288        return err;
1289}
1290
1291static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd)
1292{
1293
1294        dd->caps.has_dma = 0;
1295        dd->caps.has_cfb_3keys = 0;
1296
1297        /* keep only major version number */
1298        switch (dd->hw_version & 0xf00) {
1299        case 0x700:
1300                dd->caps.has_dma = 1;
1301                dd->caps.has_cfb_3keys = 1;
1302                break;
1303        case 0x600:
1304                break;
1305        default:
1306                dev_warn(dd->dev,
1307                                "Unmanaged tdes version, set minimum capabilities\n");
1308                break;
1309        }
1310}
1311
1312#if defined(CONFIG_OF)
1313static const struct of_device_id atmel_tdes_dt_ids[] = {
1314        { .compatible = "atmel,at91sam9g46-tdes" },
1315        { /* sentinel */ }
1316};
1317MODULE_DEVICE_TABLE(of, atmel_tdes_dt_ids);
1318
1319static struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *pdev)
1320{
1321        struct device_node *np = pdev->dev.of_node;
1322        struct crypto_platform_data *pdata;
1323
1324        if (!np) {
1325                dev_err(&pdev->dev, "device node not found\n");
1326                return ERR_PTR(-EINVAL);
1327        }
1328
1329        pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1330        if (!pdata) {
1331                dev_err(&pdev->dev, "could not allocate memory for pdata\n");
1332                return ERR_PTR(-ENOMEM);
1333        }
1334
1335        pdata->dma_slave = devm_kzalloc(&pdev->dev,
1336                                        sizeof(*(pdata->dma_slave)),
1337                                        GFP_KERNEL);
1338        if (!pdata->dma_slave) {
1339                dev_err(&pdev->dev, "could not allocate memory for dma_slave\n");
1340                devm_kfree(&pdev->dev, pdata);
1341                return ERR_PTR(-ENOMEM);
1342        }
1343
1344        return pdata;
1345}
1346#else /* CONFIG_OF */
1347static inline struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *pdev)
1348{
1349        return ERR_PTR(-EINVAL);
1350}
1351#endif
1352
1353static int atmel_tdes_probe(struct platform_device *pdev)
1354{
1355        struct atmel_tdes_dev *tdes_dd;
1356        struct crypto_platform_data     *pdata;
1357        struct device *dev = &pdev->dev;
1358        struct resource *tdes_res;
1359        unsigned long tdes_phys_size;
1360        int err;
1361
1362        tdes_dd = kzalloc(sizeof(struct atmel_tdes_dev), GFP_KERNEL);
1363        if (tdes_dd == NULL) {
1364                dev_err(dev, "unable to alloc data struct.\n");
1365                err = -ENOMEM;
1366                goto tdes_dd_err;
1367        }
1368
1369        tdes_dd->dev = dev;
1370
1371        platform_set_drvdata(pdev, tdes_dd);
1372
1373        INIT_LIST_HEAD(&tdes_dd->list);
1374
1375        tasklet_init(&tdes_dd->done_task, atmel_tdes_done_task,
1376                                        (unsigned long)tdes_dd);
1377        tasklet_init(&tdes_dd->queue_task, atmel_tdes_queue_task,
1378                                        (unsigned long)tdes_dd);
1379
1380        crypto_init_queue(&tdes_dd->queue, ATMEL_TDES_QUEUE_LENGTH);
1381
1382        tdes_dd->irq = -1;
1383
1384        /* Get the base address */
1385        tdes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1386        if (!tdes_res) {
1387                dev_err(dev, "no MEM resource info\n");
1388                err = -ENODEV;
1389                goto res_err;
1390        }
1391        tdes_dd->phys_base = tdes_res->start;
1392        tdes_phys_size = resource_size(tdes_res);
1393
1394        /* Get the IRQ */
1395        tdes_dd->irq = platform_get_irq(pdev,  0);
1396        if (tdes_dd->irq < 0) {
1397                dev_err(dev, "no IRQ resource info\n");
1398                err = tdes_dd->irq;
1399                goto res_err;
1400        }
1401
1402        err = request_irq(tdes_dd->irq, atmel_tdes_irq, IRQF_SHARED,
1403                        "atmel-tdes", tdes_dd);
1404        if (err) {
1405                dev_err(dev, "unable to request tdes irq.\n");
1406                goto tdes_irq_err;
1407        }
1408
1409        /* Initializing the clock */
1410        tdes_dd->iclk = clk_get(&pdev->dev, "tdes_clk");
1411        if (IS_ERR(tdes_dd->iclk)) {
1412                dev_err(dev, "clock intialization failed.\n");
1413                err = PTR_ERR(tdes_dd->iclk);
1414                goto clk_err;
1415        }
1416
1417        tdes_dd->io_base = ioremap(tdes_dd->phys_base, tdes_phys_size);
1418        if (!tdes_dd->io_base) {
1419                dev_err(dev, "can't ioremap\n");
1420                err = -ENOMEM;
1421                goto tdes_io_err;
1422        }
1423
1424        atmel_tdes_hw_version_init(tdes_dd);
1425
1426        atmel_tdes_get_cap(tdes_dd);
1427
1428        err = atmel_tdes_buff_init(tdes_dd);
1429        if (err)
1430                goto err_tdes_buff;
1431
1432        if (tdes_dd->caps.has_dma) {
1433                pdata = pdev->dev.platform_data;
1434                if (!pdata) {
1435                        pdata = atmel_tdes_of_init(pdev);
1436                        if (IS_ERR(pdata)) {
1437                                dev_err(&pdev->dev, "platform data not available\n");
1438                                err = PTR_ERR(pdata);
1439                                goto err_pdata;
1440                        }
1441                }
1442                if (!pdata->dma_slave) {
1443                        err = -ENXIO;
1444                        goto err_pdata;
1445                }
1446                err = atmel_tdes_dma_init(tdes_dd, pdata);
1447                if (err)
1448                        goto err_tdes_dma;
1449
1450                dev_info(dev, "using %s, %s for DMA transfers\n",
1451                                dma_chan_name(tdes_dd->dma_lch_in.chan),
1452                                dma_chan_name(tdes_dd->dma_lch_out.chan));
1453        }
1454
1455        spin_lock(&atmel_tdes.lock);
1456        list_add_tail(&tdes_dd->list, &atmel_tdes.dev_list);
1457        spin_unlock(&atmel_tdes.lock);
1458
1459        err = atmel_tdes_register_algs(tdes_dd);
1460        if (err)
1461                goto err_algs;
1462
1463        dev_info(dev, "Atmel DES/TDES\n");
1464
1465        return 0;
1466
1467err_algs:
1468        spin_lock(&atmel_tdes.lock);
1469        list_del(&tdes_dd->list);
1470        spin_unlock(&atmel_tdes.lock);
1471        if (tdes_dd->caps.has_dma)
1472                atmel_tdes_dma_cleanup(tdes_dd);
1473err_tdes_dma:
1474err_pdata:
1475        atmel_tdes_buff_cleanup(tdes_dd);
1476err_tdes_buff:
1477        iounmap(tdes_dd->io_base);
1478tdes_io_err:
1479        clk_put(tdes_dd->iclk);
1480clk_err:
1481        free_irq(tdes_dd->irq, tdes_dd);
1482tdes_irq_err:
1483res_err:
1484        tasklet_kill(&tdes_dd->done_task);
1485        tasklet_kill(&tdes_dd->queue_task);
1486        kfree(tdes_dd);
1487        tdes_dd = NULL;
1488tdes_dd_err:
1489        dev_err(dev, "initialization failed.\n");
1490
1491        return err;
1492}
1493
1494static int atmel_tdes_remove(struct platform_device *pdev)
1495{
1496        static struct atmel_tdes_dev *tdes_dd;
1497
1498        tdes_dd = platform_get_drvdata(pdev);
1499        if (!tdes_dd)
1500                return -ENODEV;
1501        spin_lock(&atmel_tdes.lock);
1502        list_del(&tdes_dd->list);
1503        spin_unlock(&atmel_tdes.lock);
1504
1505        atmel_tdes_unregister_algs(tdes_dd);
1506
1507        tasklet_kill(&tdes_dd->done_task);
1508        tasklet_kill(&tdes_dd->queue_task);
1509
1510        if (tdes_dd->caps.has_dma)
1511                atmel_tdes_dma_cleanup(tdes_dd);
1512
1513        atmel_tdes_buff_cleanup(tdes_dd);
1514
1515        iounmap(tdes_dd->io_base);
1516
1517        clk_put(tdes_dd->iclk);
1518
1519        if (tdes_dd->irq >= 0)
1520                free_irq(tdes_dd->irq, tdes_dd);
1521
1522        kfree(tdes_dd);
1523        tdes_dd = NULL;
1524
1525        return 0;
1526}
1527
1528static struct platform_driver atmel_tdes_driver = {
1529        .probe          = atmel_tdes_probe,
1530        .remove         = atmel_tdes_remove,
1531        .driver         = {
1532                .name   = "atmel_tdes",
1533                .owner  = THIS_MODULE,
1534                .of_match_table = of_match_ptr(atmel_tdes_dt_ids),
1535        },
1536};
1537
1538module_platform_driver(atmel_tdes_driver);
1539
1540MODULE_DESCRIPTION("Atmel DES/TDES hw acceleration support.");
1541MODULE_LICENSE("GPL v2");
1542MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique");
1543
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.