linux/arch/cris/arch-v32/drivers/cryptocop.c
<<
>>
Prefs
   1/*
   2 * Stream co-processor driver for the ETRAX FS
   3 *
   4 *    Copyright (C) 2003-2007  Axis Communications AB
   5 */
   6
   7#include <linux/init.h>
   8#include <linux/sched.h>
   9#include <linux/module.h>
  10#include <linux/slab.h>
  11#include <linux/string.h>
  12#include <linux/fs.h>
  13#include <linux/mm.h>
  14#include <linux/spinlock.h>
  15#include <linux/stddef.h>
  16
  17#include <asm/uaccess.h>
  18#include <asm/io.h>
  19#include <linux/atomic.h>
  20
  21#include <linux/list.h>
  22#include <linux/interrupt.h>
  23
  24#include <asm/signal.h>
  25#include <asm/irq.h>
  26
  27#include <dma.h>
  28#include <hwregs/dma.h>
  29#include <hwregs/reg_map.h>
  30#include <hwregs/reg_rdwr.h>
  31#include <hwregs/intr_vect_defs.h>
  32
  33#include <hwregs/strcop.h>
  34#include <hwregs/strcop_defs.h>
  35#include <cryptocop.h>
  36
  37#ifdef CONFIG_ETRAXFS
  38#define IN_DMA 9
  39#define OUT_DMA 8
  40#define IN_DMA_INST regi_dma9
  41#define OUT_DMA_INST regi_dma8
  42#define DMA_IRQ DMA9_INTR_VECT
  43#else
  44#define IN_DMA 3
  45#define OUT_DMA 2
  46#define IN_DMA_INST regi_dma3
  47#define OUT_DMA_INST regi_dma2
  48#define DMA_IRQ DMA3_INTR_VECT
  49#endif
  50
  51#define DESCR_ALLOC_PAD  (31)
  52
  53struct cryptocop_dma_desc {
  54        char *free_buf; /* If non-null will be kfreed in free_cdesc() */
  55        dma_descr_data *dma_descr;
  56
  57        unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
  58
  59        unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
  60        struct cryptocop_dma_desc *next;
  61};
  62
  63
  64struct cryptocop_int_operation{
  65        void                        *alloc_ptr;
  66        cryptocop_session_id        sid;
  67
  68        dma_descr_context           ctx_out;
  69        dma_descr_context           ctx_in;
  70
  71        /* DMA descriptors allocated by driver. */
  72        struct cryptocop_dma_desc   *cdesc_out;
  73        struct cryptocop_dma_desc   *cdesc_in;
  74
  75        /* Strcop config to use. */
  76        cryptocop_3des_mode         tdes_mode;
  77        cryptocop_csum_type         csum_mode;
  78
  79        /* DMA descrs provided by consumer. */
  80        dma_descr_data              *ddesc_out;
  81        dma_descr_data              *ddesc_in;
  82};
  83
  84
  85struct cryptocop_tfrm_ctx {
  86        cryptocop_tfrm_id tid;
  87        unsigned int blocklength;
  88
  89        unsigned int start_ix;
  90
  91        struct cryptocop_tfrm_cfg *tcfg;
  92        struct cryptocop_transform_ctx *tctx;
  93
  94        unsigned char previous_src;
  95        unsigned char current_src;
  96
  97        /* Values to use in metadata out. */
  98        unsigned char hash_conf;
  99        unsigned char hash_mode;
 100        unsigned char ciph_conf;
 101        unsigned char cbcmode;
 102        unsigned char decrypt;
 103
 104        unsigned int requires_padding:1;
 105        unsigned int strict_block_length:1;
 106        unsigned int active:1;
 107        unsigned int done:1;
 108        size_t consumed;
 109        size_t produced;
 110
 111        /* Pad (input) descriptors to put in the DMA out list when the transform
 112         * output is put on the DMA in list. */
 113        struct cryptocop_dma_desc *pad_descs;
 114
 115        struct cryptocop_tfrm_ctx *prev_src;
 116        struct cryptocop_tfrm_ctx *curr_src;
 117
 118        /* Mapping to HW. */
 119        unsigned char unit_no;
 120};
 121
 122
 123struct cryptocop_private{
 124        cryptocop_session_id sid;
 125        struct cryptocop_private *next;
 126};
 127
 128/* Session list. */
 129
 130struct cryptocop_transform_ctx{
 131        struct cryptocop_transform_init init;
 132        unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
 133        unsigned int dec_key_set:1;
 134
 135        struct cryptocop_transform_ctx *next;
 136};
 137
 138
 139struct cryptocop_session{
 140        cryptocop_session_id sid;
 141
 142        struct cryptocop_transform_ctx *tfrm_ctx;
 143
 144        struct cryptocop_session *next;
 145};
 146
 147/* Priority levels for jobs sent to the cryptocop.  Checksum operations from
 148   kernel have highest priority since TCPIP stack processing must not
 149   be a bottleneck. */
 150typedef enum {
 151        cryptocop_prio_kernel_csum = 0,
 152        cryptocop_prio_kernel = 1,
 153        cryptocop_prio_user = 2,
 154        cryptocop_prio_no_prios = 3
 155} cryptocop_queue_priority;
 156
 157struct cryptocop_prio_queue{
 158        struct list_head jobs;
 159        cryptocop_queue_priority prio;
 160};
 161
 162struct cryptocop_prio_job{
 163        struct list_head node;
 164        cryptocop_queue_priority prio;
 165
 166        struct cryptocop_operation *oper;
 167        struct cryptocop_int_operation *iop;
 168};
 169
 170struct ioctl_job_cb_ctx {
 171        unsigned int processed:1;
 172};
 173
 174
 175static struct cryptocop_session *cryptocop_sessions = NULL;
 176spinlock_t cryptocop_sessions_lock;
 177
 178/* Next Session ID to assign. */
 179static cryptocop_session_id next_sid = 1;
 180
 181/* Pad for checksum. */
 182static const char csum_zero_pad[1] = {0x00};
 183
 184/* Trash buffer for mem2mem operations. */
 185#define MEM2MEM_DISCARD_BUF_LENGTH  (512)
 186static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
 187
 188/* Descriptor pool. */
 189/* FIXME Tweak this value. */
 190#define CRYPTOCOP_DESCRIPTOR_POOL_SIZE   (100)
 191static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
 192static struct cryptocop_dma_desc *descr_pool_free_list;
 193static int descr_pool_no_free;
 194static spinlock_t descr_pool_lock;
 195
 196/* Lock to stop cryptocop to start processing of a new operation. The holder
 197   of this lock MUST call cryptocop_start_job() after it is unlocked. */
 198spinlock_t cryptocop_process_lock;
 199
 200static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
 201static spinlock_t cryptocop_job_queue_lock;
 202static struct cryptocop_prio_job *cryptocop_running_job = NULL;
 203static spinlock_t running_job_lock;
 204
 205/* The interrupt handler appends completed jobs to this list. The scehduled
 206 * tasklet removes them upon sending the response to the crypto consumer. */
 207static struct list_head cryptocop_completed_jobs;
 208static spinlock_t cryptocop_completed_jobs_lock;
 209
 210DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
 211
 212
 213/** Local functions. **/
 214
 215static int cryptocop_open(struct inode *, struct file *);
 216
 217static int cryptocop_release(struct inode *, struct file *);
 218
 219static long cryptocop_ioctl(struct file *file,
 220                           unsigned int cmd, unsigned long arg);
 221
 222static void cryptocop_start_job(void);
 223
 224static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
 225static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
 226
 227static int cryptocop_job_queue_init(void);
 228static void cryptocop_job_queue_close(void);
 229
 230static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 231
 232static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 233
 234static int transform_ok(struct cryptocop_transform_init *tinit);
 235
 236static struct cryptocop_session *get_session(cryptocop_session_id sid);
 237
 238static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
 239
 240static void delete_internal_operation(struct cryptocop_int_operation *iop);
 241
 242static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength);
 243
 244static int init_stream_coprocessor(void);
 245
 246static void __exit exit_stream_coprocessor(void);
 247
 248/*#define LDEBUG*/
 249#ifdef LDEBUG
 250#define DEBUG(s) s
 251#define DEBUG_API(s) s
 252static void print_cryptocop_operation(struct cryptocop_operation *cop);
 253static void print_dma_descriptors(struct cryptocop_int_operation *iop);
 254static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
 255static void print_lock_status(void);
 256static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
 257#define assert(s) do{if (!(s)) panic(#s);} while(0);
 258#else
 259#define DEBUG(s)
 260#define DEBUG_API(s)
 261#define assert(s)
 262#endif
 263
 264
 265/* Transform constants. */
 266#define DES_BLOCK_LENGTH   (8)
 267#define AES_BLOCK_LENGTH   (16)
 268#define MD5_BLOCK_LENGTH   (64)
 269#define SHA1_BLOCK_LENGTH  (64)
 270#define CSUM_BLOCK_LENGTH  (2)
 271#define MD5_STATE_LENGTH   (16)
 272#define SHA1_STATE_LENGTH  (20)
 273
 274/* The device number. */
 275#define CRYPTOCOP_MAJOR    (254)
 276#define CRYPTOCOP_MINOR    (0)
 277
 278
 279
 280const struct file_operations cryptocop_fops = {
 281        .owner          = THIS_MODULE,
 282        .open           = cryptocop_open,
 283        .release        = cryptocop_release,
 284        .unlocked_ioctl = cryptocop_ioctl,
 285        .llseek         = noop_llseek,
 286};
 287
 288
 289static void free_cdesc(struct cryptocop_dma_desc *cdesc)
 290{
 291        DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
 292        kfree(cdesc->free_buf);
 293
 294        if (cdesc->from_pool) {
 295                unsigned long int flags;
 296                spin_lock_irqsave(&descr_pool_lock, flags);
 297                cdesc->next = descr_pool_free_list;
 298                descr_pool_free_list = cdesc;
 299                ++descr_pool_no_free;
 300                spin_unlock_irqrestore(&descr_pool_lock, flags);
 301        } else {
 302                kfree(cdesc);
 303        }
 304}
 305
 306
 307static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
 308{
 309        int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
 310        struct cryptocop_dma_desc *cdesc;
 311
 312        if (use_pool) {
 313                unsigned long int flags;
 314                spin_lock_irqsave(&descr_pool_lock, flags);
 315                if (!descr_pool_free_list) {
 316                        spin_unlock_irqrestore(&descr_pool_lock, flags);
 317                        DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
 318                        return NULL;
 319                }
 320                cdesc = descr_pool_free_list;
 321                descr_pool_free_list = descr_pool_free_list->next;
 322                --descr_pool_no_free;
 323                spin_unlock_irqrestore(&descr_pool_lock, flags);
 324                cdesc->from_pool = 1;
 325        } else {
 326                cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
 327                if (!cdesc) {
 328                        DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
 329                        return NULL;
 330                }
 331                cdesc->from_pool = 0;
 332        }
 333        cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
 334
 335        cdesc->next = NULL;
 336
 337        cdesc->free_buf = NULL;
 338        cdesc->dma_descr->out_eop = 0;
 339        cdesc->dma_descr->in_eop = 0;
 340        cdesc->dma_descr->intr = 0;
 341        cdesc->dma_descr->eol = 0;
 342        cdesc->dma_descr->wait = 0;
 343        cdesc->dma_descr->buf = NULL;
 344        cdesc->dma_descr->after = NULL;
 345
 346        DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
 347        return cdesc;
 348}
 349
 350
 351static void setup_descr_chain(struct cryptocop_dma_desc *cd)
 352{
 353        DEBUG(printk("setup_descr_chain: entering\n"));
 354        while (cd) {
 355                if (cd->next) {
 356                        cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
 357                } else {
 358                        cd->dma_descr->next = NULL;
 359                }
 360                cd = cd->next;
 361        }
 362        DEBUG(printk("setup_descr_chain: exit\n"));
 363}
 364
 365
 366/* Create a pad descriptor for the transform.
 367 * Return -1 for error, 0 if pad created. */
 368static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
 369{
 370        struct cryptocop_dma_desc        *cdesc = NULL;
 371        int                              error = 0;
 372        struct strcop_meta_out           mo = {
 373                .ciphsel = src_none,
 374                .hashsel = src_none,
 375                .csumsel = src_none
 376        };
 377        char                             *pad;
 378        size_t                           plen;
 379
 380        DEBUG(printk("create_pad_descriptor: start.\n"));
 381        /* Setup pad descriptor. */
 382
 383        DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
 384        cdesc = alloc_cdesc(alloc_flag);
 385        if (!cdesc){
 386                DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
 387                goto error_cleanup;
 388        }
 389        switch (tc->unit_no) {
 390        case src_md5:
 391                error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
 392                if (error){
 393                        DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
 394                        goto error_cleanup;
 395                }
 396                cdesc->free_buf = pad;
 397                mo.hashsel = src_dma;
 398                mo.hashconf = tc->hash_conf;
 399                mo.hashmode = tc->hash_mode;
 400                break;
 401        case src_sha1:
 402                error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
 403                if (error){
 404                        DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
 405                        goto error_cleanup;
 406                }
 407                cdesc->free_buf = pad;
 408                mo.hashsel = src_dma;
 409                mo.hashconf = tc->hash_conf;
 410                mo.hashmode = tc->hash_mode;
 411                break;
 412        case src_csum:
 413                if (tc->consumed % tc->blocklength){
 414                        pad = (char*)csum_zero_pad;
 415                        plen = 1;
 416                } else {
 417                        pad = (char*)cdesc; /* Use any pointer. */
 418                        plen = 0;
 419                }
 420                mo.csumsel = src_dma;
 421                break;
 422        }
 423        cdesc->dma_descr->wait = 1;
 424        cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
 425        cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
 426        cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
 427
 428        cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 429        *pad_desc = cdesc;
 430
 431        return 0;
 432
 433 error_cleanup:
 434        if (cdesc) free_cdesc(cdesc);
 435        return -1;
 436}
 437
 438
 439static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
 440{
 441        struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
 442        struct strcop_meta_out     mo = {0};
 443
 444        DEBUG(printk("setup_key_dl_desc\n"));
 445
 446        if (!key_desc) {
 447                DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
 448                return -ENOMEM;
 449        }
 450
 451        /* Download key. */
 452        if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
 453                /* Precook the AES decrypt key. */
 454                if (!tc->tctx->dec_key_set){
 455                        get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
 456                        tc->tctx->dec_key_set = 1;
 457                }
 458                key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
 459                key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 460        } else {
 461                key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
 462                key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 463        }
 464        /* Setup metadata. */
 465        mo.dlkey = 1;
 466        switch (tc->tctx->init.keylen) {
 467        case 64:
 468                mo.decrypt = 0;
 469                mo.hashmode = 0;
 470                break;
 471        case 128:
 472                mo.decrypt = 0;
 473                mo.hashmode = 1;
 474                break;
 475        case 192:
 476                mo.decrypt = 1;
 477                mo.hashmode = 0;
 478                break;
 479        case 256:
 480                mo.decrypt = 1;
 481                mo.hashmode = 1;
 482                break;
 483        default:
 484                break;
 485        }
 486        mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
 487        key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 488
 489        key_desc->dma_descr->out_eop = 1;
 490        key_desc->dma_descr->wait = 1;
 491        key_desc->dma_descr->intr = 0;
 492
 493        *kd = key_desc;
 494        return 0;
 495}
 496
 497static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 498{
 499        struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
 500        struct strcop_meta_out     mo = {0};
 501
 502        DEBUG(printk("setup_cipher_iv_desc\n"));
 503
 504        if (!iv_desc) {
 505                DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
 506                return -ENOMEM;
 507        }
 508        /* Download IV. */
 509        iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
 510        iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
 511
 512        /* Setup metadata. */
 513        mo.hashsel = mo.csumsel = src_none;
 514        mo.ciphsel = src_dma;
 515        mo.ciphconf = tc->ciph_conf;
 516        mo.cbcmode = tc->cbcmode;
 517
 518        iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 519
 520        iv_desc->dma_descr->out_eop = 0;
 521        iv_desc->dma_descr->wait = 1;
 522        iv_desc->dma_descr->intr = 0;
 523
 524        *id = iv_desc;
 525        return 0;
 526}
 527
 528/* Map the ouput length of the transform to operation output starting on the inject index. */
 529static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 530{
 531        int                        err = 0;
 532        struct cryptocop_dma_desc  head = {0};
 533        struct cryptocop_dma_desc  *outdesc = &head;
 534        size_t                     iov_offset = 0;
 535        size_t                     out_ix = 0;
 536        int                        outiov_ix = 0;
 537        struct strcop_meta_in      mi = {0};
 538
 539        size_t                     out_length = tc->produced;
 540        int                        rem_length;
 541        int                        dlength;
 542
 543        assert(out_length != 0);
 544        if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
 545                DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 546                return -EINVAL;
 547        }
 548        /* Traverse the out iovec until the result inject index is reached. */
 549        while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
 550                out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
 551                outiov_ix++;
 552        }
 553        if (outiov_ix >= operation->tfrm_op.outcount){
 554                DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 555                return -EINVAL;
 556        }
 557        iov_offset = tc->tcfg->inject_ix - out_ix;
 558        mi.dmasel = tc->unit_no;
 559
 560        /* Setup the output descriptors. */
 561        while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
 562                outdesc->next = alloc_cdesc(alloc_flag);
 563                if (!outdesc->next) {
 564                        DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
 565                        err = -ENOMEM;
 566                        goto error_cleanup;
 567                }
 568                outdesc = outdesc->next;
 569                rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
 570                dlength = (out_length < rem_length) ? out_length : rem_length;
 571
 572                DEBUG(printk("create_input_descriptors:\n"
 573                             "outiov_ix=%d, rem_length=%d, dlength=%d\n"
 574                             "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
 575                             "outcount=%d, outiov_ix=%d\n",
 576                             outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
 577
 578                outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
 579                outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
 580                outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 581
 582                out_length -= dlength;
 583                iov_offset += dlength;
 584                if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
 585                        iov_offset = 0;
 586                        ++outiov_ix;
 587                }
 588        }
 589        if (out_length > 0){
 590                DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
 591                err = -EINVAL;
 592                goto error_cleanup;
 593        }
 594        /* Set sync in last descriptor. */
 595        mi.sync = 1;
 596        outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 597
 598        *id = head.next;
 599        return 0;
 600
 601 error_cleanup:
 602        while (head.next) {
 603                outdesc = head.next->next;
 604                free_cdesc(head.next);
 605                head.next = outdesc;
 606        }
 607        return err;
 608}
 609
 610
 611static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
 612{
 613        while (desc_len != 0) {
 614                struct cryptocop_dma_desc  *cdesc;
 615                int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
 616                int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;
 617
 618                cdesc = alloc_cdesc(alloc_flag);
 619                if (!cdesc) {
 620                        DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
 621                        return -ENOMEM;
 622                }
 623                (*current_out_cdesc)->next = cdesc;
 624                (*current_out_cdesc) = cdesc;
 625
 626                cdesc->free_buf = NULL;
 627
 628                cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
 629                cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
 630
 631                assert(desc_len >= dlength);
 632                desc_len -= dlength;
 633                *iniov_offset += dlength;
 634                if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
 635                        *iniov_offset = 0;
 636                        ++(*iniov_ix);
 637                        if (*iniov_ix > operation->tfrm_op.incount) {
 638                                DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
 639                                return  -EINVAL;
 640                        }
 641                }
 642                cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
 643        } /* while (desc_len != 0) */
 644        /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
 645        (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
 646
 647        return 0;
 648}
 649
 650
 651static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
 652{
 653        DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
 654        if (tc->tcfg) {
 655                int                        failed = 0;
 656                struct cryptocop_dma_desc  *idescs = NULL;
 657                DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
 658                if (tc->pad_descs) {
 659                        DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
 660                        while (tc->pad_descs) {
 661                                DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
 662                                (*current_out_cdesc)->next = tc->pad_descs;
 663                                tc->pad_descs = tc->pad_descs->next;
 664                                (*current_out_cdesc) = (*current_out_cdesc)->next;
 665                        }
 666                }
 667
 668                /* Setup and append output descriptors to DMA in list. */
 669                if (tc->unit_no == src_dma){
 670                        /* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
 671                        struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
 672                        unsigned int start_ix = tc->start_ix;
 673                        while (start_ix){
 674                                unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
 675                                (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
 676                                if (!(*current_in_cdesc)->next){
 677                                        DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
 678                                        return -ENOMEM;
 679                                }
 680                                (*current_in_cdesc) = (*current_in_cdesc)->next;
 681                                (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
 682                                (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
 683                                (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 684                                start_ix -= desclen;
 685                        }
 686                        mi.sync = 1;
 687                        (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 688                }
 689
 690                failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
 691                if (failed){
 692                        DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
 693                        return failed;
 694                }
 695                DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
 696                while (idescs) {
 697                        DEBUG(printk("append descriptor 0x%p\n", idescs));
 698                        (*current_in_cdesc)->next = idescs;
 699                        idescs = idescs->next;
 700                        (*current_in_cdesc) = (*current_in_cdesc)->next;
 701                }
 702        }
 703        return 0;
 704}
 705
 706
 707
 708static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
 709{
 710        struct cryptocop_session *sess;
 711        struct cryptocop_transform_ctx *tctx;
 712
 713        struct cryptocop_tfrm_ctx digest_ctx = {
 714                .previous_src = src_none,
 715                .current_src = src_none,
 716                .start_ix = 0,
 717                .requires_padding = 1,
 718                .strict_block_length = 0,
 719                .hash_conf = 0,
 720                .hash_mode = 0,
 721                .ciph_conf = 0,
 722                .cbcmode = 0,
 723                .decrypt = 0,
 724                .consumed = 0,
 725                .produced = 0,
 726                .pad_descs = NULL,
 727                .active = 0,
 728                .done = 0,
 729                .prev_src = NULL,
 730                .curr_src = NULL,
 731                .tcfg = NULL};
 732        struct cryptocop_tfrm_ctx cipher_ctx = {
 733                .previous_src = src_none,
 734                .current_src = src_none,
 735                .start_ix = 0,
 736                .requires_padding = 0,
 737                .strict_block_length = 1,
 738                .hash_conf = 0,
 739                .hash_mode = 0,
 740                .ciph_conf = 0,
 741                .cbcmode = 0,
 742                .decrypt = 0,
 743                .consumed = 0,
 744                .produced = 0,
 745                .pad_descs = NULL,
 746                .active = 0,
 747                .done = 0,
 748                .prev_src = NULL,
 749                .curr_src = NULL,
 750                .tcfg = NULL};
 751        struct cryptocop_tfrm_ctx csum_ctx = {
 752                .previous_src = src_none,
 753                .current_src = src_none,
 754                .start_ix = 0,
 755                .blocklength = 2,
 756                .requires_padding = 1,
 757                .strict_block_length = 0,
 758                .hash_conf = 0,
 759                .hash_mode = 0,
 760                .ciph_conf = 0,
 761                .cbcmode = 0,
 762                .decrypt = 0,
 763                .consumed = 0,
 764                .produced = 0,
 765                .pad_descs = NULL,
 766                .active = 0,
 767                .done = 0,
 768                .tcfg = NULL,
 769                .prev_src = NULL,
 770                .curr_src = NULL,
 771                .unit_no = src_csum};
 772        struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
 773
 774        unsigned int indata_ix = 0;
 775
 776        /* iovec accounting. */
 777        int iniov_ix = 0;
 778        int iniov_offset = 0;
 779
 780        /* Operation descriptor cfg traversal pointer. */
 781        struct cryptocop_desc *odsc;
 782
 783        int failed = 0;
 784        /* List heads for allocated descriptors. */
 785        struct cryptocop_dma_desc out_cdesc_head = {0};
 786        struct cryptocop_dma_desc in_cdesc_head = {0};
 787
 788        struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
 789        struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
 790
 791        struct cryptocop_tfrm_ctx *output_tc = NULL;
 792        void                      *iop_alloc_ptr;
 793
 794        assert(operation != NULL);
 795        assert(int_op != NULL);
 796
 797        DEBUG(printk("cryptocop_setup_dma_list: start\n"));
 798        DEBUG(print_cryptocop_operation(operation));
 799
 800        sess = get_session(operation->sid);
 801        if (!sess) {
 802                DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
 803                failed = -EINVAL;
 804                goto error_cleanup;
 805        }
 806        iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
 807        if (!iop_alloc_ptr) {
 808                DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
 809                failed = -ENOMEM;
 810                goto error_cleanup;
 811        }
 812        (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
 813        DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
 814        (*int_op)->alloc_ptr = iop_alloc_ptr;
 815        DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
 816
 817        (*int_op)->sid = operation->sid;
 818        (*int_op)->cdesc_out = NULL;
 819        (*int_op)->cdesc_in = NULL;
 820        (*int_op)->tdes_mode = cryptocop_3des_ede;
 821        (*int_op)->csum_mode = cryptocop_csum_le;
 822        (*int_op)->ddesc_out = NULL;
 823        (*int_op)->ddesc_in = NULL;
 824
 825        /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
 826        if (!tcfg) {
 827                DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
 828                failed = -EINVAL;
 829                goto error_cleanup;
 830        }
 831        while (tcfg) {
 832                tctx = get_transform_ctx(sess, tcfg->tid);
 833                if (!tctx) {
 834                        DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
 835                        failed = -EINVAL;
 836                        goto error_cleanup;
 837                }
 838                if (tcfg->inject_ix > operation->tfrm_op.outlen){
 839                        DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
 840                        failed = -EINVAL;
 841                        goto error_cleanup;
 842                }
 843                switch (tctx->init.alg){
 844                case cryptocop_alg_mem2mem:
 845                        if (cipher_ctx.tcfg != NULL){
 846                                DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 847                                failed = -EINVAL;
 848                                goto error_cleanup;
 849                        }
 850                        /* mem2mem is handled as a NULL cipher. */
 851                        cipher_ctx.cbcmode = 0;
 852                        cipher_ctx.decrypt = 0;
 853                        cipher_ctx.blocklength = 1;
 854                        cipher_ctx.ciph_conf = 0;
 855                        cipher_ctx.unit_no = src_dma;
 856                        cipher_ctx.tcfg = tcfg;
 857                        cipher_ctx.tctx = tctx;
 858                        break;
 859                case cryptocop_alg_des:
 860                case cryptocop_alg_3des:
 861                case cryptocop_alg_aes:
 862                        /* cipher */
 863                        if (cipher_ctx.tcfg != NULL){
 864                                DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 865                                failed = -EINVAL;
 866                                goto error_cleanup;
 867                        }
 868                        cipher_ctx.tcfg = tcfg;
 869                        cipher_ctx.tctx = tctx;
 870                        if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
 871                                cipher_ctx.decrypt = 1;
 872                        }
 873                        switch (tctx->init.cipher_mode) {
 874                        case cryptocop_cipher_mode_ecb:
 875                                cipher_ctx.cbcmode = 0;
 876                                break;
 877                        case cryptocop_cipher_mode_cbc:
 878                                cipher_ctx.cbcmode = 1;
 879                                break;
 880                        default:
 881                                DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
 882                                failed = -EINVAL;
 883                                goto error_cleanup;
 884                        }
 885                        DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
 886                        switch (tctx->init.alg){
 887                        case cryptocop_alg_des:
 888                                cipher_ctx.ciph_conf = 0;
 889                                cipher_ctx.unit_no = src_des;
 890                                cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 891                                break;
 892                        case cryptocop_alg_3des:
 893                                cipher_ctx.ciph_conf = 1;
 894                                cipher_ctx.unit_no = src_des;
 895                                cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 896                                break;
 897                        case cryptocop_alg_aes:
 898                                cipher_ctx.ciph_conf = 2;
 899                                cipher_ctx.unit_no = src_aes;
 900                                cipher_ctx.blocklength = AES_BLOCK_LENGTH;
 901                                break;
 902                        default:
 903                                panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
 904                        }
 905                        (*int_op)->tdes_mode = tctx->init.tdes_mode;
 906                        break;
 907                case cryptocop_alg_md5:
 908                case cryptocop_alg_sha1:
 909                        /* digest */
 910                        if (digest_ctx.tcfg != NULL){
 911                                DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
 912                                failed = -EINVAL;
 913                                goto error_cleanup;
 914                        }
 915                        digest_ctx.tcfg = tcfg;
 916                        digest_ctx.tctx = tctx;
 917                        digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
 918                        switch (tctx->init.alg){
 919                        case cryptocop_alg_md5:
 920                                digest_ctx.blocklength = MD5_BLOCK_LENGTH;
 921                                digest_ctx.unit_no = src_md5;
 922                                digest_ctx.hash_conf = 1; /* 1 => MD-5 */
 923                                break;
 924                        case cryptocop_alg_sha1:
 925                                digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
 926                                digest_ctx.unit_no = src_sha1;
 927                                digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
 928                                break;
 929                        default:
 930                                panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
 931                        }
 932                        break;
 933                case cryptocop_alg_csum:
 934                        /* digest */
 935                        if (csum_ctx.tcfg != NULL){
 936                                DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
 937                                failed = -EINVAL;
 938                                goto error_cleanup;
 939                        }
 940                        (*int_op)->csum_mode = tctx->init.csum_mode;
 941                        csum_ctx.tcfg = tcfg;
 942                        csum_ctx.tctx = tctx;
 943                        break;
 944                default:
 945                        /* no algorithm. */
 946                        DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
 947                        failed = -EINVAL;
 948                        goto error_cleanup;
 949                }
 950                tcfg = tcfg->next;
 951        }
 952        /* Download key if a cipher is used. */
 953        if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
 954                struct cryptocop_dma_desc  *key_desc = NULL;
 955
 956                failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
 957                if (failed) {
 958                        DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
 959                        goto error_cleanup;
 960                }
 961                current_out_cdesc->next = key_desc;
 962                current_out_cdesc = key_desc;
 963                indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
 964
 965                /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
 966                if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
 967                        struct cryptocop_dma_desc  *iv_desc = NULL;
 968
 969                        DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
 970
 971                        failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
 972                        if (failed) {
 973                                DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
 974                                goto error_cleanup;
 975                        }
 976                        current_out_cdesc->next = iv_desc;
 977                        current_out_cdesc = iv_desc;
 978                        indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
 979                }
 980        }
 981
 982        /* Process descriptors. */
 983        odsc = operation->tfrm_op.desc;
 984        while (odsc) {
 985                struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
 986                struct strcop_meta_out      meta_out = {0};
 987                size_t                      desc_len = odsc->length;
 988                int                         active_count, eop_needed_count;
 989
 990                output_tc = NULL;
 991
 992                DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
 993
 994                while (dcfg) {
 995                        struct cryptocop_tfrm_ctx  *tc = NULL;
 996
 997                        DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
 998                        /* Get the local context for the transform and mark it as the output unit if it produces output. */
 999                        if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000                                tc = &digest_ctx;
1001                        } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002                                tc = &cipher_ctx;
1003                        } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004                                tc = &csum_ctx;
1005                        }
1006                        if (!tc) {
1007                                DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008                                failed = -EINVAL;
1009                                goto error_cleanup;
1010                        }
1011                        if (tc->done) {
1012                                DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013                                failed = -EINVAL;
1014                                goto error_cleanup;
1015                        }
1016                        if (!tc->active) {
1017                                tc->start_ix = indata_ix;
1018                                tc->active = 1;
1019                        }
1020
1021                        tc->previous_src = tc->current_src;
1022                        tc->prev_src = tc->curr_src;
1023                        /* Map source unit id to DMA source config. */
1024                        switch (dcfg->src){
1025                        case cryptocop_source_dma:
1026                                tc->current_src = src_dma;
1027                                break;
1028                        case cryptocop_source_des:
1029                                tc->current_src = src_des;
1030                                break;
1031                        case cryptocop_source_3des:
1032                                tc->current_src = src_des;
1033                                break;
1034                        case cryptocop_source_aes:
1035                                tc->current_src = src_aes;
1036                                break;
1037                        case cryptocop_source_md5:
1038                        case cryptocop_source_sha1:
1039                        case cryptocop_source_csum:
1040                        case cryptocop_source_none:
1041                        default:
1042                                /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043                                 */
1044                                DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045                                failed = -EINVAL;
1046                                goto error_cleanup;
1047                        }
1048                        if (tc->current_src != src_dma) {
1049                                /* Find the unit we are sourcing from. */
1050                                if (digest_ctx.unit_no == tc->current_src){
1051                                        tc->curr_src = &digest_ctx;
1052                                } else if (cipher_ctx.unit_no == tc->current_src){
1053                                        tc->curr_src = &cipher_ctx;
1054                                } else if (csum_ctx.unit_no == tc->current_src){
1055                                        tc->curr_src = &csum_ctx;
1056                                }
1057                                if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058                                        DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059                                        failed = -EINVAL;
1060                                        goto error_cleanup;
1061                                }
1062                        } else {
1063                                tc->curr_src = NULL;
1064                        }
1065
1066                        /* Detect source switch. */
1067                        DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068                        if (tc->active && (tc->current_src != tc->previous_src)) {
1069                                /* Only allow source switch when both the old source unit and the new one have
1070                                 * no pending data to process (i.e. the consumed length must be a multiple of the
1071                                 * transform blocklength). */
1072                                /* Note: if the src == NULL we are actually sourcing from DMA out. */
1073                                if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074                                    ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075                                {
1076                                        DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077                                        failed = -EINVAL;
1078                                        goto error_cleanup;
1079                                }
1080                        }
1081                        /* Detect unit deactivation. */
1082                        if (dcfg->last) {
1083                                /* Length check of this is handled below. */
1084                                tc->done = 1;
1085                        }
1086                        dcfg = dcfg->next;
1087                } /* while (dcfg) */
1088                DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089
1090                if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091                        DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092                        failed = -EINVAL;
1093                        goto error_cleanup;
1094                }
1095                if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096                        DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097                        failed = -EINVAL;
1098                        goto error_cleanup;
1099                }
1100                if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101                        DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102                        failed = -EINVAL;
1103                        goto error_cleanup;
1104                }
1105
1106                /* Update consumed and produced lengths.
1107
1108                   The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
1109                   other unit that process data in blocks of one octet) it is correct, but if it source from a
1110                   block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
1111                   since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112                   unit has processed an exact multiple of its block length the end result will be correct.
1113                   Beware that if the source change restriction change this code will need to be (much) reworked.
1114                */
1115                DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116
1117                if (csum_ctx.active) {
1118                        csum_ctx.consumed += desc_len;
1119                        if (csum_ctx.done) {
1120                                csum_ctx.produced = 2;
1121                        }
1122                        DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123                }
1124                if (digest_ctx.active) {
1125                        digest_ctx.consumed += desc_len;
1126                        if (digest_ctx.done) {
1127                                if (digest_ctx.unit_no == src_md5) {
1128                                        digest_ctx.produced = MD5_STATE_LENGTH;
1129                                } else {
1130                                        digest_ctx.produced = SHA1_STATE_LENGTH;
1131                                }
1132                        }
1133                        DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134                }
1135                if (cipher_ctx.active) {
1136                        /* Ciphers are allowed only to source from DMA out.  That is filtered above. */
1137                        assert(cipher_ctx.current_src == src_dma);
1138                        cipher_ctx.consumed += desc_len;
1139                        cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140                        if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141                                cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142                        }
1143                        DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144                }
1145
1146                /* Setup the DMA out descriptors. */
1147                /* Configure the metadata. */
1148                active_count = 0;
1149                eop_needed_count = 0;
1150                if (cipher_ctx.active) {
1151                        ++active_count;
1152                        if (cipher_ctx.unit_no == src_dma){
1153                                /* mem2mem */
1154                                meta_out.ciphsel = src_none;
1155                        } else {
1156                                meta_out.ciphsel = cipher_ctx.current_src;
1157                        }
1158                        meta_out.ciphconf = cipher_ctx.ciph_conf;
1159                        meta_out.cbcmode = cipher_ctx.cbcmode;
1160                        meta_out.decrypt = cipher_ctx.decrypt;
1161                        DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162                        if (cipher_ctx.done) ++eop_needed_count;
1163                } else {
1164                        meta_out.ciphsel = src_none;
1165                }
1166
1167                if (digest_ctx.active) {
1168                        ++active_count;
1169                        meta_out.hashsel = digest_ctx.current_src;
1170                        meta_out.hashconf = digest_ctx.hash_conf;
1171                        meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172                        DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173                        if (digest_ctx.done) {
1174                                assert(digest_ctx.pad_descs == NULL);
1175                                failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176                                if (failed) {
1177                                        DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178                                        goto error_cleanup;
1179                                }
1180                        }
1181                } else {
1182                        meta_out.hashsel = src_none;
1183                }
1184
1185                if (csum_ctx.active) {
1186                        ++active_count;
1187                        meta_out.csumsel = csum_ctx.current_src;
1188                        if (csum_ctx.done) {
1189                                assert(csum_ctx.pad_descs == NULL);
1190                                failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191                                if (failed) {
1192                                        DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193                                        goto error_cleanup;
1194                                }
1195                        }
1196                } else {
1197                        meta_out.csumsel = src_none;
1198                }
1199                DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200                /* Setup DMA out descriptors for the indata. */
1201                failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202                if (failed) {
1203                        DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204                        goto error_cleanup;
1205                }
1206                /* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
1207                 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208                 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209                 */
1210                assert(active_count >= eop_needed_count);
1211                assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212                if (eop_needed_count) {
1213                        /* This means that the bulk operation (cipeher/m2m) is terminated. */
1214                        if (active_count > 1) {
1215                                /* Use zero length EOP descriptor. */
1216                                struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217                                struct strcop_meta_out    ed_mo = {0};
1218                                if (!ed) {
1219                                        DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220                                        failed = -ENOMEM;
1221                                        goto error_cleanup;
1222                                }
1223
1224                                assert(cipher_ctx.active && cipher_ctx.done);
1225
1226                                if (cipher_ctx.unit_no == src_dma){
1227                                        /* mem2mem */
1228                                        ed_mo.ciphsel = src_none;
1229                                } else {
1230                                        ed_mo.ciphsel = cipher_ctx.current_src;
1231                                }
1232                                ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233                                ed_mo.cbcmode = cipher_ctx.cbcmode;
1234                                ed_mo.decrypt = cipher_ctx.decrypt;
1235
1236                                ed->free_buf = NULL;
1237                                ed->dma_descr->wait = 1;
1238                                ed->dma_descr->out_eop = 1;
1239
1240                                ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241                                ed->dma_descr->after = ed->dma_descr->buf;
1242                                ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243                                current_out_cdesc->next = ed;
1244                                current_out_cdesc = ed;
1245                        } else {
1246                                /* Set EOP in the current out descriptor since the only active module is
1247                                 * the one needing the EOP. */
1248
1249                                current_out_cdesc->dma_descr->out_eop = 1;
1250                        }
1251                }
1252
1253                if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254                if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255                if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256                indata_ix += odsc->length;
1257                odsc = odsc->next;
1258        } /* while (odsc) */ /* Process descriptors. */
1259        DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260        if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261                DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262                failed = -EINVAL;
1263                goto error_cleanup;
1264        }
1265        if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266                DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267                failed = -EINVAL;
1268                goto error_cleanup;
1269        }
1270        if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271                DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272                failed = -EINVAL;
1273                goto error_cleanup;
1274        }
1275
1276        failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277        if (failed){
1278                DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279                goto error_cleanup;
1280        }
1281        failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282        if (failed){
1283                DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284                goto error_cleanup;
1285        }
1286        failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287        if (failed){
1288                DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289                goto error_cleanup;
1290        }
1291
1292        DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293        (*int_op)->cdesc_out = out_cdesc_head.next;
1294        (*int_op)->cdesc_in = in_cdesc_head.next;
1295        DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296
1297        setup_descr_chain(out_cdesc_head.next);
1298        setup_descr_chain(in_cdesc_head.next);
1299
1300        /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301         * last DMA out descriptor for EOL.
1302         */
1303        current_in_cdesc->dma_descr->intr = 1;
1304        current_in_cdesc->dma_descr->eol = 1;
1305        current_out_cdesc->dma_descr->eol = 1;
1306
1307        /* Setup DMA contexts. */
1308        (*int_op)->ctx_out.next = NULL;
1309        (*int_op)->ctx_out.eol = 1;
1310        (*int_op)->ctx_out.intr = 0;
1311        (*int_op)->ctx_out.store_mode = 0;
1312        (*int_op)->ctx_out.en = 0;
1313        (*int_op)->ctx_out.dis = 0;
1314        (*int_op)->ctx_out.md0 = 0;
1315        (*int_op)->ctx_out.md1 = 0;
1316        (*int_op)->ctx_out.md2 = 0;
1317        (*int_op)->ctx_out.md3 = 0;
1318        (*int_op)->ctx_out.md4 = 0;
1319        (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320        (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321
1322        (*int_op)->ctx_in.next = NULL;
1323        (*int_op)->ctx_in.eol = 1;
1324        (*int_op)->ctx_in.intr = 0;
1325        (*int_op)->ctx_in.store_mode = 0;
1326        (*int_op)->ctx_in.en = 0;
1327        (*int_op)->ctx_in.dis = 0;
1328        (*int_op)->ctx_in.md0 = 0;
1329        (*int_op)->ctx_in.md1 = 0;
1330        (*int_op)->ctx_in.md2 = 0;
1331        (*int_op)->ctx_in.md3 = 0;
1332        (*int_op)->ctx_in.md4 = 0;
1333
1334        (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335        (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336
1337        DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338        return 0;
1339
1340error_cleanup:
1341        {
1342                /* Free all allocated resources. */
1343                struct cryptocop_dma_desc *tmp_cdesc;
1344                while (digest_ctx.pad_descs){
1345                        tmp_cdesc = digest_ctx.pad_descs->next;
1346                        free_cdesc(digest_ctx.pad_descs);
1347                        digest_ctx.pad_descs = tmp_cdesc;
1348                }
1349                while (csum_ctx.pad_descs){
1350                        tmp_cdesc = csum_ctx.pad_descs->next;
1351                        free_cdesc(csum_ctx.pad_descs);
1352                        csum_ctx.pad_descs = tmp_cdesc;
1353                }
1354                assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355
1356                if (*int_op != NULL) delete_internal_operation(*int_op);
1357        }
1358        DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359        return failed;
1360}
1361
1362
1363static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364{
1365        void                      *ptr = iop->alloc_ptr;
1366        struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367        struct cryptocop_dma_desc *next;
1368
1369        DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370
1371        while (cd) {
1372                next = cd->next;
1373                free_cdesc(cd);
1374                cd = next;
1375        }
1376        cd = iop->cdesc_in;
1377        while (cd) {
1378                next = cd->next;
1379                free_cdesc(cd);
1380                cd = next;
1381        }
1382        kfree(ptr);
1383}
1384
1385#define MD5_MIN_PAD_LENGTH (9)
1386#define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387
1388static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389{
1390        size_t                  padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391        unsigned char           *p;
1392        int                     i;
1393        unsigned long long int  bit_length = hashed_length << 3;
1394
1395        if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396
1397        p = kzalloc(padlen, alloc_flag);
1398        if (!p) return -ENOMEM;
1399
1400        *p = 0x80;
1401
1402        DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403
1404        i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405        while (bit_length != 0){
1406                p[i++] = bit_length % 0x100;
1407                bit_length >>= 8;
1408        }
1409
1410        *pad = (char*)p;
1411        *pad_length = padlen;
1412
1413        return 0;
1414}
1415
1416#define SHA1_MIN_PAD_LENGTH (9)
1417#define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418
1419static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420{
1421        size_t                  padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422        unsigned char           *p;
1423        int                     i;
1424        unsigned long long int  bit_length = hashed_length << 3;
1425
1426        if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427
1428        p = kzalloc(padlen, alloc_flag);
1429        if (!p) return -ENOMEM;
1430
1431        *p = 0x80;
1432
1433        DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1434
1435        i = padlen - 1;
1436        while (bit_length != 0){
1437                p[i--] = bit_length % 0x100;
1438                bit_length >>= 8;
1439        }
1440
1441        *pad = (char*)p;
1442        *pad_length = padlen;
1443
1444        return 0;
1445}
1446
1447
1448static int transform_ok(struct cryptocop_transform_init *tinit)
1449{
1450        switch (tinit->alg){
1451        case cryptocop_alg_csum:
1452                switch (tinit->csum_mode){
1453                case cryptocop_csum_le:
1454                case cryptocop_csum_be:
1455                        break;
1456                default:
1457                        DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1458                        return -EINVAL;
1459                }
1460        case cryptocop_alg_mem2mem:
1461        case cryptocop_alg_md5:
1462        case cryptocop_alg_sha1:
1463                if (tinit->keylen != 0) {
1464                        DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1465                        return -EINVAL; /* This check is a bit strict. */
1466                }
1467                break;
1468        case cryptocop_alg_des:
1469                if (tinit->keylen != 64) {
1470                        DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1471                        return -EINVAL;
1472                }
1473                break;
1474        case cryptocop_alg_3des:
1475                if (tinit->keylen != 192) {
1476                        DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1477                        return -EINVAL;
1478                }
1479                break;
1480        case cryptocop_alg_aes:
1481                if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1482                        DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1483                        return -EINVAL;
1484                }
1485                break;
1486        case cryptocop_no_alg:
1487        default:
1488                DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1489                return -EINVAL;
1490        }
1491
1492        switch (tinit->alg){
1493        case cryptocop_alg_des:
1494        case cryptocop_alg_3des:
1495        case cryptocop_alg_aes:
1496                if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1497        default:
1498                 break;
1499        }
1500        return 0;
1501}
1502
1503
1504int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1505{
1506        struct cryptocop_session         *sess;
1507        struct cryptocop_transform_init  *tfrm_in = tinit;
1508        struct cryptocop_transform_init  *tmp_in;
1509        int                              no_tfrms = 0;
1510        int                              i;
1511        unsigned long int                flags;
1512
1513        init_stream_coprocessor(); /* For safety if we are called early */
1514
1515        while (tfrm_in){
1516                int err;
1517                ++no_tfrms;
1518                if ((err = transform_ok(tfrm_in))) {
1519                        DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1520                        return err;
1521                }
1522                tfrm_in = tfrm_in->next;
1523        }
1524        if (0 == no_tfrms) {
1525                DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1526                return -EINVAL;
1527        }
1528
1529        sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1530        if (!sess){
1531                DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1532                return -ENOMEM;
1533        }
1534
1535        sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1536        if (!sess->tfrm_ctx) {
1537                DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1538                kfree(sess);
1539                return -ENOMEM;
1540        }
1541
1542        tfrm_in = tinit;
1543        for (i = 0; i < no_tfrms; i++){
1544                tmp_in = tfrm_in->next;
1545                while (tmp_in){
1546                        if (tmp_in->tid == tfrm_in->tid) {
1547                                DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1548                                kfree(sess->tfrm_ctx);
1549                                kfree(sess);
1550                                return -EINVAL;
1551                        }
1552                        tmp_in = tmp_in->next;
1553                }
1554                memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1555                sess->tfrm_ctx[i].dec_key_set = 0;
1556                sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1557
1558                tfrm_in = tfrm_in->next;
1559        }
1560        sess->tfrm_ctx[i-1].next = NULL;
1561
1562        spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1563        sess->sid = next_sid;
1564        next_sid++;
1565        /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1566         *      OTOH 2^64 is a really large number of session. */
1567        if (next_sid == 0) next_sid = 1;
1568
1569        /* Prepend to session list. */
1570        sess->next = cryptocop_sessions;
1571        cryptocop_sessions = sess;
1572        spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1573        *sid = sess->sid;
1574        return 0;
1575}
1576
1577
1578int cryptocop_free_session(cryptocop_session_id sid)
1579{
1580        struct cryptocop_transform_ctx    *tc;
1581        struct cryptocop_session          *sess = NULL;
1582        struct cryptocop_session          *psess = NULL;
1583        unsigned long int                 flags;
1584        int                               i;
1585        LIST_HEAD(remove_list);
1586        struct list_head                  *node, *tmp;
1587        struct cryptocop_prio_job         *pj;
1588
1589        DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1590
1591        spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1592        sess = cryptocop_sessions;
1593        while (sess && sess->sid != sid){
1594                psess = sess;
1595                sess = sess->next;
1596        }
1597        if (sess){
1598                if (psess){
1599                        psess->next = sess->next;
1600                } else {
1601                        cryptocop_sessions = sess->next;
1602                }
1603        }
1604        spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1605
1606        if (!sess) return -EINVAL;
1607
1608        /* Remove queued jobs. */
1609        spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1610
1611        for (i = 0; i < cryptocop_prio_no_prios; i++){
1612                if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1613                        list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1614                                pj = list_entry(node, struct cryptocop_prio_job, node);
1615                                if (pj->oper->sid == sid) {
1616                                        list_move_tail(node, &remove_list);
1617                                }
1618                        }
1619                }
1620        }
1621        spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1622
1623        list_for_each_safe(node, tmp, &remove_list) {
1624                list_del(node);
1625                pj = list_entry(node, struct cryptocop_prio_job, node);
1626                pj->oper->operation_status = -EAGAIN;  /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1627                DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1628                pj->oper->cb(pj->oper, pj->oper->cb_data);
1629                delete_internal_operation(pj->iop);
1630                kfree(pj);
1631        }
1632
1633        tc = sess->tfrm_ctx;
1634        /* Erase keying data. */
1635        while (tc){
1636                DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1637                memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1638                memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639                tc = tc->next;
1640        }
1641        kfree(sess->tfrm_ctx);
1642        kfree(sess);
1643
1644        return 0;
1645}
1646
1647static struct cryptocop_session *get_session(cryptocop_session_id sid)
1648{
1649        struct cryptocop_session    *sess;
1650        unsigned long int           flags;
1651
1652        spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1653        sess = cryptocop_sessions;
1654        while (sess && (sess->sid != sid)){
1655                sess = sess->next;
1656        }
1657        spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1658
1659        return sess;
1660}
1661
1662static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1663{
1664        struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1665
1666        DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1667        assert(sess != NULL);
1668        while (tc && tc->init.tid != tid){
1669                DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1670                tc = tc->next;
1671        }
1672        DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1673        return tc;
1674}
1675
1676
1677
1678/* The AES s-transform matrix (s-box). */
1679static const u8 aes_sbox[256] = {
1680        99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
1681        202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
1682        183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
1683        4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
1684        9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
1685        83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
1686        208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
1687        81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
1688        205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
1689        96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
1690        224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
1691        231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
1692        186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
1693        112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
1694        225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
1695        140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
1696};
1697
1698/* AES has a 32 bit word round constants for each round in the
1699 * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
1700 */
1701static u32 round_constant[11] = {
1702        0x01000000, 0x02000000, 0x04000000, 0x08000000,
1703        0x10000000, 0x20000000, 0x40000000, 0x80000000,
1704        0x1B000000, 0x36000000, 0x6C000000
1705};
1706
1707/* Apply the s-box to each of the four occtets in w. */
1708static u32 aes_ks_subword(const u32 w)
1709{
1710        u8 bytes[4];
1711
1712        *(u32*)(&bytes[0]) = w;
1713        bytes[0] = aes_sbox[bytes[0]];
1714        bytes[1] = aes_sbox[bytes[1]];
1715        bytes[2] = aes_sbox[bytes[2]];
1716        bytes[3] = aes_sbox[bytes[3]];
1717        return *(u32*)(&bytes[0]);
1718}
1719
1720/* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1721 * (Note that AES words are 32 bit long)
1722 *
1723 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1724 * word temp
1725 * i = 0
1726 * while (i < Nk) {
1727 *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1728 *   i = i + 1
1729 * }
1730 * i = Nk
1731 *
1732 * while (i < (Nb * (Nr + 1))) {
1733 *   temp = w[i - 1]
1734 *   if ((i mod Nk) == 0) {
1735 *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1736 *   }
1737 *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
1738 *     temp = SubWord(temp)
1739 *   }
1740 *   w[i] = w[i - Nk] xor temp
1741 * }
1742 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1743 * SubWord(t) applies the AES s-box individually to each octet
1744 * in a 32 bit word.
1745 *
1746 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1747 * values for Nr of 10, 12, and 14).  Nb is always 4.
1748 *
1749 * To construct w[i], w[i - 1] and w[i - Nk] must be
1750 * available.  Consequently we must keep a state of the last Nk words
1751 * to be able to create the last round keys.
1752 */
1753static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
1754{
1755        u32 temp;
1756        u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1757        u8  w_last_ix;
1758        int i;
1759        u8  nr, nk;
1760
1761        switch (keylength){
1762        case 128:
1763                nk = 4;
1764                nr = 10;
1765                break;
1766        case 192:
1767                nk = 6;
1768                nr = 12;
1769                break;
1770        case 256:
1771                nk = 8;
1772                nr = 14;
1773                break;
1774        default:
1775                panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1776        };
1777
1778        /* Need to do host byte order correction here since key is byte oriented and the
1779         * kx algorithm is word (u32) oriented. */
1780        for (i = 0; i < nk; i+=1) {
1781                w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1782        }
1783
1784        i = (int)nk;
1785        w_last_ix = i - 1;
1786        while (i < (4 * (nr + 2))) {
1787                temp = w_ring[w_last_ix];
1788                if (!(i % nk)) {
1789                        /* RotWord(temp) */
1790                        temp = (temp << 8) | (temp >> 24);
1791                        temp = aes_ks_subword(temp);
1792                        temp ^= round_constant[i/nk - 1];
1793                } else if ((nk > 6) && ((i % nk) == 4)) {
1794                        temp = aes_ks_subword(temp);
1795                }
1796                w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1797                temp ^= w_ring[w_last_ix];
1798                w_ring[w_last_ix] = temp;
1799
1800                /* We need the round keys for round Nr+1 and Nr+2 (round key
1801                 * Nr+2 is the round key beyond the last one used when
1802                 * encrypting).  Rounds are numbered starting from 0, Nr=10
1803                 * implies 11 rounds are used in encryption/decryption.
1804                 */
1805                if (i >= (4 * nr)) {
1806                        /* Need to do host byte order correction here, the key
1807                         * is byte oriented. */
1808                        *(u32*)dec_key = cpu_to_be32(temp);
1809                        dec_key += 4;
1810                }
1811                ++i;
1812        }
1813}
1814
1815
1816/**** Job/operation management. ****/
1817
1818int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1819{
1820        return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1821}
1822
1823int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1824{
1825        return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1826}
1827
1828int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1829{
1830        return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1831}
1832
1833static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1834{
1835        int                           ret;
1836        struct cryptocop_prio_job     *pj = NULL;
1837        unsigned long int             flags;
1838
1839        DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1840
1841        if (!operation || !operation->cb){
1842                DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1843                return -EINVAL;
1844        }
1845
1846        if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1847                DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1848                return ret;
1849        }
1850        assert(pj != NULL);
1851
1852        spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1853        list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1854        spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1855
1856        /* Make sure a job is running */
1857        cryptocop_start_job();
1858        return 0;
1859}
1860
1861static void cryptocop_do_tasklet(unsigned long unused);
1862DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1863
1864static void cryptocop_do_tasklet(unsigned long unused)
1865{
1866        struct list_head             *node;
1867        struct cryptocop_prio_job    *pj = NULL;
1868        unsigned long                flags;
1869
1870        DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1871
1872        do {
1873                spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1874                if (!list_empty(&cryptocop_completed_jobs)){
1875                        node = cryptocop_completed_jobs.next;
1876                        list_del(node);
1877                        pj = list_entry(node, struct cryptocop_prio_job, node);
1878                } else {
1879                        pj = NULL;
1880                }
1881                spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1882                if (pj) {
1883                        assert(pj->oper != NULL);
1884
1885                        /* Notify consumer of operation completeness. */
1886                        DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1887
1888                        pj->oper->operation_status = 0; /* Job is completed. */
1889                        pj->oper->cb(pj->oper, pj->oper->cb_data);
1890                        delete_internal_operation(pj->iop);
1891                        kfree(pj);
1892                }
1893        } while (pj != NULL);
1894
1895        DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1896}
1897
1898static irqreturn_t
1899dma_done_interrupt(int irq, void *dev_id)
1900{
1901        struct cryptocop_prio_job *done_job;
1902        reg_dma_rw_ack_intr ack_intr = {
1903                .data = 1,
1904        };
1905
1906        REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1907
1908        DEBUG(printk("cryptocop DMA done\n"));
1909
1910        spin_lock(&running_job_lock);
1911        if (cryptocop_running_job == NULL){
1912                printk("stream co-processor got interrupt when not busy\n");
1913                spin_unlock(&running_job_lock);
1914                return IRQ_HANDLED;
1915        }
1916        done_job = cryptocop_running_job;
1917        cryptocop_running_job = NULL;
1918        spin_unlock(&running_job_lock);
1919
1920        /* Start processing a job. */
1921        if (!spin_trylock(&cryptocop_process_lock)){
1922                DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1923        } else {
1924                cryptocop_start_job();
1925                spin_unlock(&cryptocop_process_lock);
1926        }
1927
1928        done_job->oper->operation_status = 0; /* Job is completed. */
1929        if (done_job->oper->fast_callback){
1930                /* This operation wants callback from interrupt. */
1931                done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1932                delete_internal_operation(done_job->iop);
1933                kfree(done_job);
1934        } else {
1935                spin_lock(&cryptocop_completed_jobs_lock);
1936                list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1937                spin_unlock(&cryptocop_completed_jobs_lock);
1938                tasklet_schedule(&cryptocop_tasklet);
1939        }
1940
1941        DEBUG(printk("cryptocop leave irq handler\n"));
1942        return IRQ_HANDLED;
1943}
1944
1945
1946/* Setup interrupts and DMA channels. */
1947static int init_cryptocop(void)
1948{
1949        unsigned long          flags;
1950        reg_dma_rw_cfg         dma_cfg = {.en = 1};
1951        reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1952        reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1953        reg_strcop_rw_cfg      strcop_cfg = {
1954                .ipend = regk_strcop_little,
1955                .td1 = regk_strcop_e,
1956                .td2 = regk_strcop_d,
1957                .td3 = regk_strcop_e,
1958                .ignore_sync = 0,
1959                .en = 1
1960        };
1961
1962        if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1963                        "stream co-processor DMA", NULL))
1964                panic("request_irq stream co-processor irq dma9");
1965
1966        (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1967                0, dma_strp);
1968        (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969                0, dma_strp);
1970
1971        local_irq_save(flags);
1972
1973        /* Reset and enable the cryptocop. */
1974        strcop_cfg.en = 0;
1975        REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1976        strcop_cfg.en = 1;
1977        REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978
1979        /* Enable DMAs. */
1980        REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1981        REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1982
1983        /* Set up wordsize = 4 for DMAs. */
1984        DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1985        DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1986
1987        /* Enable interrupts. */
1988        REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1989
1990        /* Clear intr ack. */
1991        REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1992
1993        local_irq_restore(flags);
1994
1995        return 0;
1996}
1997
1998/* Free used cryptocop hw resources (interrupt and DMA channels). */
1999static void release_cryptocop(void)
2000{
2001        unsigned long          flags;
2002        reg_dma_rw_cfg         dma_cfg = {.en = 0};
2003        reg_dma_rw_intr_mask   intr_mask_in = {0};
2004        reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
2005
2006        local_irq_save(flags);
2007
2008        /* Clear intr ack. */
2009        REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2010
2011        /* Disable DMAs. */
2012        REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2013        REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2014
2015        /* Disable interrupts. */
2016        REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2017
2018        local_irq_restore(flags);
2019
2020        free_irq(DMA_IRQ, NULL);
2021
2022        (void)crisv32_free_dma(OUT_DMA);
2023        (void)crisv32_free_dma(IN_DMA);
2024}
2025
2026
2027/* Init job queue. */
2028static int cryptocop_job_queue_init(void)
2029{
2030        int i;
2031
2032        INIT_LIST_HEAD(&cryptocop_completed_jobs);
2033
2034        for (i = 0; i < cryptocop_prio_no_prios; i++){
2035                cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2036                INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2037        }
2038        return 0;
2039}
2040
2041
2042static void cryptocop_job_queue_close(void)
2043{
2044        struct list_head               *node, *tmp;
2045        struct cryptocop_prio_job      *pj = NULL;
2046        unsigned long int              process_flags, flags;
2047        int                            i;
2048
2049        /* FIXME: This is as yet untested code. */
2050
2051        /* Stop strcop from getting an operation to process while we are closing the
2052           module. */
2053        spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2054
2055        /* Empty the job queue. */
2056        for (i = 0; i < cryptocop_prio_no_prios; i++){
2057                if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2058                        list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2059                                pj = list_entry(node, struct cryptocop_prio_job, node);
2060                                list_del(node);
2061
2062                                /* Call callback to notify consumer of job removal. */
2063                                DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2064                                pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2065                                pj->oper->cb(pj->oper, pj->oper->cb_data);
2066
2067                                delete_internal_operation(pj->iop);
2068                                kfree(pj);
2069                        }
2070                }
2071        }
2072        spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2073
2074        /* Remove the running job, if any. */
2075        spin_lock_irqsave(&running_job_lock, flags);
2076        if (cryptocop_running_job){
2077                reg_strcop_rw_cfg rw_cfg;
2078                reg_dma_rw_cfg    dma_out_cfg, dma_in_cfg;
2079
2080                /* Stop DMA. */
2081                dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2082                dma_out_cfg.en = regk_dma_no;
2083                REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2084
2085                dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2086                dma_in_cfg.en = regk_dma_no;
2087                REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2088
2089                /* Disble the cryptocop. */
2090                rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2091                rw_cfg.en = 0;
2092                REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2093
2094                pj = cryptocop_running_job;
2095                cryptocop_running_job = NULL;
2096
2097                /* Call callback to notify consumer of job removal. */
2098                DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2099                pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2100                pj->oper->cb(pj->oper, pj->oper->cb_data);
2101
2102                delete_internal_operation(pj->iop);
2103                kfree(pj);
2104        }
2105        spin_unlock_irqrestore(&running_job_lock, flags);
2106
2107        /* Remove completed jobs, if any. */
2108        spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2109
2110        list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2111                pj = list_entry(node, struct cryptocop_prio_job, node);
2112                list_del(node);
2113                /* Call callback to notify consumer of job removal. */
2114                DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2115                pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2116                pj->oper->cb(pj->oper, pj->oper->cb_data);
2117
2118                delete_internal_operation(pj->iop);
2119                kfree(pj);
2120        }
2121        spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2122}
2123
2124
2125static void cryptocop_start_job(void)
2126{
2127        int                          i;
2128        struct cryptocop_prio_job    *pj;
2129        unsigned long int            flags;
2130        unsigned long int            running_job_flags;
2131        reg_strcop_rw_cfg            rw_cfg = {.en = 1, .ignore_sync = 0};
2132
2133        DEBUG(printk("cryptocop_start_job: entering\n"));
2134
2135        spin_lock_irqsave(&running_job_lock, running_job_flags);
2136        if (cryptocop_running_job != NULL){
2137                /* Already running. */
2138                DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2139                spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2140                return;
2141        }
2142        spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2143
2144        /* Check the queues in priority order. */
2145        for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2146        if (i == cryptocop_prio_no_prios) {
2147                spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2148                spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2149                DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2150                return; /* No jobs to run */
2151        }
2152        DEBUG(printk("starting job for prio %d\n", i));
2153
2154        /* TODO: Do not starve lower priority jobs.  Let in a lower
2155         * prio job for every N-th processed higher prio job or some
2156         * other scheduling policy.  This could reasonably be
2157         * tweakable since the optimal balance would depend on the
2158         * type of load on the system. */
2159
2160        /* Pull the DMA lists from the job and start the DMA client. */
2161        pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2162        list_del(&pj->node);
2163        spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2164        cryptocop_running_job = pj;
2165
2166        /* Set config register (3DES and CSUM modes). */
2167        switch (pj->iop->tdes_mode){
2168        case cryptocop_3des_eee:
2169                rw_cfg.td1 = regk_strcop_e;
2170                rw_cfg.td2 = regk_strcop_e;
2171                rw_cfg.td3 = regk_strcop_e;
2172                break;
2173        case cryptocop_3des_eed:
2174                rw_cfg.td1 = regk_strcop_e;
2175                rw_cfg.td2 = regk_strcop_e;
2176                rw_cfg.td3 = regk_strcop_d;
2177                break;
2178        case cryptocop_3des_ede:
2179                rw_cfg.td1 = regk_strcop_e;
2180                rw_cfg.td2 = regk_strcop_d;
2181                rw_cfg.td3 = regk_strcop_e;
2182                break;
2183        case cryptocop_3des_edd:
2184                rw_cfg.td1 = regk_strcop_e;
2185                rw_cfg.td2 = regk_strcop_d;
2186                rw_cfg.td3 = regk_strcop_d;
2187                break;
2188        case cryptocop_3des_dee:
2189                rw_cfg.td1 = regk_strcop_d;
2190                rw_cfg.td2 = regk_strcop_e;
2191                rw_cfg.td3 = regk_strcop_e;
2192                break;
2193        case cryptocop_3des_ded:
2194                rw_cfg.td1 = regk_strcop_d;
2195                rw_cfg.td2 = regk_strcop_e;
2196                rw_cfg.td3 = regk_strcop_d;
2197                break;
2198        case cryptocop_3des_dde:
2199                rw_cfg.td1 = regk_strcop_d;
2200                rw_cfg.td2 = regk_strcop_d;
2201                rw_cfg.td3 = regk_strcop_e;
2202                break;
2203        case cryptocop_3des_ddd:
2204                rw_cfg.td1 = regk_strcop_d;
2205                rw_cfg.td2 = regk_strcop_d;
2206                rw_cfg.td3 = regk_strcop_d;
2207                break;
2208        default:
2209                DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2210        }
2211        switch (pj->iop->csum_mode){
2212        case cryptocop_csum_le:
2213                rw_cfg.ipend = regk_strcop_little;
2214                break;
2215        case cryptocop_csum_be:
2216                rw_cfg.ipend = regk_strcop_big;
2217                break;
2218        default:
2219                DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2220        }
2221        REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2222
2223        DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2224                     "ctx_in: 0x%p, phys: 0x%p\n"
2225                     "ctx_out: 0x%p, phys: 0x%p\n",
2226                     pj,
2227                     &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2228                     &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2229
2230        /* Start input DMA. */
2231        flush_dma_context(&pj->iop->ctx_in);
2232        DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2233
2234        /* Start output DMA. */
2235        DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2236
2237        spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2238        DEBUG(printk("cryptocop_start_job: exiting\n"));
2239}
2240
2241
2242static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2243{
2244        int  err;
2245        int  alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2246        void *iop_alloc_ptr = NULL;
2247
2248        *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2249        if (!*pj) return -ENOMEM;
2250
2251        DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2252
2253        (*pj)->oper = operation;
2254        DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n",  (*pj)->oper->cb, (*pj)->oper->cb_data));
2255
2256        if (operation->use_dmalists) {
2257                DEBUG(print_user_dma_lists(&operation->list_op));
2258                if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2259                        DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2260                        kfree(*pj);
2261                        return -EINVAL;
2262                }
2263                iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2264                if (!iop_alloc_ptr) {
2265                        DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2266                        kfree(*pj);
2267                        return -ENOMEM;
2268                }
2269                (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2270                DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2271                (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2272                (*pj)->iop->sid = operation->sid;
2273                (*pj)->iop->cdesc_out = NULL;
2274                (*pj)->iop->cdesc_in = NULL;
2275                (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2276                (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2277                (*pj)->iop->ddesc_out = operation->list_op.outlist;
2278                (*pj)->iop->ddesc_in = operation->list_op.inlist;
2279
2280                /* Setup DMA contexts. */
2281                (*pj)->iop->ctx_out.next = NULL;
2282                (*pj)->iop->ctx_out.eol = 1;
2283                (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2284                (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2285
2286                (*pj)->iop->ctx_in.next = NULL;
2287                (*pj)->iop->ctx_in.eol = 1;
2288                (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2289                (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2290        } else {
2291                if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2292                        DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2293                        kfree(*pj);
2294                        return err;
2295                }
2296        }
2297        DEBUG(print_dma_descriptors((*pj)->iop));
2298
2299        DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2300
2301        return 0;
2302}
2303
2304static int cryptocop_open(struct inode *inode, struct file *filp)
2305{
2306        int p = iminor(inode);
2307
2308        if (p != CRYPTOCOP_MINOR) return -EINVAL;
2309
2310        filp->private_data = NULL;
2311        return 0;
2312}
2313
2314
2315static int cryptocop_release(struct inode *inode, struct file *filp)
2316{
2317        struct cryptocop_private *dev = filp->private_data;
2318        struct cryptocop_private *dev_next;
2319
2320        while (dev){
2321                dev_next = dev->next;
2322                if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2323                        (void)cryptocop_free_session(dev->sid);
2324                }
2325                kfree(dev);
2326                dev = dev_next;
2327        }
2328
2329        return 0;
2330}
2331
2332
2333static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2334                                         unsigned int cmd, unsigned long arg)
2335{
2336        struct cryptocop_private  *dev = filp->private_data;
2337        struct cryptocop_private  *prev_dev = NULL;
2338        struct strcop_session_op  *sess_op = (struct strcop_session_op *)arg;
2339        struct strcop_session_op  sop;
2340        int                       err;
2341
2342        DEBUG(printk("cryptocop_ioctl_close_session\n"));
2343
2344        if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2345                return -EFAULT;
2346        err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2347        if (err) return -EFAULT;
2348
2349        while (dev && (dev->sid != sop.ses_id)) {
2350                prev_dev = dev;
2351                dev = dev->next;
2352        }
2353        if (dev){
2354                if (prev_dev){
2355                        prev_dev->next = dev->next;
2356                } else {
2357                        filp->private_data = dev->next;
2358                }
2359                err = cryptocop_free_session(dev->sid);
2360                if (err) return -EFAULT;
2361        } else {
2362                DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2363                return -EINVAL;
2364        }
2365        return 0;
2366}
2367
2368
2369static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2370{
2371        struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2372
2373        DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2374
2375        jc->processed = 1;
2376        wake_up(&cryptocop_ioc_process_wq);
2377}
2378
2379
2380#define CRYPTOCOP_IOCTL_CIPHER_TID  (1)
2381#define CRYPTOCOP_IOCTL_DIGEST_TID  (2)
2382#define CRYPTOCOP_IOCTL_CSUM_TID    (3)
2383
2384static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2385{
2386        size_t ch_ix = 0;
2387
2388        if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2389        if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2390        if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2391
2392        DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2393        return ch_ix;
2394}
2395
2396
2397static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2398{
2399        size_t ch_ix = INT_MAX;
2400        size_t tmp_ix = 0;
2401
2402        if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2403                if (crp_op->cipher_start > ix) {
2404                        ch_ix = crp_op->cipher_start;
2405                } else {
2406                        ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2407                }
2408        }
2409        if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2410                if (crp_op->digest_start > ix) {
2411                        tmp_ix = crp_op->digest_start;
2412                } else {
2413                        tmp_ix = crp_op->digest_start + crp_op->digest_len;
2414                }
2415                if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2416        }
2417        if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2418                if (crp_op->csum_start > ix) {
2419                        tmp_ix = crp_op->csum_start;
2420                } else {
2421                        tmp_ix = crp_op->csum_start + crp_op->csum_len;
2422                }
2423                if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2424        }
2425        if (ch_ix == INT_MAX) ch_ix = ix;
2426        DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2427        return ch_ix;
2428}
2429
2430
2431/* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2432 * Return -1 for ok, 0 for fail. */
2433static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2434{
2435        int tmplen;
2436
2437        assert(iov != NULL);
2438        assert(iovix != NULL);
2439        assert(pages != NULL);
2440        assert(pageix != NULL);
2441        assert(pageoffset != NULL);
2442
2443        DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2444
2445        while (map_length > 0){
2446                DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2447                if (*iovix >= iovlen){
2448                        DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2449                        return 0;
2450                }
2451                if (*pageix >= nopages){
2452                        DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2453                        return 0;
2454                }
2455                iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2456                tmplen = PAGE_SIZE - *pageoffset;
2457                if (tmplen < map_length){
2458                        (*pageoffset) = 0;
2459                        (*pageix)++;
2460                } else {
2461                        tmplen = map_length;
2462                        (*pageoffset) += map_length;
2463                }
2464                DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2465                iov[*iovix].iov_len = tmplen;
2466                map_length -= tmplen;
2467                (*iovix)++;
2468        }
2469        DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2470        return -1;
2471}
2472
2473
2474
2475static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2476{
2477        int                             i;
2478        struct cryptocop_private        *dev = filp->private_data;
2479        struct strcop_crypto_op         *crp_oper = (struct strcop_crypto_op *)arg;
2480        struct strcop_crypto_op         oper = {0};
2481        int                             err = 0;
2482        struct cryptocop_operation      *cop = NULL;
2483
2484        struct ioctl_job_cb_ctx         *jc = NULL;
2485
2486        struct page                     **inpages = NULL;
2487        struct page                     **outpages = NULL;
2488        int                             noinpages = 0;
2489        int                             nooutpages = 0;
2490
2491        struct cryptocop_desc           descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2492                                                   * can get connected/disconnected on different places in the indata. */
2493        struct cryptocop_desc_cfg       dcfgs[5*3];
2494        int                             desc_ix = 0;
2495        int                             dcfg_ix = 0;
2496        struct cryptocop_tfrm_cfg       ciph_tcfg = {0};
2497        struct cryptocop_tfrm_cfg       digest_tcfg = {0};
2498        struct cryptocop_tfrm_cfg       csum_tcfg = {0};
2499
2500        unsigned char                   *digest_result = NULL;
2501        int                             digest_length = 0;
2502        int                             cblocklen = 0;
2503        unsigned char                   csum_result[CSUM_BLOCK_LENGTH];
2504        struct cryptocop_session        *sess;
2505
2506        int    iovlen = 0;
2507        int    iovix = 0;
2508        int    pageix = 0;
2509        int    pageoffset = 0;
2510
2511        size_t prev_ix = 0;
2512        size_t next_ix;
2513
2514        int    cipher_active, digest_active, csum_active;
2515        int    end_digest, end_csum;
2516        int    digest_done = 0;
2517        int    cipher_done = 0;
2518        int    csum_done = 0;
2519
2520        DEBUG(printk("cryptocop_ioctl_process\n"));
2521
2522        if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2523                DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2524                return -EFAULT;
2525        }
2526        if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2527                DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2528                return -EFAULT;
2529        }
2530        DEBUG(print_strcop_crypto_op(&oper));
2531
2532        while (dev && dev->sid != oper.ses_id) dev = dev->next;
2533        if (!dev){
2534                DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2535                return -EINVAL;
2536        }
2537
2538        /* Check buffers. */
2539        if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2540                DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2541                return -EINVAL;
2542        }
2543
2544        if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2545                DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2546                return -EFAULT;
2547        }
2548        if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2549                DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2550                return -EFAULT;
2551        }
2552
2553        cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2554        if (!cop) {
2555                DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2556                return -ENOMEM;
2557        }
2558        jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2559        if (!jc) {
2560                DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2561                err = -ENOMEM;
2562                goto error_cleanup;
2563        }
2564        jc->processed = 0;
2565
2566        cop->cb_data = jc;
2567        cop->cb = ioctl_process_job_callback;
2568        cop->operation_status = 0;
2569        cop->use_dmalists = 0;
2570        cop->in_interrupt = 0;
2571        cop->fast_callback = 0;
2572        cop->tfrm_op.tfrm_cfg = NULL;
2573        cop->tfrm_op.desc = NULL;
2574        cop->tfrm_op.indata = NULL;
2575        cop->tfrm_op.incount = 0;
2576        cop->tfrm_op.inlen = 0;
2577        cop->tfrm_op.outdata = NULL;
2578        cop->tfrm_op.outcount = 0;
2579        cop->tfrm_op.outlen = 0;
2580
2581        sess = get_session(oper.ses_id);
2582        if (!sess){
2583                DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2584                kfree(cop);
2585                kfree(jc);
2586                return -EINVAL;
2587        }
2588
2589        if (oper.do_cipher) {
2590                unsigned int                    cipher_outlen = 0;
2591                struct cryptocop_transform_ctx  *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2592                if (!tc) {
2593                        DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2594                        err = -EINVAL;
2595                        goto error_cleanup;
2596                }
2597                ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2598                ciph_tcfg.inject_ix = 0;
2599                ciph_tcfg.flags = 0;
2600                if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2601                        DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2602                        kfree(cop);
2603                        kfree(jc);
2604                        return -EINVAL;
2605                }
2606                cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2607                if (oper.cipher_len % cblocklen) {
2608                        kfree(cop);
2609                        kfree(jc);
2610                        DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2611                        return -EINVAL;
2612                }
2613                cipher_outlen = oper.cipher_len;
2614                if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2615                        if (oper.cipher_explicit) {
2616                                ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2617                                memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2618                        } else {
2619                                cipher_outlen = oper.cipher_len - cblocklen;
2620                        }
2621                } else {
2622                        if (oper.cipher_explicit){
2623                                kfree(cop);
2624                                kfree(jc);
2625                                DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2626                                return -EINVAL;
2627                        }
2628                }
2629                if (oper.cipher_outlen != cipher_outlen) {
2630                        kfree(cop);
2631                        kfree(jc);
2632                        DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2633                        return -EINVAL;
2634                }
2635
2636                if (oper.decrypt){
2637                        ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2638                } else {
2639                        ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2640                }
2641                ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2642                cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2643        }
2644        if (oper.do_digest){
2645                struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2646                if (!tc) {
2647                        DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2648                        err = -EINVAL;
2649                        goto error_cleanup;
2650                }
2651                digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2652                digest_result = kmalloc(digest_length, GFP_KERNEL);
2653                if (!digest_result) {
2654                        DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2655                        err = -EINVAL;
2656                        goto error_cleanup;
2657                }
2658                DEBUG(memset(digest_result, 0xff, digest_length));
2659
2660                digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2661                digest_tcfg.inject_ix = 0;
2662                ciph_tcfg.inject_ix += digest_length;
2663                if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2664                        DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2665                        err = -EINVAL;
2666                        goto error_cleanup;
2667                }
2668
2669                digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2670                cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2671        }
2672        if (oper.do_csum){
2673                csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2674                csum_tcfg.inject_ix = digest_length;
2675                ciph_tcfg.inject_ix += 2;
2676
2677                if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2678                        DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2679                        kfree(cop);
2680                        kfree(jc);
2681                        return -EINVAL;
2682                }
2683
2684                csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2685                cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2686        }
2687
2688        prev_ix = first_cfg_change_ix(&oper);
2689        if (prev_ix > oper.inlen) {
2690                DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2691                nooutpages = noinpages = 0;
2692                err = -EINVAL;
2693                goto error_cleanup;
2694        }
2695        DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2696
2697        /* Map user pages for in and out data of the operation. */
2698        noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2699        DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2700        inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2701        if (!inpages){
2702                DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2703                nooutpages = noinpages = 0;
2704                err = -ENOMEM;
2705                goto error_cleanup;
2706        }
2707        if (oper.do_cipher){
2708                nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2709                DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2710                outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2711                if (!outpages){
2712                        DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2713                        nooutpages = noinpages = 0;
2714                        err = -ENOMEM;
2715                        goto error_cleanup;
2716                }
2717        }
2718
2719        /* Acquire the mm page semaphore. */
2720        down_read(&current->mm->mmap_sem);
2721
2722        err = get_user_pages(current,
2723                             current->mm,
2724                             (unsigned long int)(oper.indata + prev_ix),
2725                             noinpages,
2726                             0,  /* read access only for in data */
2727                             0, /* no force */
2728                             inpages,
2729                             NULL);
2730
2731        if (err < 0) {
2732                up_read(&current->mm->mmap_sem);
2733                nooutpages = noinpages = 0;
2734                DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2735                goto error_cleanup;
2736        }
2737        noinpages = err;
2738        if (oper.do_cipher){
2739                err = get_user_pages(current,
2740                                     current->mm,
2741                                     (unsigned long int)oper.cipher_outdata,
2742                                     nooutpages,
2743                                     1, /* write access for out data */
2744                                     0, /* no force */
2745                                     outpages,
2746                                     NULL);
2747                up_read(&current->mm->mmap_sem);
2748                if (err < 0) {
2749                        nooutpages = 0;
2750                        DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2751                        goto error_cleanup;
2752                }
2753                nooutpages = err;
2754        } else {
2755                up_read(&current->mm->mmap_sem);
2756        }
2757
2758        /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2759         * csum output and splits when units are (dis-)connected. */
2760        cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2761        cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2762        if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2763                DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2764                err = -ENOMEM;
2765                goto error_cleanup;
2766        }
2767
2768        cop->tfrm_op.inlen = oper.inlen - prev_ix;
2769        cop->tfrm_op.outlen = 0;
2770        if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2771        if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2772        if (oper.do_csum) cop->tfrm_op.outlen += 2;
2773
2774        /* Setup the in iovecs. */
2775        cop->tfrm_op.incount = noinpages;
2776        if (noinpages > 1){
2777                size_t tmplen = cop->tfrm_op.inlen;
2778
2779                cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2780                cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2781                tmplen -= cop->tfrm_op.indata[0].iov_len;
2782                for (i = 1; i<noinpages; i++){
2783                        cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2784                        cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2785                        tmplen -= PAGE_SIZE;
2786                }
2787        } else {
2788                cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2789                cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2790        }
2791
2792        iovlen = nooutpages + 6;
2793        pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2794
2795        next_ix = next_cfg_change_ix(&oper, prev_ix);
2796        if (prev_ix == next_ix){
2797                DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2798                err = -EINVAL;  /* This should be impossible barring bugs. */
2799                goto error_cleanup;
2800        }
2801        while (prev_ix != next_ix){
2802                end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2803                descs[desc_ix].cfg = NULL;
2804                descs[desc_ix].length = next_ix - prev_ix;
2805
2806                if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2807                        dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2808                        dcfgs[dcfg_ix].src = cryptocop_source_dma;
2809                        cipher_active = 1;
2810
2811                        if (next_ix == (oper.cipher_start + oper.cipher_len)){
2812                                cipher_done = 1;
2813                                dcfgs[dcfg_ix].last = 1;
2814                        } else {
2815                                dcfgs[dcfg_ix].last = 0;
2816                        }
2817                        dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2818                        descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2819                        ++dcfg_ix;
2820                }
2821                if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2822                        digest_active = 1;
2823                        dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2824                        dcfgs[dcfg_ix].src = cryptocop_source_dma;
2825                        if (next_ix == (oper.digest_start + oper.digest_len)){
2826                                assert(!digest_done);
2827                                digest_done = 1;
2828                                dcfgs[dcfg_ix].last = 1;
2829                        } else {
2830                                dcfgs[dcfg_ix].last = 0;
2831                        }
2832                        dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2833                        descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2834                        ++dcfg_ix;
2835                }
2836                if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2837                        csum_active = 1;
2838                        dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2839                        dcfgs[dcfg_ix].src = cryptocop_source_dma;
2840                        if (next_ix == (oper.csum_start + oper.csum_len)){
2841                                csum_done = 1;
2842                                dcfgs[dcfg_ix].last = 1;
2843                        } else {
2844                                dcfgs[dcfg_ix].last = 0;
2845                        }
2846                        dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2847                        descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2848                        ++dcfg_ix;
2849                }
2850                if (!descs[desc_ix].cfg){
2851                        DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2852                        err = -EINVAL;
2853                        goto error_cleanup;
2854                }
2855                descs[desc_ix].next = &(descs[desc_ix]) + 1;
2856                ++desc_ix;
2857                prev_ix = next_ix;
2858                next_ix = next_cfg_change_ix(&oper, prev_ix);
2859        }
2860        if (desc_ix > 0){
2861                descs[desc_ix-1].next = NULL;
2862        } else {
2863                descs[0].next = NULL;
2864        }
2865        if (oper.do_digest) {
2866                DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2867                /* Add outdata iovec, length == <length of type of digest> */
2868                cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2869                cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2870                ++iovix;
2871        }
2872        if (oper.do_csum) {
2873                /* Add outdata iovec, length == 2, the length of csum. */
2874                DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2875                /* Add outdata iovec, length == <length of type of digest> */
2876                cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2877                cop->tfrm_op.outdata[iovix].iov_len = 2;
2878                ++iovix;
2879        }
2880        if (oper.do_cipher) {
2881                if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2882                        DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2883                        err = -ENOSYS; /* This should be impossible barring bugs. */
2884                        goto error_cleanup;
2885                }
2886        }
2887        DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2888        cop->tfrm_op.outcount = iovix;
2889        assert(iovix <= (nooutpages + 6));
2890
2891        cop->sid = oper.ses_id;
2892        cop->tfrm_op.desc = &descs[0];
2893
2894        DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2895
2896        if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2897                DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2898                err = -EINVAL;
2899                goto error_cleanup;
2900        }
2901
2902        DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2903
2904        wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2905        DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2906        if (!jc->processed){
2907                printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2908                err = -EIO;
2909                goto error_cleanup;
2910        }
2911
2912        /* Job process done.  Cipher output should already be correct in job so no post processing of outdata. */
2913        DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2914        if (cop->operation_status == 0){
2915                if (oper.do_digest){
2916                        DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2917                        err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2918                        if (0 != err){
2919                                DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2920                                err = -EFAULT;
2921                                goto error_cleanup;
2922                        }
2923                }
2924                if (oper.do_csum){
2925                        DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2926                        err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2927                        if (0 != err){
2928                                DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2929                                err = -EFAULT;
2930                                goto error_cleanup;
2931                        }
2932                }
2933                err = 0;
2934        } else {
2935                DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2936                err = cop->operation_status;
2937        }
2938
2939 error_cleanup:
2940        /* Release page caches. */
2941        for (i = 0; i < noinpages; i++){
2942                put_page(inpages[i]);
2943        }
2944        for (i = 0; i < nooutpages; i++){
2945                int spdl_err;
2946                /* Mark output pages dirty. */
2947                spdl_err = set_page_dirty_lock(outpages[i]);
2948                DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2949        }
2950        for (i = 0; i < nooutpages; i++){
2951                put_page(outpages[i]);
2952        }
2953
2954        kfree(digest_result);
2955        kfree(inpages);
2956        kfree(outpages);
2957        if (cop){
2958                kfree(cop->tfrm_op.indata);
2959                kfree(cop->tfrm_op.outdata);
2960                kfree(cop);
2961        }
2962        kfree(jc);
2963
2964        DEBUG(print_lock_status());
2965
2966        return err;
2967}
2968
2969
2970static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2971{
2972        cryptocop_session_id             sid;
2973        int                              err;
2974        struct cryptocop_private         *dev;
2975        struct strcop_session_op         *sess_op = (struct strcop_session_op *)arg;
2976        struct strcop_session_op         sop;
2977        struct cryptocop_transform_init  *tis = NULL;
2978        struct cryptocop_transform_init  ti_cipher = {0};
2979        struct cryptocop_transform_init  ti_digest = {0};
2980        struct cryptocop_transform_init  ti_csum = {0};
2981
2982        if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2983                return -EFAULT;
2984        err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2985        if (err) return -EFAULT;
2986        if (sop.cipher != cryptocop_cipher_none) {
2987                if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2988        }
2989        DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2990
2991        DEBUG(printk("\tcipher:%d\n"
2992                     "\tcipher_mode:%d\n"
2993                     "\tdigest:%d\n"
2994                     "\tcsum:%d\n",
2995                     (int)sop.cipher,
2996                     (int)sop.cmode,
2997                     (int)sop.digest,
2998                     (int)sop.csum));
2999
3000        if (sop.cipher != cryptocop_cipher_none){
3001                /* Init the cipher. */
3002                switch (sop.cipher){
3003                case cryptocop_cipher_des:
3004                        ti_cipher.alg = cryptocop_alg_des;
3005                        break;
3006                case cryptocop_cipher_3des:
3007                        ti_cipher.alg = cryptocop_alg_3des;
3008                        break;
3009                case cryptocop_cipher_aes:
3010                        ti_cipher.alg = cryptocop_alg_aes;
3011                        break;
3012                default:
3013                        DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3014                        return -EINVAL;
3015                };
3016                DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3017                copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3018                ti_cipher.keylen = sop.keylen;
3019                switch (sop.cmode){
3020                case cryptocop_cipher_mode_cbc:
3021                case cryptocop_cipher_mode_ecb:
3022                        ti_cipher.cipher_mode = sop.cmode;
3023                        break;
3024                default:
3025                        DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3026                        return -EINVAL;
3027                }
3028                DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3029                switch (sop.des3_mode){
3030                case cryptocop_3des_eee:
3031                case cryptocop_3des_eed:
3032                case cryptocop_3des_ede:
3033                case cryptocop_3des_edd:
3034                case cryptocop_3des_dee:
3035                case cryptocop_3des_ded:
3036                case cryptocop_3des_dde:
3037                case cryptocop_3des_ddd:
3038                        ti_cipher.tdes_mode = sop.des3_mode;
3039                        break;
3040                default:
3041                        DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3042                        return -EINVAL;
3043                }
3044                ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3045                ti_cipher.next = tis;
3046                tis = &ti_cipher;
3047        } /* if (sop.cipher != cryptocop_cipher_none) */
3048        if (sop.digest != cryptocop_digest_none){
3049                DEBUG(printk("setting digest transform\n"));
3050                switch (sop.digest){
3051                case cryptocop_digest_md5:
3052                        ti_digest.alg = cryptocop_alg_md5;
3053                        break;
3054                case cryptocop_digest_sha1:
3055                        ti_digest.alg = cryptocop_alg_sha1;
3056                        break;
3057                default:
3058                        DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3059                        return -EINVAL;
3060                }
3061                ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3062                ti_digest.next = tis;
3063                tis = &ti_digest;
3064        } /* if (sop.digest != cryptocop_digest_none) */
3065        if (sop.csum != cryptocop_csum_none){
3066                DEBUG(printk("setting csum transform\n"));
3067                switch (sop.csum){
3068                case cryptocop_csum_le:
3069                case cryptocop_csum_be:
3070                        ti_csum.csum_mode = sop.csum;
3071                        break;
3072                default:
3073                        DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3074                        return -EINVAL;
3075                }
3076                ti_csum.alg = cryptocop_alg_csum;
3077                ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3078                ti_csum.next = tis;
3079                tis = &ti_csum;
3080        } /* (sop.csum != cryptocop_csum_none) */
3081        dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3082        if (!dev){
3083                DEBUG_API(printk("create session, alloc dev\n"));
3084                return -ENOMEM;
3085        }
3086
3087        err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3088        DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3089
3090        if (err) {
3091                kfree(dev);
3092                return err;
3093        }
3094        sess_op->ses_id = sid;
3095        dev->sid = sid;
3096        dev->next = filp->private_data;
3097        filp->private_data = dev;
3098
3099        return 0;
3100}
3101
3102static long cryptocop_ioctl_unlocked(struct inode *inode,
3103        struct file *filp, unsigned int cmd, unsigned long arg)
3104{
3105        int err = 0;
3106        if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3107                DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3108                return -ENOTTY;
3109        }
3110        if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3111                return -ENOTTY;
3112        }
3113        /* Access check of the argument.  Some commands, e.g. create session and process op,
3114           needs additional checks.  Those are handled in the command handling functions. */
3115        if (_IOC_DIR(cmd) & _IOC_READ)
3116                err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3117        else if (_IOC_DIR(cmd) & _IOC_WRITE)
3118                err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3119        if (err) return -EFAULT;
3120
3121        switch (cmd) {
3122        case CRYPTOCOP_IO_CREATE_SESSION:
3123                return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3124        case CRYPTOCOP_IO_CLOSE_SESSION:
3125                return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3126        case CRYPTOCOP_IO_PROCESS_OP:
3127                return cryptocop_ioctl_process(inode, filp, cmd, arg);
3128        default:
3129                DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3130                return -ENOTTY;
3131        }
3132        return 0;
3133}
3134
3135static long
3136cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3137{
3138       struct inode *inode = file->f_path.dentry->d_inode;
3139       long ret;
3140
3141       mutex_lock(&cryptocop_mutex);
3142       ret = cryptocop_ioctl_unlocked(inode, filp, cmd, arg);
3143       mutex_unlock(&cryptocop_mutex);
3144
3145       return ret;
3146}
3147
3148
3149#ifdef LDEBUG
3150static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3151{
3152        struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3153        struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3154        int                       i;
3155
3156        printk("print_dma_descriptors start\n");
3157
3158        printk("iop:\n");
3159        printk("\tsid: 0x%lld\n", iop->sid);
3160
3161        printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3162        printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3163        printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3164        printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3165
3166        printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3167        printk("\tnext: 0x%p\n"
3168               "\tsaved_data: 0x%p\n"
3169               "\tsaved_data_buf: 0x%p\n",
3170               iop->ctx_out.next,
3171               iop->ctx_out.saved_data,
3172               iop->ctx_out.saved_data_buf);
3173
3174        printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3175        printk("\tnext: 0x%p\n"
3176               "\tsaved_data: 0x%p\n"
3177               "\tsaved_data_buf: 0x%p\n",
3178               iop->ctx_in.next,
3179               iop->ctx_in.saved_data,
3180               iop->ctx_in.saved_data_buf);
3181
3182        i = 0;
3183        while (cdesc_out) {
3184                dma_descr_data *td;
3185                printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3186                printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3187                td = cdesc_out->dma_descr;
3188                printk("\n\tbuf: 0x%p\n"
3189                       "\tafter: 0x%p\n"
3190                       "\tmd: 0x%04x\n"
3191                       "\tnext: 0x%p\n",
3192                       td->buf,
3193                       td->after,
3194                       td->md,
3195                       td->next);
3196                printk("flags:\n"
3197                       "\twait:\t%d\n"
3198                       "\teol:\t%d\n"
3199                       "\touteop:\t%d\n"
3200                       "\tineop:\t%d\n"
3201                       "\tintr:\t%d\n",
3202                       td->wait,
3203                       td->eol,
3204                       td->out_eop,
3205                       td->in_eop,
3206                       td->intr);
3207                cdesc_out = cdesc_out->next;
3208                i++;
3209        }
3210        i = 0;
3211        while (cdesc_in) {
3212                dma_descr_data *td;
3213                printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3214                printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3215                td = cdesc_in->dma_descr;
3216                printk("\n\tbuf: 0x%p\n"
3217                       "\tafter: 0x%p\n"
3218                       "\tmd: 0x%04x\n"
3219                       "\tnext: 0x%p\n",
3220                       td->buf,
3221                       td->after,
3222                       td->md,
3223                       td->next);
3224                printk("flags:\n"
3225                       "\twait:\t%d\n"
3226                       "\teol:\t%d\n"
3227                       "\touteop:\t%d\n"
3228                       "\tineop:\t%d\n"
3229                       "\tintr:\t%d\n",
3230                       td->wait,
3231                       td->eol,
3232                       td->out_eop,
3233                       td->in_eop,
3234                       td->intr);
3235                cdesc_in = cdesc_in->next;
3236                i++;
3237        }
3238
3239        printk("print_dma_descriptors end\n");
3240}
3241
3242
3243static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3244{
3245        printk("print_strcop_crypto_op, 0x%p\n", cop);
3246
3247        /* Indata. */
3248        printk("indata=0x%p\n"
3249               "inlen=%d\n"
3250               "do_cipher=%d\n"
3251               "decrypt=%d\n"
3252               "cipher_explicit=%d\n"
3253               "cipher_start=%d\n"
3254               "cipher_len=%d\n"
3255               "outdata=0x%p\n"
3256               "outlen=%d\n",
3257               cop->indata,
3258               cop->inlen,
3259               cop->do_cipher,
3260               cop->decrypt,
3261               cop->cipher_explicit,
3262               cop->cipher_start,
3263               cop->cipher_len,
3264               cop->cipher_outdata,
3265               cop->cipher_outlen);
3266
3267        printk("do_digest=%d\n"
3268               "digest_start=%d\n"
3269               "digest_len=%d\n",
3270               cop->do_digest,
3271               cop->digest_start,
3272               cop->digest_len);
3273
3274        printk("do_csum=%d\n"
3275               "csum_start=%d\n"
3276               "csum_len=%d\n",
3277               cop->do_csum,
3278               cop->csum_start,
3279               cop->csum_len);
3280}
3281
3282static void print_cryptocop_operation(struct cryptocop_operation *cop)
3283{
3284        struct cryptocop_desc      *d;
3285        struct cryptocop_tfrm_cfg  *tc;
3286        struct cryptocop_desc_cfg  *dc;
3287        int                        i;
3288
3289        printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3290        printk("sid: %lld\n", cop->sid);
3291        printk("operation_status=%d\n"
3292               "use_dmalists=%d\n"
3293               "in_interrupt=%d\n"
3294               "fast_callback=%d\n",
3295               cop->operation_status,
3296               cop->use_dmalists,
3297               cop->in_interrupt,
3298               cop->fast_callback);
3299
3300        if (cop->use_dmalists){
3301                print_user_dma_lists(&cop->list_op);
3302        } else {
3303                printk("cop->tfrm_op\n"
3304                       "tfrm_cfg=0x%p\n"
3305                       "desc=0x%p\n"
3306                       "indata=0x%p\n"
3307                       "incount=%d\n"
3308                       "inlen=%d\n"
3309                       "outdata=0x%p\n"
3310                       "outcount=%d\n"
3311                       "outlen=%d\n\n",
3312                       cop->tfrm_op.tfrm_cfg,
3313                       cop->tfrm_op.desc,
3314                       cop->tfrm_op.indata,
3315                       cop->tfrm_op.incount,
3316                       cop->tfrm_op.inlen,
3317                       cop->tfrm_op.outdata,
3318                       cop->tfrm_op.outcount,
3319                       cop->tfrm_op.outlen);
3320
3321                tc = cop->tfrm_op.tfrm_cfg;
3322                while (tc){
3323                        printk("tfrm_cfg, 0x%p\n"
3324                               "tid=%d\n"
3325                               "flags=%d\n"
3326                               "inject_ix=%d\n"
3327                               "next=0x%p\n",
3328                               tc,
3329                               tc->tid,
3330                               tc->flags,
3331                               tc->inject_ix,
3332                               tc->next);
3333                        tc = tc->next;
3334                }
3335                d = cop->tfrm_op.desc;
3336                while (d){
3337                        printk("\n======================desc, 0x%p\n"
3338                               "length=%d\n"
3339                               "cfg=0x%p\n"
3340                               "next=0x%p\n",
3341                               d,
3342                               d->length,
3343                               d->cfg,
3344                               d->next);
3345                        dc = d->cfg;
3346                        while (dc){
3347                                printk("=========desc_cfg, 0x%p\n"
3348                                       "tid=%d\n"
3349                                       "src=%d\n"
3350                                       "last=%d\n"
3351                                       "next=0x%p\n",
3352                                       dc,
3353                                       dc->tid,
3354                                       dc->src,
3355                                       dc->last,
3356                                       dc->next);
3357                                dc = dc->next;
3358                        }
3359                        d = d->next;
3360                }
3361                printk("\n====iniov\n");
3362                for (i = 0; i < cop->tfrm_op.incount; i++){
3363                        printk("indata[%d]\n"
3364                               "base=0x%p\n"
3365                               "len=%d\n",
3366                               i,
3367                               cop->tfrm_op.indata[i].iov_base,
3368                               cop->tfrm_op.indata[i].iov_len);
3369                }
3370                printk("\n====outiov\n");
3371                for (i = 0; i < cop->tfrm_op.outcount; i++){
3372                        printk("outdata[%d]\n"
3373                               "base=0x%p\n"
3374                               "len=%d\n",
3375                               i,
3376                               cop->tfrm_op.outdata[i].iov_base,
3377                               cop->tfrm_op.outdata[i].iov_len);
3378                }
3379        }
3380        printk("------------end print_cryptocop_operation\n");
3381}
3382
3383
3384static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3385{
3386        dma_descr_data *dd;
3387        int i;
3388
3389        printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3390
3391        printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3392        printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3393
3394        printk("##############outlist\n");
3395        dd = phys_to_virt((unsigned long int)dma_op->outlist);
3396        i = 0;
3397        while (dd != NULL) {
3398                printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3399                printk("\n\tbuf: 0x%p\n"
3400                       "\tafter: 0x%p\n"
3401                       "\tmd: 0x%04x\n"
3402                       "\tnext: 0x%p\n",
3403                       dd->buf,
3404                       dd->after,
3405                       dd->md,
3406                       dd->next);
3407                printk("flags:\n"
3408                       "\twait:\t%d\n"
3409                       "\teol:\t%d\n"
3410                       "\touteop:\t%d\n"
3411                       "\tineop:\t%d\n"
3412                       "\tintr:\t%d\n",
3413                       dd->wait,
3414                       dd->eol,
3415                       dd->out_eop,
3416                       dd->in_eop,
3417                       dd->intr);
3418                if (dd->eol)
3419                        dd = NULL;
3420                else
3421                        dd = phys_to_virt((unsigned long int)dd->next);
3422                ++i;
3423        }
3424
3425        printk("##############inlist\n");
3426        dd = phys_to_virt((unsigned long int)dma_op->inlist);
3427        i = 0;
3428        while (dd != NULL) {
3429                printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3430                printk("\n\tbuf: 0x%p\n"
3431                       "\tafter: 0x%p\n"
3432                       "\tmd: 0x%04x\n"
3433                       "\tnext: 0x%p\n",
3434                       dd->buf,
3435                       dd->after,
3436                       dd->md,
3437                       dd->next);
3438                printk("flags:\n"
3439                       "\twait:\t%d\n"
3440                       "\teol:\t%d\n"
3441                       "\touteop:\t%d\n"
3442                       "\tineop:\t%d\n"
3443                       "\tintr:\t%d\n",
3444                       dd->wait,
3445                       dd->eol,
3446                       dd->out_eop,
3447                       dd->in_eop,
3448                       dd->intr);
3449                if (dd->eol)
3450                        dd = NULL;
3451                else
3452                        dd = phys_to_virt((unsigned long int)dd->next);
3453                ++i;
3454        }
3455}
3456
3457
3458static void print_lock_status(void)
3459{
3460        printk("**********************print_lock_status\n");
3461        printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3462        printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3463        printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3464        printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3465        printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3466        printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3467}
3468#endif /* LDEBUG */
3469
3470
3471static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3472
3473static int init_stream_coprocessor(void)
3474{
3475        int err;
3476        int i;
3477        static int initialized = 0;
3478
3479        if (initialized)
3480                return 0;
3481
3482        initialized = 1;
3483
3484        printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3485
3486        err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3487        if (err < 0) {
3488                printk(KERN_ERR "stream co-processor: could not get major number.\n");
3489                return err;
3490        }
3491
3492        err = init_cryptocop();
3493        if (err) {
3494                (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3495                return err;
3496        }
3497        err = cryptocop_job_queue_init();
3498        if (err) {
3499                release_cryptocop();
3500                (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3501                return err;
3502        }
3503        /* Init the descriptor pool. */
3504        for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3505                descr_pool[i].from_pool = 1;
3506                descr_pool[i].next = &descr_pool[i + 1];
3507        }
3508        descr_pool[i].from_pool = 1;
3509        descr_pool[i].next = NULL;
3510        descr_pool_free_list = &descr_pool[0];
3511        descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3512
3513        spin_lock_init(&cryptocop_completed_jobs_lock);
3514        spin_lock_init(&cryptocop_job_queue_lock);
3515        spin_lock_init(&descr_pool_lock);
3516        spin_lock_init(&cryptocop_sessions_lock);
3517        spin_lock_init(&running_job_lock);
3518        spin_lock_init(&cryptocop_process_lock);
3519
3520        cryptocop_sessions = NULL;
3521        next_sid = 1;
3522
3523        cryptocop_running_job = NULL;
3524
3525        printk("stream co-processor: init done.\n");
3526        return 0;
3527}
3528
3529static void __exit exit_stream_coprocessor(void)
3530{
3531        release_cryptocop();
3532        cryptocop_job_queue_close();
3533}
3534
3535module_init(init_stream_coprocessor);
3536module_exit(exit_stream_coprocessor);
3537
3538