linux/arch/cris/arch-v32/drivers/cryptocop.c
<<
>>
Prefs
   1/*
   2 * Stream co-processor driver for the ETRAX FS
   3 *
   4 *    Copyright (C) 2003-2007  Axis Communications AB
   5 */
   6
   7#include <linux/init.h>
   8#include <linux/sched.h>
   9#include <linux/module.h>
  10#include <linux/slab.h>
  11#include <linux/string.h>
  12#include <linux/fs.h>
  13#include <linux/mm.h>
  14#include <linux/smp_lock.h>
  15#include <linux/spinlock.h>
  16#include <linux/stddef.h>
  17
  18#include <asm/uaccess.h>
  19#include <asm/io.h>
  20#include <asm/atomic.h>
  21
  22#include <linux/list.h>
  23#include <linux/interrupt.h>
  24
  25#include <asm/signal.h>
  26#include <asm/irq.h>
  27
  28#include <dma.h>
  29#include <hwregs/dma.h>
  30#include <hwregs/reg_map.h>
  31#include <hwregs/reg_rdwr.h>
  32#include <hwregs/intr_vect_defs.h>
  33
  34#include <hwregs/strcop.h>
  35#include <hwregs/strcop_defs.h>
  36#include <cryptocop.h>
  37
  38#ifdef CONFIG_ETRAXFS
  39#define IN_DMA 9
  40#define OUT_DMA 8
  41#define IN_DMA_INST regi_dma9
  42#define OUT_DMA_INST regi_dma8
  43#define DMA_IRQ DMA9_INTR_VECT
  44#else
  45#define IN_DMA 3
  46#define OUT_DMA 2
  47#define IN_DMA_INST regi_dma3
  48#define OUT_DMA_INST regi_dma2
  49#define DMA_IRQ DMA3_INTR_VECT
  50#endif
  51
  52#define DESCR_ALLOC_PAD  (31)
  53
  54struct cryptocop_dma_desc {
  55        char *free_buf; /* If non-null will be kfreed in free_cdesc() */
  56        dma_descr_data *dma_descr;
  57
  58        unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
  59
  60        unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
  61        struct cryptocop_dma_desc *next;
  62};
  63
  64
  65struct cryptocop_int_operation{
  66        void                        *alloc_ptr;
  67        cryptocop_session_id        sid;
  68
  69        dma_descr_context           ctx_out;
  70        dma_descr_context           ctx_in;
  71
  72        /* DMA descriptors allocated by driver. */
  73        struct cryptocop_dma_desc   *cdesc_out;
  74        struct cryptocop_dma_desc   *cdesc_in;
  75
  76        /* Strcop config to use. */
  77        cryptocop_3des_mode         tdes_mode;
  78        cryptocop_csum_type         csum_mode;
  79
  80        /* DMA descrs provided by consumer. */
  81        dma_descr_data              *ddesc_out;
  82        dma_descr_data              *ddesc_in;
  83};
  84
  85
  86struct cryptocop_tfrm_ctx {
  87        cryptocop_tfrm_id tid;
  88        unsigned int blocklength;
  89
  90        unsigned int start_ix;
  91
  92        struct cryptocop_tfrm_cfg *tcfg;
  93        struct cryptocop_transform_ctx *tctx;
  94
  95        unsigned char previous_src;
  96        unsigned char current_src;
  97
  98        /* Values to use in metadata out. */
  99        unsigned char hash_conf;
 100        unsigned char hash_mode;
 101        unsigned char ciph_conf;
 102        unsigned char cbcmode;
 103        unsigned char decrypt;
 104
 105        unsigned int requires_padding:1;
 106        unsigned int strict_block_length:1;
 107        unsigned int active:1;
 108        unsigned int done:1;
 109        size_t consumed;
 110        size_t produced;
 111
 112        /* Pad (input) descriptors to put in the DMA out list when the transform
 113         * output is put on the DMA in list. */
 114        struct cryptocop_dma_desc *pad_descs;
 115
 116        struct cryptocop_tfrm_ctx *prev_src;
 117        struct cryptocop_tfrm_ctx *curr_src;
 118
 119        /* Mapping to HW. */
 120        unsigned char unit_no;
 121};
 122
 123
 124struct cryptocop_private{
 125        cryptocop_session_id sid;
 126        struct cryptocop_private *next;
 127};
 128
 129/* Session list. */
 130
 131struct cryptocop_transform_ctx{
 132        struct cryptocop_transform_init init;
 133        unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
 134        unsigned int dec_key_set:1;
 135
 136        struct cryptocop_transform_ctx *next;
 137};
 138
 139
 140struct cryptocop_session{
 141        cryptocop_session_id sid;
 142
 143        struct cryptocop_transform_ctx *tfrm_ctx;
 144
 145        struct cryptocop_session *next;
 146};
 147
 148/* Priority levels for jobs sent to the cryptocop.  Checksum operations from
 149   kernel have highest priority since TCPIP stack processing must not
 150   be a bottleneck. */
 151typedef enum {
 152        cryptocop_prio_kernel_csum = 0,
 153        cryptocop_prio_kernel = 1,
 154        cryptocop_prio_user = 2,
 155        cryptocop_prio_no_prios = 3
 156} cryptocop_queue_priority;
 157
 158struct cryptocop_prio_queue{
 159        struct list_head jobs;
 160        cryptocop_queue_priority prio;
 161};
 162
 163struct cryptocop_prio_job{
 164        struct list_head node;
 165        cryptocop_queue_priority prio;
 166
 167        struct cryptocop_operation *oper;
 168        struct cryptocop_int_operation *iop;
 169};
 170
 171struct ioctl_job_cb_ctx {
 172        unsigned int processed:1;
 173};
 174
 175
 176static struct cryptocop_session *cryptocop_sessions = NULL;
 177spinlock_t cryptocop_sessions_lock;
 178
 179/* Next Session ID to assign. */
 180static cryptocop_session_id next_sid = 1;
 181
 182/* Pad for checksum. */
 183static const char csum_zero_pad[1] = {0x00};
 184
 185/* Trash buffer for mem2mem operations. */
 186#define MEM2MEM_DISCARD_BUF_LENGTH  (512)
 187static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
 188
 189/* Descriptor pool. */
 190/* FIXME Tweak this value. */
 191#define CRYPTOCOP_DESCRIPTOR_POOL_SIZE   (100)
 192static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
 193static struct cryptocop_dma_desc *descr_pool_free_list;
 194static int descr_pool_no_free;
 195static spinlock_t descr_pool_lock;
 196
 197/* Lock to stop cryptocop to start processing of a new operation. The holder
 198   of this lock MUST call cryptocop_start_job() after it is unlocked. */
 199spinlock_t cryptocop_process_lock;
 200
 201static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
 202static spinlock_t cryptocop_job_queue_lock;
 203static struct cryptocop_prio_job *cryptocop_running_job = NULL;
 204static spinlock_t running_job_lock;
 205
 206/* The interrupt handler appends completed jobs to this list. The scehduled
 207 * tasklet removes them upon sending the response to the crypto consumer. */
 208static struct list_head cryptocop_completed_jobs;
 209static spinlock_t cryptocop_completed_jobs_lock;
 210
 211DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
 212
 213
 214/** Local functions. **/
 215
 216static int cryptocop_open(struct inode *, struct file *);
 217
 218static int cryptocop_release(struct inode *, struct file *);
 219
 220static int cryptocop_ioctl(struct inode *inode, struct file *file,
 221                           unsigned int cmd, unsigned long arg);
 222
 223static void cryptocop_start_job(void);
 224
 225static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
 226static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
 227
 228static int cryptocop_job_queue_init(void);
 229static void cryptocop_job_queue_close(void);
 230
 231static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 232
 233static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 234
 235static int transform_ok(struct cryptocop_transform_init *tinit);
 236
 237static struct cryptocop_session *get_session(cryptocop_session_id sid);
 238
 239static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
 240
 241static void delete_internal_operation(struct cryptocop_int_operation *iop);
 242
 243static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength);
 244
 245static int init_stream_coprocessor(void);
 246
 247static void __exit exit_stream_coprocessor(void);
 248
 249/*#define LDEBUG*/
 250#ifdef LDEBUG
 251#define DEBUG(s) s
 252#define DEBUG_API(s) s
 253static void print_cryptocop_operation(struct cryptocop_operation *cop);
 254static void print_dma_descriptors(struct cryptocop_int_operation *iop);
 255static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
 256static void print_lock_status(void);
 257static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
 258#define assert(s) do{if (!(s)) panic(#s);} while(0);
 259#else
 260#define DEBUG(s)
 261#define DEBUG_API(s)
 262#define assert(s)
 263#endif
 264
 265
 266/* Transform constants. */
 267#define DES_BLOCK_LENGTH   (8)
 268#define AES_BLOCK_LENGTH   (16)
 269#define MD5_BLOCK_LENGTH   (64)
 270#define SHA1_BLOCK_LENGTH  (64)
 271#define CSUM_BLOCK_LENGTH  (2)
 272#define MD5_STATE_LENGTH   (16)
 273#define SHA1_STATE_LENGTH  (20)
 274
 275/* The device number. */
 276#define CRYPTOCOP_MAJOR    (254)
 277#define CRYPTOCOP_MINOR    (0)
 278
 279
 280
 281const struct file_operations cryptocop_fops = {
 282        .owner =        THIS_MODULE,
 283        .open =         cryptocop_open,
 284        .release =      cryptocop_release,
 285        .ioctl =        cryptocop_ioctl
 286};
 287
 288
 289static void free_cdesc(struct cryptocop_dma_desc *cdesc)
 290{
 291        DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
 292        kfree(cdesc->free_buf);
 293
 294        if (cdesc->from_pool) {
 295                unsigned long int flags;
 296                spin_lock_irqsave(&descr_pool_lock, flags);
 297                cdesc->next = descr_pool_free_list;
 298                descr_pool_free_list = cdesc;
 299                ++descr_pool_no_free;
 300                spin_unlock_irqrestore(&descr_pool_lock, flags);
 301        } else {
 302                kfree(cdesc);
 303        }
 304}
 305
 306
 307static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
 308{
 309        int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
 310        struct cryptocop_dma_desc *cdesc;
 311
 312        if (use_pool) {
 313                unsigned long int flags;
 314                spin_lock_irqsave(&descr_pool_lock, flags);
 315                if (!descr_pool_free_list) {
 316                        spin_unlock_irqrestore(&descr_pool_lock, flags);
 317                        DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
 318                        return NULL;
 319                }
 320                cdesc = descr_pool_free_list;
 321                descr_pool_free_list = descr_pool_free_list->next;
 322                --descr_pool_no_free;
 323                spin_unlock_irqrestore(&descr_pool_lock, flags);
 324                cdesc->from_pool = 1;
 325        } else {
 326                cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
 327                if (!cdesc) {
 328                        DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
 329                        return NULL;
 330                }
 331                cdesc->from_pool = 0;
 332        }
 333        cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
 334
 335        cdesc->next = NULL;
 336
 337        cdesc->free_buf = NULL;
 338        cdesc->dma_descr->out_eop = 0;
 339        cdesc->dma_descr->in_eop = 0;
 340        cdesc->dma_descr->intr = 0;
 341        cdesc->dma_descr->eol = 0;
 342        cdesc->dma_descr->wait = 0;
 343        cdesc->dma_descr->buf = NULL;
 344        cdesc->dma_descr->after = NULL;
 345
 346        DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
 347        return cdesc;
 348}
 349
 350
 351static void setup_descr_chain(struct cryptocop_dma_desc *cd)
 352{
 353        DEBUG(printk("setup_descr_chain: entering\n"));
 354        while (cd) {
 355                if (cd->next) {
 356                        cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
 357                } else {
 358                        cd->dma_descr->next = NULL;
 359                }
 360                cd = cd->next;
 361        }
 362        DEBUG(printk("setup_descr_chain: exit\n"));
 363}
 364
 365
 366/* Create a pad descriptor for the transform.
 367 * Return -1 for error, 0 if pad created. */
 368static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
 369{
 370        struct cryptocop_dma_desc        *cdesc = NULL;
 371        int                              error = 0;
 372        struct strcop_meta_out           mo = {
 373                .ciphsel = src_none,
 374                .hashsel = src_none,
 375                .csumsel = src_none
 376        };
 377        char                             *pad;
 378        size_t                           plen;
 379
 380        DEBUG(printk("create_pad_descriptor: start.\n"));
 381        /* Setup pad descriptor. */
 382
 383        DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
 384        cdesc = alloc_cdesc(alloc_flag);
 385        if (!cdesc){
 386                DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
 387                goto error_cleanup;
 388        }
 389        switch (tc->unit_no) {
 390        case src_md5:
 391                error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
 392                if (error){
 393                        DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
 394                        goto error_cleanup;
 395                }
 396                cdesc->free_buf = pad;
 397                mo.hashsel = src_dma;
 398                mo.hashconf = tc->hash_conf;
 399                mo.hashmode = tc->hash_mode;
 400                break;
 401        case src_sha1:
 402                error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
 403                if (error){
 404                        DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
 405                        goto error_cleanup;
 406                }
 407                cdesc->free_buf = pad;
 408                mo.hashsel = src_dma;
 409                mo.hashconf = tc->hash_conf;
 410                mo.hashmode = tc->hash_mode;
 411                break;
 412        case src_csum:
 413                if (tc->consumed % tc->blocklength){
 414                        pad = (char*)csum_zero_pad;
 415                        plen = 1;
 416                } else {
 417                        pad = (char*)cdesc; /* Use any pointer. */
 418                        plen = 0;
 419                }
 420                mo.csumsel = src_dma;
 421                break;
 422        }
 423        cdesc->dma_descr->wait = 1;
 424        cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
 425        cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
 426        cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
 427
 428        cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 429        *pad_desc = cdesc;
 430
 431        return 0;
 432
 433 error_cleanup:
 434        if (cdesc) free_cdesc(cdesc);
 435        return -1;
 436}
 437
 438
 439static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
 440{
 441        struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
 442        struct strcop_meta_out     mo = {0};
 443
 444        DEBUG(printk("setup_key_dl_desc\n"));
 445
 446        if (!key_desc) {
 447                DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
 448                return -ENOMEM;
 449        }
 450
 451        /* Download key. */
 452        if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
 453                /* Precook the AES decrypt key. */
 454                if (!tc->tctx->dec_key_set){
 455                        get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
 456                        tc->tctx->dec_key_set = 1;
 457                }
 458                key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
 459                key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 460        } else {
 461                key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
 462                key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 463        }
 464        /* Setup metadata. */
 465        mo.dlkey = 1;
 466        switch (tc->tctx->init.keylen) {
 467        case 64:
 468                mo.decrypt = 0;
 469                mo.hashmode = 0;
 470                break;
 471        case 128:
 472                mo.decrypt = 0;
 473                mo.hashmode = 1;
 474                break;
 475        case 192:
 476                mo.decrypt = 1;
 477                mo.hashmode = 0;
 478                break;
 479        case 256:
 480                mo.decrypt = 1;
 481                mo.hashmode = 1;
 482                break;
 483        default:
 484                break;
 485        }
 486        mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
 487        key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 488
 489        key_desc->dma_descr->out_eop = 1;
 490        key_desc->dma_descr->wait = 1;
 491        key_desc->dma_descr->intr = 0;
 492
 493        *kd = key_desc;
 494        return 0;
 495}
 496
 497static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 498{
 499        struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
 500        struct strcop_meta_out     mo = {0};
 501
 502        DEBUG(printk("setup_cipher_iv_desc\n"));
 503
 504        if (!iv_desc) {
 505                DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
 506                return -ENOMEM;
 507        }
 508        /* Download IV. */
 509        iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
 510        iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
 511
 512        /* Setup metadata. */
 513        mo.hashsel = mo.csumsel = src_none;
 514        mo.ciphsel = src_dma;
 515        mo.ciphconf = tc->ciph_conf;
 516        mo.cbcmode = tc->cbcmode;
 517
 518        iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 519
 520        iv_desc->dma_descr->out_eop = 0;
 521        iv_desc->dma_descr->wait = 1;
 522        iv_desc->dma_descr->intr = 0;
 523
 524        *id = iv_desc;
 525        return 0;
 526}
 527
 528/* Map the ouput length of the transform to operation output starting on the inject index. */
 529static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 530{
 531        int                        err = 0;
 532        struct cryptocop_dma_desc  head = {0};
 533        struct cryptocop_dma_desc  *outdesc = &head;
 534        size_t                     iov_offset = 0;
 535        size_t                     out_ix = 0;
 536        int                        outiov_ix = 0;
 537        struct strcop_meta_in      mi = {0};
 538
 539        size_t                     out_length = tc->produced;
 540        int                        rem_length;
 541        int                        dlength;
 542
 543        assert(out_length != 0);
 544        if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
 545                DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 546                return -EINVAL;
 547        }
 548        /* Traverse the out iovec until the result inject index is reached. */
 549        while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
 550                out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
 551                outiov_ix++;
 552        }
 553        if (outiov_ix >= operation->tfrm_op.outcount){
 554                DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 555                return -EINVAL;
 556        }
 557        iov_offset = tc->tcfg->inject_ix - out_ix;
 558        mi.dmasel = tc->unit_no;
 559
 560        /* Setup the output descriptors. */
 561        while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
 562                outdesc->next = alloc_cdesc(alloc_flag);
 563                if (!outdesc->next) {
 564                        DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
 565                        err = -ENOMEM;
 566                        goto error_cleanup;
 567                }
 568                outdesc = outdesc->next;
 569                rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
 570                dlength = (out_length < rem_length) ? out_length : rem_length;
 571
 572                DEBUG(printk("create_input_descriptors:\n"
 573                             "outiov_ix=%d, rem_length=%d, dlength=%d\n"
 574                             "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
 575                             "outcount=%d, outiov_ix=%d\n",
 576                             outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
 577
 578                outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
 579                outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
 580                outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 581
 582                out_length -= dlength;
 583                iov_offset += dlength;
 584                if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
 585                        iov_offset = 0;
 586                        ++outiov_ix;
 587                }
 588        }
 589        if (out_length > 0){
 590                DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
 591                err = -EINVAL;
 592                goto error_cleanup;
 593        }
 594        /* Set sync in last descriptor. */
 595        mi.sync = 1;
 596        outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 597
 598        *id = head.next;
 599        return 0;
 600
 601 error_cleanup:
 602        while (head.next) {
 603                outdesc = head.next->next;
 604                free_cdesc(head.next);
 605                head.next = outdesc;
 606        }
 607        return err;
 608}
 609
 610
 611static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
 612{
 613        while (desc_len != 0) {
 614                struct cryptocop_dma_desc  *cdesc;
 615                int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
 616                int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;
 617
 618                cdesc = alloc_cdesc(alloc_flag);
 619                if (!cdesc) {
 620                        DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
 621                        return -ENOMEM;
 622                }
 623                (*current_out_cdesc)->next = cdesc;
 624                (*current_out_cdesc) = cdesc;
 625
 626                cdesc->free_buf = NULL;
 627
 628                cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
 629                cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
 630
 631                desc_len -= dlength;
 632                *iniov_offset += dlength;
 633                assert(desc_len >= 0);
 634                if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
 635                        *iniov_offset = 0;
 636                        ++(*iniov_ix);
 637                        if (*iniov_ix > operation->tfrm_op.incount) {
 638                                DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
 639                                return  -EINVAL;
 640                        }
 641                }
 642                cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
 643        } /* while (desc_len != 0) */
 644        /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
 645        (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
 646
 647        return 0;
 648}
 649
 650
 651static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
 652{
 653        DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
 654        if (tc->tcfg) {
 655                int                        failed = 0;
 656                struct cryptocop_dma_desc  *idescs = NULL;
 657                DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
 658                if (tc->pad_descs) {
 659                        DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
 660                        while (tc->pad_descs) {
 661                                DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
 662                                (*current_out_cdesc)->next = tc->pad_descs;
 663                                tc->pad_descs = tc->pad_descs->next;
 664                                (*current_out_cdesc) = (*current_out_cdesc)->next;
 665                        }
 666                }
 667
 668                /* Setup and append output descriptors to DMA in list. */
 669                if (tc->unit_no == src_dma){
 670                        /* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
 671                        struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
 672                        unsigned int start_ix = tc->start_ix;
 673                        while (start_ix){
 674                                unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
 675                                (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
 676                                if (!(*current_in_cdesc)->next){
 677                                        DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
 678                                        return -ENOMEM;
 679                                }
 680                                (*current_in_cdesc) = (*current_in_cdesc)->next;
 681                                (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
 682                                (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
 683                                (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 684                                start_ix -= desclen;
 685                        }
 686                        mi.sync = 1;
 687                        (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 688                }
 689
 690                failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
 691                if (failed){
 692                        DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
 693                        return failed;
 694                }
 695                DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
 696                while (idescs) {
 697                        DEBUG(printk("append descriptor 0x%p\n", idescs));
 698                        (*current_in_cdesc)->next = idescs;
 699                        idescs = idescs->next;
 700                        (*current_in_cdesc) = (*current_in_cdesc)->next;
 701                }
 702        }
 703        return 0;
 704}
 705
 706
 707
 708static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
 709{
 710        struct cryptocop_session *sess;
 711        struct cryptocop_transform_ctx *tctx;
 712
 713        struct cryptocop_tfrm_ctx digest_ctx = {
 714                .previous_src = src_none,
 715                .current_src = src_none,
 716                .start_ix = 0,
 717                .requires_padding = 1,
 718                .strict_block_length = 0,
 719                .hash_conf = 0,
 720                .hash_mode = 0,
 721                .ciph_conf = 0,
 722                .cbcmode = 0,
 723                .decrypt = 0,
 724                .consumed = 0,
 725                .produced = 0,
 726                .pad_descs = NULL,
 727                .active = 0,
 728                .done = 0,
 729                .prev_src = NULL,
 730                .curr_src = NULL,
 731                .tcfg = NULL};
 732        struct cryptocop_tfrm_ctx cipher_ctx = {
 733                .previous_src = src_none,
 734                .current_src = src_none,
 735                .start_ix = 0,
 736                .requires_padding = 0,
 737                .strict_block_length = 1,
 738                .hash_conf = 0,
 739                .hash_mode = 0,
 740                .ciph_conf = 0,
 741                .cbcmode = 0,
 742                .decrypt = 0,
 743                .consumed = 0,
 744                .produced = 0,
 745                .pad_descs = NULL,
 746                .active = 0,
 747                .done = 0,
 748                .prev_src = NULL,
 749                .curr_src = NULL,
 750                .tcfg = NULL};
 751        struct cryptocop_tfrm_ctx csum_ctx = {
 752                .previous_src = src_none,
 753                .current_src = src_none,
 754                .start_ix = 0,
 755                .blocklength = 2,
 756                .requires_padding = 1,
 757                .strict_block_length = 0,
 758                .hash_conf = 0,
 759                .hash_mode = 0,
 760                .ciph_conf = 0,
 761                .cbcmode = 0,
 762                .decrypt = 0,
 763                .consumed = 0,
 764                .produced = 0,
 765                .pad_descs = NULL,
 766                .active = 0,
 767                .done = 0,
 768                .tcfg = NULL,
 769                .prev_src = NULL,
 770                .curr_src = NULL,
 771                .unit_no = src_csum};
 772        struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
 773
 774        unsigned int indata_ix = 0;
 775
 776        /* iovec accounting. */
 777        int iniov_ix = 0;
 778        int iniov_offset = 0;
 779
 780        /* Operation descriptor cfg traversal pointer. */
 781        struct cryptocop_desc *odsc;
 782
 783        int failed = 0;
 784        /* List heads for allocated descriptors. */
 785        struct cryptocop_dma_desc out_cdesc_head = {0};
 786        struct cryptocop_dma_desc in_cdesc_head = {0};
 787
 788        struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
 789        struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
 790
 791        struct cryptocop_tfrm_ctx *output_tc = NULL;
 792        void                      *iop_alloc_ptr;
 793
 794        assert(operation != NULL);
 795        assert(int_op != NULL);
 796
 797        DEBUG(printk("cryptocop_setup_dma_list: start\n"));
 798        DEBUG(print_cryptocop_operation(operation));
 799
 800        sess = get_session(operation->sid);
 801        if (!sess) {
 802                DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
 803                failed = -EINVAL;
 804                goto error_cleanup;
 805        }
 806        iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
 807        if (!iop_alloc_ptr) {
 808                DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
 809                failed = -ENOMEM;
 810                goto error_cleanup;
 811        }
 812        (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
 813        DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
 814        (*int_op)->alloc_ptr = iop_alloc_ptr;
 815        DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
 816
 817        (*int_op)->sid = operation->sid;
 818        (*int_op)->cdesc_out = NULL;
 819        (*int_op)->cdesc_in = NULL;
 820        (*int_op)->tdes_mode = cryptocop_3des_ede;
 821        (*int_op)->csum_mode = cryptocop_csum_le;
 822        (*int_op)->ddesc_out = NULL;
 823        (*int_op)->ddesc_in = NULL;
 824
 825        /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
 826        if (!tcfg) {
 827                DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
 828                failed = -EINVAL;
 829                goto error_cleanup;
 830        }
 831        while (tcfg) {
 832                tctx = get_transform_ctx(sess, tcfg->tid);
 833                if (!tctx) {
 834                        DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
 835                        failed = -EINVAL;
 836                        goto error_cleanup;
 837                }
 838                if (tcfg->inject_ix > operation->tfrm_op.outlen){
 839                        DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
 840                        failed = -EINVAL;
 841                        goto error_cleanup;
 842                }
 843                switch (tctx->init.alg){
 844                case cryptocop_alg_mem2mem:
 845                        if (cipher_ctx.tcfg != NULL){
 846                                DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 847                                failed = -EINVAL;
 848                                goto error_cleanup;
 849                        }
 850                        /* mem2mem is handled as a NULL cipher. */
 851                        cipher_ctx.cbcmode = 0;
 852                        cipher_ctx.decrypt = 0;
 853                        cipher_ctx.blocklength = 1;
 854                        cipher_ctx.ciph_conf = 0;
 855                        cipher_ctx.unit_no = src_dma;
 856                        cipher_ctx.tcfg = tcfg;
 857                        cipher_ctx.tctx = tctx;
 858                        break;
 859                case cryptocop_alg_des:
 860                case cryptocop_alg_3des:
 861                case cryptocop_alg_aes:
 862                        /* cipher */
 863                        if (cipher_ctx.tcfg != NULL){
 864                                DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 865                                failed = -EINVAL;
 866                                goto error_cleanup;
 867                        }
 868                        cipher_ctx.tcfg = tcfg;
 869                        cipher_ctx.tctx = tctx;
 870                        if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
 871                                cipher_ctx.decrypt = 1;
 872                        }
 873                        switch (tctx->init.cipher_mode) {
 874                        case cryptocop_cipher_mode_ecb:
 875                                cipher_ctx.cbcmode = 0;
 876                                break;
 877                        case cryptocop_cipher_mode_cbc:
 878                                cipher_ctx.cbcmode = 1;
 879                                break;
 880                        default:
 881                                DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
 882                                failed = -EINVAL;
 883                                goto error_cleanup;
 884                        }
 885                        DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
 886                        switch (tctx->init.alg){
 887                        case cryptocop_alg_des:
 888                                cipher_ctx.ciph_conf = 0;
 889                                cipher_ctx.unit_no = src_des;
 890                                cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 891                                break;
 892                        case cryptocop_alg_3des:
 893                                cipher_ctx.ciph_conf = 1;
 894                                cipher_ctx.unit_no = src_des;
 895                                cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 896                                break;
 897                        case cryptocop_alg_aes:
 898                                cipher_ctx.ciph_conf = 2;
 899                                cipher_ctx.unit_no = src_aes;
 900                                cipher_ctx.blocklength = AES_BLOCK_LENGTH;
 901                                break;
 902                        default:
 903                                panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
 904                        }
 905                        (*int_op)->tdes_mode = tctx->init.tdes_mode;
 906                        break;
 907                case cryptocop_alg_md5:
 908                case cryptocop_alg_sha1:
 909                        /* digest */
 910                        if (digest_ctx.tcfg != NULL){
 911                                DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
 912                                failed = -EINVAL;
 913                                goto error_cleanup;
 914                        }
 915                        digest_ctx.tcfg = tcfg;
 916                        digest_ctx.tctx = tctx;
 917                        digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
 918                        switch (tctx->init.alg){
 919                        case cryptocop_alg_md5:
 920                                digest_ctx.blocklength = MD5_BLOCK_LENGTH;
 921                                digest_ctx.unit_no = src_md5;
 922                                digest_ctx.hash_conf = 1; /* 1 => MD-5 */
 923                                break;
 924                        case cryptocop_alg_sha1:
 925                                digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
 926                                digest_ctx.unit_no = src_sha1;
 927                                digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
 928                                break;
 929                        default:
 930                                panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
 931                        }
 932                        break;
 933                case cryptocop_alg_csum:
 934                        /* digest */
 935                        if (csum_ctx.tcfg != NULL){
 936                                DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
 937                                failed = -EINVAL;
 938                                goto error_cleanup;
 939                        }
 940                        (*int_op)->csum_mode = tctx->init.csum_mode;
 941                        csum_ctx.tcfg = tcfg;
 942                        csum_ctx.tctx = tctx;
 943                        break;
 944                default:
 945                        /* no algorithm. */
 946                        DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
 947                        failed = -EINVAL;
 948                        goto error_cleanup;
 949                }
 950                tcfg = tcfg->next;
 951        }
 952        /* Download key if a cipher is used. */
 953        if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
 954                struct cryptocop_dma_desc  *key_desc = NULL;
 955
 956                failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
 957                if (failed) {
 958                        DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
 959                        goto error_cleanup;
 960                }
 961                current_out_cdesc->next = key_desc;
 962                current_out_cdesc = key_desc;
 963                indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
 964
 965                /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
 966                if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
 967                        struct cryptocop_dma_desc  *iv_desc = NULL;
 968
 969                        DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
 970
 971                        failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
 972                        if (failed) {
 973                                DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
 974                                goto error_cleanup;
 975                        }
 976                        current_out_cdesc->next = iv_desc;
 977                        current_out_cdesc = iv_desc;
 978                        indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
 979                }
 980        }
 981
 982        /* Process descriptors. */
 983        odsc = operation->tfrm_op.desc;
 984        while (odsc) {
 985                struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
 986                struct strcop_meta_out      meta_out = {0};
 987                size_t                      desc_len = odsc->length;
 988                int                         active_count, eop_needed_count;
 989
 990                output_tc = NULL;
 991
 992                DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
 993
 994                while (dcfg) {
 995                        struct cryptocop_tfrm_ctx  *tc = NULL;
 996
 997                        DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
 998                        /* Get the local context for the transform and mark it as the output unit if it produces output. */
 999                        if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000                                tc = &digest_ctx;
1001                        } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002                                tc = &cipher_ctx;
1003                        } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004                                tc = &csum_ctx;
1005                        }
1006                        if (!tc) {
1007                                DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008                                failed = -EINVAL;
1009                                goto error_cleanup;
1010                        }
1011                        if (tc->done) {
1012                                DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013                                failed = -EINVAL;
1014                                goto error_cleanup;
1015                        }
1016                        if (!tc->active) {
1017                                tc->start_ix = indata_ix;
1018                                tc->active = 1;
1019                        }
1020
1021                        tc->previous_src = tc->current_src;
1022                        tc->prev_src = tc->curr_src;
1023                        /* Map source unit id to DMA source config. */
1024                        switch (dcfg->src){
1025                        case cryptocop_source_dma:
1026                                tc->current_src = src_dma;
1027                                break;
1028                        case cryptocop_source_des:
1029                                tc->current_src = src_des;
1030                                break;
1031                        case cryptocop_source_3des:
1032                                tc->current_src = src_des;
1033                                break;
1034                        case cryptocop_source_aes:
1035                                tc->current_src = src_aes;
1036                                break;
1037                        case cryptocop_source_md5:
1038                        case cryptocop_source_sha1:
1039                        case cryptocop_source_csum:
1040                        case cryptocop_source_none:
1041                        default:
1042                                /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043                                 */
1044                                DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045                                failed = -EINVAL;
1046                                goto error_cleanup;
1047                        }
1048                        if (tc->current_src != src_dma) {
1049                                /* Find the unit we are sourcing from. */
1050                                if (digest_ctx.unit_no == tc->current_src){
1051                                        tc->curr_src = &digest_ctx;
1052                                } else if (cipher_ctx.unit_no == tc->current_src){
1053                                        tc->curr_src = &cipher_ctx;
1054                                } else if (csum_ctx.unit_no == tc->current_src){
1055                                        tc->curr_src = &csum_ctx;
1056                                }
1057                                if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058                                        DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059                                        failed = -EINVAL;
1060                                        goto error_cleanup;
1061                                }
1062                        } else {
1063                                tc->curr_src = NULL;
1064                        }
1065
1066                        /* Detect source switch. */
1067                        DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068                        if (tc->active && (tc->current_src != tc->previous_src)) {
1069                                /* Only allow source switch when both the old source unit and the new one have
1070                                 * no pending data to process (i.e. the consumed length must be a multiple of the
1071                                 * transform blocklength). */
1072                                /* Note: if the src == NULL we are actually sourcing from DMA out. */
1073                                if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074                                    ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075                                {
1076                                        DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077                                        failed = -EINVAL;
1078                                        goto error_cleanup;
1079                                }
1080                        }
1081                        /* Detect unit deactivation. */
1082                        if (dcfg->last) {
1083                                /* Length check of this is handled below. */
1084                                tc->done = 1;
1085                        }
1086                        dcfg = dcfg->next;
1087                } /* while (dcfg) */
1088                DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089
1090                if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091                        DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092                        failed = -EINVAL;
1093                        goto error_cleanup;
1094                }
1095                if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096                        DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097                        failed = -EINVAL;
1098                        goto error_cleanup;
1099                }
1100                if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101                        DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102                        failed = -EINVAL;
1103                        goto error_cleanup;
1104                }
1105
1106                /* Update consumed and produced lengths.
1107
1108                   The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
1109                   other unit that process data in blocks of one octet) it is correct, but if it source from a
1110                   block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
1111                   since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112                   unit has processed an exact multiple of its block length the end result will be correct.
1113                   Beware that if the source change restriction change this code will need to be (much) reworked.
1114                */
1115                DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116
1117                if (csum_ctx.active) {
1118                        csum_ctx.consumed += desc_len;
1119                        if (csum_ctx.done) {
1120                                csum_ctx.produced = 2;
1121                        }
1122                        DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123                }
1124                if (digest_ctx.active) {
1125                        digest_ctx.consumed += desc_len;
1126                        if (digest_ctx.done) {
1127                                if (digest_ctx.unit_no == src_md5) {
1128                                        digest_ctx.produced = MD5_STATE_LENGTH;
1129                                } else {
1130                                        digest_ctx.produced = SHA1_STATE_LENGTH;
1131                                }
1132                        }
1133                        DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134                }
1135                if (cipher_ctx.active) {
1136                        /* Ciphers are allowed only to source from DMA out.  That is filtered above. */
1137                        assert(cipher_ctx.current_src == src_dma);
1138                        cipher_ctx.consumed += desc_len;
1139                        cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140                        if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141                                cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142                        }
1143                        DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144                }
1145
1146                /* Setup the DMA out descriptors. */
1147                /* Configure the metadata. */
1148                active_count = 0;
1149                eop_needed_count = 0;
1150                if (cipher_ctx.active) {
1151                        ++active_count;
1152                        if (cipher_ctx.unit_no == src_dma){
1153                                /* mem2mem */
1154                                meta_out.ciphsel = src_none;
1155                        } else {
1156                                meta_out.ciphsel = cipher_ctx.current_src;
1157                        }
1158                        meta_out.ciphconf = cipher_ctx.ciph_conf;
1159                        meta_out.cbcmode = cipher_ctx.cbcmode;
1160                        meta_out.decrypt = cipher_ctx.decrypt;
1161                        DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162                        if (cipher_ctx.done) ++eop_needed_count;
1163                } else {
1164                        meta_out.ciphsel = src_none;
1165                }
1166
1167                if (digest_ctx.active) {
1168                        ++active_count;
1169                        meta_out.hashsel = digest_ctx.current_src;
1170                        meta_out.hashconf = digest_ctx.hash_conf;
1171                        meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172                        DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173                        if (digest_ctx.done) {
1174                                assert(digest_ctx.pad_descs == NULL);
1175                                failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176                                if (failed) {
1177                                        DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178                                        goto error_cleanup;
1179                                }
1180                        }
1181                } else {
1182                        meta_out.hashsel = src_none;
1183                }
1184
1185                if (csum_ctx.active) {
1186                        ++active_count;
1187                        meta_out.csumsel = csum_ctx.current_src;
1188                        if (csum_ctx.done) {
1189                                assert(csum_ctx.pad_descs == NULL);
1190                                failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191                                if (failed) {
1192                                        DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193                                        goto error_cleanup;
1194                                }
1195                        }
1196                } else {
1197                        meta_out.csumsel = src_none;
1198                }
1199                DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200                /* Setup DMA out descriptors for the indata. */
1201                failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202                if (failed) {
1203                        DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204                        goto error_cleanup;
1205                }
1206                /* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
1207                 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208                 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209                 */
1210                assert(active_count >= eop_needed_count);
1211                assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212                if (eop_needed_count) {
1213                        /* This means that the bulk operation (cipeher/m2m) is terminated. */
1214                        if (active_count > 1) {
1215                                /* Use zero length EOP descriptor. */
1216                                struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217                                struct strcop_meta_out    ed_mo = {0};
1218                                if (!ed) {
1219                                        DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220                                        failed = -ENOMEM;
1221                                        goto error_cleanup;
1222                                }
1223
1224                                assert(cipher_ctx.active && cipher_ctx.done);
1225
1226                                if (cipher_ctx.unit_no == src_dma){
1227                                        /* mem2mem */
1228                                        ed_mo.ciphsel = src_none;
1229                                } else {
1230                                        ed_mo.ciphsel = cipher_ctx.current_src;
1231                                }
1232                                ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233                                ed_mo.cbcmode = cipher_ctx.cbcmode;
1234                                ed_mo.decrypt = cipher_ctx.decrypt;
1235
1236                                ed->free_buf = NULL;
1237                                ed->dma_descr->wait = 1;
1238                                ed->dma_descr->out_eop = 1;
1239
1240                                ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241                                ed->dma_descr->after = ed->dma_descr->buf;
1242                                ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243                                current_out_cdesc->next = ed;
1244                                current_out_cdesc = ed;
1245                        } else {
1246                                /* Set EOP in the current out descriptor since the only active module is
1247                                 * the one needing the EOP. */
1248
1249                                current_out_cdesc->dma_descr->out_eop = 1;
1250                        }
1251                }
1252
1253                if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254                if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255                if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256                indata_ix += odsc->length;
1257                odsc = odsc->next;
1258        } /* while (odsc) */ /* Process descriptors. */
1259        DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260        if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261                DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262                failed = -EINVAL;
1263                goto error_cleanup;
1264        }
1265        if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266                DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267                failed = -EINVAL;
1268                goto error_cleanup;
1269        }
1270        if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271                DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272                failed = -EINVAL;
1273                goto error_cleanup;
1274        }
1275
1276        failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277        if (failed){
1278                DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279                goto error_cleanup;
1280        }
1281        failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282        if (failed){
1283                DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284                goto error_cleanup;
1285        }
1286        failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287        if (failed){
1288                DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289                goto error_cleanup;
1290        }
1291
1292        DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293        (*int_op)->cdesc_out = out_cdesc_head.next;
1294        (*int_op)->cdesc_in = in_cdesc_head.next;
1295        DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296
1297        setup_descr_chain(out_cdesc_head.next);
1298        setup_descr_chain(in_cdesc_head.next);
1299
1300        /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301         * last DMA out descriptor for EOL.
1302         */
1303        current_in_cdesc->dma_descr->intr = 1;
1304        current_in_cdesc->dma_descr->eol = 1;
1305        current_out_cdesc->dma_descr->eol = 1;
1306
1307        /* Setup DMA contexts. */
1308        (*int_op)->ctx_out.next = NULL;
1309        (*int_op)->ctx_out.eol = 1;
1310        (*int_op)->ctx_out.intr = 0;
1311        (*int_op)->ctx_out.store_mode = 0;
1312        (*int_op)->ctx_out.en = 0;
1313        (*int_op)->ctx_out.dis = 0;
1314        (*int_op)->ctx_out.md0 = 0;
1315        (*int_op)->ctx_out.md1 = 0;
1316        (*int_op)->ctx_out.md2 = 0;
1317        (*int_op)->ctx_out.md3 = 0;
1318        (*int_op)->ctx_out.md4 = 0;
1319        (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320        (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321
1322        (*int_op)->ctx_in.next = NULL;
1323        (*int_op)->ctx_in.eol = 1;
1324        (*int_op)->ctx_in.intr = 0;
1325        (*int_op)->ctx_in.store_mode = 0;
1326        (*int_op)->ctx_in.en = 0;
1327        (*int_op)->ctx_in.dis = 0;
1328        (*int_op)->ctx_in.md0 = 0;
1329        (*int_op)->ctx_in.md1 = 0;
1330        (*int_op)->ctx_in.md2 = 0;
1331        (*int_op)->ctx_in.md3 = 0;
1332        (*int_op)->ctx_in.md4 = 0;
1333
1334        (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335        (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336
1337        DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338        return 0;
1339
1340error_cleanup:
1341        {
1342                /* Free all allocated resources. */
1343                struct cryptocop_dma_desc *tmp_cdesc;
1344                while (digest_ctx.pad_descs){
1345                        tmp_cdesc = digest_ctx.pad_descs->next;
1346                        free_cdesc(digest_ctx.pad_descs);
1347                        digest_ctx.pad_descs = tmp_cdesc;
1348                }
1349                while (csum_ctx.pad_descs){
1350                        tmp_cdesc = csum_ctx.pad_descs->next;
1351                        free_cdesc(csum_ctx.pad_descs);
1352                        csum_ctx.pad_descs = tmp_cdesc;
1353                }
1354                assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355
1356                if (*int_op != NULL) delete_internal_operation(*int_op);
1357        }
1358        DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359        return failed;
1360}
1361
1362
1363static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364{
1365        void                      *ptr = iop->alloc_ptr;
1366        struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367        struct cryptocop_dma_desc *next;
1368
1369        DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370
1371        while (cd) {
1372                next = cd->next;
1373                free_cdesc(cd);
1374                cd = next;
1375        }
1376        cd = iop->cdesc_in;
1377        while (cd) {
1378                next = cd->next;
1379                free_cdesc(cd);
1380                cd = next;
1381        }
1382        kfree(ptr);
1383}
1384
1385#define MD5_MIN_PAD_LENGTH (9)
1386#define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387
1388static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389{
1390        size_t                  padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391        unsigned char           *p;
1392        int                     i;
1393        unsigned long long int  bit_length = hashed_length << 3;
1394
1395        if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396
1397        p = kmalloc(padlen, alloc_flag);
1398        if (!pad) return -ENOMEM;
1399
1400        *p = 0x80;
1401        memset(p+1, 0, padlen - 1);
1402
1403        DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1404
1405        i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1406        while (bit_length != 0){
1407                p[i++] = bit_length % 0x100;
1408                bit_length >>= 8;
1409        }
1410
1411        *pad = (char*)p;
1412        *pad_length = padlen;
1413
1414        return 0;
1415}
1416
1417#define SHA1_MIN_PAD_LENGTH (9)
1418#define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1419
1420static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1421{
1422        size_t                  padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1423        unsigned char           *p;
1424        int                     i;
1425        unsigned long long int  bit_length = hashed_length << 3;
1426
1427        if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1428
1429        p = kmalloc(padlen, alloc_flag);
1430        if (!pad) return -ENOMEM;
1431
1432        *p = 0x80;
1433        memset(p+1, 0, padlen - 1);
1434
1435        DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1436
1437        i = padlen - 1;
1438        while (bit_length != 0){
1439                p[i--] = bit_length % 0x100;
1440                bit_length >>= 8;
1441        }
1442
1443        *pad = (char*)p;
1444        *pad_length = padlen;
1445
1446        return 0;
1447}
1448
1449
1450static int transform_ok(struct cryptocop_transform_init *tinit)
1451{
1452        switch (tinit->alg){
1453        case cryptocop_alg_csum:
1454                switch (tinit->csum_mode){
1455                case cryptocop_csum_le:
1456                case cryptocop_csum_be:
1457                        break;
1458                default:
1459                        DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1460                        return -EINVAL;
1461                }
1462        case cryptocop_alg_mem2mem:
1463        case cryptocop_alg_md5:
1464        case cryptocop_alg_sha1:
1465                if (tinit->keylen != 0) {
1466                        DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1467                        return -EINVAL; /* This check is a bit strict. */
1468                }
1469                break;
1470        case cryptocop_alg_des:
1471                if (tinit->keylen != 64) {
1472                        DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1473                        return -EINVAL;
1474                }
1475                break;
1476        case cryptocop_alg_3des:
1477                if (tinit->keylen != 192) {
1478                        DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1479                        return -EINVAL;
1480                }
1481                break;
1482        case cryptocop_alg_aes:
1483                if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1484                        DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1485                        return -EINVAL;
1486                }
1487                break;
1488        case cryptocop_no_alg:
1489        default:
1490                DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1491                return -EINVAL;
1492        }
1493
1494        switch (tinit->alg){
1495        case cryptocop_alg_des:
1496        case cryptocop_alg_3des:
1497        case cryptocop_alg_aes:
1498                if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1499        default:
1500                 break;
1501        }
1502        return 0;
1503}
1504
1505
1506int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1507{
1508        struct cryptocop_session         *sess;
1509        struct cryptocop_transform_init  *tfrm_in = tinit;
1510        struct cryptocop_transform_init  *tmp_in;
1511        int                              no_tfrms = 0;
1512        int                              i;
1513        unsigned long int                flags;
1514
1515        init_stream_coprocessor(); /* For safety if we are called early */
1516
1517        while (tfrm_in){
1518                int err;
1519                ++no_tfrms;
1520                if ((err = transform_ok(tfrm_in))) {
1521                        DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1522                        return err;
1523                }
1524                tfrm_in = tfrm_in->next;
1525        }
1526        if (0 == no_tfrms) {
1527                DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1528                return -EINVAL;
1529        }
1530
1531        sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1532        if (!sess){
1533                DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1534                return -ENOMEM;
1535        }
1536
1537        sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1538        if (!sess->tfrm_ctx) {
1539                DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1540                kfree(sess);
1541                return -ENOMEM;
1542        }
1543
1544        tfrm_in = tinit;
1545        for (i = 0; i < no_tfrms; i++){
1546                tmp_in = tfrm_in->next;
1547                while (tmp_in){
1548                        if (tmp_in->tid == tfrm_in->tid) {
1549                                DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1550                                kfree(sess->tfrm_ctx);
1551                                kfree(sess);
1552                                return -EINVAL;
1553                        }
1554                        tmp_in = tmp_in->next;
1555                }
1556                memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1557                sess->tfrm_ctx[i].dec_key_set = 0;
1558                sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1559
1560                tfrm_in = tfrm_in->next;
1561        }
1562        sess->tfrm_ctx[i-1].next = NULL;
1563
1564        spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1565        sess->sid = next_sid;
1566        next_sid++;
1567        /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1568         *      OTOH 2^64 is a really large number of session. */
1569        if (next_sid == 0) next_sid = 1;
1570
1571        /* Prepend to session list. */
1572        sess->next = cryptocop_sessions;
1573        cryptocop_sessions = sess;
1574        spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1575        *sid = sess->sid;
1576        return 0;
1577}
1578
1579
1580int cryptocop_free_session(cryptocop_session_id sid)
1581{
1582        struct cryptocop_transform_ctx    *tc;
1583        struct cryptocop_session          *sess = NULL;
1584        struct cryptocop_session          *psess = NULL;
1585        unsigned long int                 flags;
1586        int                               i;
1587        LIST_HEAD(remove_list);
1588        struct list_head                  *node, *tmp;
1589        struct cryptocop_prio_job         *pj;
1590
1591        DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1592
1593        spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1594        sess = cryptocop_sessions;
1595        while (sess && sess->sid != sid){
1596                psess = sess;
1597                sess = sess->next;
1598        }
1599        if (sess){
1600                if (psess){
1601                        psess->next = sess->next;
1602                } else {
1603                        cryptocop_sessions = sess->next;
1604                }
1605        }
1606        spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1607
1608        if (!sess) return -EINVAL;
1609
1610        /* Remove queued jobs. */
1611        spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1612
1613        for (i = 0; i < cryptocop_prio_no_prios; i++){
1614                if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1615                        list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1616                                pj = list_entry(node, struct cryptocop_prio_job, node);
1617                                if (pj->oper->sid == sid) {
1618                                        list_move_tail(node, &remove_list);
1619                                }
1620                        }
1621                }
1622        }
1623        spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1624
1625        list_for_each_safe(node, tmp, &remove_list) {
1626                list_del(node);
1627                pj = list_entry(node, struct cryptocop_prio_job, node);
1628                pj->oper->operation_status = -EAGAIN;  /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1629                DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1630                pj->oper->cb(pj->oper, pj->oper->cb_data);
1631                delete_internal_operation(pj->iop);
1632                kfree(pj);
1633        }
1634
1635        tc = sess->tfrm_ctx;
1636        /* Erase keying data. */
1637        while (tc){
1638                DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1639                memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1640                memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1641                tc = tc->next;
1642        }
1643        kfree(sess->tfrm_ctx);
1644        kfree(sess);
1645
1646        return 0;
1647}
1648
1649static struct cryptocop_session *get_session(cryptocop_session_id sid)
1650{
1651        struct cryptocop_session    *sess;
1652        unsigned long int           flags;
1653
1654        spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1655        sess = cryptocop_sessions;
1656        while (sess && (sess->sid != sid)){
1657                sess = sess->next;
1658        }
1659        spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1660
1661        return sess;
1662}
1663
1664static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1665{
1666        struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1667
1668        DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1669        assert(sess != NULL);
1670        while (tc && tc->init.tid != tid){
1671                DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1672                tc = tc->next;
1673        }
1674        DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1675        return tc;
1676}
1677
1678
1679
1680/* The AES s-transform matrix (s-box). */
1681static const u8 aes_sbox[256] = {
1682        99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
1683        202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
1684        183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
1685        4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
1686        9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
1687        83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
1688        208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
1689        81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
1690        205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
1691        96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
1692        224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
1693        231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
1694        186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
1695        112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
1696        225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
1697        140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
1698};
1699
1700/* AES has a 32 bit word round constants for each round in the
1701 * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
1702 */
1703static u32 round_constant[11] = {
1704        0x01000000, 0x02000000, 0x04000000, 0x08000000,
1705        0x10000000, 0x20000000, 0x40000000, 0x80000000,
1706        0x1B000000, 0x36000000, 0x6C000000
1707};
1708
1709/* Apply the s-box to each of the four occtets in w. */
1710static u32 aes_ks_subword(const u32 w)
1711{
1712        u8 bytes[4];
1713
1714        *(u32*)(&bytes[0]) = w;
1715        bytes[0] = aes_sbox[bytes[0]];
1716        bytes[1] = aes_sbox[bytes[1]];
1717        bytes[2] = aes_sbox[bytes[2]];
1718        bytes[3] = aes_sbox[bytes[3]];
1719        return *(u32*)(&bytes[0]);
1720}
1721
1722/* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1723 * (Note that AES words are 32 bit long)
1724 *
1725 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1726 * word temp
1727 * i = 0
1728 * while (i < Nk) {
1729 *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1730 *   i = i + 1
1731 * }
1732 * i = Nk
1733 *
1734 * while (i < (Nb * (Nr + 1))) {
1735 *   temp = w[i - 1]
1736 *   if ((i mod Nk) == 0) {
1737 *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1738 *   }
1739 *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
1740 *     temp = SubWord(temp)
1741 *   }
1742 *   w[i] = w[i - Nk] xor temp
1743 * }
1744 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1745 * SubWord(t) applies the AES s-box individually to each octet
1746 * in a 32 bit word.
1747 *
1748 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1749 * values for Nr of 10, 12, and 14).  Nb is always 4.
1750 *
1751 * To construct w[i], w[i - 1] and w[i - Nk] must be
1752 * available.  Consequently we must keep a state of the last Nk words
1753 * to be able to create the last round keys.
1754 */
1755static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
1756{
1757        u32 temp;
1758        u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1759        u8  w_last_ix;
1760        int i;
1761        u8  nr, nk;
1762
1763        switch (keylength){
1764        case 128:
1765                nk = 4;
1766                nr = 10;
1767                break;
1768        case 192:
1769                nk = 6;
1770                nr = 12;
1771                break;
1772        case 256:
1773                nk = 8;
1774                nr = 14;
1775                break;
1776        default:
1777                panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1778        };
1779
1780        /* Need to do host byte order correction here since key is byte oriented and the
1781         * kx algorithm is word (u32) oriented. */
1782        for (i = 0; i < nk; i+=1) {
1783                w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1784        }
1785
1786        i = (int)nk;
1787        w_last_ix = i - 1;
1788        while (i < (4 * (nr + 2))) {
1789                temp = w_ring[w_last_ix];
1790                if (!(i % nk)) {
1791                        /* RotWord(temp) */
1792                        temp = (temp << 8) | (temp >> 24);
1793                        temp = aes_ks_subword(temp);
1794                        temp ^= round_constant[i/nk - 1];
1795                } else if ((nk > 6) && ((i % nk) == 4)) {
1796                        temp = aes_ks_subword(temp);
1797                }
1798                w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1799                temp ^= w_ring[w_last_ix];
1800                w_ring[w_last_ix] = temp;
1801
1802                /* We need the round keys for round Nr+1 and Nr+2 (round key
1803                 * Nr+2 is the round key beyond the last one used when
1804                 * encrypting).  Rounds are numbered starting from 0, Nr=10
1805                 * implies 11 rounds are used in encryption/decryption.
1806                 */
1807                if (i >= (4 * nr)) {
1808                        /* Need to do host byte order correction here, the key
1809                         * is byte oriented. */
1810                        *(u32*)dec_key = cpu_to_be32(temp);
1811                        dec_key += 4;
1812                }
1813                ++i;
1814        }
1815}
1816
1817
1818/**** Job/operation management. ****/
1819
1820int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1821{
1822        return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1823}
1824
1825int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1826{
1827        return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1828}
1829
1830int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1831{
1832        return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1833}
1834
1835static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1836{
1837        int                           ret;
1838        struct cryptocop_prio_job     *pj = NULL;
1839        unsigned long int             flags;
1840
1841        DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1842
1843        if (!operation || !operation->cb){
1844                DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1845                return -EINVAL;
1846        }
1847
1848        if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1849                DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1850                return ret;
1851        }
1852        assert(pj != NULL);
1853
1854        spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1855        list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1856        spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1857
1858        /* Make sure a job is running */
1859        cryptocop_start_job();
1860        return 0;
1861}
1862
1863static void cryptocop_do_tasklet(unsigned long unused);
1864DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1865
1866static void cryptocop_do_tasklet(unsigned long unused)
1867{
1868        struct list_head             *node;
1869        struct cryptocop_prio_job    *pj = NULL;
1870        unsigned long                flags;
1871
1872        DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1873
1874        do {
1875                spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1876                if (!list_empty(&cryptocop_completed_jobs)){
1877                        node = cryptocop_completed_jobs.next;
1878                        list_del(node);
1879                        pj = list_entry(node, struct cryptocop_prio_job, node);
1880                } else {
1881                        pj = NULL;
1882                }
1883                spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1884                if (pj) {
1885                        assert(pj->oper != NULL);
1886
1887                        /* Notify consumer of operation completeness. */
1888                        DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1889
1890                        pj->oper->operation_status = 0; /* Job is completed. */
1891                        pj->oper->cb(pj->oper, pj->oper->cb_data);
1892                        delete_internal_operation(pj->iop);
1893                        kfree(pj);
1894                }
1895        } while (pj != NULL);
1896
1897        DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1898}
1899
1900static irqreturn_t
1901dma_done_interrupt(int irq, void *dev_id)
1902{
1903        struct cryptocop_prio_job *done_job;
1904        reg_dma_rw_ack_intr ack_intr = {
1905                .data = 1,
1906        };
1907
1908        REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1909
1910        DEBUG(printk("cryptocop DMA done\n"));
1911
1912        spin_lock(&running_job_lock);
1913        if (cryptocop_running_job == NULL){
1914                printk("stream co-processor got interrupt when not busy\n");
1915                spin_unlock(&running_job_lock);
1916                return IRQ_HANDLED;
1917        }
1918        done_job = cryptocop_running_job;
1919        cryptocop_running_job = NULL;
1920        spin_unlock(&running_job_lock);
1921
1922        /* Start processing a job. */
1923        if (!spin_trylock(&cryptocop_process_lock)){
1924                DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1925        } else {
1926                cryptocop_start_job();
1927                spin_unlock(&cryptocop_process_lock);
1928        }
1929
1930        done_job->oper->operation_status = 0; /* Job is completed. */
1931        if (done_job->oper->fast_callback){
1932                /* This operation wants callback from interrupt. */
1933                done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1934                delete_internal_operation(done_job->iop);
1935                kfree(done_job);
1936        } else {
1937                spin_lock(&cryptocop_completed_jobs_lock);
1938                list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1939                spin_unlock(&cryptocop_completed_jobs_lock);
1940                tasklet_schedule(&cryptocop_tasklet);
1941        }
1942
1943        DEBUG(printk("cryptocop leave irq handler\n"));
1944        return IRQ_HANDLED;
1945}
1946
1947
1948/* Setup interrupts and DMA channels. */
1949static int init_cryptocop(void)
1950{
1951        unsigned long          flags;
1952        reg_dma_rw_cfg         dma_cfg = {.en = 1};
1953        reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1954        reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1955        reg_strcop_rw_cfg      strcop_cfg = {
1956                .ipend = regk_strcop_little,
1957                .td1 = regk_strcop_e,
1958                .td2 = regk_strcop_d,
1959                .td3 = regk_strcop_e,
1960                .ignore_sync = 0,
1961                .en = 1
1962        };
1963
1964        if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1965                        "stream co-processor DMA", NULL))
1966                panic("request_irq stream co-processor irq dma9");
1967
1968        (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969                0, dma_strp);
1970        (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1971                0, dma_strp);
1972
1973        local_irq_save(flags);
1974
1975        /* Reset and enable the cryptocop. */
1976        strcop_cfg.en = 0;
1977        REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978        strcop_cfg.en = 1;
1979        REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1980
1981        /* Enable DMAs. */
1982        REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1983        REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1984
1985        /* Set up wordsize = 4 for DMAs. */
1986        DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1987        DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1988
1989        /* Enable interrupts. */
1990        REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1991
1992        /* Clear intr ack. */
1993        REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1994
1995        local_irq_restore(flags);
1996
1997        return 0;
1998}
1999
2000/* Free used cryptocop hw resources (interrupt and DMA channels). */
2001static void release_cryptocop(void)
2002{
2003        unsigned long          flags;
2004        reg_dma_rw_cfg         dma_cfg = {.en = 0};
2005        reg_dma_rw_intr_mask   intr_mask_in = {0};
2006        reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
2007
2008        local_irq_save(flags);
2009
2010        /* Clear intr ack. */
2011        REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2012
2013        /* Disable DMAs. */
2014        REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2015        REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2016
2017        /* Disable interrupts. */
2018        REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2019
2020        local_irq_restore(flags);
2021
2022        free_irq(DMA_IRQ, NULL);
2023
2024        (void)crisv32_free_dma(OUT_DMA);
2025        (void)crisv32_free_dma(IN_DMA);
2026}
2027
2028
2029/* Init job queue. */
2030static int cryptocop_job_queue_init(void)
2031{
2032        int i;
2033
2034        INIT_LIST_HEAD(&cryptocop_completed_jobs);
2035
2036        for (i = 0; i < cryptocop_prio_no_prios; i++){
2037                cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2038                INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2039        }
2040        return 0;
2041}
2042
2043
2044static void cryptocop_job_queue_close(void)
2045{
2046        struct list_head               *node, *tmp;
2047        struct cryptocop_prio_job      *pj = NULL;
2048        unsigned long int              process_flags, flags;
2049        int                            i;
2050
2051        /* FIXME: This is as yet untested code. */
2052
2053        /* Stop strcop from getting an operation to process while we are closing the
2054           module. */
2055        spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2056
2057        /* Empty the job queue. */
2058        for (i = 0; i < cryptocop_prio_no_prios; i++){
2059                if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2060                        list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2061                                pj = list_entry(node, struct cryptocop_prio_job, node);
2062                                list_del(node);
2063
2064                                /* Call callback to notify consumer of job removal. */
2065                                DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2066                                pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2067                                pj->oper->cb(pj->oper, pj->oper->cb_data);
2068
2069                                delete_internal_operation(pj->iop);
2070                                kfree(pj);
2071                        }
2072                }
2073        }
2074        spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2075
2076        /* Remove the running job, if any. */
2077        spin_lock_irqsave(&running_job_lock, flags);
2078        if (cryptocop_running_job){
2079                reg_strcop_rw_cfg rw_cfg;
2080                reg_dma_rw_cfg    dma_out_cfg, dma_in_cfg;
2081
2082                /* Stop DMA. */
2083                dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2084                dma_out_cfg.en = regk_dma_no;
2085                REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2086
2087                dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2088                dma_in_cfg.en = regk_dma_no;
2089                REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2090
2091                /* Disble the cryptocop. */
2092                rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2093                rw_cfg.en = 0;
2094                REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2095
2096                pj = cryptocop_running_job;
2097                cryptocop_running_job = NULL;
2098
2099                /* Call callback to notify consumer of job removal. */
2100                DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2101                pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2102                pj->oper->cb(pj->oper, pj->oper->cb_data);
2103
2104                delete_internal_operation(pj->iop);
2105                kfree(pj);
2106        }
2107        spin_unlock_irqrestore(&running_job_lock, flags);
2108
2109        /* Remove completed jobs, if any. */
2110        spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2111
2112        list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2113                pj = list_entry(node, struct cryptocop_prio_job, node);
2114                list_del(node);
2115                /* Call callback to notify consumer of job removal. */
2116                DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2117                pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2118                pj->oper->cb(pj->oper, pj->oper->cb_data);
2119
2120                delete_internal_operation(pj->iop);
2121                kfree(pj);
2122        }
2123        spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2124}
2125
2126
2127static void cryptocop_start_job(void)
2128{
2129        int                          i;
2130        struct cryptocop_prio_job    *pj;
2131        unsigned long int            flags;
2132        unsigned long int            running_job_flags;
2133        reg_strcop_rw_cfg            rw_cfg = {.en = 1, .ignore_sync = 0};
2134
2135        DEBUG(printk("cryptocop_start_job: entering\n"));
2136
2137        spin_lock_irqsave(&running_job_lock, running_job_flags);
2138        if (cryptocop_running_job != NULL){
2139                /* Already running. */
2140                DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2141                spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2142                return;
2143        }
2144        spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2145
2146        /* Check the queues in priority order. */
2147        for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2148        if (i == cryptocop_prio_no_prios) {
2149                spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2150                spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2151                DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2152                return; /* No jobs to run */
2153        }
2154        DEBUG(printk("starting job for prio %d\n", i));
2155
2156        /* TODO: Do not starve lower priority jobs.  Let in a lower
2157         * prio job for every N-th processed higher prio job or some
2158         * other scheduling policy.  This could reasonably be
2159         * tweakable since the optimal balance would depend on the
2160         * type of load on the system. */
2161
2162        /* Pull the DMA lists from the job and start the DMA client. */
2163        pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2164        list_del(&pj->node);
2165        spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2166        cryptocop_running_job = pj;
2167
2168        /* Set config register (3DES and CSUM modes). */
2169        switch (pj->iop->tdes_mode){
2170        case cryptocop_3des_eee:
2171                rw_cfg.td1 = regk_strcop_e;
2172                rw_cfg.td2 = regk_strcop_e;
2173                rw_cfg.td3 = regk_strcop_e;
2174                break;
2175        case cryptocop_3des_eed:
2176                rw_cfg.td1 = regk_strcop_e;
2177                rw_cfg.td2 = regk_strcop_e;
2178                rw_cfg.td3 = regk_strcop_d;
2179                break;
2180        case cryptocop_3des_ede:
2181                rw_cfg.td1 = regk_strcop_e;
2182                rw_cfg.td2 = regk_strcop_d;
2183                rw_cfg.td3 = regk_strcop_e;
2184                break;
2185        case cryptocop_3des_edd:
2186                rw_cfg.td1 = regk_strcop_e;
2187                rw_cfg.td2 = regk_strcop_d;
2188                rw_cfg.td3 = regk_strcop_d;
2189                break;
2190        case cryptocop_3des_dee:
2191                rw_cfg.td1 = regk_strcop_d;
2192                rw_cfg.td2 = regk_strcop_e;
2193                rw_cfg.td3 = regk_strcop_e;
2194                break;
2195        case cryptocop_3des_ded:
2196                rw_cfg.td1 = regk_strcop_d;
2197                rw_cfg.td2 = regk_strcop_e;
2198                rw_cfg.td3 = regk_strcop_d;
2199                break;
2200        case cryptocop_3des_dde:
2201                rw_cfg.td1 = regk_strcop_d;
2202                rw_cfg.td2 = regk_strcop_d;
2203                rw_cfg.td3 = regk_strcop_e;
2204                break;
2205        case cryptocop_3des_ddd:
2206                rw_cfg.td1 = regk_strcop_d;
2207                rw_cfg.td2 = regk_strcop_d;
2208                rw_cfg.td3 = regk_strcop_d;
2209                break;
2210        default:
2211                DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2212        }
2213        switch (pj->iop->csum_mode){
2214        case cryptocop_csum_le:
2215                rw_cfg.ipend = regk_strcop_little;
2216                break;
2217        case cryptocop_csum_be:
2218                rw_cfg.ipend = regk_strcop_big;
2219                break;
2220        default:
2221                DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2222        }
2223        REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2224
2225        DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2226                     "ctx_in: 0x%p, phys: 0x%p\n"
2227                     "ctx_out: 0x%p, phys: 0x%p\n",
2228                     pj,
2229                     &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2230                     &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2231
2232        /* Start input DMA. */
2233        flush_dma_context(&pj->iop->ctx_in);
2234        DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2235
2236        /* Start output DMA. */
2237        DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2238
2239        spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2240        DEBUG(printk("cryptocop_start_job: exiting\n"));
2241}
2242
2243
2244static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2245{
2246        int  err;
2247        int  alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2248        void *iop_alloc_ptr = NULL;
2249
2250        *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2251        if (!*pj) return -ENOMEM;
2252
2253        DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2254
2255        (*pj)->oper = operation;
2256        DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n",  (*pj)->oper->cb, (*pj)->oper->cb_data));
2257
2258        if (operation->use_dmalists) {
2259                DEBUG(print_user_dma_lists(&operation->list_op));
2260                if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2261                        DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2262                        kfree(*pj);
2263                        return -EINVAL;
2264                }
2265                iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2266                if (!iop_alloc_ptr) {
2267                        DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2268                        kfree(*pj);
2269                        return -ENOMEM;
2270                }
2271                (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2272                DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2273                (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2274                (*pj)->iop->sid = operation->sid;
2275                (*pj)->iop->cdesc_out = NULL;
2276                (*pj)->iop->cdesc_in = NULL;
2277                (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2278                (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2279                (*pj)->iop->ddesc_out = operation->list_op.outlist;
2280                (*pj)->iop->ddesc_in = operation->list_op.inlist;
2281
2282                /* Setup DMA contexts. */
2283                (*pj)->iop->ctx_out.next = NULL;
2284                (*pj)->iop->ctx_out.eol = 1;
2285                (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2286                (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2287
2288                (*pj)->iop->ctx_in.next = NULL;
2289                (*pj)->iop->ctx_in.eol = 1;
2290                (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2291                (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2292        } else {
2293                if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2294                        DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2295                        kfree(*pj);
2296                        return err;
2297                }
2298        }
2299        DEBUG(print_dma_descriptors((*pj)->iop));
2300
2301        DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2302
2303        return 0;
2304}
2305
2306static int cryptocop_open(struct inode *inode, struct file *filp)
2307{
2308        int p = iminor(inode);
2309
2310        cycle_kernel_lock();
2311        if (p != CRYPTOCOP_MINOR) return -EINVAL;
2312
2313        filp->private_data = NULL;
2314        return 0;
2315}
2316
2317
2318static int cryptocop_release(struct inode *inode, struct file *filp)
2319{
2320        struct cryptocop_private *dev = filp->private_data;
2321        struct cryptocop_private *dev_next;
2322
2323        while (dev){
2324                dev_next = dev->next;
2325                if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2326                        (void)cryptocop_free_session(dev->sid);
2327                }
2328                kfree(dev);
2329                dev = dev_next;
2330        }
2331
2332        return 0;
2333}
2334
2335
2336static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2337                                         unsigned int cmd, unsigned long arg)
2338{
2339        struct cryptocop_private  *dev = filp->private_data;
2340        struct cryptocop_private  *prev_dev = NULL;
2341        struct strcop_session_op  *sess_op = (struct strcop_session_op *)arg;
2342        struct strcop_session_op  sop;
2343        int                       err;
2344
2345        DEBUG(printk("cryptocop_ioctl_close_session\n"));
2346
2347        if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2348                return -EFAULT;
2349        err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2350        if (err) return -EFAULT;
2351
2352        while (dev && (dev->sid != sop.ses_id)) {
2353                prev_dev = dev;
2354                dev = dev->next;
2355        }
2356        if (dev){
2357                if (prev_dev){
2358                        prev_dev->next = dev->next;
2359                } else {
2360                        filp->private_data = dev->next;
2361                }
2362                err = cryptocop_free_session(dev->sid);
2363                if (err) return -EFAULT;
2364        } else {
2365                DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2366                return -EINVAL;
2367        }
2368        return 0;
2369}
2370
2371
2372static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2373{
2374        struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2375
2376        DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2377
2378        jc->processed = 1;
2379        wake_up(&cryptocop_ioc_process_wq);
2380}
2381
2382
2383#define CRYPTOCOP_IOCTL_CIPHER_TID  (1)
2384#define CRYPTOCOP_IOCTL_DIGEST_TID  (2)
2385#define CRYPTOCOP_IOCTL_CSUM_TID    (3)
2386
2387static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2388{
2389        size_t ch_ix = 0;
2390
2391        if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2392        if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2393        if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2394
2395        DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2396        return ch_ix;
2397}
2398
2399
2400static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2401{
2402        size_t ch_ix = INT_MAX;
2403        size_t tmp_ix = 0;
2404
2405        if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2406                if (crp_op->cipher_start > ix) {
2407                        ch_ix = crp_op->cipher_start;
2408                } else {
2409                        ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2410                }
2411        }
2412        if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2413                if (crp_op->digest_start > ix) {
2414                        tmp_ix = crp_op->digest_start;
2415                } else {
2416                        tmp_ix = crp_op->digest_start + crp_op->digest_len;
2417                }
2418                if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2419        }
2420        if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2421                if (crp_op->csum_start > ix) {
2422                        tmp_ix = crp_op->csum_start;
2423                } else {
2424                        tmp_ix = crp_op->csum_start + crp_op->csum_len;
2425                }
2426                if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2427        }
2428        if (ch_ix == INT_MAX) ch_ix = ix;
2429        DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2430        return ch_ix;
2431}
2432
2433
2434/* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2435 * Return -1 for ok, 0 for fail. */
2436static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2437{
2438        int tmplen;
2439
2440        assert(iov != NULL);
2441        assert(iovix != NULL);
2442        assert(pages != NULL);
2443        assert(pageix != NULL);
2444        assert(pageoffset != NULL);
2445
2446        DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2447
2448        while (map_length > 0){
2449                DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2450                if (*iovix >= iovlen){
2451                        DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2452                        return 0;
2453                }
2454                if (*pageix >= nopages){
2455                        DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2456                        return 0;
2457                }
2458                iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2459                tmplen = PAGE_SIZE - *pageoffset;
2460                if (tmplen < map_length){
2461                        (*pageoffset) = 0;
2462                        (*pageix)++;
2463                } else {
2464                        tmplen = map_length;
2465                        (*pageoffset) += map_length;
2466                }
2467                DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2468                iov[*iovix].iov_len = tmplen;
2469                map_length -= tmplen;
2470                (*iovix)++;
2471        }
2472        DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2473        return -1;
2474}
2475
2476
2477
2478static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2479{
2480        int                             i;
2481        struct cryptocop_private        *dev = filp->private_data;
2482        struct strcop_crypto_op         *crp_oper = (struct strcop_crypto_op *)arg;
2483        struct strcop_crypto_op         oper = {0};
2484        int                             err = 0;
2485        struct cryptocop_operation      *cop = NULL;
2486
2487        struct ioctl_job_cb_ctx         *jc = NULL;
2488
2489        struct page                     **inpages = NULL;
2490        struct page                     **outpages = NULL;
2491        int                             noinpages = 0;
2492        int                             nooutpages = 0;
2493
2494        struct cryptocop_desc           descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2495                                                   * can get connected/disconnected on different places in the indata. */
2496        struct cryptocop_desc_cfg       dcfgs[5*3];
2497        int                             desc_ix = 0;
2498        int                             dcfg_ix = 0;
2499        struct cryptocop_tfrm_cfg       ciph_tcfg = {0};
2500        struct cryptocop_tfrm_cfg       digest_tcfg = {0};
2501        struct cryptocop_tfrm_cfg       csum_tcfg = {0};
2502
2503        unsigned char                   *digest_result = NULL;
2504        int                             digest_length = 0;
2505        int                             cblocklen = 0;
2506        unsigned char                   csum_result[CSUM_BLOCK_LENGTH];
2507        struct cryptocop_session        *sess;
2508
2509        int    iovlen = 0;
2510        int    iovix = 0;
2511        int    pageix = 0;
2512        int    pageoffset = 0;
2513
2514        size_t prev_ix = 0;
2515        size_t next_ix;
2516
2517        int    cipher_active, digest_active, csum_active;
2518        int    end_digest, end_csum;
2519        int    digest_done = 0;
2520        int    cipher_done = 0;
2521        int    csum_done = 0;
2522
2523        DEBUG(printk("cryptocop_ioctl_process\n"));
2524
2525        if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2526                DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2527                return -EFAULT;
2528        }
2529        if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2530                DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2531                return -EFAULT;
2532        }
2533        DEBUG(print_strcop_crypto_op(&oper));
2534
2535        while (dev && dev->sid != oper.ses_id) dev = dev->next;
2536        if (!dev){
2537                DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2538                return -EINVAL;
2539        }
2540
2541        /* Check buffers. */
2542        if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2543                DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2544                return -EINVAL;
2545        }
2546
2547        if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2548                DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2549                return -EFAULT;
2550        }
2551        if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2552                DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2553                return -EFAULT;
2554        }
2555
2556        cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2557        if (!cop) {
2558                DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2559                return -ENOMEM;
2560        }
2561        jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2562        if (!jc) {
2563                DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2564                err = -ENOMEM;
2565                goto error_cleanup;
2566        }
2567        jc->processed = 0;
2568
2569        cop->cb_data = jc;
2570        cop->cb = ioctl_process_job_callback;
2571        cop->operation_status = 0;
2572        cop->use_dmalists = 0;
2573        cop->in_interrupt = 0;
2574        cop->fast_callback = 0;
2575        cop->tfrm_op.tfrm_cfg = NULL;
2576        cop->tfrm_op.desc = NULL;
2577        cop->tfrm_op.indata = NULL;
2578        cop->tfrm_op.incount = 0;
2579        cop->tfrm_op.inlen = 0;
2580        cop->tfrm_op.outdata = NULL;
2581        cop->tfrm_op.outcount = 0;
2582        cop->tfrm_op.outlen = 0;
2583
2584        sess = get_session(oper.ses_id);
2585        if (!sess){
2586                DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2587                kfree(cop);
2588                kfree(jc);
2589                return -EINVAL;
2590        }
2591
2592        if (oper.do_cipher) {
2593                unsigned int                    cipher_outlen = 0;
2594                struct cryptocop_transform_ctx  *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2595                if (!tc) {
2596                        DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2597                        err = -EINVAL;
2598                        goto error_cleanup;
2599                }
2600                ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2601                ciph_tcfg.inject_ix = 0;
2602                ciph_tcfg.flags = 0;
2603                if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2604                        DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2605                        kfree(cop);
2606                        kfree(jc);
2607                        return -EINVAL;
2608                }
2609                cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2610                if (oper.cipher_len % cblocklen) {
2611                        kfree(cop);
2612                        kfree(jc);
2613                        DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2614                        return -EINVAL;
2615                }
2616                cipher_outlen = oper.cipher_len;
2617                if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2618                        if (oper.cipher_explicit) {
2619                                ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2620                                memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2621                        } else {
2622                                cipher_outlen = oper.cipher_len - cblocklen;
2623                        }
2624                } else {
2625                        if (oper.cipher_explicit){
2626                                kfree(cop);
2627                                kfree(jc);
2628                                DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2629                                return -EINVAL;
2630                        }
2631                }
2632                if (oper.cipher_outlen != cipher_outlen) {
2633                        kfree(cop);
2634                        kfree(jc);
2635                        DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2636                        return -EINVAL;
2637                }
2638
2639                if (oper.decrypt){
2640                        ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2641                } else {
2642                        ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2643                }
2644                ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2645                cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2646        }
2647        if (oper.do_digest){
2648                struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2649                if (!tc) {
2650                        DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2651                        err = -EINVAL;
2652                        goto error_cleanup;
2653                }
2654                digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2655                digest_result = kmalloc(digest_length, GFP_KERNEL);
2656                if (!digest_result) {
2657                        DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2658                        err = -EINVAL;
2659                        goto error_cleanup;
2660                }
2661                DEBUG(memset(digest_result, 0xff, digest_length));
2662
2663                digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2664                digest_tcfg.inject_ix = 0;
2665                ciph_tcfg.inject_ix += digest_length;
2666                if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2667                        DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2668                        err = -EINVAL;
2669                        goto error_cleanup;
2670                }
2671
2672                digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2673                cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2674        }
2675        if (oper.do_csum){
2676                csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2677                csum_tcfg.inject_ix = digest_length;
2678                ciph_tcfg.inject_ix += 2;
2679
2680                if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2681                        DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2682                        kfree(cop);
2683                        kfree(jc);
2684                        return -EINVAL;
2685                }
2686
2687                csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2688                cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2689        }
2690
2691        prev_ix = first_cfg_change_ix(&oper);
2692        if (prev_ix > oper.inlen) {
2693                DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2694                nooutpages = noinpages = 0;
2695                err = -EINVAL;
2696                goto error_cleanup;
2697        }
2698        DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2699
2700        /* Map user pages for in and out data of the operation. */
2701        noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2702        DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2703        inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2704        if (!inpages){
2705                DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2706                nooutpages = noinpages = 0;
2707                err = -ENOMEM;
2708                goto error_cleanup;
2709        }
2710        if (oper.do_cipher){
2711                nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2712                DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2713                outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2714                if (!outpages){
2715                        DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2716                        nooutpages = noinpages = 0;
2717                        err = -ENOMEM;
2718                        goto error_cleanup;
2719                }
2720        }
2721
2722        /* Acquire the mm page semaphore. */
2723        down_read(&current->mm->mmap_sem);
2724
2725        err = get_user_pages(current,
2726                             current->mm,
2727                             (unsigned long int)(oper.indata + prev_ix),
2728                             noinpages,
2729                             0,  /* read access only for in data */
2730                             0, /* no force */
2731                             inpages,
2732                             NULL);
2733
2734        if (err < 0) {
2735                up_read(&current->mm->mmap_sem);
2736                nooutpages = noinpages = 0;
2737                DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2738                goto error_cleanup;
2739        }
2740        noinpages = err;
2741        if (oper.do_cipher){
2742                err = get_user_pages(current,
2743                                     current->mm,
2744                                     (unsigned long int)oper.cipher_outdata,
2745                                     nooutpages,
2746                                     1, /* write access for out data */
2747                                     0, /* no force */
2748                                     outpages,
2749                                     NULL);
2750                up_read(&current->mm->mmap_sem);
2751                if (err < 0) {
2752                        nooutpages = 0;
2753                        DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2754                        goto error_cleanup;
2755                }
2756                nooutpages = err;
2757        } else {
2758                up_read(&current->mm->mmap_sem);
2759        }
2760
2761        /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2762         * csum output and splits when units are (dis-)connected. */
2763        cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2764        cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2765        if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2766                DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2767                err = -ENOMEM;
2768                goto error_cleanup;
2769        }
2770
2771        cop->tfrm_op.inlen = oper.inlen - prev_ix;
2772        cop->tfrm_op.outlen = 0;
2773        if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2774        if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2775        if (oper.do_csum) cop->tfrm_op.outlen += 2;
2776
2777        /* Setup the in iovecs. */
2778        cop->tfrm_op.incount = noinpages;
2779        if (noinpages > 1){
2780                size_t tmplen = cop->tfrm_op.inlen;
2781
2782                cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2783                cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2784                tmplen -= cop->tfrm_op.indata[0].iov_len;
2785                for (i = 1; i<noinpages; i++){
2786                        cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2787                        cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2788                        tmplen -= PAGE_SIZE;
2789                }
2790        } else {
2791                cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2792                cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2793        }
2794
2795        iovlen = nooutpages + 6;
2796        pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2797
2798        next_ix = next_cfg_change_ix(&oper, prev_ix);
2799        if (prev_ix == next_ix){
2800                DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2801                err = -EINVAL;  /* This should be impossible barring bugs. */
2802                goto error_cleanup;
2803        }
2804        while (prev_ix != next_ix){
2805                end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2806                descs[desc_ix].cfg = NULL;
2807                descs[desc_ix].length = next_ix - prev_ix;
2808
2809                if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2810                        dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2811                        dcfgs[dcfg_ix].src = cryptocop_source_dma;
2812                        cipher_active = 1;
2813
2814                        if (next_ix == (oper.cipher_start + oper.cipher_len)){
2815                                cipher_done = 1;
2816                                dcfgs[dcfg_ix].last = 1;
2817                        } else {
2818                                dcfgs[dcfg_ix].last = 0;
2819                        }
2820                        dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2821                        descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2822                        ++dcfg_ix;
2823                }
2824                if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2825                        digest_active = 1;
2826                        dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2827                        dcfgs[dcfg_ix].src = cryptocop_source_dma;
2828                        if (next_ix == (oper.digest_start + oper.digest_len)){
2829                                assert(!digest_done);
2830                                digest_done = 1;
2831                                dcfgs[dcfg_ix].last = 1;
2832                        } else {
2833                                dcfgs[dcfg_ix].last = 0;
2834                        }
2835                        dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2836                        descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2837                        ++dcfg_ix;
2838                }
2839                if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2840                        csum_active = 1;
2841                        dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2842                        dcfgs[dcfg_ix].src = cryptocop_source_dma;
2843                        if (next_ix == (oper.csum_start + oper.csum_len)){
2844                                csum_done = 1;
2845                                dcfgs[dcfg_ix].last = 1;
2846                        } else {
2847                                dcfgs[dcfg_ix].last = 0;
2848                        }
2849                        dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2850                        descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2851                        ++dcfg_ix;
2852                }
2853                if (!descs[desc_ix].cfg){
2854                        DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2855                        err = -EINVAL;
2856                        goto error_cleanup;
2857                }
2858                descs[desc_ix].next = &(descs[desc_ix]) + 1;
2859                ++desc_ix;
2860                prev_ix = next_ix;
2861                next_ix = next_cfg_change_ix(&oper, prev_ix);
2862        }
2863        if (desc_ix > 0){
2864                descs[desc_ix-1].next = NULL;
2865        } else {
2866                descs[0].next = NULL;
2867        }
2868        if (oper.do_digest) {
2869                DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2870                /* Add outdata iovec, length == <length of type of digest> */
2871                cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2872                cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2873                ++iovix;
2874        }
2875        if (oper.do_csum) {
2876                /* Add outdata iovec, length == 2, the length of csum. */
2877                DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2878                /* Add outdata iovec, length == <length of type of digest> */
2879                cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2880                cop->tfrm_op.outdata[iovix].iov_len = 2;
2881                ++iovix;
2882        }
2883        if (oper.do_cipher) {
2884                if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2885                        DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2886                        err = -ENOSYS; /* This should be impossible barring bugs. */
2887                        goto error_cleanup;
2888                }
2889        }
2890        DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2891        cop->tfrm_op.outcount = iovix;
2892        assert(iovix <= (nooutpages + 6));
2893
2894        cop->sid = oper.ses_id;
2895        cop->tfrm_op.desc = &descs[0];
2896
2897        DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2898
2899        if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2900                DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2901                err = -EINVAL;
2902                goto error_cleanup;
2903        }
2904
2905        DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2906
2907        wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2908        DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2909        if (!jc->processed){
2910                printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2911                err = -EIO;
2912                goto error_cleanup;
2913        }
2914
2915        /* Job process done.  Cipher output should already be correct in job so no post processing of outdata. */
2916        DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2917        if (cop->operation_status == 0){
2918                if (oper.do_digest){
2919                        DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2920                        err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2921                        if (0 != err){
2922                                DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2923                                err = -EFAULT;
2924                                goto error_cleanup;
2925                        }
2926                }
2927                if (oper.do_csum){
2928                        DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2929                        err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2930                        if (0 != err){
2931                                DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2932                                err = -EFAULT;
2933                                goto error_cleanup;
2934                        }
2935                }
2936                err = 0;
2937        } else {
2938                DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2939                err = cop->operation_status;
2940        }
2941
2942 error_cleanup:
2943        /* Release page caches. */
2944        for (i = 0; i < noinpages; i++){
2945                put_page(inpages[i]);
2946        }
2947        for (i = 0; i < nooutpages; i++){
2948                int spdl_err;
2949                /* Mark output pages dirty. */
2950                spdl_err = set_page_dirty_lock(outpages[i]);
2951                DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2952        }
2953        for (i = 0; i < nooutpages; i++){
2954                put_page(outpages[i]);
2955        }
2956
2957        kfree(digest_result);
2958        kfree(inpages);
2959        kfree(outpages);
2960        if (cop){
2961                kfree(cop->tfrm_op.indata);
2962                kfree(cop->tfrm_op.outdata);
2963                kfree(cop);
2964        }
2965        kfree(jc);
2966
2967        DEBUG(print_lock_status());
2968
2969        return err;
2970}
2971
2972
2973static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2974{
2975        cryptocop_session_id             sid;
2976        int                              err;
2977        struct cryptocop_private         *dev;
2978        struct strcop_session_op         *sess_op = (struct strcop_session_op *)arg;
2979        struct strcop_session_op         sop;
2980        struct cryptocop_transform_init  *tis = NULL;
2981        struct cryptocop_transform_init  ti_cipher = {0};
2982        struct cryptocop_transform_init  ti_digest = {0};
2983        struct cryptocop_transform_init  ti_csum = {0};
2984
2985        if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2986                return -EFAULT;
2987        err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2988        if (err) return -EFAULT;
2989        if (sop.cipher != cryptocop_cipher_none) {
2990                if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2991        }
2992        DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2993
2994        DEBUG(printk("\tcipher:%d\n"
2995                     "\tcipher_mode:%d\n"
2996                     "\tdigest:%d\n"
2997                     "\tcsum:%d\n",
2998                     (int)sop.cipher,
2999                     (int)sop.cmode,
3000                     (int)sop.digest,
3001                     (int)sop.csum));
3002
3003        if (sop.cipher != cryptocop_cipher_none){
3004                /* Init the cipher. */
3005                switch (sop.cipher){
3006                case cryptocop_cipher_des:
3007                        ti_cipher.alg = cryptocop_alg_des;
3008                        break;
3009                case cryptocop_cipher_3des:
3010                        ti_cipher.alg = cryptocop_alg_3des;
3011                        break;
3012                case cryptocop_cipher_aes:
3013                        ti_cipher.alg = cryptocop_alg_aes;
3014                        break;
3015                default:
3016                        DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3017                        return -EINVAL;
3018                };
3019                DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3020                copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3021                ti_cipher.keylen = sop.keylen;
3022                switch (sop.cmode){
3023                case cryptocop_cipher_mode_cbc:
3024                case cryptocop_cipher_mode_ecb:
3025                        ti_cipher.cipher_mode = sop.cmode;
3026                        break;
3027                default:
3028                        DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3029                        return -EINVAL;
3030                }
3031                DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3032                switch (sop.des3_mode){
3033                case cryptocop_3des_eee:
3034                case cryptocop_3des_eed:
3035                case cryptocop_3des_ede:
3036                case cryptocop_3des_edd:
3037                case cryptocop_3des_dee:
3038                case cryptocop_3des_ded:
3039                case cryptocop_3des_dde:
3040                case cryptocop_3des_ddd:
3041                        ti_cipher.tdes_mode = sop.des3_mode;
3042                        break;
3043                default:
3044                        DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3045                        return -EINVAL;
3046                }
3047                ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3048                ti_cipher.next = tis;
3049                tis = &ti_cipher;
3050        } /* if (sop.cipher != cryptocop_cipher_none) */
3051        if (sop.digest != cryptocop_digest_none){
3052                DEBUG(printk("setting digest transform\n"));
3053                switch (sop.digest){
3054                case cryptocop_digest_md5:
3055                        ti_digest.alg = cryptocop_alg_md5;
3056                        break;
3057                case cryptocop_digest_sha1:
3058                        ti_digest.alg = cryptocop_alg_sha1;
3059                        break;
3060                default:
3061                        DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3062                        return -EINVAL;
3063                }
3064                ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3065                ti_digest.next = tis;
3066                tis = &ti_digest;
3067        } /* if (sop.digest != cryptocop_digest_none) */
3068        if (sop.csum != cryptocop_csum_none){
3069                DEBUG(printk("setting csum transform\n"));
3070                switch (sop.csum){
3071                case cryptocop_csum_le:
3072                case cryptocop_csum_be:
3073                        ti_csum.csum_mode = sop.csum;
3074                        break;
3075                default:
3076                        DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3077                        return -EINVAL;
3078                }
3079                ti_csum.alg = cryptocop_alg_csum;
3080                ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3081                ti_csum.next = tis;
3082                tis = &ti_csum;
3083        } /* (sop.csum != cryptocop_csum_none) */
3084        dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3085        if (!dev){
3086                DEBUG_API(printk("create session, alloc dev\n"));
3087                return -ENOMEM;
3088        }
3089
3090        err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3091        DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3092
3093        if (err) {
3094                kfree(dev);
3095                return err;
3096        }
3097        sess_op->ses_id = sid;
3098        dev->sid = sid;
3099        dev->next = filp->private_data;
3100        filp->private_data = dev;
3101
3102        return 0;
3103}
3104
3105static int cryptocop_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
3106{
3107        int err = 0;
3108        if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3109                DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3110                return -ENOTTY;
3111        }
3112        if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3113                return -ENOTTY;
3114        }
3115        /* Access check of the argument.  Some commands, e.g. create session and process op,
3116           needs additional checks.  Those are handled in the command handling functions. */
3117        if (_IOC_DIR(cmd) & _IOC_READ)
3118                err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3119        else if (_IOC_DIR(cmd) & _IOC_WRITE)
3120                err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3121        if (err) return -EFAULT;
3122
3123        switch (cmd) {
3124        case CRYPTOCOP_IO_CREATE_SESSION:
3125                return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3126        case CRYPTOCOP_IO_CLOSE_SESSION:
3127                return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3128        case CRYPTOCOP_IO_PROCESS_OP:
3129                return cryptocop_ioctl_process(inode, filp, cmd, arg);
3130        default:
3131                DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3132                return -ENOTTY;
3133        }
3134        return 0;
3135}
3136
3137
3138#ifdef LDEBUG
3139static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3140{
3141        struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3142        struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3143        int                       i;
3144
3145        printk("print_dma_descriptors start\n");
3146
3147        printk("iop:\n");
3148        printk("\tsid: 0x%lld\n", iop->sid);
3149
3150        printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3151        printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3152        printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3153        printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3154
3155        printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3156        printk("\tnext: 0x%p\n"
3157               "\tsaved_data: 0x%p\n"
3158               "\tsaved_data_buf: 0x%p\n",
3159               iop->ctx_out.next,
3160               iop->ctx_out.saved_data,
3161               iop->ctx_out.saved_data_buf);
3162
3163        printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3164        printk("\tnext: 0x%p\n"
3165               "\tsaved_data: 0x%p\n"
3166               "\tsaved_data_buf: 0x%p\n",
3167               iop->ctx_in.next,
3168               iop->ctx_in.saved_data,
3169               iop->ctx_in.saved_data_buf);
3170
3171        i = 0;
3172        while (cdesc_out) {
3173                dma_descr_data *td;
3174                printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3175                printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3176                td = cdesc_out->dma_descr;
3177                printk("\n\tbuf: 0x%p\n"
3178                       "\tafter: 0x%p\n"
3179                       "\tmd: 0x%04x\n"
3180                       "\tnext: 0x%p\n",
3181                       td->buf,
3182                       td->after,
3183                       td->md,
3184                       td->next);
3185                printk("flags:\n"
3186                       "\twait:\t%d\n"
3187                       "\teol:\t%d\n"
3188                       "\touteop:\t%d\n"
3189                       "\tineop:\t%d\n"
3190                       "\tintr:\t%d\n",
3191                       td->wait,
3192                       td->eol,
3193                       td->out_eop,
3194                       td->in_eop,
3195                       td->intr);
3196                cdesc_out = cdesc_out->next;
3197                i++;
3198        }
3199        i = 0;
3200        while (cdesc_in) {
3201                dma_descr_data *td;
3202                printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3203                printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3204                td = cdesc_in->dma_descr;
3205                printk("\n\tbuf: 0x%p\n"
3206                       "\tafter: 0x%p\n"
3207                       "\tmd: 0x%04x\n"
3208                       "\tnext: 0x%p\n",
3209                       td->buf,
3210                       td->after,
3211                       td->md,
3212                       td->next);
3213                printk("flags:\n"
3214                       "\twait:\t%d\n"
3215                       "\teol:\t%d\n"
3216                       "\touteop:\t%d\n"
3217                       "\tineop:\t%d\n"
3218                       "\tintr:\t%d\n",
3219                       td->wait,
3220                       td->eol,
3221                       td->out_eop,
3222                       td->in_eop,
3223                       td->intr);
3224                cdesc_in = cdesc_in->next;
3225                i++;
3226        }
3227
3228        printk("print_dma_descriptors end\n");
3229}
3230
3231
3232static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3233{
3234        printk("print_strcop_crypto_op, 0x%p\n", cop);
3235
3236        /* Indata. */
3237        printk("indata=0x%p\n"
3238               "inlen=%d\n"
3239               "do_cipher=%d\n"
3240               "decrypt=%d\n"
3241               "cipher_explicit=%d\n"
3242               "cipher_start=%d\n"
3243               "cipher_len=%d\n"
3244               "outdata=0x%p\n"
3245               "outlen=%d\n",
3246               cop->indata,
3247               cop->inlen,
3248               cop->do_cipher,
3249               cop->decrypt,
3250               cop->cipher_explicit,
3251               cop->cipher_start,
3252               cop->cipher_len,
3253               cop->cipher_outdata,
3254               cop->cipher_outlen);
3255
3256        printk("do_digest=%d\n"
3257               "digest_start=%d\n"
3258               "digest_len=%d\n",
3259               cop->do_digest,
3260               cop->digest_start,
3261               cop->digest_len);
3262
3263        printk("do_csum=%d\n"
3264               "csum_start=%d\n"
3265               "csum_len=%d\n",
3266               cop->do_csum,
3267               cop->csum_start,
3268               cop->csum_len);
3269}
3270
3271static void print_cryptocop_operation(struct cryptocop_operation *cop)
3272{
3273        struct cryptocop_desc      *d;
3274        struct cryptocop_tfrm_cfg  *tc;
3275        struct cryptocop_desc_cfg  *dc;
3276        int                        i;
3277
3278        printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3279        printk("sid: %lld\n", cop->sid);
3280        printk("operation_status=%d\n"
3281               "use_dmalists=%d\n"
3282               "in_interrupt=%d\n"
3283               "fast_callback=%d\n",
3284               cop->operation_status,
3285               cop->use_dmalists,
3286               cop->in_interrupt,
3287               cop->fast_callback);
3288
3289        if (cop->use_dmalists){
3290                print_user_dma_lists(&cop->list_op);
3291        } else {
3292                printk("cop->tfrm_op\n"
3293                       "tfrm_cfg=0x%p\n"
3294                       "desc=0x%p\n"
3295                       "indata=0x%p\n"
3296                       "incount=%d\n"
3297                       "inlen=%d\n"
3298                       "outdata=0x%p\n"
3299                       "outcount=%d\n"
3300                       "outlen=%d\n\n",
3301                       cop->tfrm_op.tfrm_cfg,
3302                       cop->tfrm_op.desc,
3303                       cop->tfrm_op.indata,
3304                       cop->tfrm_op.incount,
3305                       cop->tfrm_op.inlen,
3306                       cop->tfrm_op.outdata,
3307                       cop->tfrm_op.outcount,
3308                       cop->tfrm_op.outlen);
3309
3310                tc = cop->tfrm_op.tfrm_cfg;
3311                while (tc){
3312                        printk("tfrm_cfg, 0x%p\n"
3313                               "tid=%d\n"
3314                               "flags=%d\n"
3315                               "inject_ix=%d\n"
3316                               "next=0x%p\n",
3317                               tc,
3318                               tc->tid,
3319                               tc->flags,
3320                               tc->inject_ix,
3321                               tc->next);
3322                        tc = tc->next;
3323                }
3324                d = cop->tfrm_op.desc;
3325                while (d){
3326                        printk("\n======================desc, 0x%p\n"
3327                               "length=%d\n"
3328                               "cfg=0x%p\n"
3329                               "next=0x%p\n",
3330                               d,
3331                               d->length,
3332                               d->cfg,
3333                               d->next);
3334                        dc = d->cfg;
3335                        while (dc){
3336                                printk("=========desc_cfg, 0x%p\n"
3337                                       "tid=%d\n"
3338                                       "src=%d\n"
3339                                       "last=%d\n"
3340                                       "next=0x%p\n",
3341                                       dc,
3342                                       dc->tid,
3343                                       dc->src,
3344                                       dc->last,
3345                                       dc->next);
3346                                dc = dc->next;
3347                        }
3348                        d = d->next;
3349                }
3350                printk("\n====iniov\n");
3351                for (i = 0; i < cop->tfrm_op.incount; i++){
3352                        printk("indata[%d]\n"
3353                               "base=0x%p\n"
3354                               "len=%d\n",
3355                               i,
3356                               cop->tfrm_op.indata[i].iov_base,
3357                               cop->tfrm_op.indata[i].iov_len);
3358                }
3359                printk("\n====outiov\n");
3360                for (i = 0; i < cop->tfrm_op.outcount; i++){
3361                        printk("outdata[%d]\n"
3362                               "base=0x%p\n"
3363                               "len=%d\n",
3364                               i,
3365                               cop->tfrm_op.outdata[i].iov_base,
3366                               cop->tfrm_op.outdata[i].iov_len);
3367                }
3368        }
3369        printk("------------end print_cryptocop_operation\n");
3370}
3371
3372
3373static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3374{
3375        dma_descr_data *dd;
3376        int i;
3377
3378        printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3379
3380        printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3381        printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3382
3383        printk("##############outlist\n");
3384        dd = phys_to_virt((unsigned long int)dma_op->outlist);
3385        i = 0;
3386        while (dd != NULL) {
3387                printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3388                printk("\n\tbuf: 0x%p\n"
3389                       "\tafter: 0x%p\n"
3390                       "\tmd: 0x%04x\n"
3391                       "\tnext: 0x%p\n",
3392                       dd->buf,
3393                       dd->after,
3394                       dd->md,
3395                       dd->next);
3396                printk("flags:\n"
3397                       "\twait:\t%d\n"
3398                       "\teol:\t%d\n"
3399                       "\touteop:\t%d\n"
3400                       "\tineop:\t%d\n"
3401                       "\tintr:\t%d\n",
3402                       dd->wait,
3403                       dd->eol,
3404                       dd->out_eop,
3405                       dd->in_eop,
3406                       dd->intr);
3407                if (dd->eol)
3408                        dd = NULL;
3409                else
3410                        dd = phys_to_virt((unsigned long int)dd->next);
3411                ++i;
3412        }
3413
3414        printk("##############inlist\n");
3415        dd = phys_to_virt((unsigned long int)dma_op->inlist);
3416        i = 0;
3417        while (dd != NULL) {
3418                printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3419                printk("\n\tbuf: 0x%p\n"
3420                       "\tafter: 0x%p\n"
3421                       "\tmd: 0x%04x\n"
3422                       "\tnext: 0x%p\n",
3423                       dd->buf,
3424                       dd->after,
3425                       dd->md,
3426                       dd->next);
3427                printk("flags:\n"
3428                       "\twait:\t%d\n"
3429                       "\teol:\t%d\n"
3430                       "\touteop:\t%d\n"
3431                       "\tineop:\t%d\n"
3432                       "\tintr:\t%d\n",
3433                       dd->wait,
3434                       dd->eol,
3435                       dd->out_eop,
3436                       dd->in_eop,
3437                       dd->intr);
3438                if (dd->eol)
3439                        dd = NULL;
3440                else
3441                        dd = phys_to_virt((unsigned long int)dd->next);
3442                ++i;
3443        }
3444}
3445
3446
3447static void print_lock_status(void)
3448{
3449        printk("**********************print_lock_status\n");
3450        printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3451        printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3452        printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3453        printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3454        printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3455        printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3456}
3457#endif /* LDEBUG */
3458
3459
3460static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3461
3462static int init_stream_coprocessor(void)
3463{
3464        int err;
3465        int i;
3466        static int initialized = 0;
3467
3468        if (initialized)
3469                return 0;
3470
3471        initialized = 1;
3472
3473        printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3474
3475        err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3476        if (err < 0) {
3477                printk(KERN_ERR "stream co-processor: could not get major number.\n");
3478                return err;
3479        }
3480
3481        err = init_cryptocop();
3482        if (err) {
3483                (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3484                return err;
3485        }
3486        err = cryptocop_job_queue_init();
3487        if (err) {
3488                release_cryptocop();
3489                (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3490                return err;
3491        }
3492        /* Init the descriptor pool. */
3493        for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3494                descr_pool[i].from_pool = 1;
3495                descr_pool[i].next = &descr_pool[i + 1];
3496        }
3497        descr_pool[i].from_pool = 1;
3498        descr_pool[i].next = NULL;
3499        descr_pool_free_list = &descr_pool[0];
3500        descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3501
3502        spin_lock_init(&cryptocop_completed_jobs_lock);
3503        spin_lock_init(&cryptocop_job_queue_lock);
3504        spin_lock_init(&descr_pool_lock);
3505        spin_lock_init(&cryptocop_sessions_lock);
3506        spin_lock_init(&running_job_lock);
3507        spin_lock_init(&cryptocop_process_lock);
3508
3509        cryptocop_sessions = NULL;
3510        next_sid = 1;
3511
3512        cryptocop_running_job = NULL;
3513
3514        printk("stream co-processor: init done.\n");
3515        return 0;
3516}
3517
3518static void __exit exit_stream_coprocessor(void)
3519{
3520        release_cryptocop();
3521        cryptocop_job_queue_close();
3522}
3523
3524module_init(init_stream_coprocessor);
3525module_exit(exit_stream_coprocessor);
3526
3527