linux/drivers/crypto/bcm/cipher.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright 2016 Broadcom
   4 */
   5
   6#include <linux/err.h>
   7#include <linux/module.h>
   8#include <linux/init.h>
   9#include <linux/errno.h>
  10#include <linux/kernel.h>
  11#include <linux/interrupt.h>
  12#include <linux/platform_device.h>
  13#include <linux/scatterlist.h>
  14#include <linux/crypto.h>
  15#include <linux/kthread.h>
  16#include <linux/rtnetlink.h>
  17#include <linux/sched.h>
  18#include <linux/of_address.h>
  19#include <linux/of_device.h>
  20#include <linux/io.h>
  21#include <linux/bitops.h>
  22
  23#include <crypto/algapi.h>
  24#include <crypto/aead.h>
  25#include <crypto/internal/aead.h>
  26#include <crypto/aes.h>
  27#include <crypto/internal/des.h>
  28#include <crypto/hmac.h>
  29#include <crypto/md5.h>
  30#include <crypto/authenc.h>
  31#include <crypto/skcipher.h>
  32#include <crypto/hash.h>
  33#include <crypto/sha1.h>
  34#include <crypto/sha2.h>
  35#include <crypto/sha3.h>
  36
  37#include "util.h"
  38#include "cipher.h"
  39#include "spu.h"
  40#include "spum.h"
  41#include "spu2.h"
  42
  43/* ================= Device Structure ================== */
  44
  45struct bcm_device_private iproc_priv;
  46
  47/* ==================== Parameters ===================== */
  48
  49int flow_debug_logging;
  50module_param(flow_debug_logging, int, 0644);
  51MODULE_PARM_DESC(flow_debug_logging, "Enable Flow Debug Logging");
  52
  53int packet_debug_logging;
  54module_param(packet_debug_logging, int, 0644);
  55MODULE_PARM_DESC(packet_debug_logging, "Enable Packet Debug Logging");
  56
  57int debug_logging_sleep;
  58module_param(debug_logging_sleep, int, 0644);
  59MODULE_PARM_DESC(debug_logging_sleep, "Packet Debug Logging Sleep");
  60
  61/*
  62 * The value of these module parameters is used to set the priority for each
  63 * algo type when this driver registers algos with the kernel crypto API.
  64 * To use a priority other than the default, set the priority in the insmod or
  65 * modprobe. Changing the module priority after init time has no effect.
  66 *
  67 * The default priorities are chosen to be lower (less preferred) than ARMv8 CE
  68 * algos, but more preferred than generic software algos.
  69 */
  70static int cipher_pri = 150;
  71module_param(cipher_pri, int, 0644);
  72MODULE_PARM_DESC(cipher_pri, "Priority for cipher algos");
  73
  74static int hash_pri = 100;
  75module_param(hash_pri, int, 0644);
  76MODULE_PARM_DESC(hash_pri, "Priority for hash algos");
  77
  78static int aead_pri = 150;
  79module_param(aead_pri, int, 0644);
  80MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
  81
  82/* A type 3 BCM header, expected to precede the SPU header for SPU-M.
  83 * Bits 3 and 4 in the first byte encode the channel number (the dma ringset).
  84 * 0x60 - ring 0
  85 * 0x68 - ring 1
  86 * 0x70 - ring 2
  87 * 0x78 - ring 3
  88 */
  89static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
  90/*
  91 * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN
  92 * is set dynamically after reading SPU type from device tree.
  93 */
  94#define BCM_HDR_LEN  iproc_priv.bcm_hdr_len
  95
  96/* min and max time to sleep before retrying when mbox queue is full. usec */
  97#define MBOX_SLEEP_MIN  800
  98#define MBOX_SLEEP_MAX 1000
  99
 100/**
 101 * select_channel() - Select a SPU channel to handle a crypto request. Selects
 102 * channel in round robin order.
 103 *
 104 * Return:  channel index
 105 */
 106static u8 select_channel(void)
 107{
 108        u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan);
 109
 110        return chan_idx % iproc_priv.spu.num_chan;
 111}
 112
 113/**
 114 * spu_skcipher_rx_sg_create() - Build up the scatterlist of buffers used to
 115 * receive a SPU response message for an skcipher request. Includes buffers to
 116 * catch SPU message headers and the response data.
 117 * @mssg:       mailbox message containing the receive sg
 118 * @rctx:       crypto request context
 119 * @rx_frag_num: number of scatterlist elements required to hold the
 120 *              SPU response message
 121 * @chunksize:  Number of bytes of response data expected
 122 * @stat_pad_len: Number of bytes required to pad the STAT field to
 123 *              a 4-byte boundary
 124 *
 125 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 126 * when the request completes, whether the request is handled successfully or
 127 * there is an error.
 128 *
 129 * Returns:
 130 *   0 if successful
 131 *   < 0 if an error
 132 */
 133static int
 134spu_skcipher_rx_sg_create(struct brcm_message *mssg,
 135                            struct iproc_reqctx_s *rctx,
 136                            u8 rx_frag_num,
 137                            unsigned int chunksize, u32 stat_pad_len)
 138{
 139        struct spu_hw *spu = &iproc_priv.spu;
 140        struct scatterlist *sg; /* used to build sgs in mbox message */
 141        struct iproc_ctx_s *ctx = rctx->ctx;
 142        u32 datalen;            /* Number of bytes of response data expected */
 143
 144        mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
 145                                rctx->gfp);
 146        if (!mssg->spu.dst)
 147                return -ENOMEM;
 148
 149        sg = mssg->spu.dst;
 150        sg_init_table(sg, rx_frag_num);
 151        /* Space for SPU message header */
 152        sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
 153
 154        /* If XTS tweak in payload, add buffer to receive encrypted tweak */
 155        if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 156            spu->spu_xts_tweak_in_payload())
 157                sg_set_buf(sg++, rctx->msg_buf.c.supdt_tweak,
 158                           SPU_XTS_TWEAK_SIZE);
 159
 160        /* Copy in each dst sg entry from request, up to chunksize */
 161        datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
 162                                 rctx->dst_nents, chunksize);
 163        if (datalen < chunksize) {
 164                pr_err("%s(): failed to copy dst sg to mbox msg. chunksize %u, datalen %u",
 165                       __func__, chunksize, datalen);
 166                return -EFAULT;
 167        }
 168
 169        if (stat_pad_len)
 170                sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
 171
 172        memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
 173        sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
 174
 175        return 0;
 176}
 177
 178/**
 179 * spu_skcipher_tx_sg_create() - Build up the scatterlist of buffers used to
 180 * send a SPU request message for an skcipher request. Includes SPU message
 181 * headers and the request data.
 182 * @mssg:       mailbox message containing the transmit sg
 183 * @rctx:       crypto request context
 184 * @tx_frag_num: number of scatterlist elements required to construct the
 185 *              SPU request message
 186 * @chunksize:  Number of bytes of request data
 187 * @pad_len:    Number of pad bytes
 188 *
 189 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 190 * when the request completes, whether the request is handled successfully or
 191 * there is an error.
 192 *
 193 * Returns:
 194 *   0 if successful
 195 *   < 0 if an error
 196 */
 197static int
 198spu_skcipher_tx_sg_create(struct brcm_message *mssg,
 199                            struct iproc_reqctx_s *rctx,
 200                            u8 tx_frag_num, unsigned int chunksize, u32 pad_len)
 201{
 202        struct spu_hw *spu = &iproc_priv.spu;
 203        struct scatterlist *sg; /* used to build sgs in mbox message */
 204        struct iproc_ctx_s *ctx = rctx->ctx;
 205        u32 datalen;            /* Number of bytes of response data expected */
 206        u32 stat_len;
 207
 208        mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
 209                                rctx->gfp);
 210        if (unlikely(!mssg->spu.src))
 211                return -ENOMEM;
 212
 213        sg = mssg->spu.src;
 214        sg_init_table(sg, tx_frag_num);
 215
 216        sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
 217                   BCM_HDR_LEN + ctx->spu_req_hdr_len);
 218
 219        /* if XTS tweak in payload, copy from IV (where crypto API puts it) */
 220        if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 221            spu->spu_xts_tweak_in_payload())
 222                sg_set_buf(sg++, rctx->msg_buf.iv_ctr, SPU_XTS_TWEAK_SIZE);
 223
 224        /* Copy in each src sg entry from request, up to chunksize */
 225        datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
 226                                 rctx->src_nents, chunksize);
 227        if (unlikely(datalen < chunksize)) {
 228                pr_err("%s(): failed to copy src sg to mbox msg",
 229                       __func__);
 230                return -EFAULT;
 231        }
 232
 233        if (pad_len)
 234                sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
 235
 236        stat_len = spu->spu_tx_status_len();
 237        if (stat_len) {
 238                memset(rctx->msg_buf.tx_stat, 0, stat_len);
 239                sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
 240        }
 241        return 0;
 242}
 243
 244static int mailbox_send_message(struct brcm_message *mssg, u32 flags,
 245                                u8 chan_idx)
 246{
 247        int err;
 248        int retry_cnt = 0;
 249        struct device *dev = &(iproc_priv.pdev->dev);
 250
 251        err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg);
 252        if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) {
 253                while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) {
 254                        /*
 255                         * Mailbox queue is full. Since MAY_SLEEP is set, assume
 256                         * not in atomic context and we can wait and try again.
 257                         */
 258                        retry_cnt++;
 259                        usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX);
 260                        err = mbox_send_message(iproc_priv.mbox[chan_idx],
 261                                                mssg);
 262                        atomic_inc(&iproc_priv.mb_no_spc);
 263                }
 264        }
 265        if (err < 0) {
 266                atomic_inc(&iproc_priv.mb_send_fail);
 267                return err;
 268        }
 269
 270        /* Check error returned by mailbox controller */
 271        err = mssg->error;
 272        if (unlikely(err < 0)) {
 273                dev_err(dev, "message error %d", err);
 274                /* Signal txdone for mailbox channel */
 275        }
 276
 277        /* Signal txdone for mailbox channel */
 278        mbox_client_txdone(iproc_priv.mbox[chan_idx], err);
 279        return err;
 280}
 281
 282/**
 283 * handle_skcipher_req() - Submit as much of a block cipher request as fits in
 284 * a single SPU request message, starting at the current position in the request
 285 * data.
 286 * @rctx:       Crypto request context
 287 *
 288 * This may be called on the crypto API thread, or, when a request is so large
 289 * it must be broken into multiple SPU messages, on the thread used to invoke
 290 * the response callback. When requests are broken into multiple SPU
 291 * messages, we assume subsequent messages depend on previous results, and
 292 * thus always wait for previous results before submitting the next message.
 293 * Because requests are submitted in lock step like this, there is no need
 294 * to synchronize access to request data structures.
 295 *
 296 * Return: -EINPROGRESS: request has been accepted and result will be returned
 297 *                       asynchronously
 298 *         Any other value indicates an error
 299 */
 300static int handle_skcipher_req(struct iproc_reqctx_s *rctx)
 301{
 302        struct spu_hw *spu = &iproc_priv.spu;
 303        struct crypto_async_request *areq = rctx->parent;
 304        struct skcipher_request *req =
 305            container_of(areq, struct skcipher_request, base);
 306        struct iproc_ctx_s *ctx = rctx->ctx;
 307        struct spu_cipher_parms cipher_parms;
 308        int err;
 309        unsigned int chunksize; /* Num bytes of request to submit */
 310        int remaining;  /* Bytes of request still to process */
 311        int chunk_start;        /* Beginning of data for current SPU msg */
 312
 313        /* IV or ctr value to use in this SPU msg */
 314        u8 local_iv_ctr[MAX_IV_SIZE];
 315        u32 stat_pad_len;       /* num bytes to align status field */
 316        u32 pad_len;            /* total length of all padding */
 317        struct brcm_message *mssg;      /* mailbox message */
 318
 319        /* number of entries in src and dst sg in mailbox message. */
 320        u8 rx_frag_num = 2;     /* response header and STATUS */
 321        u8 tx_frag_num = 1;     /* request header */
 322
 323        flow_log("%s\n", __func__);
 324
 325        cipher_parms.alg = ctx->cipher.alg;
 326        cipher_parms.mode = ctx->cipher.mode;
 327        cipher_parms.type = ctx->cipher_type;
 328        cipher_parms.key_len = ctx->enckeylen;
 329        cipher_parms.key_buf = ctx->enckey;
 330        cipher_parms.iv_buf = local_iv_ctr;
 331        cipher_parms.iv_len = rctx->iv_ctr_len;
 332
 333        mssg = &rctx->mb_mssg;
 334        chunk_start = rctx->src_sent;
 335        remaining = rctx->total_todo - chunk_start;
 336
 337        /* determine the chunk we are breaking off and update the indexes */
 338        if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
 339            (remaining > ctx->max_payload))
 340                chunksize = ctx->max_payload;
 341        else
 342                chunksize = remaining;
 343
 344        rctx->src_sent += chunksize;
 345        rctx->total_sent = rctx->src_sent;
 346
 347        /* Count number of sg entries to be included in this request */
 348        rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
 349        rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
 350
 351        if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
 352            rctx->is_encrypt && chunk_start)
 353                /*
 354                 * Encrypting non-first first chunk. Copy last block of
 355                 * previous result to IV for this chunk.
 356                 */
 357                sg_copy_part_to_buf(req->dst, rctx->msg_buf.iv_ctr,
 358                                    rctx->iv_ctr_len,
 359                                    chunk_start - rctx->iv_ctr_len);
 360
 361        if (rctx->iv_ctr_len) {
 362                /* get our local copy of the iv */
 363                __builtin_memcpy(local_iv_ctr, rctx->msg_buf.iv_ctr,
 364                                 rctx->iv_ctr_len);
 365
 366                /* generate the next IV if possible */
 367                if ((ctx->cipher.mode == CIPHER_MODE_CBC) &&
 368                    !rctx->is_encrypt) {
 369                        /*
 370                         * CBC Decrypt: next IV is the last ciphertext block in
 371                         * this chunk
 372                         */
 373                        sg_copy_part_to_buf(req->src, rctx->msg_buf.iv_ctr,
 374                                            rctx->iv_ctr_len,
 375                                            rctx->src_sent - rctx->iv_ctr_len);
 376                } else if (ctx->cipher.mode == CIPHER_MODE_CTR) {
 377                        /*
 378                         * The SPU hardware increments the counter once for
 379                         * each AES block of 16 bytes. So update the counter
 380                         * for the next chunk, if there is one. Note that for
 381                         * this chunk, the counter has already been copied to
 382                         * local_iv_ctr. We can assume a block size of 16,
 383                         * because we only support CTR mode for AES, not for
 384                         * any other cipher alg.
 385                         */
 386                        add_to_ctr(rctx->msg_buf.iv_ctr, chunksize >> 4);
 387                }
 388        }
 389
 390        if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
 391                flow_log("max_payload infinite\n");
 392        else
 393                flow_log("max_payload %u\n", ctx->max_payload);
 394
 395        flow_log("sent:%u start:%u remains:%u size:%u\n",
 396                 rctx->src_sent, chunk_start, remaining, chunksize);
 397
 398        /* Copy SPU header template created at setkey time */
 399        memcpy(rctx->msg_buf.bcm_spu_req_hdr, ctx->bcm_spu_req_hdr,
 400               sizeof(rctx->msg_buf.bcm_spu_req_hdr));
 401
 402        spu->spu_cipher_req_finish(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
 403                                   ctx->spu_req_hdr_len, !(rctx->is_encrypt),
 404                                   &cipher_parms, chunksize);
 405
 406        atomic64_add(chunksize, &iproc_priv.bytes_out);
 407
 408        stat_pad_len = spu->spu_wordalign_padlen(chunksize);
 409        if (stat_pad_len)
 410                rx_frag_num++;
 411        pad_len = stat_pad_len;
 412        if (pad_len) {
 413                tx_frag_num++;
 414                spu->spu_request_pad(rctx->msg_buf.spu_req_pad, 0,
 415                                     0, ctx->auth.alg, ctx->auth.mode,
 416                                     rctx->total_sent, stat_pad_len);
 417        }
 418
 419        spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
 420                              ctx->spu_req_hdr_len);
 421        packet_log("payload:\n");
 422        dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
 423        packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
 424
 425        /*
 426         * Build mailbox message containing SPU request msg and rx buffers
 427         * to catch response message
 428         */
 429        memset(mssg, 0, sizeof(*mssg));
 430        mssg->type = BRCM_MESSAGE_SPU;
 431        mssg->ctx = rctx;       /* Will be returned in response */
 432
 433        /* Create rx scatterlist to catch result */
 434        rx_frag_num += rctx->dst_nents;
 435
 436        if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 437            spu->spu_xts_tweak_in_payload())
 438                rx_frag_num++;  /* extra sg to insert tweak */
 439
 440        err = spu_skcipher_rx_sg_create(mssg, rctx, rx_frag_num, chunksize,
 441                                          stat_pad_len);
 442        if (err)
 443                return err;
 444
 445        /* Create tx scatterlist containing SPU request message */
 446        tx_frag_num += rctx->src_nents;
 447        if (spu->spu_tx_status_len())
 448                tx_frag_num++;
 449
 450        if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 451            spu->spu_xts_tweak_in_payload())
 452                tx_frag_num++;  /* extra sg to insert tweak */
 453
 454        err = spu_skcipher_tx_sg_create(mssg, rctx, tx_frag_num, chunksize,
 455                                          pad_len);
 456        if (err)
 457                return err;
 458
 459        err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
 460        if (unlikely(err < 0))
 461                return err;
 462
 463        return -EINPROGRESS;
 464}
 465
 466/**
 467 * handle_skcipher_resp() - Process a block cipher SPU response. Updates the
 468 * total received count for the request and updates global stats.
 469 * @rctx:       Crypto request context
 470 */
 471static void handle_skcipher_resp(struct iproc_reqctx_s *rctx)
 472{
 473        struct spu_hw *spu = &iproc_priv.spu;
 474        struct crypto_async_request *areq = rctx->parent;
 475        struct skcipher_request *req = skcipher_request_cast(areq);
 476        struct iproc_ctx_s *ctx = rctx->ctx;
 477        u32 payload_len;
 478
 479        /* See how much data was returned */
 480        payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
 481
 482        /*
 483         * In XTS mode, the first SPU_XTS_TWEAK_SIZE bytes may be the
 484         * encrypted tweak ("i") value; we don't count those.
 485         */
 486        if ((ctx->cipher.mode == CIPHER_MODE_XTS) &&
 487            spu->spu_xts_tweak_in_payload() &&
 488            (payload_len >= SPU_XTS_TWEAK_SIZE))
 489                payload_len -= SPU_XTS_TWEAK_SIZE;
 490
 491        atomic64_add(payload_len, &iproc_priv.bytes_in);
 492
 493        flow_log("%s() offset: %u, bd_len: %u BD:\n",
 494                 __func__, rctx->total_received, payload_len);
 495
 496        dump_sg(req->dst, rctx->total_received, payload_len);
 497
 498        rctx->total_received += payload_len;
 499        if (rctx->total_received == rctx->total_todo) {
 500                atomic_inc(&iproc_priv.op_counts[SPU_OP_CIPHER]);
 501                atomic_inc(
 502                   &iproc_priv.cipher_cnt[ctx->cipher.alg][ctx->cipher.mode]);
 503        }
 504}
 505
 506/**
 507 * spu_ahash_rx_sg_create() - Build up the scatterlist of buffers used to
 508 * receive a SPU response message for an ahash request.
 509 * @mssg:       mailbox message containing the receive sg
 510 * @rctx:       crypto request context
 511 * @rx_frag_num: number of scatterlist elements required to hold the
 512 *              SPU response message
 513 * @digestsize: length of hash digest, in bytes
 514 * @stat_pad_len: Number of bytes required to pad the STAT field to
 515 *              a 4-byte boundary
 516 *
 517 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 518 * when the request completes, whether the request is handled successfully or
 519 * there is an error.
 520 *
 521 * Return:
 522 *   0 if successful
 523 *   < 0 if an error
 524 */
 525static int
 526spu_ahash_rx_sg_create(struct brcm_message *mssg,
 527                       struct iproc_reqctx_s *rctx,
 528                       u8 rx_frag_num, unsigned int digestsize,
 529                       u32 stat_pad_len)
 530{
 531        struct spu_hw *spu = &iproc_priv.spu;
 532        struct scatterlist *sg; /* used to build sgs in mbox message */
 533        struct iproc_ctx_s *ctx = rctx->ctx;
 534
 535        mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
 536                                rctx->gfp);
 537        if (!mssg->spu.dst)
 538                return -ENOMEM;
 539
 540        sg = mssg->spu.dst;
 541        sg_init_table(sg, rx_frag_num);
 542        /* Space for SPU message header */
 543        sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
 544
 545        /* Space for digest */
 546        sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
 547
 548        if (stat_pad_len)
 549                sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
 550
 551        memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
 552        sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
 553        return 0;
 554}
 555
 556/**
 557 * spu_ahash_tx_sg_create() -  Build up the scatterlist of buffers used to send
 558 * a SPU request message for an ahash request. Includes SPU message headers and
 559 * the request data.
 560 * @mssg:       mailbox message containing the transmit sg
 561 * @rctx:       crypto request context
 562 * @tx_frag_num: number of scatterlist elements required to construct the
 563 *              SPU request message
 564 * @spu_hdr_len: length in bytes of SPU message header
 565 * @hash_carry_len: Number of bytes of data carried over from previous req
 566 * @new_data_len: Number of bytes of new request data
 567 * @pad_len:    Number of pad bytes
 568 *
 569 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
 570 * when the request completes, whether the request is handled successfully or
 571 * there is an error.
 572 *
 573 * Return:
 574 *   0 if successful
 575 *   < 0 if an error
 576 */
 577static int
 578spu_ahash_tx_sg_create(struct brcm_message *mssg,
 579                       struct iproc_reqctx_s *rctx,
 580                       u8 tx_frag_num,
 581                       u32 spu_hdr_len,
 582                       unsigned int hash_carry_len,
 583                       unsigned int new_data_len, u32 pad_len)
 584{
 585        struct spu_hw *spu = &iproc_priv.spu;
 586        struct scatterlist *sg; /* used to build sgs in mbox message */
 587        u32 datalen;            /* Number of bytes of response data expected */
 588        u32 stat_len;
 589
 590        mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
 591                                rctx->gfp);
 592        if (!mssg->spu.src)
 593                return -ENOMEM;
 594
 595        sg = mssg->spu.src;
 596        sg_init_table(sg, tx_frag_num);
 597
 598        sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
 599                   BCM_HDR_LEN + spu_hdr_len);
 600
 601        if (hash_carry_len)
 602                sg_set_buf(sg++, rctx->hash_carry, hash_carry_len);
 603
 604        if (new_data_len) {
 605                /* Copy in each src sg entry from request, up to chunksize */
 606                datalen = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
 607                                         rctx->src_nents, new_data_len);
 608                if (datalen < new_data_len) {
 609                        pr_err("%s(): failed to copy src sg to mbox msg",
 610                               __func__);
 611                        return -EFAULT;
 612                }
 613        }
 614
 615        if (pad_len)
 616                sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
 617
 618        stat_len = spu->spu_tx_status_len();
 619        if (stat_len) {
 620                memset(rctx->msg_buf.tx_stat, 0, stat_len);
 621                sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
 622        }
 623
 624        return 0;
 625}
 626
 627/**
 628 * handle_ahash_req() - Process an asynchronous hash request from the crypto
 629 * API.
 630 * @rctx:  Crypto request context
 631 *
 632 * Builds a SPU request message embedded in a mailbox message and submits the
 633 * mailbox message on a selected mailbox channel. The SPU request message is
 634 * constructed as a scatterlist, including entries from the crypto API's
 635 * src scatterlist to avoid copying the data to be hashed. This function is
 636 * called either on the thread from the crypto API, or, in the case that the
 637 * crypto API request is too large to fit in a single SPU request message,
 638 * on the thread that invokes the receive callback with a response message.
 639 * Because some operations require the response from one chunk before the next
 640 * chunk can be submitted, we always wait for the response for the previous
 641 * chunk before submitting the next chunk. Because requests are submitted in
 642 * lock step like this, there is no need to synchronize access to request data
 643 * structures.
 644 *
 645 * Return:
 646 *   -EINPROGRESS: request has been submitted to SPU and response will be
 647 *                 returned asynchronously
 648 *   -EAGAIN:      non-final request included a small amount of data, which for
 649 *                 efficiency we did not submit to the SPU, but instead stored
 650 *                 to be submitted to the SPU with the next part of the request
 651 *   other:        an error code
 652 */
 653static int handle_ahash_req(struct iproc_reqctx_s *rctx)
 654{
 655        struct spu_hw *spu = &iproc_priv.spu;
 656        struct crypto_async_request *areq = rctx->parent;
 657        struct ahash_request *req = ahash_request_cast(areq);
 658        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 659        struct crypto_tfm *tfm = crypto_ahash_tfm(ahash);
 660        unsigned int blocksize = crypto_tfm_alg_blocksize(tfm);
 661        struct iproc_ctx_s *ctx = rctx->ctx;
 662
 663        /* number of bytes still to be hashed in this req */
 664        unsigned int nbytes_to_hash = 0;
 665        int err;
 666        unsigned int chunksize = 0;     /* length of hash carry + new data */
 667        /*
 668         * length of new data, not from hash carry, to be submitted in
 669         * this hw request
 670         */
 671        unsigned int new_data_len;
 672
 673        unsigned int __maybe_unused chunk_start = 0;
 674        u32 db_size;     /* Length of data field, incl gcm and hash padding */
 675        int pad_len = 0; /* total pad len, including gcm, hash, stat padding */
 676        u32 data_pad_len = 0;   /* length of GCM/CCM padding */
 677        u32 stat_pad_len = 0;   /* length of padding to align STATUS word */
 678        struct brcm_message *mssg;      /* mailbox message */
 679        struct spu_request_opts req_opts;
 680        struct spu_cipher_parms cipher_parms;
 681        struct spu_hash_parms hash_parms;
 682        struct spu_aead_parms aead_parms;
 683        unsigned int local_nbuf;
 684        u32 spu_hdr_len;
 685        unsigned int digestsize;
 686        u16 rem = 0;
 687
 688        /*
 689         * number of entries in src and dst sg. Always includes SPU msg header.
 690         * rx always includes a buffer to catch digest and STATUS.
 691         */
 692        u8 rx_frag_num = 3;
 693        u8 tx_frag_num = 1;
 694
 695        flow_log("total_todo %u, total_sent %u\n",
 696                 rctx->total_todo, rctx->total_sent);
 697
 698        memset(&req_opts, 0, sizeof(req_opts));
 699        memset(&cipher_parms, 0, sizeof(cipher_parms));
 700        memset(&hash_parms, 0, sizeof(hash_parms));
 701        memset(&aead_parms, 0, sizeof(aead_parms));
 702
 703        req_opts.bd_suppress = true;
 704        hash_parms.alg = ctx->auth.alg;
 705        hash_parms.mode = ctx->auth.mode;
 706        hash_parms.type = HASH_TYPE_NONE;
 707        hash_parms.key_buf = (u8 *)ctx->authkey;
 708        hash_parms.key_len = ctx->authkeylen;
 709
 710        /*
 711         * For hash algorithms below assignment looks bit odd but
 712         * it's needed for AES-XCBC and AES-CMAC hash algorithms
 713         * to differentiate between 128, 192, 256 bit key values.
 714         * Based on the key values, hash algorithm is selected.
 715         * For example for 128 bit key, hash algorithm is AES-128.
 716         */
 717        cipher_parms.type = ctx->cipher_type;
 718
 719        mssg = &rctx->mb_mssg;
 720        chunk_start = rctx->src_sent;
 721
 722        /*
 723         * Compute the amount remaining to hash. This may include data
 724         * carried over from previous requests.
 725         */
 726        nbytes_to_hash = rctx->total_todo - rctx->total_sent;
 727        chunksize = nbytes_to_hash;
 728        if ((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
 729            (chunksize > ctx->max_payload))
 730                chunksize = ctx->max_payload;
 731
 732        /*
 733         * If this is not a final request and the request data is not a multiple
 734         * of a full block, then simply park the extra data and prefix it to the
 735         * data for the next request.
 736         */
 737        if (!rctx->is_final) {
 738                u8 *dest = rctx->hash_carry + rctx->hash_carry_len;
 739                u16 new_len;  /* len of data to add to hash carry */
 740
 741                rem = chunksize % blocksize;   /* remainder */
 742                if (rem) {
 743                        /* chunksize not a multiple of blocksize */
 744                        chunksize -= rem;
 745                        if (chunksize == 0) {
 746                                /* Don't have a full block to submit to hw */
 747                                new_len = rem - rctx->hash_carry_len;
 748                                sg_copy_part_to_buf(req->src, dest, new_len,
 749                                                    rctx->src_sent);
 750                                rctx->hash_carry_len = rem;
 751                                flow_log("Exiting with hash carry len: %u\n",
 752                                         rctx->hash_carry_len);
 753                                packet_dump("  buf: ",
 754                                            rctx->hash_carry,
 755                                            rctx->hash_carry_len);
 756                                return -EAGAIN;
 757                        }
 758                }
 759        }
 760
 761        /* if we have hash carry, then prefix it to the data in this request */
 762        local_nbuf = rctx->hash_carry_len;
 763        rctx->hash_carry_len = 0;
 764        if (local_nbuf)
 765                tx_frag_num++;
 766        new_data_len = chunksize - local_nbuf;
 767
 768        /* Count number of sg entries to be used in this request */
 769        rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip,
 770                                       new_data_len);
 771
 772        /* AES hashing keeps key size in type field, so need to copy it here */
 773        if (hash_parms.alg == HASH_ALG_AES)
 774                hash_parms.type = (enum hash_type)cipher_parms.type;
 775        else
 776                hash_parms.type = spu->spu_hash_type(rctx->total_sent);
 777
 778        digestsize = spu->spu_digest_size(ctx->digestsize, ctx->auth.alg,
 779                                          hash_parms.type);
 780        hash_parms.digestsize = digestsize;
 781
 782        /* update the indexes */
 783        rctx->total_sent += chunksize;
 784        /* if you sent a prebuf then that wasn't from this req->src */
 785        rctx->src_sent += new_data_len;
 786
 787        if ((rctx->total_sent == rctx->total_todo) && rctx->is_final)
 788                hash_parms.pad_len = spu->spu_hash_pad_len(hash_parms.alg,
 789                                                           hash_parms.mode,
 790                                                           chunksize,
 791                                                           blocksize);
 792
 793        /*
 794         * If a non-first chunk, then include the digest returned from the
 795         * previous chunk so that hw can add to it (except for AES types).
 796         */
 797        if ((hash_parms.type == HASH_TYPE_UPDT) &&
 798            (hash_parms.alg != HASH_ALG_AES)) {
 799                hash_parms.key_buf = rctx->incr_hash;
 800                hash_parms.key_len = digestsize;
 801        }
 802
 803        atomic64_add(chunksize, &iproc_priv.bytes_out);
 804
 805        flow_log("%s() final: %u nbuf: %u ",
 806                 __func__, rctx->is_final, local_nbuf);
 807
 808        if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
 809                flow_log("max_payload infinite\n");
 810        else
 811                flow_log("max_payload %u\n", ctx->max_payload);
 812
 813        flow_log("chunk_start: %u chunk_size: %u\n", chunk_start, chunksize);
 814
 815        /* Prepend SPU header with type 3 BCM header */
 816        memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
 817
 818        hash_parms.prebuf_len = local_nbuf;
 819        spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
 820                                              BCM_HDR_LEN,
 821                                              &req_opts, &cipher_parms,
 822                                              &hash_parms, &aead_parms,
 823                                              new_data_len);
 824
 825        if (spu_hdr_len == 0) {
 826                pr_err("Failed to create SPU request header\n");
 827                return -EFAULT;
 828        }
 829
 830        /*
 831         * Determine total length of padding required. Put all padding in one
 832         * buffer.
 833         */
 834        data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode, chunksize);
 835        db_size = spu_real_db_size(0, 0, local_nbuf, new_data_len,
 836                                   0, 0, hash_parms.pad_len);
 837        if (spu->spu_tx_status_len())
 838                stat_pad_len = spu->spu_wordalign_padlen(db_size);
 839        if (stat_pad_len)
 840                rx_frag_num++;
 841        pad_len = hash_parms.pad_len + data_pad_len + stat_pad_len;
 842        if (pad_len) {
 843                tx_frag_num++;
 844                spu->spu_request_pad(rctx->msg_buf.spu_req_pad, data_pad_len,
 845                                     hash_parms.pad_len, ctx->auth.alg,
 846                                     ctx->auth.mode, rctx->total_sent,
 847                                     stat_pad_len);
 848        }
 849
 850        spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
 851                              spu_hdr_len);
 852        packet_dump("    prebuf: ", rctx->hash_carry, local_nbuf);
 853        flow_log("Data:\n");
 854        dump_sg(rctx->src_sg, rctx->src_skip, new_data_len);
 855        packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
 856
 857        /*
 858         * Build mailbox message containing SPU request msg and rx buffers
 859         * to catch response message
 860         */
 861        memset(mssg, 0, sizeof(*mssg));
 862        mssg->type = BRCM_MESSAGE_SPU;
 863        mssg->ctx = rctx;       /* Will be returned in response */
 864
 865        /* Create rx scatterlist to catch result */
 866        err = spu_ahash_rx_sg_create(mssg, rctx, rx_frag_num, digestsize,
 867                                     stat_pad_len);
 868        if (err)
 869                return err;
 870
 871        /* Create tx scatterlist containing SPU request message */
 872        tx_frag_num += rctx->src_nents;
 873        if (spu->spu_tx_status_len())
 874                tx_frag_num++;
 875        err = spu_ahash_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
 876                                     local_nbuf, new_data_len, pad_len);
 877        if (err)
 878                return err;
 879
 880        err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
 881        if (unlikely(err < 0))
 882                return err;
 883
 884        return -EINPROGRESS;
 885}
 886
 887/**
 888 * spu_hmac_outer_hash() - Request synchonous software compute of the outer hash
 889 * for an HMAC request.
 890 * @req:  The HMAC request from the crypto API
 891 * @ctx:  The session context
 892 *
 893 * Return: 0 if synchronous hash operation successful
 894 *         -EINVAL if the hash algo is unrecognized
 895 *         any other value indicates an error
 896 */
 897static int spu_hmac_outer_hash(struct ahash_request *req,
 898                               struct iproc_ctx_s *ctx)
 899{
 900        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
 901        unsigned int blocksize =
 902                crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
 903        int rc;
 904
 905        switch (ctx->auth.alg) {
 906        case HASH_ALG_MD5:
 907                rc = do_shash("md5", req->result, ctx->opad, blocksize,
 908                              req->result, ctx->digestsize, NULL, 0);
 909                break;
 910        case HASH_ALG_SHA1:
 911                rc = do_shash("sha1", req->result, ctx->opad, blocksize,
 912                              req->result, ctx->digestsize, NULL, 0);
 913                break;
 914        case HASH_ALG_SHA224:
 915                rc = do_shash("sha224", req->result, ctx->opad, blocksize,
 916                              req->result, ctx->digestsize, NULL, 0);
 917                break;
 918        case HASH_ALG_SHA256:
 919                rc = do_shash("sha256", req->result, ctx->opad, blocksize,
 920                              req->result, ctx->digestsize, NULL, 0);
 921                break;
 922        case HASH_ALG_SHA384:
 923                rc = do_shash("sha384", req->result, ctx->opad, blocksize,
 924                              req->result, ctx->digestsize, NULL, 0);
 925                break;
 926        case HASH_ALG_SHA512:
 927                rc = do_shash("sha512", req->result, ctx->opad, blocksize,
 928                              req->result, ctx->digestsize, NULL, 0);
 929                break;
 930        default:
 931                pr_err("%s() Error : unknown hmac type\n", __func__);
 932                rc = -EINVAL;
 933        }
 934        return rc;
 935}
 936
 937/**
 938 * ahash_req_done() - Process a hash result from the SPU hardware.
 939 * @rctx: Crypto request context
 940 *
 941 * Return: 0 if successful
 942 *         < 0 if an error
 943 */
 944static int ahash_req_done(struct iproc_reqctx_s *rctx)
 945{
 946        struct spu_hw *spu = &iproc_priv.spu;
 947        struct crypto_async_request *areq = rctx->parent;
 948        struct ahash_request *req = ahash_request_cast(areq);
 949        struct iproc_ctx_s *ctx = rctx->ctx;
 950        int err;
 951
 952        memcpy(req->result, rctx->msg_buf.digest, ctx->digestsize);
 953
 954        if (spu->spu_type == SPU_TYPE_SPUM) {
 955                /* byte swap the output from the UPDT function to network byte
 956                 * order
 957                 */
 958                if (ctx->auth.alg == HASH_ALG_MD5) {
 959                        __swab32s((u32 *)req->result);
 960                        __swab32s(((u32 *)req->result) + 1);
 961                        __swab32s(((u32 *)req->result) + 2);
 962                        __swab32s(((u32 *)req->result) + 3);
 963                        __swab32s(((u32 *)req->result) + 4);
 964                }
 965        }
 966
 967        flow_dump("  digest ", req->result, ctx->digestsize);
 968
 969        /* if this an HMAC then do the outer hash */
 970        if (rctx->is_sw_hmac) {
 971                err = spu_hmac_outer_hash(req, ctx);
 972                if (err < 0)
 973                        return err;
 974                flow_dump("  hmac: ", req->result, ctx->digestsize);
 975        }
 976
 977        if (rctx->is_sw_hmac || ctx->auth.mode == HASH_MODE_HMAC) {
 978                atomic_inc(&iproc_priv.op_counts[SPU_OP_HMAC]);
 979                atomic_inc(&iproc_priv.hmac_cnt[ctx->auth.alg]);
 980        } else {
 981                atomic_inc(&iproc_priv.op_counts[SPU_OP_HASH]);
 982                atomic_inc(&iproc_priv.hash_cnt[ctx->auth.alg]);
 983        }
 984
 985        return 0;
 986}
 987
 988/**
 989 * handle_ahash_resp() - Process a SPU response message for a hash request.
 990 * Checks if the entire crypto API request has been processed, and if so,
 991 * invokes post processing on the result.
 992 * @rctx: Crypto request context
 993 */
 994static void handle_ahash_resp(struct iproc_reqctx_s *rctx)
 995{
 996        struct iproc_ctx_s *ctx = rctx->ctx;
 997        struct crypto_async_request *areq = rctx->parent;
 998        struct ahash_request *req = ahash_request_cast(areq);
 999        struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1000        unsigned int blocksize =
1001                crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
1002        /*
1003         * Save hash to use as input to next op if incremental. Might be copying
1004         * too much, but that's easier than figuring out actual digest size here
1005         */
1006        memcpy(rctx->incr_hash, rctx->msg_buf.digest, MAX_DIGEST_SIZE);
1007
1008        flow_log("%s() blocksize:%u digestsize:%u\n",
1009                 __func__, blocksize, ctx->digestsize);
1010
1011        atomic64_add(ctx->digestsize, &iproc_priv.bytes_in);
1012
1013        if (rctx->is_final && (rctx->total_sent == rctx->total_todo))
1014                ahash_req_done(rctx);
1015}
1016
1017/**
1018 * spu_aead_rx_sg_create() - Build up the scatterlist of buffers used to receive
1019 * a SPU response message for an AEAD request. Includes buffers to catch SPU
1020 * message headers and the response data.
1021 * @mssg:       mailbox message containing the receive sg
1022 * @req:        Crypto API request
1023 * @rctx:       crypto request context
1024 * @rx_frag_num: number of scatterlist elements required to hold the
1025 *              SPU response message
1026 * @assoc_len:  Length of associated data included in the crypto request
1027 * @ret_iv_len: Length of IV returned in response
1028 * @resp_len:   Number of bytes of response data expected to be written to
1029 *              dst buffer from crypto API
1030 * @digestsize: Length of hash digest, in bytes
1031 * @stat_pad_len: Number of bytes required to pad the STAT field to
1032 *              a 4-byte boundary
1033 *
1034 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1035 * when the request completes, whether the request is handled successfully or
1036 * there is an error.
1037 *
1038 * Returns:
1039 *   0 if successful
1040 *   < 0 if an error
1041 */
1042static int spu_aead_rx_sg_create(struct brcm_message *mssg,
1043                                 struct aead_request *req,
1044                                 struct iproc_reqctx_s *rctx,
1045                                 u8 rx_frag_num,
1046                                 unsigned int assoc_len,
1047                                 u32 ret_iv_len, unsigned int resp_len,
1048                                 unsigned int digestsize, u32 stat_pad_len)
1049{
1050        struct spu_hw *spu = &iproc_priv.spu;
1051        struct scatterlist *sg; /* used to build sgs in mbox message */
1052        struct iproc_ctx_s *ctx = rctx->ctx;
1053        u32 datalen;            /* Number of bytes of response data expected */
1054        u32 assoc_buf_len;
1055        u8 data_padlen = 0;
1056
1057        if (ctx->is_rfc4543) {
1058                /* RFC4543: only pad after data, not after AAD */
1059                data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1060                                                          assoc_len + resp_len);
1061                assoc_buf_len = assoc_len;
1062        } else {
1063                data_padlen = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1064                                                          resp_len);
1065                assoc_buf_len = spu->spu_assoc_resp_len(ctx->cipher.mode,
1066                                                assoc_len, ret_iv_len,
1067                                                rctx->is_encrypt);
1068        }
1069
1070        if (ctx->cipher.mode == CIPHER_MODE_CCM)
1071                /* ICV (after data) must be in the next 32-bit word for CCM */
1072                data_padlen += spu->spu_wordalign_padlen(assoc_buf_len +
1073                                                         resp_len +
1074                                                         data_padlen);
1075
1076        if (data_padlen)
1077                /* have to catch gcm pad in separate buffer */
1078                rx_frag_num++;
1079
1080        mssg->spu.dst = kcalloc(rx_frag_num, sizeof(struct scatterlist),
1081                                rctx->gfp);
1082        if (!mssg->spu.dst)
1083                return -ENOMEM;
1084
1085        sg = mssg->spu.dst;
1086        sg_init_table(sg, rx_frag_num);
1087
1088        /* Space for SPU message header */
1089        sg_set_buf(sg++, rctx->msg_buf.spu_resp_hdr, ctx->spu_resp_hdr_len);
1090
1091        if (assoc_buf_len) {
1092                /*
1093                 * Don't write directly to req->dst, because SPU may pad the
1094                 * assoc data in the response
1095                 */
1096                memset(rctx->msg_buf.a.resp_aad, 0, assoc_buf_len);
1097                sg_set_buf(sg++, rctx->msg_buf.a.resp_aad, assoc_buf_len);
1098        }
1099
1100        if (resp_len) {
1101                /*
1102                 * Copy in each dst sg entry from request, up to chunksize.
1103                 * dst sg catches just the data. digest caught in separate buf.
1104                 */
1105                datalen = spu_msg_sg_add(&sg, &rctx->dst_sg, &rctx->dst_skip,
1106                                         rctx->dst_nents, resp_len);
1107                if (datalen < (resp_len)) {
1108                        pr_err("%s(): failed to copy dst sg to mbox msg. expected len %u, datalen %u",
1109                               __func__, resp_len, datalen);
1110                        return -EFAULT;
1111                }
1112        }
1113
1114        /* If GCM/CCM data is padded, catch padding in separate buffer */
1115        if (data_padlen) {
1116                memset(rctx->msg_buf.a.gcmpad, 0, data_padlen);
1117                sg_set_buf(sg++, rctx->msg_buf.a.gcmpad, data_padlen);
1118        }
1119
1120        /* Always catch ICV in separate buffer */
1121        sg_set_buf(sg++, rctx->msg_buf.digest, digestsize);
1122
1123        flow_log("stat_pad_len %u\n", stat_pad_len);
1124        if (stat_pad_len) {
1125                memset(rctx->msg_buf.rx_stat_pad, 0, stat_pad_len);
1126                sg_set_buf(sg++, rctx->msg_buf.rx_stat_pad, stat_pad_len);
1127        }
1128
1129        memset(rctx->msg_buf.rx_stat, 0, SPU_RX_STATUS_LEN);
1130        sg_set_buf(sg, rctx->msg_buf.rx_stat, spu->spu_rx_status_len());
1131
1132        return 0;
1133}
1134
1135/**
1136 * spu_aead_tx_sg_create() - Build up the scatterlist of buffers used to send a
1137 * SPU request message for an AEAD request. Includes SPU message headers and the
1138 * request data.
1139 * @mssg:       mailbox message containing the transmit sg
1140 * @rctx:       crypto request context
1141 * @tx_frag_num: number of scatterlist elements required to construct the
1142 *              SPU request message
1143 * @spu_hdr_len: length of SPU message header in bytes
1144 * @assoc:      crypto API associated data scatterlist
1145 * @assoc_len:  length of associated data
1146 * @assoc_nents: number of scatterlist entries containing assoc data
1147 * @aead_iv_len: length of AEAD IV, if included
1148 * @chunksize:  Number of bytes of request data
1149 * @aad_pad_len: Number of bytes of padding at end of AAD. For GCM/CCM.
1150 * @pad_len:    Number of pad bytes
1151 * @incl_icv:   If true, write separate ICV buffer after data and
1152 *              any padding
1153 *
1154 * The scatterlist that gets allocated here is freed in spu_chunk_cleanup()
1155 * when the request completes, whether the request is handled successfully or
1156 * there is an error.
1157 *
1158 * Return:
1159 *   0 if successful
1160 *   < 0 if an error
1161 */
1162static int spu_aead_tx_sg_create(struct brcm_message *mssg,
1163                                 struct iproc_reqctx_s *rctx,
1164                                 u8 tx_frag_num,
1165                                 u32 spu_hdr_len,
1166                                 struct scatterlist *assoc,
1167                                 unsigned int assoc_len,
1168                                 int assoc_nents,
1169                                 unsigned int aead_iv_len,
1170                                 unsigned int chunksize,
1171                                 u32 aad_pad_len, u32 pad_len, bool incl_icv)
1172{
1173        struct spu_hw *spu = &iproc_priv.spu;
1174        struct scatterlist *sg; /* used to build sgs in mbox message */
1175        struct scatterlist *assoc_sg = assoc;
1176        struct iproc_ctx_s *ctx = rctx->ctx;
1177        u32 datalen;            /* Number of bytes of data to write */
1178        u32 written;            /* Number of bytes of data written */
1179        u32 assoc_offset = 0;
1180        u32 stat_len;
1181
1182        mssg->spu.src = kcalloc(tx_frag_num, sizeof(struct scatterlist),
1183                                rctx->gfp);
1184        if (!mssg->spu.src)
1185                return -ENOMEM;
1186
1187        sg = mssg->spu.src;
1188        sg_init_table(sg, tx_frag_num);
1189
1190        sg_set_buf(sg++, rctx->msg_buf.bcm_spu_req_hdr,
1191                   BCM_HDR_LEN + spu_hdr_len);
1192
1193        if (assoc_len) {
1194                /* Copy in each associated data sg entry from request */
1195                written = spu_msg_sg_add(&sg, &assoc_sg, &assoc_offset,
1196                                         assoc_nents, assoc_len);
1197                if (written < assoc_len) {
1198                        pr_err("%s(): failed to copy assoc sg to mbox msg",
1199                               __func__);
1200                        return -EFAULT;
1201                }
1202        }
1203
1204        if (aead_iv_len)
1205                sg_set_buf(sg++, rctx->msg_buf.iv_ctr, aead_iv_len);
1206
1207        if (aad_pad_len) {
1208                memset(rctx->msg_buf.a.req_aad_pad, 0, aad_pad_len);
1209                sg_set_buf(sg++, rctx->msg_buf.a.req_aad_pad, aad_pad_len);
1210        }
1211
1212        datalen = chunksize;
1213        if ((chunksize > ctx->digestsize) && incl_icv)
1214                datalen -= ctx->digestsize;
1215        if (datalen) {
1216                /* For aead, a single msg should consume the entire src sg */
1217                written = spu_msg_sg_add(&sg, &rctx->src_sg, &rctx->src_skip,
1218                                         rctx->src_nents, datalen);
1219                if (written < datalen) {
1220                        pr_err("%s(): failed to copy src sg to mbox msg",
1221                               __func__);
1222                        return -EFAULT;
1223                }
1224        }
1225
1226        if (pad_len) {
1227                memset(rctx->msg_buf.spu_req_pad, 0, pad_len);
1228                sg_set_buf(sg++, rctx->msg_buf.spu_req_pad, pad_len);
1229        }
1230
1231        if (incl_icv)
1232                sg_set_buf(sg++, rctx->msg_buf.digest, ctx->digestsize);
1233
1234        stat_len = spu->spu_tx_status_len();
1235        if (stat_len) {
1236                memset(rctx->msg_buf.tx_stat, 0, stat_len);
1237                sg_set_buf(sg, rctx->msg_buf.tx_stat, stat_len);
1238        }
1239        return 0;
1240}
1241
1242/**
1243 * handle_aead_req() - Submit a SPU request message for the next chunk of the
1244 * current AEAD request.
1245 * @rctx:  Crypto request context
1246 *
1247 * Unlike other operation types, we assume the length of the request fits in
1248 * a single SPU request message. aead_enqueue() makes sure this is true.
1249 * Comments for other op types regarding threads applies here as well.
1250 *
1251 * Unlike incremental hash ops, where the spu returns the entire hash for
1252 * truncated algs like sha-224, the SPU returns just the truncated hash in
1253 * response to aead requests. So digestsize is always ctx->digestsize here.
1254 *
1255 * Return: -EINPROGRESS: crypto request has been accepted and result will be
1256 *                       returned asynchronously
1257 *         Any other value indicates an error
1258 */
1259static int handle_aead_req(struct iproc_reqctx_s *rctx)
1260{
1261        struct spu_hw *spu = &iproc_priv.spu;
1262        struct crypto_async_request *areq = rctx->parent;
1263        struct aead_request *req = container_of(areq,
1264                                                struct aead_request, base);
1265        struct iproc_ctx_s *ctx = rctx->ctx;
1266        int err;
1267        unsigned int chunksize;
1268        unsigned int resp_len;
1269        u32 spu_hdr_len;
1270        u32 db_size;
1271        u32 stat_pad_len;
1272        u32 pad_len;
1273        struct brcm_message *mssg;      /* mailbox message */
1274        struct spu_request_opts req_opts;
1275        struct spu_cipher_parms cipher_parms;
1276        struct spu_hash_parms hash_parms;
1277        struct spu_aead_parms aead_parms;
1278        int assoc_nents = 0;
1279        bool incl_icv = false;
1280        unsigned int digestsize = ctx->digestsize;
1281
1282        /* number of entries in src and dst sg. Always includes SPU msg header.
1283         */
1284        u8 rx_frag_num = 2;     /* and STATUS */
1285        u8 tx_frag_num = 1;
1286
1287        /* doing the whole thing at once */
1288        chunksize = rctx->total_todo;
1289
1290        flow_log("%s: chunksize %u\n", __func__, chunksize);
1291
1292        memset(&req_opts, 0, sizeof(req_opts));
1293        memset(&hash_parms, 0, sizeof(hash_parms));
1294        memset(&aead_parms, 0, sizeof(aead_parms));
1295
1296        req_opts.is_inbound = !(rctx->is_encrypt);
1297        req_opts.auth_first = ctx->auth_first;
1298        req_opts.is_aead = true;
1299        req_opts.is_esp = ctx->is_esp;
1300
1301        cipher_parms.alg = ctx->cipher.alg;
1302        cipher_parms.mode = ctx->cipher.mode;
1303        cipher_parms.type = ctx->cipher_type;
1304        cipher_parms.key_buf = ctx->enckey;
1305        cipher_parms.key_len = ctx->enckeylen;
1306        cipher_parms.iv_buf = rctx->msg_buf.iv_ctr;
1307        cipher_parms.iv_len = rctx->iv_ctr_len;
1308
1309        hash_parms.alg = ctx->auth.alg;
1310        hash_parms.mode = ctx->auth.mode;
1311        hash_parms.type = HASH_TYPE_NONE;
1312        hash_parms.key_buf = (u8 *)ctx->authkey;
1313        hash_parms.key_len = ctx->authkeylen;
1314        hash_parms.digestsize = digestsize;
1315
1316        if ((ctx->auth.alg == HASH_ALG_SHA224) &&
1317            (ctx->authkeylen < SHA224_DIGEST_SIZE))
1318                hash_parms.key_len = SHA224_DIGEST_SIZE;
1319
1320        aead_parms.assoc_size = req->assoclen;
1321        if (ctx->is_esp && !ctx->is_rfc4543) {
1322                /*
1323                 * 8-byte IV is included assoc data in request. SPU2
1324                 * expects AAD to include just SPI and seqno. So
1325                 * subtract off the IV len.
1326                 */
1327                aead_parms.assoc_size -= GCM_RFC4106_IV_SIZE;
1328
1329                if (rctx->is_encrypt) {
1330                        aead_parms.return_iv = true;
1331                        aead_parms.ret_iv_len = GCM_RFC4106_IV_SIZE;
1332                        aead_parms.ret_iv_off = GCM_ESP_SALT_SIZE;
1333                }
1334        } else {
1335                aead_parms.ret_iv_len = 0;
1336        }
1337
1338        /*
1339         * Count number of sg entries from the crypto API request that are to
1340         * be included in this mailbox message. For dst sg, don't count space
1341         * for digest. Digest gets caught in a separate buffer and copied back
1342         * to dst sg when processing response.
1343         */
1344        rctx->src_nents = spu_sg_count(rctx->src_sg, rctx->src_skip, chunksize);
1345        rctx->dst_nents = spu_sg_count(rctx->dst_sg, rctx->dst_skip, chunksize);
1346        if (aead_parms.assoc_size)
1347                assoc_nents = spu_sg_count(rctx->assoc, 0,
1348                                           aead_parms.assoc_size);
1349
1350        mssg = &rctx->mb_mssg;
1351
1352        rctx->total_sent = chunksize;
1353        rctx->src_sent = chunksize;
1354        if (spu->spu_assoc_resp_len(ctx->cipher.mode,
1355                                    aead_parms.assoc_size,
1356                                    aead_parms.ret_iv_len,
1357                                    rctx->is_encrypt))
1358                rx_frag_num++;
1359
1360        aead_parms.iv_len = spu->spu_aead_ivlen(ctx->cipher.mode,
1361                                                rctx->iv_ctr_len);
1362
1363        if (ctx->auth.alg == HASH_ALG_AES)
1364                hash_parms.type = (enum hash_type)ctx->cipher_type;
1365
1366        /* General case AAD padding (CCM and RFC4543 special cases below) */
1367        aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1368                                                 aead_parms.assoc_size);
1369
1370        /* General case data padding (CCM decrypt special case below) */
1371        aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1372                                                           chunksize);
1373
1374        if (ctx->cipher.mode == CIPHER_MODE_CCM) {
1375                /*
1376                 * for CCM, AAD len + 2 (rather than AAD len) needs to be
1377                 * 128-bit aligned
1378                 */
1379                aead_parms.aad_pad_len = spu->spu_gcm_ccm_pad_len(
1380                                         ctx->cipher.mode,
1381                                         aead_parms.assoc_size + 2);
1382
1383                /*
1384                 * And when decrypting CCM, need to pad without including
1385                 * size of ICV which is tacked on to end of chunk
1386                 */
1387                if (!rctx->is_encrypt)
1388                        aead_parms.data_pad_len =
1389                                spu->spu_gcm_ccm_pad_len(ctx->cipher.mode,
1390                                                        chunksize - digestsize);
1391
1392                /* CCM also requires software to rewrite portions of IV: */
1393                spu->spu_ccm_update_iv(digestsize, &cipher_parms, req->assoclen,
1394                                       chunksize, rctx->is_encrypt,
1395                                       ctx->is_esp);
1396        }
1397
1398        if (ctx->is_rfc4543) {
1399                /*
1400                 * RFC4543: data is included in AAD, so don't pad after AAD
1401                 * and pad data based on both AAD + data size
1402                 */
1403                aead_parms.aad_pad_len = 0;
1404                if (!rctx->is_encrypt)
1405                        aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1406                                        ctx->cipher.mode,
1407                                        aead_parms.assoc_size + chunksize -
1408                                        digestsize);
1409                else
1410                        aead_parms.data_pad_len = spu->spu_gcm_ccm_pad_len(
1411                                        ctx->cipher.mode,
1412                                        aead_parms.assoc_size + chunksize);
1413
1414                req_opts.is_rfc4543 = true;
1415        }
1416
1417        if (spu_req_incl_icv(ctx->cipher.mode, rctx->is_encrypt)) {
1418                incl_icv = true;
1419                tx_frag_num++;
1420                /* Copy ICV from end of src scatterlist to digest buf */
1421                sg_copy_part_to_buf(req->src, rctx->msg_buf.digest, digestsize,
1422                                    req->assoclen + rctx->total_sent -
1423                                    digestsize);
1424        }
1425
1426        atomic64_add(chunksize, &iproc_priv.bytes_out);
1427
1428        flow_log("%s()-sent chunksize:%u\n", __func__, chunksize);
1429
1430        /* Prepend SPU header with type 3 BCM header */
1431        memcpy(rctx->msg_buf.bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1432
1433        spu_hdr_len = spu->spu_create_request(rctx->msg_buf.bcm_spu_req_hdr +
1434                                              BCM_HDR_LEN, &req_opts,
1435                                              &cipher_parms, &hash_parms,
1436                                              &aead_parms, chunksize);
1437
1438        /* Determine total length of padding. Put all padding in one buffer. */
1439        db_size = spu_real_db_size(aead_parms.assoc_size, aead_parms.iv_len, 0,
1440                                   chunksize, aead_parms.aad_pad_len,
1441                                   aead_parms.data_pad_len, 0);
1442
1443        stat_pad_len = spu->spu_wordalign_padlen(db_size);
1444
1445        if (stat_pad_len)
1446                rx_frag_num++;
1447        pad_len = aead_parms.data_pad_len + stat_pad_len;
1448        if (pad_len) {
1449                tx_frag_num++;
1450                spu->spu_request_pad(rctx->msg_buf.spu_req_pad,
1451                                     aead_parms.data_pad_len, 0,
1452                                     ctx->auth.alg, ctx->auth.mode,
1453                                     rctx->total_sent, stat_pad_len);
1454        }
1455
1456        spu->spu_dump_msg_hdr(rctx->msg_buf.bcm_spu_req_hdr + BCM_HDR_LEN,
1457                              spu_hdr_len);
1458        dump_sg(rctx->assoc, 0, aead_parms.assoc_size);
1459        packet_dump("    aead iv: ", rctx->msg_buf.iv_ctr, aead_parms.iv_len);
1460        packet_log("BD:\n");
1461        dump_sg(rctx->src_sg, rctx->src_skip, chunksize);
1462        packet_dump("   pad: ", rctx->msg_buf.spu_req_pad, pad_len);
1463
1464        /*
1465         * Build mailbox message containing SPU request msg and rx buffers
1466         * to catch response message
1467         */
1468        memset(mssg, 0, sizeof(*mssg));
1469        mssg->type = BRCM_MESSAGE_SPU;
1470        mssg->ctx = rctx;       /* Will be returned in response */
1471
1472        /* Create rx scatterlist to catch result */
1473        rx_frag_num += rctx->dst_nents;
1474        resp_len = chunksize;
1475
1476        /*
1477         * Always catch ICV in separate buffer. Have to for GCM/CCM because of
1478         * padding. Have to for SHA-224 and other truncated SHAs because SPU
1479         * sends entire digest back.
1480         */
1481        rx_frag_num++;
1482
1483        if (((ctx->cipher.mode == CIPHER_MODE_GCM) ||
1484             (ctx->cipher.mode == CIPHER_MODE_CCM)) && !rctx->is_encrypt) {
1485                /*
1486                 * Input is ciphertxt plus ICV, but ICV not incl
1487                 * in output.
1488                 */
1489                resp_len -= ctx->digestsize;
1490                if (resp_len == 0)
1491                        /* no rx frags to catch output data */
1492                        rx_frag_num -= rctx->dst_nents;
1493        }
1494
1495        err = spu_aead_rx_sg_create(mssg, req, rctx, rx_frag_num,
1496                                    aead_parms.assoc_size,
1497                                    aead_parms.ret_iv_len, resp_len, digestsize,
1498                                    stat_pad_len);
1499        if (err)
1500                return err;
1501
1502        /* Create tx scatterlist containing SPU request message */
1503        tx_frag_num += rctx->src_nents;
1504        tx_frag_num += assoc_nents;
1505        if (aead_parms.aad_pad_len)
1506                tx_frag_num++;
1507        if (aead_parms.iv_len)
1508                tx_frag_num++;
1509        if (spu->spu_tx_status_len())
1510                tx_frag_num++;
1511        err = spu_aead_tx_sg_create(mssg, rctx, tx_frag_num, spu_hdr_len,
1512                                    rctx->assoc, aead_parms.assoc_size,
1513                                    assoc_nents, aead_parms.iv_len, chunksize,
1514                                    aead_parms.aad_pad_len, pad_len, incl_icv);
1515        if (err)
1516                return err;
1517
1518        err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx);
1519        if (unlikely(err < 0))
1520                return err;
1521
1522        return -EINPROGRESS;
1523}
1524
1525/**
1526 * handle_aead_resp() - Process a SPU response message for an AEAD request.
1527 * @rctx:  Crypto request context
1528 */
1529static void handle_aead_resp(struct iproc_reqctx_s *rctx)
1530{
1531        struct spu_hw *spu = &iproc_priv.spu;
1532        struct crypto_async_request *areq = rctx->parent;
1533        struct aead_request *req = container_of(areq,
1534                                                struct aead_request, base);
1535        struct iproc_ctx_s *ctx = rctx->ctx;
1536        u32 payload_len;
1537        unsigned int icv_offset;
1538        u32 result_len;
1539
1540        /* See how much data was returned */
1541        payload_len = spu->spu_payload_length(rctx->msg_buf.spu_resp_hdr);
1542        flow_log("payload_len %u\n", payload_len);
1543
1544        /* only count payload */
1545        atomic64_add(payload_len, &iproc_priv.bytes_in);
1546
1547        if (req->assoclen)
1548                packet_dump("  assoc_data ", rctx->msg_buf.a.resp_aad,
1549                            req->assoclen);
1550
1551        /*
1552         * Copy the ICV back to the destination
1553         * buffer. In decrypt case, SPU gives us back the digest, but crypto
1554         * API doesn't expect ICV in dst buffer.
1555         */
1556        result_len = req->cryptlen;
1557        if (rctx->is_encrypt) {
1558                icv_offset = req->assoclen + rctx->total_sent;
1559                packet_dump("  ICV: ", rctx->msg_buf.digest, ctx->digestsize);
1560                flow_log("copying ICV to dst sg at offset %u\n", icv_offset);
1561                sg_copy_part_from_buf(req->dst, rctx->msg_buf.digest,
1562                                      ctx->digestsize, icv_offset);
1563                result_len += ctx->digestsize;
1564        }
1565
1566        packet_log("response data:  ");
1567        dump_sg(req->dst, req->assoclen, result_len);
1568
1569        atomic_inc(&iproc_priv.op_counts[SPU_OP_AEAD]);
1570        if (ctx->cipher.alg == CIPHER_ALG_AES) {
1571                if (ctx->cipher.mode == CIPHER_MODE_CCM)
1572                        atomic_inc(&iproc_priv.aead_cnt[AES_CCM]);
1573                else if (ctx->cipher.mode == CIPHER_MODE_GCM)
1574                        atomic_inc(&iproc_priv.aead_cnt[AES_GCM]);
1575                else
1576                        atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1577        } else {
1578                atomic_inc(&iproc_priv.aead_cnt[AUTHENC]);
1579        }
1580}
1581
1582/**
1583 * spu_chunk_cleanup() - Do cleanup after processing one chunk of a request
1584 * @rctx:  request context
1585 *
1586 * Mailbox scatterlists are allocated for each chunk. So free them after
1587 * processing each chunk.
1588 */
1589static void spu_chunk_cleanup(struct iproc_reqctx_s *rctx)
1590{
1591        /* mailbox message used to tx request */
1592        struct brcm_message *mssg = &rctx->mb_mssg;
1593
1594        kfree(mssg->spu.src);
1595        kfree(mssg->spu.dst);
1596        memset(mssg, 0, sizeof(struct brcm_message));
1597}
1598
1599/**
1600 * finish_req() - Used to invoke the complete callback from the requester when
1601 * a request has been handled asynchronously.
1602 * @rctx:  Request context
1603 * @err:   Indicates whether the request was successful or not
1604 *
1605 * Ensures that cleanup has been done for request
1606 */
1607static void finish_req(struct iproc_reqctx_s *rctx, int err)
1608{
1609        struct crypto_async_request *areq = rctx->parent;
1610
1611        flow_log("%s() err:%d\n\n", __func__, err);
1612
1613        /* No harm done if already called */
1614        spu_chunk_cleanup(rctx);
1615
1616        if (areq)
1617                areq->complete(areq, err);
1618}
1619
1620/**
1621 * spu_rx_callback() - Callback from mailbox framework with a SPU response.
1622 * @cl:         mailbox client structure for SPU driver
1623 * @msg:        mailbox message containing SPU response
1624 */
1625static void spu_rx_callback(struct mbox_client *cl, void *msg)
1626{
1627        struct spu_hw *spu = &iproc_priv.spu;
1628        struct brcm_message *mssg = msg;
1629        struct iproc_reqctx_s *rctx;
1630        int err;
1631
1632        rctx = mssg->ctx;
1633        if (unlikely(!rctx)) {
1634                /* This is fatal */
1635                pr_err("%s(): no request context", __func__);
1636                err = -EFAULT;
1637                goto cb_finish;
1638        }
1639
1640        /* process the SPU status */
1641        err = spu->spu_status_process(rctx->msg_buf.rx_stat);
1642        if (err != 0) {
1643                if (err == SPU_INVALID_ICV)
1644                        atomic_inc(&iproc_priv.bad_icv);
1645                err = -EBADMSG;
1646                goto cb_finish;
1647        }
1648
1649        /* Process the SPU response message */
1650        switch (rctx->ctx->alg->type) {
1651        case CRYPTO_ALG_TYPE_SKCIPHER:
1652                handle_skcipher_resp(rctx);
1653                break;
1654        case CRYPTO_ALG_TYPE_AHASH:
1655                handle_ahash_resp(rctx);
1656                break;
1657        case CRYPTO_ALG_TYPE_AEAD:
1658                handle_aead_resp(rctx);
1659                break;
1660        default:
1661                err = -EINVAL;
1662                goto cb_finish;
1663        }
1664
1665        /*
1666         * If this response does not complete the request, then send the next
1667         * request chunk.
1668         */
1669        if (rctx->total_sent < rctx->total_todo) {
1670                /* Deallocate anything specific to previous chunk */
1671                spu_chunk_cleanup(rctx);
1672
1673                switch (rctx->ctx->alg->type) {
1674                case CRYPTO_ALG_TYPE_SKCIPHER:
1675                        err = handle_skcipher_req(rctx);
1676                        break;
1677                case CRYPTO_ALG_TYPE_AHASH:
1678                        err = handle_ahash_req(rctx);
1679                        if (err == -EAGAIN)
1680                                /*
1681                                 * we saved data in hash carry, but tell crypto
1682                                 * API we successfully completed request.
1683                                 */
1684                                err = 0;
1685                        break;
1686                case CRYPTO_ALG_TYPE_AEAD:
1687                        err = handle_aead_req(rctx);
1688                        break;
1689                default:
1690                        err = -EINVAL;
1691                }
1692
1693                if (err == -EINPROGRESS)
1694                        /* Successfully submitted request for next chunk */
1695                        return;
1696        }
1697
1698cb_finish:
1699        finish_req(rctx, err);
1700}
1701
1702/* ==================== Kernel Cryptographic API ==================== */
1703
1704/**
1705 * skcipher_enqueue() - Handle skcipher encrypt or decrypt request.
1706 * @req:        Crypto API request
1707 * @encrypt:    true if encrypting; false if decrypting
1708 *
1709 * Return: -EINPROGRESS if request accepted and result will be returned
1710 *                      asynchronously
1711 *         < 0 if an error
1712 */
1713static int skcipher_enqueue(struct skcipher_request *req, bool encrypt)
1714{
1715        struct iproc_reqctx_s *rctx = skcipher_request_ctx(req);
1716        struct iproc_ctx_s *ctx =
1717            crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1718        int err;
1719
1720        flow_log("%s() enc:%u\n", __func__, encrypt);
1721
1722        rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1723                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1724        rctx->parent = &req->base;
1725        rctx->is_encrypt = encrypt;
1726        rctx->bd_suppress = false;
1727        rctx->total_todo = req->cryptlen;
1728        rctx->src_sent = 0;
1729        rctx->total_sent = 0;
1730        rctx->total_received = 0;
1731        rctx->ctx = ctx;
1732
1733        /* Initialize current position in src and dst scatterlists */
1734        rctx->src_sg = req->src;
1735        rctx->src_nents = 0;
1736        rctx->src_skip = 0;
1737        rctx->dst_sg = req->dst;
1738        rctx->dst_nents = 0;
1739        rctx->dst_skip = 0;
1740
1741        if (ctx->cipher.mode == CIPHER_MODE_CBC ||
1742            ctx->cipher.mode == CIPHER_MODE_CTR ||
1743            ctx->cipher.mode == CIPHER_MODE_OFB ||
1744            ctx->cipher.mode == CIPHER_MODE_XTS ||
1745            ctx->cipher.mode == CIPHER_MODE_GCM ||
1746            ctx->cipher.mode == CIPHER_MODE_CCM) {
1747                rctx->iv_ctr_len =
1748                    crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req));
1749                memcpy(rctx->msg_buf.iv_ctr, req->iv, rctx->iv_ctr_len);
1750        } else {
1751                rctx->iv_ctr_len = 0;
1752        }
1753
1754        /* Choose a SPU to process this request */
1755        rctx->chan_idx = select_channel();
1756        err = handle_skcipher_req(rctx);
1757        if (err != -EINPROGRESS)
1758                /* synchronous result */
1759                spu_chunk_cleanup(rctx);
1760
1761        return err;
1762}
1763
1764static int des_setkey(struct crypto_skcipher *cipher, const u8 *key,
1765                      unsigned int keylen)
1766{
1767        struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1768        int err;
1769
1770        err = verify_skcipher_des_key(cipher, key);
1771        if (err)
1772                return err;
1773
1774        ctx->cipher_type = CIPHER_TYPE_DES;
1775        return 0;
1776}
1777
1778static int threedes_setkey(struct crypto_skcipher *cipher, const u8 *key,
1779                           unsigned int keylen)
1780{
1781        struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1782        int err;
1783
1784        err = verify_skcipher_des3_key(cipher, key);
1785        if (err)
1786                return err;
1787
1788        ctx->cipher_type = CIPHER_TYPE_3DES;
1789        return 0;
1790}
1791
1792static int aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
1793                      unsigned int keylen)
1794{
1795        struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1796
1797        if (ctx->cipher.mode == CIPHER_MODE_XTS)
1798                /* XTS includes two keys of equal length */
1799                keylen = keylen / 2;
1800
1801        switch (keylen) {
1802        case AES_KEYSIZE_128:
1803                ctx->cipher_type = CIPHER_TYPE_AES128;
1804                break;
1805        case AES_KEYSIZE_192:
1806                ctx->cipher_type = CIPHER_TYPE_AES192;
1807                break;
1808        case AES_KEYSIZE_256:
1809                ctx->cipher_type = CIPHER_TYPE_AES256;
1810                break;
1811        default:
1812                return -EINVAL;
1813        }
1814        WARN_ON((ctx->max_payload != SPU_MAX_PAYLOAD_INF) &&
1815                ((ctx->max_payload % AES_BLOCK_SIZE) != 0));
1816        return 0;
1817}
1818
1819static int skcipher_setkey(struct crypto_skcipher *cipher, const u8 *key,
1820                             unsigned int keylen)
1821{
1822        struct spu_hw *spu = &iproc_priv.spu;
1823        struct iproc_ctx_s *ctx = crypto_skcipher_ctx(cipher);
1824        struct spu_cipher_parms cipher_parms;
1825        u32 alloc_len = 0;
1826        int err;
1827
1828        flow_log("skcipher_setkey() keylen: %d\n", keylen);
1829        flow_dump("  key: ", key, keylen);
1830
1831        switch (ctx->cipher.alg) {
1832        case CIPHER_ALG_DES:
1833                err = des_setkey(cipher, key, keylen);
1834                break;
1835        case CIPHER_ALG_3DES:
1836                err = threedes_setkey(cipher, key, keylen);
1837                break;
1838        case CIPHER_ALG_AES:
1839                err = aes_setkey(cipher, key, keylen);
1840                break;
1841        default:
1842                pr_err("%s() Error: unknown cipher alg\n", __func__);
1843                err = -EINVAL;
1844        }
1845        if (err)
1846                return err;
1847
1848        memcpy(ctx->enckey, key, keylen);
1849        ctx->enckeylen = keylen;
1850
1851        /* SPU needs XTS keys in the reverse order the crypto API presents */
1852        if ((ctx->cipher.alg == CIPHER_ALG_AES) &&
1853            (ctx->cipher.mode == CIPHER_MODE_XTS)) {
1854                unsigned int xts_keylen = keylen / 2;
1855
1856                memcpy(ctx->enckey, key + xts_keylen, xts_keylen);
1857                memcpy(ctx->enckey + xts_keylen, key, xts_keylen);
1858        }
1859
1860        if (spu->spu_type == SPU_TYPE_SPUM)
1861                alloc_len = BCM_HDR_LEN + SPU_HEADER_ALLOC_LEN;
1862        else if (spu->spu_type == SPU_TYPE_SPU2)
1863                alloc_len = BCM_HDR_LEN + SPU2_HEADER_ALLOC_LEN;
1864        memset(ctx->bcm_spu_req_hdr, 0, alloc_len);
1865        cipher_parms.iv_buf = NULL;
1866        cipher_parms.iv_len = crypto_skcipher_ivsize(cipher);
1867        flow_log("%s: iv_len %u\n", __func__, cipher_parms.iv_len);
1868
1869        cipher_parms.alg = ctx->cipher.alg;
1870        cipher_parms.mode = ctx->cipher.mode;
1871        cipher_parms.type = ctx->cipher_type;
1872        cipher_parms.key_buf = ctx->enckey;
1873        cipher_parms.key_len = ctx->enckeylen;
1874
1875        /* Prepend SPU request message with BCM header */
1876        memcpy(ctx->bcm_spu_req_hdr, BCMHEADER, BCM_HDR_LEN);
1877        ctx->spu_req_hdr_len =
1878            spu->spu_cipher_req_init(ctx->bcm_spu_req_hdr + BCM_HDR_LEN,
1879                                     &cipher_parms);
1880
1881        ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
1882                                                          ctx->enckeylen,
1883                                                          false);
1884
1885        atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_CIPHER]);
1886
1887        return 0;
1888}
1889
1890static int skcipher_encrypt(struct skcipher_request *req)
1891{
1892        flow_log("skcipher_encrypt() nbytes:%u\n", req->cryptlen);
1893
1894        return skcipher_enqueue(req, true);
1895}
1896
1897static int skcipher_decrypt(struct skcipher_request *req)
1898{
1899        flow_log("skcipher_decrypt() nbytes:%u\n", req->cryptlen);
1900        return skcipher_enqueue(req, false);
1901}
1902
1903static int ahash_enqueue(struct ahash_request *req)
1904{
1905        struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
1906        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1907        struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
1908        int err;
1909        const char *alg_name;
1910
1911        flow_log("ahash_enqueue() nbytes:%u\n", req->nbytes);
1912
1913        rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1914                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1915        rctx->parent = &req->base;
1916        rctx->ctx = ctx;
1917        rctx->bd_suppress = true;
1918        memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
1919
1920        /* Initialize position in src scatterlist */
1921        rctx->src_sg = req->src;
1922        rctx->src_skip = 0;
1923        rctx->src_nents = 0;
1924        rctx->dst_sg = NULL;
1925        rctx->dst_skip = 0;
1926        rctx->dst_nents = 0;
1927
1928        /* SPU2 hardware does not compute hash of zero length data */
1929        if ((rctx->is_final == 1) && (rctx->total_todo == 0) &&
1930            (iproc_priv.spu.spu_type == SPU_TYPE_SPU2)) {
1931                alg_name = crypto_tfm_alg_name(crypto_ahash_tfm(tfm));
1932                flow_log("Doing %sfinal %s zero-len hash request in software\n",
1933                         rctx->is_final ? "" : "non-", alg_name);
1934                err = do_shash((unsigned char *)alg_name, req->result,
1935                               NULL, 0, NULL, 0, ctx->authkey,
1936                               ctx->authkeylen);
1937                if (err < 0)
1938                        flow_log("Hash request failed with error %d\n", err);
1939                return err;
1940        }
1941        /* Choose a SPU to process this request */
1942        rctx->chan_idx = select_channel();
1943
1944        err = handle_ahash_req(rctx);
1945        if (err != -EINPROGRESS)
1946                /* synchronous result */
1947                spu_chunk_cleanup(rctx);
1948
1949        if (err == -EAGAIN)
1950                /*
1951                 * we saved data in hash carry, but tell crypto API
1952                 * we successfully completed request.
1953                 */
1954                err = 0;
1955
1956        return err;
1957}
1958
1959static int __ahash_init(struct ahash_request *req)
1960{
1961        struct spu_hw *spu = &iproc_priv.spu;
1962        struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
1963        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1964        struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
1965
1966        flow_log("%s()\n", __func__);
1967
1968        /* Initialize the context */
1969        rctx->hash_carry_len = 0;
1970        rctx->is_final = 0;
1971
1972        rctx->total_todo = 0;
1973        rctx->src_sent = 0;
1974        rctx->total_sent = 0;
1975        rctx->total_received = 0;
1976
1977        ctx->digestsize = crypto_ahash_digestsize(tfm);
1978        /* If we add a hash whose digest is larger, catch it here. */
1979        WARN_ON(ctx->digestsize > MAX_DIGEST_SIZE);
1980
1981        rctx->is_sw_hmac = false;
1982
1983        ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen, 0,
1984                                                          true);
1985
1986        return 0;
1987}
1988
1989/**
1990 * spu_no_incr_hash() - Determine whether incremental hashing is supported.
1991 * @ctx:  Crypto session context
1992 *
1993 * SPU-2 does not support incremental hashing (we'll have to revisit and
1994 * condition based on chip revision or device tree entry if future versions do
1995 * support incremental hash)
1996 *
1997 * SPU-M also doesn't support incremental hashing of AES-XCBC
1998 *
1999 * Return: true if incremental hashing is not supported
2000 *         false otherwise
2001 */
2002static bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
2003{
2004        struct spu_hw *spu = &iproc_priv.spu;
2005
2006        if (spu->spu_type == SPU_TYPE_SPU2)
2007                return true;
2008
2009        if ((ctx->auth.alg == HASH_ALG_AES) &&
2010            (ctx->auth.mode == HASH_MODE_XCBC))
2011                return true;
2012
2013        /* Otherwise, incremental hashing is supported */
2014        return false;
2015}
2016
2017static int ahash_init(struct ahash_request *req)
2018{
2019        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2020        struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2021        const char *alg_name;
2022        struct crypto_shash *hash;
2023        int ret;
2024        gfp_t gfp;
2025
2026        if (spu_no_incr_hash(ctx)) {
2027                /*
2028                 * If we get an incremental hashing request and it's not
2029                 * supported by the hardware, we need to handle it in software
2030                 * by calling synchronous hash functions.
2031                 */
2032                alg_name = crypto_tfm_alg_name(crypto_ahash_tfm(tfm));
2033                hash = crypto_alloc_shash(alg_name, 0, 0);
2034                if (IS_ERR(hash)) {
2035                        ret = PTR_ERR(hash);
2036                        goto err;
2037                }
2038
2039                gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2040                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2041                ctx->shash = kmalloc(sizeof(*ctx->shash) +
2042                                     crypto_shash_descsize(hash), gfp);
2043                if (!ctx->shash) {
2044                        ret = -ENOMEM;
2045                        goto err_hash;
2046                }
2047                ctx->shash->tfm = hash;
2048
2049                /* Set the key using data we already have from setkey */
2050                if (ctx->authkeylen > 0) {
2051                        ret = crypto_shash_setkey(hash, ctx->authkey,
2052                                                  ctx->authkeylen);
2053                        if (ret)
2054                                goto err_shash;
2055                }
2056
2057                /* Initialize hash w/ this key and other params */
2058                ret = crypto_shash_init(ctx->shash);
2059                if (ret)
2060                        goto err_shash;
2061        } else {
2062                /* Otherwise call the internal function which uses SPU hw */
2063                ret = __ahash_init(req);
2064        }
2065
2066        return ret;
2067
2068err_shash:
2069        kfree(ctx->shash);
2070err_hash:
2071        crypto_free_shash(hash);
2072err:
2073        return ret;
2074}
2075
2076static int __ahash_update(struct ahash_request *req)
2077{
2078        struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2079
2080        flow_log("ahash_update() nbytes:%u\n", req->nbytes);
2081
2082        if (!req->nbytes)
2083                return 0;
2084        rctx->total_todo += req->nbytes;
2085        rctx->src_sent = 0;
2086
2087        return ahash_enqueue(req);
2088}
2089
2090static int ahash_update(struct ahash_request *req)
2091{
2092        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2093        struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2094        u8 *tmpbuf;
2095        int ret;
2096        int nents;
2097        gfp_t gfp;
2098
2099        if (spu_no_incr_hash(ctx)) {
2100                /*
2101                 * If we get an incremental hashing request and it's not
2102                 * supported by the hardware, we need to handle it in software
2103                 * by calling synchronous hash functions.
2104                 */
2105                if (req->src)
2106                        nents = sg_nents(req->src);
2107                else
2108                        return -EINVAL;
2109
2110                /* Copy data from req scatterlist to tmp buffer */
2111                gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2112                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2113                tmpbuf = kmalloc(req->nbytes, gfp);
2114                if (!tmpbuf)
2115                        return -ENOMEM;
2116
2117                if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2118                                req->nbytes) {
2119                        kfree(tmpbuf);
2120                        return -EINVAL;
2121                }
2122
2123                /* Call synchronous update */
2124                ret = crypto_shash_update(ctx->shash, tmpbuf, req->nbytes);
2125                kfree(tmpbuf);
2126        } else {
2127                /* Otherwise call the internal function which uses SPU hw */
2128                ret = __ahash_update(req);
2129        }
2130
2131        return ret;
2132}
2133
2134static int __ahash_final(struct ahash_request *req)
2135{
2136        struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2137
2138        flow_log("ahash_final() nbytes:%u\n", req->nbytes);
2139
2140        rctx->is_final = 1;
2141
2142        return ahash_enqueue(req);
2143}
2144
2145static int ahash_final(struct ahash_request *req)
2146{
2147        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2148        struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2149        int ret;
2150
2151        if (spu_no_incr_hash(ctx)) {
2152                /*
2153                 * If we get an incremental hashing request and it's not
2154                 * supported by the hardware, we need to handle it in software
2155                 * by calling synchronous hash functions.
2156                 */
2157                ret = crypto_shash_final(ctx->shash, req->result);
2158
2159                /* Done with hash, can deallocate it now */
2160                crypto_free_shash(ctx->shash->tfm);
2161                kfree(ctx->shash);
2162
2163        } else {
2164                /* Otherwise call the internal function which uses SPU hw */
2165                ret = __ahash_final(req);
2166        }
2167
2168        return ret;
2169}
2170
2171static int __ahash_finup(struct ahash_request *req)
2172{
2173        struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2174
2175        flow_log("ahash_finup() nbytes:%u\n", req->nbytes);
2176
2177        rctx->total_todo += req->nbytes;
2178        rctx->src_sent = 0;
2179        rctx->is_final = 1;
2180
2181        return ahash_enqueue(req);
2182}
2183
2184static int ahash_finup(struct ahash_request *req)
2185{
2186        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2187        struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2188        u8 *tmpbuf;
2189        int ret;
2190        int nents;
2191        gfp_t gfp;
2192
2193        if (spu_no_incr_hash(ctx)) {
2194                /*
2195                 * If we get an incremental hashing request and it's not
2196                 * supported by the hardware, we need to handle it in software
2197                 * by calling synchronous hash functions.
2198                 */
2199                if (req->src) {
2200                        nents = sg_nents(req->src);
2201                } else {
2202                        ret = -EINVAL;
2203                        goto ahash_finup_exit;
2204                }
2205
2206                /* Copy data from req scatterlist to tmp buffer */
2207                gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2208                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2209                tmpbuf = kmalloc(req->nbytes, gfp);
2210                if (!tmpbuf) {
2211                        ret = -ENOMEM;
2212                        goto ahash_finup_exit;
2213                }
2214
2215                if (sg_copy_to_buffer(req->src, nents, tmpbuf, req->nbytes) !=
2216                                req->nbytes) {
2217                        ret = -EINVAL;
2218                        goto ahash_finup_free;
2219                }
2220
2221                /* Call synchronous update */
2222                ret = crypto_shash_finup(ctx->shash, tmpbuf, req->nbytes,
2223                                         req->result);
2224        } else {
2225                /* Otherwise call the internal function which uses SPU hw */
2226                return __ahash_finup(req);
2227        }
2228ahash_finup_free:
2229        kfree(tmpbuf);
2230
2231ahash_finup_exit:
2232        /* Done with hash, can deallocate it now */
2233        crypto_free_shash(ctx->shash->tfm);
2234        kfree(ctx->shash);
2235        return ret;
2236}
2237
2238static int ahash_digest(struct ahash_request *req)
2239{
2240        int err;
2241
2242        flow_log("ahash_digest() nbytes:%u\n", req->nbytes);
2243
2244        /* whole thing at once */
2245        err = __ahash_init(req);
2246        if (!err)
2247                err = __ahash_finup(req);
2248
2249        return err;
2250}
2251
2252static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key,
2253                        unsigned int keylen)
2254{
2255        struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2256
2257        flow_log("%s() ahash:%p key:%p keylen:%u\n",
2258                 __func__, ahash, key, keylen);
2259        flow_dump("  key: ", key, keylen);
2260
2261        if (ctx->auth.alg == HASH_ALG_AES) {
2262                switch (keylen) {
2263                case AES_KEYSIZE_128:
2264                        ctx->cipher_type = CIPHER_TYPE_AES128;
2265                        break;
2266                case AES_KEYSIZE_192:
2267                        ctx->cipher_type = CIPHER_TYPE_AES192;
2268                        break;
2269                case AES_KEYSIZE_256:
2270                        ctx->cipher_type = CIPHER_TYPE_AES256;
2271                        break;
2272                default:
2273                        pr_err("%s() Error: Invalid key length\n", __func__);
2274                        return -EINVAL;
2275                }
2276        } else {
2277                pr_err("%s() Error: unknown hash alg\n", __func__);
2278                return -EINVAL;
2279        }
2280        memcpy(ctx->authkey, key, keylen);
2281        ctx->authkeylen = keylen;
2282
2283        return 0;
2284}
2285
2286static int ahash_export(struct ahash_request *req, void *out)
2287{
2288        const struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2289        struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)out;
2290
2291        spu_exp->total_todo = rctx->total_todo;
2292        spu_exp->total_sent = rctx->total_sent;
2293        spu_exp->is_sw_hmac = rctx->is_sw_hmac;
2294        memcpy(spu_exp->hash_carry, rctx->hash_carry, sizeof(rctx->hash_carry));
2295        spu_exp->hash_carry_len = rctx->hash_carry_len;
2296        memcpy(spu_exp->incr_hash, rctx->incr_hash, sizeof(rctx->incr_hash));
2297
2298        return 0;
2299}
2300
2301static int ahash_import(struct ahash_request *req, const void *in)
2302{
2303        struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2304        struct spu_hash_export_s *spu_exp = (struct spu_hash_export_s *)in;
2305
2306        rctx->total_todo = spu_exp->total_todo;
2307        rctx->total_sent = spu_exp->total_sent;
2308        rctx->is_sw_hmac = spu_exp->is_sw_hmac;
2309        memcpy(rctx->hash_carry, spu_exp->hash_carry, sizeof(rctx->hash_carry));
2310        rctx->hash_carry_len = spu_exp->hash_carry_len;
2311        memcpy(rctx->incr_hash, spu_exp->incr_hash, sizeof(rctx->incr_hash));
2312
2313        return 0;
2314}
2315
2316static int ahash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key,
2317                             unsigned int keylen)
2318{
2319        struct iproc_ctx_s *ctx = crypto_ahash_ctx(ahash);
2320        unsigned int blocksize =
2321                crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash));
2322        unsigned int digestsize = crypto_ahash_digestsize(ahash);
2323        unsigned int index;
2324        int rc;
2325
2326        flow_log("%s() ahash:%p key:%p keylen:%u blksz:%u digestsz:%u\n",
2327                 __func__, ahash, key, keylen, blocksize, digestsize);
2328        flow_dump("  key: ", key, keylen);
2329
2330        if (keylen > blocksize) {
2331                switch (ctx->auth.alg) {
2332                case HASH_ALG_MD5:
2333                        rc = do_shash("md5", ctx->authkey, key, keylen, NULL,
2334                                      0, NULL, 0);
2335                        break;
2336                case HASH_ALG_SHA1:
2337                        rc = do_shash("sha1", ctx->authkey, key, keylen, NULL,
2338                                      0, NULL, 0);
2339                        break;
2340                case HASH_ALG_SHA224:
2341                        rc = do_shash("sha224", ctx->authkey, key, keylen, NULL,
2342                                      0, NULL, 0);
2343                        break;
2344                case HASH_ALG_SHA256:
2345                        rc = do_shash("sha256", ctx->authkey, key, keylen, NULL,
2346                                      0, NULL, 0);
2347                        break;
2348                case HASH_ALG_SHA384:
2349                        rc = do_shash("sha384", ctx->authkey, key, keylen, NULL,
2350                                      0, NULL, 0);
2351                        break;
2352                case HASH_ALG_SHA512:
2353                        rc = do_shash("sha512", ctx->authkey, key, keylen, NULL,
2354                                      0, NULL, 0);
2355                        break;
2356                case HASH_ALG_SHA3_224:
2357                        rc = do_shash("sha3-224", ctx->authkey, key, keylen,
2358                                      NULL, 0, NULL, 0);
2359                        break;
2360                case HASH_ALG_SHA3_256:
2361                        rc = do_shash("sha3-256", ctx->authkey, key, keylen,
2362                                      NULL, 0, NULL, 0);
2363                        break;
2364                case HASH_ALG_SHA3_384:
2365                        rc = do_shash("sha3-384", ctx->authkey, key, keylen,
2366                                      NULL, 0, NULL, 0);
2367                        break;
2368                case HASH_ALG_SHA3_512:
2369                        rc = do_shash("sha3-512", ctx->authkey, key, keylen,
2370                                      NULL, 0, NULL, 0);
2371                        break;
2372                default:
2373                        pr_err("%s() Error: unknown hash alg\n", __func__);
2374                        return -EINVAL;
2375                }
2376                if (rc < 0) {
2377                        pr_err("%s() Error %d computing shash for %s\n",
2378                               __func__, rc, hash_alg_name[ctx->auth.alg]);
2379                        return rc;
2380                }
2381                ctx->authkeylen = digestsize;
2382
2383                flow_log("  keylen > digestsize... hashed\n");
2384                flow_dump("  newkey: ", ctx->authkey, ctx->authkeylen);
2385        } else {
2386                memcpy(ctx->authkey, key, keylen);
2387                ctx->authkeylen = keylen;
2388        }
2389
2390        /*
2391         * Full HMAC operation in SPUM is not verified,
2392         * So keeping the generation of IPAD, OPAD and
2393         * outer hashing in software.
2394         */
2395        if (iproc_priv.spu.spu_type == SPU_TYPE_SPUM) {
2396                memcpy(ctx->ipad, ctx->authkey, ctx->authkeylen);
2397                memset(ctx->ipad + ctx->authkeylen, 0,
2398                       blocksize - ctx->authkeylen);
2399                ctx->authkeylen = 0;
2400                memcpy(ctx->opad, ctx->ipad, blocksize);
2401
2402                for (index = 0; index < blocksize; index++) {
2403                        ctx->ipad[index] ^= HMAC_IPAD_VALUE;
2404                        ctx->opad[index] ^= HMAC_OPAD_VALUE;
2405                }
2406
2407                flow_dump("  ipad: ", ctx->ipad, blocksize);
2408                flow_dump("  opad: ", ctx->opad, blocksize);
2409        }
2410        ctx->digestsize = digestsize;
2411        atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_HMAC]);
2412
2413        return 0;
2414}
2415
2416static int ahash_hmac_init(struct ahash_request *req)
2417{
2418        struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2419        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2420        struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2421        unsigned int blocksize =
2422                        crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2423
2424        flow_log("ahash_hmac_init()\n");
2425
2426        /* init the context as a hash */
2427        ahash_init(req);
2428
2429        if (!spu_no_incr_hash(ctx)) {
2430                /* SPU-M can do incr hashing but needs sw for outer HMAC */
2431                rctx->is_sw_hmac = true;
2432                ctx->auth.mode = HASH_MODE_HASH;
2433                /* start with a prepended ipad */
2434                memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2435                rctx->hash_carry_len = blocksize;
2436                rctx->total_todo += blocksize;
2437        }
2438
2439        return 0;
2440}
2441
2442static int ahash_hmac_update(struct ahash_request *req)
2443{
2444        flow_log("ahash_hmac_update() nbytes:%u\n", req->nbytes);
2445
2446        if (!req->nbytes)
2447                return 0;
2448
2449        return ahash_update(req);
2450}
2451
2452static int ahash_hmac_final(struct ahash_request *req)
2453{
2454        flow_log("ahash_hmac_final() nbytes:%u\n", req->nbytes);
2455
2456        return ahash_final(req);
2457}
2458
2459static int ahash_hmac_finup(struct ahash_request *req)
2460{
2461        flow_log("ahash_hmac_finupl() nbytes:%u\n", req->nbytes);
2462
2463        return ahash_finup(req);
2464}
2465
2466static int ahash_hmac_digest(struct ahash_request *req)
2467{
2468        struct iproc_reqctx_s *rctx = ahash_request_ctx(req);
2469        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2470        struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm);
2471        unsigned int blocksize =
2472                        crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2473
2474        flow_log("ahash_hmac_digest() nbytes:%u\n", req->nbytes);
2475
2476        /* Perform initialization and then call finup */
2477        __ahash_init(req);
2478
2479        if (iproc_priv.spu.spu_type == SPU_TYPE_SPU2) {
2480                /*
2481                 * SPU2 supports full HMAC implementation in the
2482                 * hardware, need not to generate IPAD, OPAD and
2483                 * outer hash in software.
2484                 * Only for hash key len > hash block size, SPU2
2485                 * expects to perform hashing on the key, shorten
2486                 * it to digest size and feed it as hash key.
2487                 */
2488                rctx->is_sw_hmac = false;
2489                ctx->auth.mode = HASH_MODE_HMAC;
2490        } else {
2491                rctx->is_sw_hmac = true;
2492                ctx->auth.mode = HASH_MODE_HASH;
2493                /* start with a prepended ipad */
2494                memcpy(rctx->hash_carry, ctx->ipad, blocksize);
2495                rctx->hash_carry_len = blocksize;
2496                rctx->total_todo += blocksize;
2497        }
2498
2499        return __ahash_finup(req);
2500}
2501
2502/* aead helpers */
2503
2504static int aead_need_fallback(struct aead_request *req)
2505{
2506        struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2507        struct spu_hw *spu = &iproc_priv.spu;
2508        struct crypto_aead *aead = crypto_aead_reqtfm(req);
2509        struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2510        u32 payload_len;
2511
2512        /*
2513         * SPU hardware cannot handle the AES-GCM/CCM case where plaintext
2514         * and AAD are both 0 bytes long. So use fallback in this case.
2515         */
2516        if (((ctx->cipher.mode == CIPHER_MODE_GCM) ||
2517             (ctx->cipher.mode == CIPHER_MODE_CCM)) &&
2518            (req->assoclen == 0)) {
2519                if ((rctx->is_encrypt && (req->cryptlen == 0)) ||
2520                    (!rctx->is_encrypt && (req->cryptlen == ctx->digestsize))) {
2521                        flow_log("AES GCM/CCM needs fallback for 0 len req\n");
2522                        return 1;
2523                }
2524        }
2525
2526        /* SPU-M hardware only supports CCM digest size of 8, 12, or 16 bytes */
2527        if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2528            (spu->spu_type == SPU_TYPE_SPUM) &&
2529            (ctx->digestsize != 8) && (ctx->digestsize != 12) &&
2530            (ctx->digestsize != 16)) {
2531                flow_log("%s() AES CCM needs fallback for digest size %d\n",
2532                         __func__, ctx->digestsize);
2533                return 1;
2534        }
2535
2536        /*
2537         * SPU-M on NSP has an issue where AES-CCM hash is not correct
2538         * when AAD size is 0
2539         */
2540        if ((ctx->cipher.mode == CIPHER_MODE_CCM) &&
2541            (spu->spu_subtype == SPU_SUBTYPE_SPUM_NSP) &&
2542            (req->assoclen == 0)) {
2543                flow_log("%s() AES_CCM needs fallback for 0 len AAD on NSP\n",
2544                         __func__);
2545                return 1;
2546        }
2547
2548        /*
2549         * RFC4106 and RFC4543 cannot handle the case where AAD is other than
2550         * 16 or 20 bytes long. So use fallback in this case.
2551         */
2552        if (ctx->cipher.mode == CIPHER_MODE_GCM &&
2553            ctx->cipher.alg == CIPHER_ALG_AES &&
2554            rctx->iv_ctr_len == GCM_RFC4106_IV_SIZE &&
2555            req->assoclen != 16 && req->assoclen != 20) {
2556                flow_log("RFC4106/RFC4543 needs fallback for assoclen"
2557                         " other than 16 or 20 bytes\n");
2558                return 1;
2559        }
2560
2561        payload_len = req->cryptlen;
2562        if (spu->spu_type == SPU_TYPE_SPUM)
2563                payload_len += req->assoclen;
2564
2565        flow_log("%s() payload len: %u\n", __func__, payload_len);
2566
2567        if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2568                return 0;
2569        else
2570                return payload_len > ctx->max_payload;
2571}
2572
2573static void aead_complete(struct crypto_async_request *areq, int err)
2574{
2575        struct aead_request *req =
2576            container_of(areq, struct aead_request, base);
2577        struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2578        struct crypto_aead *aead = crypto_aead_reqtfm(req);
2579
2580        flow_log("%s() err:%d\n", __func__, err);
2581
2582        areq->tfm = crypto_aead_tfm(aead);
2583
2584        areq->complete = rctx->old_complete;
2585        areq->data = rctx->old_data;
2586
2587        areq->complete(areq, err);
2588}
2589
2590static int aead_do_fallback(struct aead_request *req, bool is_encrypt)
2591{
2592        struct crypto_aead *aead = crypto_aead_reqtfm(req);
2593        struct crypto_tfm *tfm = crypto_aead_tfm(aead);
2594        struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2595        struct iproc_ctx_s *ctx = crypto_tfm_ctx(tfm);
2596        int err;
2597        u32 req_flags;
2598
2599        flow_log("%s() enc:%u\n", __func__, is_encrypt);
2600
2601        if (ctx->fallback_cipher) {
2602                /* Store the cipher tfm and then use the fallback tfm */
2603                rctx->old_tfm = tfm;
2604                aead_request_set_tfm(req, ctx->fallback_cipher);
2605                /*
2606                 * Save the callback and chain ourselves in, so we can restore
2607                 * the tfm
2608                 */
2609                rctx->old_complete = req->base.complete;
2610                rctx->old_data = req->base.data;
2611                req_flags = aead_request_flags(req);
2612                aead_request_set_callback(req, req_flags, aead_complete, req);
2613                err = is_encrypt ? crypto_aead_encrypt(req) :
2614                    crypto_aead_decrypt(req);
2615
2616                if (err == 0) {
2617                        /*
2618                         * fallback was synchronous (did not return
2619                         * -EINPROGRESS). So restore request state here.
2620                         */
2621                        aead_request_set_callback(req, req_flags,
2622                                                  rctx->old_complete, req);
2623                        req->base.data = rctx->old_data;
2624                        aead_request_set_tfm(req, aead);
2625                        flow_log("%s() fallback completed successfully\n\n",
2626                                 __func__);
2627                }
2628        } else {
2629                err = -EINVAL;
2630        }
2631
2632        return err;
2633}
2634
2635static int aead_enqueue(struct aead_request *req, bool is_encrypt)
2636{
2637        struct iproc_reqctx_s *rctx = aead_request_ctx(req);
2638        struct crypto_aead *aead = crypto_aead_reqtfm(req);
2639        struct iproc_ctx_s *ctx = crypto_aead_ctx(aead);
2640        int err;
2641
2642        flow_log("%s() enc:%u\n", __func__, is_encrypt);
2643
2644        if (req->assoclen > MAX_ASSOC_SIZE) {
2645                pr_err
2646                    ("%s() Error: associated data too long. (%u > %u bytes)\n",
2647                     __func__, req->assoclen, MAX_ASSOC_SIZE);
2648                return -EINVAL;
2649        }
2650
2651        rctx->gfp = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2652                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2653        rctx->parent = &req->base;
2654        rctx->is_encrypt = is_encrypt;
2655        rctx->bd_suppress = false;
2656        rctx->total_todo = req->cryptlen;
2657        rctx->src_sent = 0;
2658        rctx->total_sent = 0;
2659        rctx->total_received = 0;
2660        rctx->is_sw_hmac = false;
2661        rctx->ctx = ctx;
2662        memset(&rctx->mb_mssg, 0, sizeof(struct brcm_message));
2663
2664        /* assoc data is at start of src sg */
2665        rctx->assoc = req->src;
2666
2667        /*
2668         * Init current position in src scatterlist to be after assoc data.
2669         * src_skip set to buffer offset where data begins. (Assoc data could
2670         * end in the middle of a buffer.)
2671         */
2672        if (spu_sg_at_offset(req->src, req->assoclen, &rctx->src_sg,
2673                             &rctx->src_skip) < 0) {
2674                pr_err("%s() Error: Unable to find start of src data\n",
2675                       __func__);
2676                return -EINVAL;
2677        }
2678
2679        rctx->src_nents = 0;
2680        rctx->dst_nents = 0;
2681        if (req->dst == req->src) {
2682                rctx->dst_sg = rctx->src_sg;
2683                rctx->dst_skip = rctx->src_skip;
2684        } else {
2685                /*
2686                 * Expect req->dst to have room for assoc data followed by
2687                 * output data and ICV, if encrypt. So initialize dst_sg
2688                 * to point beyond assoc len offset.
2689                 */
2690                if (spu_sg_at_offset(req->dst, req->assoclen, &rctx->dst_sg,
2691                                     &rctx->dst_skip) < 0) {
2692                        pr_err("%s() Error: Unable to find start of dst data\n",
2693                               __func__);
2694                        return -EINVAL;
2695                }
2696        }
2697
2698        if (ctx->cipher.mode == CIPHER_MODE_CBC ||
2699            ctx->cipher.mode == CIPHER_MODE_CTR ||
2700            ctx->cipher.mode == CIPHER_MODE_OFB ||
2701            ctx->cipher.mode == CIPHER_MODE_XTS ||
2702            ctx->cipher.mode == CIPHER_MODE_GCM) {
2703                rctx->iv_ctr_len =
2704                        ctx->salt_len +
2705                        crypto_aead_ivsize(crypto_aead_reqtfm(req));
2706        } else if (ctx->cipher.mode == CIPHER_MODE_CCM) {
2707                rctx->iv_ctr_len = CCM_AES_IV_SIZE;
2708        } else {
2709                rctx->iv_ctr_len = 0;
2710        }
2711
2712        rctx->hash_carry_len = 0;
2713
2714        flow_log("  src sg: %p\n", req->src);
2715        flow_log("  rctx->src_sg: %p, src_skip %u\n",
2716                 rctx->src_sg, rctx->src_skip);
2717        flow_log("  assoc:  %p, assoclen %u\n", rctx->assoc, req->assoclen);
2718        flow_log("  dst sg: %p\n", req->dst);
2719        flow_log("  rctx->dst_sg: %p, dst_skip %u\n",
2720                 rctx->dst_sg, rctx->dst_skip);
2721        flow_log("  iv_ctr_len:%u\n", rctx->iv_ctr_len);
2722        flow_dump("  iv: ", req->iv, rctx->iv_ctr_len);
2723        flow_log("  authkeylen:%u\n", ctx->authkeylen);
2724        flow_log("  is_esp: %s\n", ctx->is_esp ? "yes" : "no");
2725
2726        if (ctx->max_payload == SPU_MAX_PAYLOAD_INF)
2727                flow_log("  max_payload infinite");
2728        else
2729                flow_log("  max_payload: %u\n", ctx->max_payload);
2730
2731        if (unlikely(aead_need_fallback(req)))
2732                return aead_do_fallback(req, is_encrypt);
2733
2734        /*
2735         * Do memory allocations for request after fallback check, because if we
2736         * do fallback, we won't call finish_req() to dealloc.
2737         */
2738        if (rctx->iv_ctr_len) {
2739                if (ctx->salt_len)
2740                        memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset,
2741                               ctx->salt, ctx->salt_len);
2742                memcpy(rctx->msg_buf.iv_ctr + ctx->salt_offset + ctx->salt_len,
2743                       req->iv,
2744                       rctx->iv_ctr_len - ctx->salt_len - ctx->salt_offset);
2745        }
2746
2747        rctx->chan_idx = select_channel();
2748        err = handle_aead_req(rctx);
2749        if (err != -EINPROGRESS)
2750                /* synchronous result */
2751                spu_chunk_cleanup(rctx);
2752
2753        return err;
2754}
2755
2756static int aead_authenc_setkey(struct crypto_aead *cipher,
2757                               const u8 *key, unsigned int keylen)
2758{
2759        struct spu_hw *spu = &iproc_priv.spu;
2760        struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2761        struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2762        struct crypto_authenc_keys keys;
2763        int ret;
2764
2765        flow_log("%s() aead:%p key:%p keylen:%u\n", __func__, cipher, key,
2766                 keylen);
2767        flow_dump("  key: ", key, keylen);
2768
2769        ret = crypto_authenc_extractkeys(&keys, key, keylen);
2770        if (ret)
2771                goto badkey;
2772
2773        if (keys.enckeylen > MAX_KEY_SIZE ||
2774            keys.authkeylen > MAX_KEY_SIZE)
2775                goto badkey;
2776
2777        ctx->enckeylen = keys.enckeylen;
2778        ctx->authkeylen = keys.authkeylen;
2779
2780        memcpy(ctx->enckey, keys.enckey, keys.enckeylen);
2781        /* May end up padding auth key. So make sure it's zeroed. */
2782        memset(ctx->authkey, 0, sizeof(ctx->authkey));
2783        memcpy(ctx->authkey, keys.authkey, keys.authkeylen);
2784
2785        switch (ctx->alg->cipher_info.alg) {
2786        case CIPHER_ALG_DES:
2787                if (verify_aead_des_key(cipher, keys.enckey, keys.enckeylen))
2788                        return -EINVAL;
2789
2790                ctx->cipher_type = CIPHER_TYPE_DES;
2791                break;
2792        case CIPHER_ALG_3DES:
2793                if (verify_aead_des3_key(cipher, keys.enckey, keys.enckeylen))
2794                        return -EINVAL;
2795
2796                ctx->cipher_type = CIPHER_TYPE_3DES;
2797                break;
2798        case CIPHER_ALG_AES:
2799                switch (ctx->enckeylen) {
2800                case AES_KEYSIZE_128:
2801                        ctx->cipher_type = CIPHER_TYPE_AES128;
2802                        break;
2803                case AES_KEYSIZE_192:
2804                        ctx->cipher_type = CIPHER_TYPE_AES192;
2805                        break;
2806                case AES_KEYSIZE_256:
2807                        ctx->cipher_type = CIPHER_TYPE_AES256;
2808                        break;
2809                default:
2810                        goto badkey;
2811                }
2812                break;
2813        default:
2814                pr_err("%s() Error: Unknown cipher alg\n", __func__);
2815                return -EINVAL;
2816        }
2817
2818        flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2819                 ctx->authkeylen);
2820        flow_dump("  enc: ", ctx->enckey, ctx->enckeylen);
2821        flow_dump("  auth: ", ctx->authkey, ctx->authkeylen);
2822
2823        /* setkey the fallback just in case we needto use it */
2824        if (ctx->fallback_cipher) {
2825                flow_log("  running fallback setkey()\n");
2826
2827                ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2828                ctx->fallback_cipher->base.crt_flags |=
2829                    tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2830                ret = crypto_aead_setkey(ctx->fallback_cipher, key, keylen);
2831                if (ret)
2832                        flow_log("  fallback setkey() returned:%d\n", ret);
2833        }
2834
2835        ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2836                                                          ctx->enckeylen,
2837                                                          false);
2838
2839        atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2840
2841        return ret;
2842
2843badkey:
2844        ctx->enckeylen = 0;
2845        ctx->authkeylen = 0;
2846        ctx->digestsize = 0;
2847
2848        return -EINVAL;
2849}
2850
2851static int aead_gcm_ccm_setkey(struct crypto_aead *cipher,
2852                               const u8 *key, unsigned int keylen)
2853{
2854        struct spu_hw *spu = &iproc_priv.spu;
2855        struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2856        struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
2857
2858        int ret = 0;
2859
2860        flow_log("%s() keylen:%u\n", __func__, keylen);
2861        flow_dump("  key: ", key, keylen);
2862
2863        if (!ctx->is_esp)
2864                ctx->digestsize = keylen;
2865
2866        ctx->enckeylen = keylen;
2867        ctx->authkeylen = 0;
2868
2869        switch (ctx->enckeylen) {
2870        case AES_KEYSIZE_128:
2871                ctx->cipher_type = CIPHER_TYPE_AES128;
2872                break;
2873        case AES_KEYSIZE_192:
2874                ctx->cipher_type = CIPHER_TYPE_AES192;
2875                break;
2876        case AES_KEYSIZE_256:
2877                ctx->cipher_type = CIPHER_TYPE_AES256;
2878                break;
2879        default:
2880                goto badkey;
2881        }
2882
2883        memcpy(ctx->enckey, key, ctx->enckeylen);
2884
2885        flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2886                 ctx->authkeylen);
2887        flow_dump("  enc: ", ctx->enckey, ctx->enckeylen);
2888        flow_dump("  auth: ", ctx->authkey, ctx->authkeylen);
2889
2890        /* setkey the fallback just in case we need to use it */
2891        if (ctx->fallback_cipher) {
2892                flow_log("  running fallback setkey()\n");
2893
2894                ctx->fallback_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
2895                ctx->fallback_cipher->base.crt_flags |=
2896                    tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
2897                ret = crypto_aead_setkey(ctx->fallback_cipher, key,
2898                                         keylen + ctx->salt_len);
2899                if (ret)
2900                        flow_log("  fallback setkey() returned:%d\n", ret);
2901        }
2902
2903        ctx->spu_resp_hdr_len = spu->spu_response_hdr_len(ctx->authkeylen,
2904                                                          ctx->enckeylen,
2905                                                          false);
2906
2907        atomic_inc(&iproc_priv.setkey_cnt[SPU_OP_AEAD]);
2908
2909        flow_log("  enckeylen:%u authkeylen:%u\n", ctx->enckeylen,
2910                 ctx->authkeylen);
2911
2912        return ret;
2913
2914badkey:
2915        ctx->enckeylen = 0;
2916        ctx->authkeylen = 0;
2917        ctx->digestsize = 0;
2918
2919        return -EINVAL;
2920}
2921
2922/**
2923 * aead_gcm_esp_setkey() - setkey() operation for ESP variant of GCM AES.
2924 * @cipher: AEAD structure
2925 * @key:    Key followed by 4 bytes of salt
2926 * @keylen: Length of key plus salt, in bytes
2927 *
2928 * Extracts salt from key and stores it to be prepended to IV on each request.
2929 * Digest is always 16 bytes
2930 *
2931 * Return: Value from generic gcm setkey.
2932 */
2933static int aead_gcm_esp_setkey(struct crypto_aead *cipher,
2934                               const u8 *key, unsigned int keylen)
2935{
2936        struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2937
2938        flow_log("%s\n", __func__);
2939
2940        if (keylen < GCM_ESP_SALT_SIZE)
2941                return -EINVAL;
2942
2943        ctx->salt_len = GCM_ESP_SALT_SIZE;
2944        ctx->salt_offset = GCM_ESP_SALT_OFFSET;
2945        memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
2946        keylen -= GCM_ESP_SALT_SIZE;
2947        ctx->digestsize = GCM_ESP_DIGESTSIZE;
2948        ctx->is_esp = true;
2949        flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
2950
2951        return aead_gcm_ccm_setkey(cipher, key, keylen);
2952}
2953
2954/**
2955 * rfc4543_gcm_esp_setkey() - setkey operation for RFC4543 variant of GCM/GMAC.
2956 * @cipher: AEAD structure
2957 * @key:    Key followed by 4 bytes of salt
2958 * @keylen: Length of key plus salt, in bytes
2959 *
2960 * Extracts salt from key and stores it to be prepended to IV on each request.
2961 * Digest is always 16 bytes
2962 *
2963 * Return: Value from generic gcm setkey.
2964 */
2965static int rfc4543_gcm_esp_setkey(struct crypto_aead *cipher,
2966                                  const u8 *key, unsigned int keylen)
2967{
2968        struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
2969
2970        flow_log("%s\n", __func__);
2971
2972        if (keylen < GCM_ESP_SALT_SIZE)
2973                return -EINVAL;
2974
2975        ctx->salt_len = GCM_ESP_SALT_SIZE;
2976        ctx->salt_offset = GCM_ESP_SALT_OFFSET;
2977        memcpy(ctx->salt, key + keylen - GCM_ESP_SALT_SIZE, GCM_ESP_SALT_SIZE);
2978        keylen -= GCM_ESP_SALT_SIZE;
2979        ctx->digestsize = GCM_ESP_DIGESTSIZE;
2980        ctx->is_esp = true;
2981        ctx->is_rfc4543 = true;
2982        flow_dump("salt: ", ctx->salt, GCM_ESP_SALT_SIZE);
2983
2984        return aead_gcm_ccm_setkey(cipher, key, keylen);
2985}
2986
2987/**
2988 * aead_ccm_esp_setkey() - setkey() operation for ESP variant of CCM AES.
2989 * @cipher: AEAD structure
2990 * @key:    Key followed by 4 bytes of salt
2991 * @keylen: Length of key plus salt, in bytes
2992 *
2993 * Extracts salt from key and stores it to be prepended to IV on each request.
2994 * Digest is always 16 bytes
2995 *
2996 * Return: Value from generic ccm setkey.
2997 */
2998static int aead_ccm_esp_setkey(struct crypto_aead *cipher,
2999                               const u8 *key, unsigned int keylen)
3000{
3001        struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
3002
3003        flow_log("%s\n", __func__);
3004
3005        if (keylen < CCM_ESP_SALT_SIZE)
3006                return -EINVAL;
3007
3008        ctx->salt_len = CCM_ESP_SALT_SIZE;
3009        ctx->salt_offset = CCM_ESP_SALT_OFFSET;
3010        memcpy(ctx->salt, key + keylen - CCM_ESP_SALT_SIZE, CCM_ESP_SALT_SIZE);
3011        keylen -= CCM_ESP_SALT_SIZE;
3012        ctx->is_esp = true;
3013        flow_dump("salt: ", ctx->salt, CCM_ESP_SALT_SIZE);
3014
3015        return aead_gcm_ccm_setkey(cipher, key, keylen);
3016}
3017
3018static int aead_setauthsize(struct crypto_aead *cipher, unsigned int authsize)
3019{
3020        struct iproc_ctx_s *ctx = crypto_aead_ctx(cipher);
3021        int ret = 0;
3022
3023        flow_log("%s() authkeylen:%u authsize:%u\n",
3024                 __func__, ctx->authkeylen, authsize);
3025
3026        ctx->digestsize = authsize;
3027
3028        /* setkey the fallback just in case we needto use it */
3029        if (ctx->fallback_cipher) {
3030                flow_log("  running fallback setauth()\n");
3031
3032                ret = crypto_aead_setauthsize(ctx->fallback_cipher, authsize);
3033                if (ret)
3034                        flow_log("  fallback setauth() returned:%d\n", ret);
3035        }
3036
3037        return ret;
3038}
3039
3040static int aead_encrypt(struct aead_request *req)
3041{
3042        flow_log("%s() cryptlen:%u %08x\n", __func__, req->cryptlen,
3043                 req->cryptlen);
3044        dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3045        flow_log("  assoc_len:%u\n", req->assoclen);
3046
3047        return aead_enqueue(req, true);
3048}
3049
3050static int aead_decrypt(struct aead_request *req)
3051{
3052        flow_log("%s() cryptlen:%u\n", __func__, req->cryptlen);
3053        dump_sg(req->src, 0, req->cryptlen + req->assoclen);
3054        flow_log("  assoc_len:%u\n", req->assoclen);
3055
3056        return aead_enqueue(req, false);
3057}
3058
3059/* ==================== Supported Cipher Algorithms ==================== */
3060
3061static struct iproc_alg_s driver_algs[] = {
3062        {
3063         .type = CRYPTO_ALG_TYPE_AEAD,
3064         .alg.aead = {
3065                 .base = {
3066                        .cra_name = "gcm(aes)",
3067                        .cra_driver_name = "gcm-aes-iproc",
3068                        .cra_blocksize = AES_BLOCK_SIZE,
3069                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3070                 },
3071                 .setkey = aead_gcm_ccm_setkey,
3072                 .ivsize = GCM_AES_IV_SIZE,
3073                .maxauthsize = AES_BLOCK_SIZE,
3074         },
3075         .cipher_info = {
3076                         .alg = CIPHER_ALG_AES,
3077                         .mode = CIPHER_MODE_GCM,
3078                         },
3079         .auth_info = {
3080                       .alg = HASH_ALG_AES,
3081                       .mode = HASH_MODE_GCM,
3082                       },
3083         .auth_first = 0,
3084         },
3085        {
3086         .type = CRYPTO_ALG_TYPE_AEAD,
3087         .alg.aead = {
3088                 .base = {
3089                        .cra_name = "ccm(aes)",
3090                        .cra_driver_name = "ccm-aes-iproc",
3091                        .cra_blocksize = AES_BLOCK_SIZE,
3092                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3093                 },
3094                 .setkey = aead_gcm_ccm_setkey,
3095                 .ivsize = CCM_AES_IV_SIZE,
3096                .maxauthsize = AES_BLOCK_SIZE,
3097         },
3098         .cipher_info = {
3099                         .alg = CIPHER_ALG_AES,
3100                         .mode = CIPHER_MODE_CCM,
3101                         },
3102         .auth_info = {
3103                       .alg = HASH_ALG_AES,
3104                       .mode = HASH_MODE_CCM,
3105                       },
3106         .auth_first = 0,
3107         },
3108        {
3109         .type = CRYPTO_ALG_TYPE_AEAD,
3110         .alg.aead = {
3111                 .base = {
3112                        .cra_name = "rfc4106(gcm(aes))",
3113                        .cra_driver_name = "gcm-aes-esp-iproc",
3114                        .cra_blocksize = AES_BLOCK_SIZE,
3115                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3116                 },
3117                 .setkey = aead_gcm_esp_setkey,
3118                 .ivsize = GCM_RFC4106_IV_SIZE,
3119                 .maxauthsize = AES_BLOCK_SIZE,
3120         },
3121         .cipher_info = {
3122                         .alg = CIPHER_ALG_AES,
3123                         .mode = CIPHER_MODE_GCM,
3124                         },
3125         .auth_info = {
3126                       .alg = HASH_ALG_AES,
3127                       .mode = HASH_MODE_GCM,
3128                       },
3129         .auth_first = 0,
3130         },
3131        {
3132         .type = CRYPTO_ALG_TYPE_AEAD,
3133         .alg.aead = {
3134                 .base = {
3135                        .cra_name = "rfc4309(ccm(aes))",
3136                        .cra_driver_name = "ccm-aes-esp-iproc",
3137                        .cra_blocksize = AES_BLOCK_SIZE,
3138                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3139                 },
3140                 .setkey = aead_ccm_esp_setkey,
3141                 .ivsize = CCM_AES_IV_SIZE,
3142                 .maxauthsize = AES_BLOCK_SIZE,
3143         },
3144         .cipher_info = {
3145                         .alg = CIPHER_ALG_AES,
3146                         .mode = CIPHER_MODE_CCM,
3147                         },
3148         .auth_info = {
3149                       .alg = HASH_ALG_AES,
3150                       .mode = HASH_MODE_CCM,
3151                       },
3152         .auth_first = 0,
3153         },
3154        {
3155         .type = CRYPTO_ALG_TYPE_AEAD,
3156         .alg.aead = {
3157                 .base = {
3158                        .cra_name = "rfc4543(gcm(aes))",
3159                        .cra_driver_name = "gmac-aes-esp-iproc",
3160                        .cra_blocksize = AES_BLOCK_SIZE,
3161                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK
3162                 },
3163                 .setkey = rfc4543_gcm_esp_setkey,
3164                 .ivsize = GCM_RFC4106_IV_SIZE,
3165                 .maxauthsize = AES_BLOCK_SIZE,
3166         },
3167         .cipher_info = {
3168                         .alg = CIPHER_ALG_AES,
3169                         .mode = CIPHER_MODE_GCM,
3170                         },
3171         .auth_info = {
3172                       .alg = HASH_ALG_AES,
3173                       .mode = HASH_MODE_GCM,
3174                       },
3175         .auth_first = 0,
3176         },
3177        {
3178         .type = CRYPTO_ALG_TYPE_AEAD,
3179         .alg.aead = {
3180                 .base = {
3181                        .cra_name = "authenc(hmac(md5),cbc(aes))",
3182                        .cra_driver_name = "authenc-hmac-md5-cbc-aes-iproc",
3183                        .cra_blocksize = AES_BLOCK_SIZE,
3184                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3185                                     CRYPTO_ALG_ASYNC |
3186                                     CRYPTO_ALG_ALLOCATES_MEMORY
3187                 },
3188                 .setkey = aead_authenc_setkey,
3189                .ivsize = AES_BLOCK_SIZE,
3190                .maxauthsize = MD5_DIGEST_SIZE,
3191         },
3192         .cipher_info = {
3193                         .alg = CIPHER_ALG_AES,
3194                         .mode = CIPHER_MODE_CBC,
3195                         },
3196         .auth_info = {
3197                       .alg = HASH_ALG_MD5,
3198                       .mode = HASH_MODE_HMAC,
3199                       },
3200         .auth_first = 0,
3201         },
3202        {
3203         .type = CRYPTO_ALG_TYPE_AEAD,
3204         .alg.aead = {
3205                 .base = {
3206                        .cra_name = "authenc(hmac(sha1),cbc(aes))",
3207                        .cra_driver_name = "authenc-hmac-sha1-cbc-aes-iproc",
3208                        .cra_blocksize = AES_BLOCK_SIZE,
3209                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3210                                     CRYPTO_ALG_ASYNC |
3211                                     CRYPTO_ALG_ALLOCATES_MEMORY
3212                 },
3213                 .setkey = aead_authenc_setkey,
3214                 .ivsize = AES_BLOCK_SIZE,
3215                 .maxauthsize = SHA1_DIGEST_SIZE,
3216         },
3217         .cipher_info = {
3218                         .alg = CIPHER_ALG_AES,
3219                         .mode = CIPHER_MODE_CBC,
3220                         },
3221         .auth_info = {
3222                       .alg = HASH_ALG_SHA1,
3223                       .mode = HASH_MODE_HMAC,
3224                       },
3225         .auth_first = 0,
3226         },
3227        {
3228         .type = CRYPTO_ALG_TYPE_AEAD,
3229         .alg.aead = {
3230                 .base = {
3231                        .cra_name = "authenc(hmac(sha256),cbc(aes))",
3232                        .cra_driver_name = "authenc-hmac-sha256-cbc-aes-iproc",
3233                        .cra_blocksize = AES_BLOCK_SIZE,
3234                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3235                                     CRYPTO_ALG_ASYNC |
3236                                     CRYPTO_ALG_ALLOCATES_MEMORY
3237                 },
3238                 .setkey = aead_authenc_setkey,
3239                 .ivsize = AES_BLOCK_SIZE,
3240                 .maxauthsize = SHA256_DIGEST_SIZE,
3241         },
3242         .cipher_info = {
3243                         .alg = CIPHER_ALG_AES,
3244                         .mode = CIPHER_MODE_CBC,
3245                         },
3246         .auth_info = {
3247                       .alg = HASH_ALG_SHA256,
3248                       .mode = HASH_MODE_HMAC,
3249                       },
3250         .auth_first = 0,
3251         },
3252        {
3253         .type = CRYPTO_ALG_TYPE_AEAD,
3254         .alg.aead = {
3255                 .base = {
3256                        .cra_name = "authenc(hmac(md5),cbc(des))",
3257                        .cra_driver_name = "authenc-hmac-md5-cbc-des-iproc",
3258                        .cra_blocksize = DES_BLOCK_SIZE,
3259                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3260                                     CRYPTO_ALG_ASYNC |
3261                                     CRYPTO_ALG_ALLOCATES_MEMORY
3262                 },
3263                 .setkey = aead_authenc_setkey,
3264                 .ivsize = DES_BLOCK_SIZE,
3265                 .maxauthsize = MD5_DIGEST_SIZE,
3266         },
3267         .cipher_info = {
3268                         .alg = CIPHER_ALG_DES,
3269                         .mode = CIPHER_MODE_CBC,
3270                         },
3271         .auth_info = {
3272                       .alg = HASH_ALG_MD5,
3273                       .mode = HASH_MODE_HMAC,
3274                       },
3275         .auth_first = 0,
3276         },
3277        {
3278         .type = CRYPTO_ALG_TYPE_AEAD,
3279         .alg.aead = {
3280                 .base = {
3281                        .cra_name = "authenc(hmac(sha1),cbc(des))",
3282                        .cra_driver_name = "authenc-hmac-sha1-cbc-des-iproc",
3283                        .cra_blocksize = DES_BLOCK_SIZE,
3284                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3285                                     CRYPTO_ALG_ASYNC |
3286                                     CRYPTO_ALG_ALLOCATES_MEMORY
3287                 },
3288                 .setkey = aead_authenc_setkey,
3289                 .ivsize = DES_BLOCK_SIZE,
3290                 .maxauthsize = SHA1_DIGEST_SIZE,
3291         },
3292         .cipher_info = {
3293                         .alg = CIPHER_ALG_DES,
3294                         .mode = CIPHER_MODE_CBC,
3295                         },
3296         .auth_info = {
3297                       .alg = HASH_ALG_SHA1,
3298                       .mode = HASH_MODE_HMAC,
3299                       },
3300         .auth_first = 0,
3301         },
3302        {
3303         .type = CRYPTO_ALG_TYPE_AEAD,
3304         .alg.aead = {
3305                 .base = {
3306                        .cra_name = "authenc(hmac(sha224),cbc(des))",
3307                        .cra_driver_name = "authenc-hmac-sha224-cbc-des-iproc",
3308                        .cra_blocksize = DES_BLOCK_SIZE,
3309                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3310                                     CRYPTO_ALG_ASYNC |
3311                                     CRYPTO_ALG_ALLOCATES_MEMORY
3312                 },
3313                 .setkey = aead_authenc_setkey,
3314                 .ivsize = DES_BLOCK_SIZE,
3315                 .maxauthsize = SHA224_DIGEST_SIZE,
3316         },
3317         .cipher_info = {
3318                         .alg = CIPHER_ALG_DES,
3319                         .mode = CIPHER_MODE_CBC,
3320                         },
3321         .auth_info = {
3322                       .alg = HASH_ALG_SHA224,
3323                       .mode = HASH_MODE_HMAC,
3324                       },
3325         .auth_first = 0,
3326         },
3327        {
3328         .type = CRYPTO_ALG_TYPE_AEAD,
3329         .alg.aead = {
3330                 .base = {
3331                        .cra_name = "authenc(hmac(sha256),cbc(des))",
3332                        .cra_driver_name = "authenc-hmac-sha256-cbc-des-iproc",
3333                        .cra_blocksize = DES_BLOCK_SIZE,
3334                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3335                                     CRYPTO_ALG_ASYNC |
3336                                     CRYPTO_ALG_ALLOCATES_MEMORY
3337                 },
3338                 .setkey = aead_authenc_setkey,
3339                 .ivsize = DES_BLOCK_SIZE,
3340                 .maxauthsize = SHA256_DIGEST_SIZE,
3341         },
3342         .cipher_info = {
3343                         .alg = CIPHER_ALG_DES,
3344                         .mode = CIPHER_MODE_CBC,
3345                         },
3346         .auth_info = {
3347                       .alg = HASH_ALG_SHA256,
3348                       .mode = HASH_MODE_HMAC,
3349                       },
3350         .auth_first = 0,
3351         },
3352        {
3353         .type = CRYPTO_ALG_TYPE_AEAD,
3354         .alg.aead = {
3355                 .base = {
3356                        .cra_name = "authenc(hmac(sha384),cbc(des))",
3357                        .cra_driver_name = "authenc-hmac-sha384-cbc-des-iproc",
3358                        .cra_blocksize = DES_BLOCK_SIZE,
3359                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3360                                     CRYPTO_ALG_ASYNC |
3361                                     CRYPTO_ALG_ALLOCATES_MEMORY
3362                 },
3363                 .setkey = aead_authenc_setkey,
3364                 .ivsize = DES_BLOCK_SIZE,
3365                 .maxauthsize = SHA384_DIGEST_SIZE,
3366         },
3367         .cipher_info = {
3368                         .alg = CIPHER_ALG_DES,
3369                         .mode = CIPHER_MODE_CBC,
3370                         },
3371         .auth_info = {
3372                       .alg = HASH_ALG_SHA384,
3373                       .mode = HASH_MODE_HMAC,
3374                       },
3375         .auth_first = 0,
3376         },
3377        {
3378         .type = CRYPTO_ALG_TYPE_AEAD,
3379         .alg.aead = {
3380                 .base = {
3381                        .cra_name = "authenc(hmac(sha512),cbc(des))",
3382                        .cra_driver_name = "authenc-hmac-sha512-cbc-des-iproc",
3383                        .cra_blocksize = DES_BLOCK_SIZE,
3384                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3385                                     CRYPTO_ALG_ASYNC |
3386                                     CRYPTO_ALG_ALLOCATES_MEMORY
3387                 },
3388                 .setkey = aead_authenc_setkey,
3389                 .ivsize = DES_BLOCK_SIZE,
3390                 .maxauthsize = SHA512_DIGEST_SIZE,
3391         },
3392         .cipher_info = {
3393                         .alg = CIPHER_ALG_DES,
3394                         .mode = CIPHER_MODE_CBC,
3395                         },
3396         .auth_info = {
3397                       .alg = HASH_ALG_SHA512,
3398                       .mode = HASH_MODE_HMAC,
3399                       },
3400         .auth_first = 0,
3401         },
3402        {
3403         .type = CRYPTO_ALG_TYPE_AEAD,
3404         .alg.aead = {
3405                 .base = {
3406                        .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
3407                        .cra_driver_name = "authenc-hmac-md5-cbc-des3-iproc",
3408                        .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3409                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3410                                     CRYPTO_ALG_ASYNC |
3411                                     CRYPTO_ALG_ALLOCATES_MEMORY
3412                 },
3413                 .setkey = aead_authenc_setkey,
3414                 .ivsize = DES3_EDE_BLOCK_SIZE,
3415                 .maxauthsize = MD5_DIGEST_SIZE,
3416         },
3417         .cipher_info = {
3418                         .alg = CIPHER_ALG_3DES,
3419                         .mode = CIPHER_MODE_CBC,
3420                         },
3421         .auth_info = {
3422                       .alg = HASH_ALG_MD5,
3423                       .mode = HASH_MODE_HMAC,
3424                       },
3425         .auth_first = 0,
3426         },
3427        {
3428         .type = CRYPTO_ALG_TYPE_AEAD,
3429         .alg.aead = {
3430                 .base = {
3431                        .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
3432                        .cra_driver_name = "authenc-hmac-sha1-cbc-des3-iproc",
3433                        .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3434                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3435                                     CRYPTO_ALG_ASYNC |
3436                                     CRYPTO_ALG_ALLOCATES_MEMORY
3437                 },
3438                 .setkey = aead_authenc_setkey,
3439                 .ivsize = DES3_EDE_BLOCK_SIZE,
3440                 .maxauthsize = SHA1_DIGEST_SIZE,
3441         },
3442         .cipher_info = {
3443                         .alg = CIPHER_ALG_3DES,
3444                         .mode = CIPHER_MODE_CBC,
3445                         },
3446         .auth_info = {
3447                       .alg = HASH_ALG_SHA1,
3448                       .mode = HASH_MODE_HMAC,
3449                       },
3450         .auth_first = 0,
3451         },
3452        {
3453         .type = CRYPTO_ALG_TYPE_AEAD,
3454         .alg.aead = {
3455                 .base = {
3456                        .cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
3457                        .cra_driver_name = "authenc-hmac-sha224-cbc-des3-iproc",
3458                        .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3459                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3460                                     CRYPTO_ALG_ASYNC |
3461                                     CRYPTO_ALG_ALLOCATES_MEMORY
3462                 },
3463                 .setkey = aead_authenc_setkey,
3464                 .ivsize = DES3_EDE_BLOCK_SIZE,
3465                 .maxauthsize = SHA224_DIGEST_SIZE,
3466         },
3467         .cipher_info = {
3468                         .alg = CIPHER_ALG_3DES,
3469                         .mode = CIPHER_MODE_CBC,
3470                         },
3471         .auth_info = {
3472                       .alg = HASH_ALG_SHA224,
3473                       .mode = HASH_MODE_HMAC,
3474                       },
3475         .auth_first = 0,
3476         },
3477        {
3478         .type = CRYPTO_ALG_TYPE_AEAD,
3479         .alg.aead = {
3480                 .base = {
3481                        .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
3482                        .cra_driver_name = "authenc-hmac-sha256-cbc-des3-iproc",
3483                        .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3484                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3485                                     CRYPTO_ALG_ASYNC |
3486                                     CRYPTO_ALG_ALLOCATES_MEMORY
3487                 },
3488                 .setkey = aead_authenc_setkey,
3489                 .ivsize = DES3_EDE_BLOCK_SIZE,
3490                 .maxauthsize = SHA256_DIGEST_SIZE,
3491         },
3492         .cipher_info = {
3493                         .alg = CIPHER_ALG_3DES,
3494                         .mode = CIPHER_MODE_CBC,
3495                         },
3496         .auth_info = {
3497                       .alg = HASH_ALG_SHA256,
3498                       .mode = HASH_MODE_HMAC,
3499                       },
3500         .auth_first = 0,
3501         },
3502        {
3503         .type = CRYPTO_ALG_TYPE_AEAD,
3504         .alg.aead = {
3505                 .base = {
3506                        .cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
3507                        .cra_driver_name = "authenc-hmac-sha384-cbc-des3-iproc",
3508                        .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3509                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3510                                     CRYPTO_ALG_ASYNC |
3511                                     CRYPTO_ALG_ALLOCATES_MEMORY
3512                 },
3513                 .setkey = aead_authenc_setkey,
3514                 .ivsize = DES3_EDE_BLOCK_SIZE,
3515                 .maxauthsize = SHA384_DIGEST_SIZE,
3516         },
3517         .cipher_info = {
3518                         .alg = CIPHER_ALG_3DES,
3519                         .mode = CIPHER_MODE_CBC,
3520                         },
3521         .auth_info = {
3522                       .alg = HASH_ALG_SHA384,
3523                       .mode = HASH_MODE_HMAC,
3524                       },
3525         .auth_first = 0,
3526         },
3527        {
3528         .type = CRYPTO_ALG_TYPE_AEAD,
3529         .alg.aead = {
3530                 .base = {
3531                        .cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
3532                        .cra_driver_name = "authenc-hmac-sha512-cbc-des3-iproc",
3533                        .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3534                        .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
3535                                     CRYPTO_ALG_ASYNC |
3536                                     CRYPTO_ALG_ALLOCATES_MEMORY
3537                 },
3538                 .setkey = aead_authenc_setkey,
3539                 .ivsize = DES3_EDE_BLOCK_SIZE,
3540                 .maxauthsize = SHA512_DIGEST_SIZE,
3541         },
3542         .cipher_info = {
3543                         .alg = CIPHER_ALG_3DES,
3544                         .mode = CIPHER_MODE_CBC,
3545                         },
3546         .auth_info = {
3547                       .alg = HASH_ALG_SHA512,
3548                       .mode = HASH_MODE_HMAC,
3549                       },
3550         .auth_first = 0,
3551         },
3552
3553/* SKCIPHER algorithms. */
3554        {
3555         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3556         .alg.skcipher = {
3557                        .base.cra_name = "ofb(des)",
3558                        .base.cra_driver_name = "ofb-des-iproc",
3559                        .base.cra_blocksize = DES_BLOCK_SIZE,
3560                        .min_keysize = DES_KEY_SIZE,
3561                        .max_keysize = DES_KEY_SIZE,
3562                        .ivsize = DES_BLOCK_SIZE,
3563                        },
3564         .cipher_info = {
3565                         .alg = CIPHER_ALG_DES,
3566                         .mode = CIPHER_MODE_OFB,
3567                         },
3568         .auth_info = {
3569                       .alg = HASH_ALG_NONE,
3570                       .mode = HASH_MODE_NONE,
3571                       },
3572         },
3573        {
3574         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3575         .alg.skcipher = {
3576                        .base.cra_name = "cbc(des)",
3577                        .base.cra_driver_name = "cbc-des-iproc",
3578                        .base.cra_blocksize = DES_BLOCK_SIZE,
3579                        .min_keysize = DES_KEY_SIZE,
3580                        .max_keysize = DES_KEY_SIZE,
3581                        .ivsize = DES_BLOCK_SIZE,
3582                        },
3583         .cipher_info = {
3584                         .alg = CIPHER_ALG_DES,
3585                         .mode = CIPHER_MODE_CBC,
3586                         },
3587         .auth_info = {
3588                       .alg = HASH_ALG_NONE,
3589                       .mode = HASH_MODE_NONE,
3590                       },
3591         },
3592        {
3593         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3594         .alg.skcipher = {
3595                        .base.cra_name = "ecb(des)",
3596                        .base.cra_driver_name = "ecb-des-iproc",
3597                        .base.cra_blocksize = DES_BLOCK_SIZE,
3598                        .min_keysize = DES_KEY_SIZE,
3599                        .max_keysize = DES_KEY_SIZE,
3600                        .ivsize = 0,
3601                        },
3602         .cipher_info = {
3603                         .alg = CIPHER_ALG_DES,
3604                         .mode = CIPHER_MODE_ECB,
3605                         },
3606         .auth_info = {
3607                       .alg = HASH_ALG_NONE,
3608                       .mode = HASH_MODE_NONE,
3609                       },
3610         },
3611        {
3612         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3613         .alg.skcipher = {
3614                        .base.cra_name = "ofb(des3_ede)",
3615                        .base.cra_driver_name = "ofb-des3-iproc",
3616                        .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3617                        .min_keysize = DES3_EDE_KEY_SIZE,
3618                        .max_keysize = DES3_EDE_KEY_SIZE,
3619                        .ivsize = DES3_EDE_BLOCK_SIZE,
3620                        },
3621         .cipher_info = {
3622                         .alg = CIPHER_ALG_3DES,
3623                         .mode = CIPHER_MODE_OFB,
3624                         },
3625         .auth_info = {
3626                       .alg = HASH_ALG_NONE,
3627                       .mode = HASH_MODE_NONE,
3628                       },
3629         },
3630        {
3631         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3632         .alg.skcipher = {
3633                        .base.cra_name = "cbc(des3_ede)",
3634                        .base.cra_driver_name = "cbc-des3-iproc",
3635                        .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3636                        .min_keysize = DES3_EDE_KEY_SIZE,
3637                        .max_keysize = DES3_EDE_KEY_SIZE,
3638                        .ivsize = DES3_EDE_BLOCK_SIZE,
3639                        },
3640         .cipher_info = {
3641                         .alg = CIPHER_ALG_3DES,
3642                         .mode = CIPHER_MODE_CBC,
3643                         },
3644         .auth_info = {
3645                       .alg = HASH_ALG_NONE,
3646                       .mode = HASH_MODE_NONE,
3647                       },
3648         },
3649        {
3650         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3651         .alg.skcipher = {
3652                        .base.cra_name = "ecb(des3_ede)",
3653                        .base.cra_driver_name = "ecb-des3-iproc",
3654                        .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
3655                        .min_keysize = DES3_EDE_KEY_SIZE,
3656                        .max_keysize = DES3_EDE_KEY_SIZE,
3657                        .ivsize = 0,
3658                        },
3659         .cipher_info = {
3660                         .alg = CIPHER_ALG_3DES,
3661                         .mode = CIPHER_MODE_ECB,
3662                         },
3663         .auth_info = {
3664                       .alg = HASH_ALG_NONE,
3665                       .mode = HASH_MODE_NONE,
3666                       },
3667         },
3668        {
3669         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3670         .alg.skcipher = {
3671                        .base.cra_name = "ofb(aes)",
3672                        .base.cra_driver_name = "ofb-aes-iproc",
3673                        .base.cra_blocksize = AES_BLOCK_SIZE,
3674                        .min_keysize = AES_MIN_KEY_SIZE,
3675                        .max_keysize = AES_MAX_KEY_SIZE,
3676                        .ivsize = AES_BLOCK_SIZE,
3677                        },
3678         .cipher_info = {
3679                         .alg = CIPHER_ALG_AES,
3680                         .mode = CIPHER_MODE_OFB,
3681                         },
3682         .auth_info = {
3683                       .alg = HASH_ALG_NONE,
3684                       .mode = HASH_MODE_NONE,
3685                       },
3686         },
3687        {
3688         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3689         .alg.skcipher = {
3690                        .base.cra_name = "cbc(aes)",
3691                        .base.cra_driver_name = "cbc-aes-iproc",
3692                        .base.cra_blocksize = AES_BLOCK_SIZE,
3693                        .min_keysize = AES_MIN_KEY_SIZE,
3694                        .max_keysize = AES_MAX_KEY_SIZE,
3695                        .ivsize = AES_BLOCK_SIZE,
3696                        },
3697         .cipher_info = {
3698                         .alg = CIPHER_ALG_AES,
3699                         .mode = CIPHER_MODE_CBC,
3700                         },
3701         .auth_info = {
3702                       .alg = HASH_ALG_NONE,
3703                       .mode = HASH_MODE_NONE,
3704                       },
3705         },
3706        {
3707         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3708         .alg.skcipher = {
3709                        .base.cra_name = "ecb(aes)",
3710                        .base.cra_driver_name = "ecb-aes-iproc",
3711                        .base.cra_blocksize = AES_BLOCK_SIZE,
3712                        .min_keysize = AES_MIN_KEY_SIZE,
3713                        .max_keysize = AES_MAX_KEY_SIZE,
3714                        .ivsize = 0,
3715                        },
3716         .cipher_info = {
3717                         .alg = CIPHER_ALG_AES,
3718                         .mode = CIPHER_MODE_ECB,
3719                         },
3720         .auth_info = {
3721                       .alg = HASH_ALG_NONE,
3722                       .mode = HASH_MODE_NONE,
3723                       },
3724         },
3725        {
3726         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3727         .alg.skcipher = {
3728                        .base.cra_name = "ctr(aes)",
3729                        .base.cra_driver_name = "ctr-aes-iproc",
3730                        .base.cra_blocksize = AES_BLOCK_SIZE,
3731                        .min_keysize = AES_MIN_KEY_SIZE,
3732                        .max_keysize = AES_MAX_KEY_SIZE,
3733                        .ivsize = AES_BLOCK_SIZE,
3734                        },
3735         .cipher_info = {
3736                         .alg = CIPHER_ALG_AES,
3737                         .mode = CIPHER_MODE_CTR,
3738                         },
3739         .auth_info = {
3740                       .alg = HASH_ALG_NONE,
3741                       .mode = HASH_MODE_NONE,
3742                       },
3743         },
3744{
3745         .type = CRYPTO_ALG_TYPE_SKCIPHER,
3746         .alg.skcipher = {
3747                        .base.cra_name = "xts(aes)",
3748                        .base.cra_driver_name = "xts-aes-iproc",
3749                        .base.cra_blocksize = AES_BLOCK_SIZE,
3750                        .min_keysize = 2 * AES_MIN_KEY_SIZE,
3751                        .max_keysize = 2 * AES_MAX_KEY_SIZE,
3752                        .ivsize = AES_BLOCK_SIZE,
3753                        },
3754         .cipher_info = {
3755                         .alg = CIPHER_ALG_AES,
3756                         .mode = CIPHER_MODE_XTS,
3757                         },
3758         .auth_info = {
3759                       .alg = HASH_ALG_NONE,
3760                       .mode = HASH_MODE_NONE,
3761                       },
3762         },
3763
3764/* AHASH algorithms. */
3765        {
3766         .type = CRYPTO_ALG_TYPE_AHASH,
3767         .alg.hash = {
3768                      .halg.digestsize = MD5_DIGEST_SIZE,
3769                      .halg.base = {
3770                                    .cra_name = "md5",
3771                                    .cra_driver_name = "md5-iproc",
3772                                    .cra_blocksize = MD5_BLOCK_WORDS * 4,
3773                                    .cra_flags = CRYPTO_ALG_ASYNC |
3774                                                 CRYPTO_ALG_ALLOCATES_MEMORY,
3775                                }
3776                      },
3777         .cipher_info = {
3778                         .alg = CIPHER_ALG_NONE,
3779                         .mode = CIPHER_MODE_NONE,
3780                         },
3781         .auth_info = {
3782                       .alg = HASH_ALG_MD5,
3783                       .mode = HASH_MODE_HASH,
3784                       },
3785         },
3786        {
3787         .type = CRYPTO_ALG_TYPE_AHASH,
3788         .alg.hash = {
3789                      .halg.digestsize = MD5_DIGEST_SIZE,
3790                      .halg.base = {
3791                                    .cra_name = "hmac(md5)",
3792                                    .cra_driver_name = "hmac-md5-iproc",
3793                                    .cra_blocksize = MD5_BLOCK_WORDS * 4,
3794                                }
3795                      },
3796         .cipher_info = {
3797                         .alg = CIPHER_ALG_NONE,
3798                         .mode = CIPHER_MODE_NONE,
3799                         },
3800         .auth_info = {
3801                       .alg = HASH_ALG_MD5,
3802                       .mode = HASH_MODE_HMAC,
3803                       },
3804         },
3805        {.type = CRYPTO_ALG_TYPE_AHASH,
3806         .alg.hash = {
3807                      .halg.digestsize = SHA1_DIGEST_SIZE,
3808                      .halg.base = {
3809                                    .cra_name = "sha1",
3810                                    .cra_driver_name = "sha1-iproc",
3811                                    .cra_blocksize = SHA1_BLOCK_SIZE,
3812                                }
3813                      },
3814         .cipher_info = {
3815                         .alg = CIPHER_ALG_NONE,
3816                         .mode = CIPHER_MODE_NONE,
3817                         },
3818         .auth_info = {
3819                       .alg = HASH_ALG_SHA1,
3820                       .mode = HASH_MODE_HASH,
3821                       },
3822         },
3823        {.type = CRYPTO_ALG_TYPE_AHASH,
3824         .alg.hash = {
3825                      .halg.digestsize = SHA1_DIGEST_SIZE,
3826                      .halg.base = {
3827                                    .cra_name = "hmac(sha1)",
3828                                    .cra_driver_name = "hmac-sha1-iproc",
3829                                    .cra_blocksize = SHA1_BLOCK_SIZE,
3830                                }
3831                      },
3832         .cipher_info = {
3833                         .alg = CIPHER_ALG_NONE,
3834                         .mode = CIPHER_MODE_NONE,
3835                         },
3836         .auth_info = {
3837                       .alg = HASH_ALG_SHA1,
3838                       .mode = HASH_MODE_HMAC,
3839                       },
3840         },
3841        {.type = CRYPTO_ALG_TYPE_AHASH,
3842         .alg.hash = {
3843                        .halg.digestsize = SHA224_DIGEST_SIZE,
3844                        .halg.base = {
3845                                    .cra_name = "sha224",
3846                                    .cra_driver_name = "sha224-iproc",
3847                                    .cra_blocksize = SHA224_BLOCK_SIZE,
3848                        }
3849                      },
3850         .cipher_info = {
3851                         .alg = CIPHER_ALG_NONE,
3852                         .mode = CIPHER_MODE_NONE,
3853                         },
3854         .auth_info = {
3855                       .alg = HASH_ALG_SHA224,
3856                       .mode = HASH_MODE_HASH,
3857                       },
3858         },
3859        {.type = CRYPTO_ALG_TYPE_AHASH,
3860         .alg.hash = {
3861                      .halg.digestsize = SHA224_DIGEST_SIZE,
3862                      .halg.base = {
3863                                    .cra_name = "hmac(sha224)",
3864                                    .cra_driver_name = "hmac-sha224-iproc",
3865                                    .cra_blocksize = SHA224_BLOCK_SIZE,
3866                                }
3867                      },
3868         .cipher_info = {
3869                         .alg = CIPHER_ALG_NONE,
3870                         .mode = CIPHER_MODE_NONE,
3871                         },
3872         .auth_info = {
3873                       .alg = HASH_ALG_SHA224,
3874                       .mode = HASH_MODE_HMAC,
3875                       },
3876         },
3877        {.type = CRYPTO_ALG_TYPE_AHASH,
3878         .alg.hash = {
3879                      .halg.digestsize = SHA256_DIGEST_SIZE,
3880                      .halg.base = {
3881                                    .cra_name = "sha256",
3882                                    .cra_driver_name = "sha256-iproc",
3883                                    .cra_blocksize = SHA256_BLOCK_SIZE,
3884                                }
3885                      },
3886         .cipher_info = {
3887                         .alg = CIPHER_ALG_NONE,
3888                         .mode = CIPHER_MODE_NONE,
3889                         },
3890         .auth_info = {
3891                       .alg = HASH_ALG_SHA256,
3892                       .mode = HASH_MODE_HASH,
3893                       },
3894         },
3895        {.type = CRYPTO_ALG_TYPE_AHASH,
3896         .alg.hash = {
3897                      .halg.digestsize = SHA256_DIGEST_SIZE,
3898                      .halg.base = {
3899                                    .cra_name = "hmac(sha256)",
3900                                    .cra_driver_name = "hmac-sha256-iproc",
3901                                    .cra_blocksize = SHA256_BLOCK_SIZE,
3902                                }
3903                      },
3904         .cipher_info = {
3905                         .alg = CIPHER_ALG_NONE,
3906                         .mode = CIPHER_MODE_NONE,
3907                         },
3908         .auth_info = {
3909                       .alg = HASH_ALG_SHA256,
3910                       .mode = HASH_MODE_HMAC,
3911                       },
3912         },
3913        {
3914        .type = CRYPTO_ALG_TYPE_AHASH,
3915         .alg.hash = {
3916                      .halg.digestsize = SHA384_DIGEST_SIZE,
3917                      .halg.base = {
3918                                    .cra_name = "sha384",
3919                                    .cra_driver_name = "sha384-iproc",
3920                                    .cra_blocksize = SHA384_BLOCK_SIZE,
3921                                }
3922                      },
3923         .cipher_info = {
3924                         .alg = CIPHER_ALG_NONE,
3925                         .mode = CIPHER_MODE_NONE,
3926                         },
3927         .auth_info = {
3928                       .alg = HASH_ALG_SHA384,
3929                       .mode = HASH_MODE_HASH,
3930                       },
3931         },
3932        {
3933         .type = CRYPTO_ALG_TYPE_AHASH,
3934         .alg.hash = {
3935                      .halg.digestsize = SHA384_DIGEST_SIZE,
3936                      .halg.base = {
3937                                    .cra_name = "hmac(sha384)",
3938                                    .cra_driver_name = "hmac-sha384-iproc",
3939                                    .cra_blocksize = SHA384_BLOCK_SIZE,
3940                                }
3941                      },
3942         .cipher_info = {
3943                         .alg = CIPHER_ALG_NONE,
3944                         .mode = CIPHER_MODE_NONE,
3945                         },
3946         .auth_info = {
3947                       .alg = HASH_ALG_SHA384,
3948                       .mode = HASH_MODE_HMAC,
3949                       },
3950         },
3951        {
3952         .type = CRYPTO_ALG_TYPE_AHASH,
3953         .alg.hash = {
3954                      .halg.digestsize = SHA512_DIGEST_SIZE,
3955                      .halg.base = {
3956                                    .cra_name = "sha512",
3957                                    .cra_driver_name = "sha512-iproc",
3958                                    .cra_blocksize = SHA512_BLOCK_SIZE,
3959                                }
3960                      },
3961         .cipher_info = {
3962                         .alg = CIPHER_ALG_NONE,
3963                         .mode = CIPHER_MODE_NONE,
3964                         },
3965         .auth_info = {
3966                       .alg = HASH_ALG_SHA512,
3967                       .mode = HASH_MODE_HASH,
3968                       },
3969         },
3970        {
3971         .type = CRYPTO_ALG_TYPE_AHASH,
3972         .alg.hash = {
3973                      .halg.digestsize = SHA512_DIGEST_SIZE,
3974                      .halg.base = {
3975                                    .cra_name = "hmac(sha512)",
3976                                    .cra_driver_name = "hmac-sha512-iproc",
3977                                    .cra_blocksize = SHA512_BLOCK_SIZE,
3978                                }
3979                      },
3980         .cipher_info = {
3981                         .alg = CIPHER_ALG_NONE,
3982                         .mode = CIPHER_MODE_NONE,
3983                         },
3984         .auth_info = {
3985                       .alg = HASH_ALG_SHA512,
3986                       .mode = HASH_MODE_HMAC,
3987                       },
3988         },
3989        {
3990         .type = CRYPTO_ALG_TYPE_AHASH,
3991         .alg.hash = {
3992                      .halg.digestsize = SHA3_224_DIGEST_SIZE,
3993                      .halg.base = {
3994                                    .cra_name = "sha3-224",
3995                                    .cra_driver_name = "sha3-224-iproc",
3996                                    .cra_blocksize = SHA3_224_BLOCK_SIZE,
3997                                }
3998                      },
3999         .cipher_info = {
4000                         .alg = CIPHER_ALG_NONE,
4001                         .mode = CIPHER_MODE_NONE,
4002                         },
4003         .auth_info = {
4004                       .alg = HASH_ALG_SHA3_224,
4005                       .mode = HASH_MODE_HASH,
4006                       },
4007         },
4008        {
4009         .type = CRYPTO_ALG_TYPE_AHASH,
4010         .alg.hash = {
4011                      .halg.digestsize = SHA3_224_DIGEST_SIZE,
4012                      .halg.base = {
4013                                    .cra_name = "hmac(sha3-224)",
4014                                    .cra_driver_name = "hmac-sha3-224-iproc",
4015                                    .cra_blocksize = SHA3_224_BLOCK_SIZE,
4016                                }
4017                      },
4018         .cipher_info = {
4019                         .alg = CIPHER_ALG_NONE,
4020                         .mode = CIPHER_MODE_NONE,
4021                         },
4022         .auth_info = {
4023                       .alg = HASH_ALG_SHA3_224,
4024                       .mode = HASH_MODE_HMAC
4025                       },
4026         },
4027        {
4028         .type = CRYPTO_ALG_TYPE_AHASH,
4029         .alg.hash = {
4030                      .halg.digestsize = SHA3_256_DIGEST_SIZE,
4031                      .halg.base = {
4032                                    .cra_name = "sha3-256",
4033                                    .cra_driver_name = "sha3-256-iproc",
4034                                    .cra_blocksize = SHA3_256_BLOCK_SIZE,
4035                                }
4036                      },
4037         .cipher_info = {
4038                         .alg = CIPHER_ALG_NONE,
4039                         .mode = CIPHER_MODE_NONE,
4040                         },
4041         .auth_info = {
4042                       .alg = HASH_ALG_SHA3_256,
4043                       .mode = HASH_MODE_HASH,
4044                       },
4045         },
4046        {
4047         .type = CRYPTO_ALG_TYPE_AHASH,
4048         .alg.hash = {
4049                      .halg.digestsize = SHA3_256_DIGEST_SIZE,
4050                      .halg.base = {
4051                                    .cra_name = "hmac(sha3-256)",
4052                                    .cra_driver_name = "hmac-sha3-256-iproc",
4053                                    .cra_blocksize = SHA3_256_BLOCK_SIZE,
4054                                }
4055                      },
4056         .cipher_info = {
4057                         .alg = CIPHER_ALG_NONE,
4058                         .mode = CIPHER_MODE_NONE,
4059                         },
4060         .auth_info = {
4061                       .alg = HASH_ALG_SHA3_256,
4062                       .mode = HASH_MODE_HMAC,
4063                       },
4064         },
4065        {
4066         .type = CRYPTO_ALG_TYPE_AHASH,
4067         .alg.hash = {
4068                      .halg.digestsize = SHA3_384_DIGEST_SIZE,
4069                      .halg.base = {
4070                                    .cra_name = "sha3-384",
4071                                    .cra_driver_name = "sha3-384-iproc",
4072                                    .cra_blocksize = SHA3_224_BLOCK_SIZE,
4073                                }
4074                      },
4075         .cipher_info = {
4076                         .alg = CIPHER_ALG_NONE,
4077                         .mode = CIPHER_MODE_NONE,
4078                         },
4079         .auth_info = {
4080                       .alg = HASH_ALG_SHA3_384,
4081                       .mode = HASH_MODE_HASH,
4082                       },
4083         },
4084        {
4085         .type = CRYPTO_ALG_TYPE_AHASH,
4086         .alg.hash = {
4087                      .halg.digestsize = SHA3_384_DIGEST_SIZE,
4088                      .halg.base = {
4089                                    .cra_name = "hmac(sha3-384)",
4090                                    .cra_driver_name = "hmac-sha3-384-iproc",
4091                                    .cra_blocksize = SHA3_384_BLOCK_SIZE,
4092                                }
4093                      },
4094         .cipher_info = {
4095                         .alg = CIPHER_ALG_NONE,
4096