linux/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (C) 2020 Marvell. */
   3
   4#include <crypto/aes.h>
   5#include <crypto/authenc.h>
   6#include <crypto/cryptd.h>
   7#include <crypto/des.h>
   8#include <crypto/internal/aead.h>
   9#include <crypto/sha1.h>
  10#include <crypto/sha2.h>
  11#include <crypto/xts.h>
  12#include <crypto/gcm.h>
  13#include <crypto/scatterwalk.h>
  14#include <linux/rtnetlink.h>
  15#include <linux/sort.h>
  16#include <linux/module.h>
  17#include "otx2_cptvf.h"
  18#include "otx2_cptvf_algs.h"
  19#include "otx2_cpt_reqmgr.h"
  20
  21/* Size of salt in AES GCM mode */
  22#define AES_GCM_SALT_SIZE 4
  23/* Size of IV in AES GCM mode */
  24#define AES_GCM_IV_SIZE 8
  25/* Size of ICV (Integrity Check Value) in AES GCM mode */
  26#define AES_GCM_ICV_SIZE 16
  27/* Offset of IV in AES GCM mode */
  28#define AES_GCM_IV_OFFSET 8
  29#define CONTROL_WORD_LEN 8
  30#define KEY2_OFFSET 48
  31#define DMA_MODE_FLAG(dma_mode) \
  32        (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0)
  33
  34/* Truncated SHA digest size */
  35#define SHA1_TRUNC_DIGEST_SIZE 12
  36#define SHA256_TRUNC_DIGEST_SIZE 16
  37#define SHA384_TRUNC_DIGEST_SIZE 24
  38#define SHA512_TRUNC_DIGEST_SIZE 32
  39
  40static DEFINE_MUTEX(mutex);
  41static int is_crypto_registered;
  42
  43struct cpt_device_desc {
  44        struct pci_dev *dev;
  45        int num_queues;
  46};
  47
  48struct cpt_device_table {
  49        atomic_t count;
  50        struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM];
  51};
  52
  53static struct cpt_device_table se_devices = {
  54        .count = ATOMIC_INIT(0)
  55};
  56
  57static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)
  58{
  59        int count;
  60
  61        count = atomic_read(&se_devices.count);
  62        if (count < 1)
  63                return -ENODEV;
  64
  65        *cpu_num = get_cpu();
  66        /*
  67         * On OcteonTX2 platform CPT instruction queue is bound to each
  68         * local function LF, in turn LFs can be attached to PF
  69         * or VF therefore we always use first device. We get maximum
  70         * performance if one CPT queue is available for each cpu
  71         * otherwise CPT queues need to be shared between cpus.
  72         */
  73        if (*cpu_num >= se_devices.desc[0].num_queues)
  74                *cpu_num %= se_devices.desc[0].num_queues;
  75        *pdev = se_devices.desc[0].dev;
  76
  77        put_cpu();
  78
  79        return 0;
  80}
  81
  82static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)
  83{
  84        struct otx2_cpt_req_ctx *rctx;
  85        struct aead_request *req;
  86        struct crypto_aead *tfm;
  87
  88        req = container_of(cpt_req->areq, struct aead_request, base);
  89        tfm = crypto_aead_reqtfm(req);
  90        rctx = aead_request_ctx(req);
  91        if (memcmp(rctx->fctx.hmac.s.hmac_calc,
  92                   rctx->fctx.hmac.s.hmac_recv,
  93                   crypto_aead_authsize(tfm)) != 0)
  94                return -EBADMSG;
  95
  96        return 0;
  97}
  98
  99static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2)
 100{
 101        struct otx2_cpt_inst_info *inst_info = arg2;
 102        struct crypto_async_request *areq = arg1;
 103        struct otx2_cpt_req_info *cpt_req;
 104        struct pci_dev *pdev;
 105
 106        if (inst_info) {
 107                cpt_req = inst_info->req;
 108                if (!status) {
 109                        /*
 110                         * When selected cipher is NULL we need to manually
 111                         * verify whether calculated hmac value matches
 112                         * received hmac value
 113                         */
 114                        if (cpt_req->req_type ==
 115                            OTX2_CPT_AEAD_ENC_DEC_NULL_REQ &&
 116                            !cpt_req->is_enc)
 117                                status = validate_hmac_cipher_null(cpt_req);
 118                }
 119                pdev = inst_info->pdev;
 120                otx2_cpt_info_destroy(pdev, inst_info);
 121        }
 122        if (areq)
 123                areq->complete(areq, status);
 124}
 125
 126static void output_iv_copyback(struct crypto_async_request *areq)
 127{
 128        struct otx2_cpt_req_info *req_info;
 129        struct otx2_cpt_req_ctx *rctx;
 130        struct skcipher_request *sreq;
 131        struct crypto_skcipher *stfm;
 132        struct otx2_cpt_enc_ctx *ctx;
 133        u32 start, ivsize;
 134
 135        sreq = container_of(areq, struct skcipher_request, base);
 136        stfm = crypto_skcipher_reqtfm(sreq);
 137        ctx = crypto_skcipher_ctx(stfm);
 138        if (ctx->cipher_type == OTX2_CPT_AES_CBC ||
 139            ctx->cipher_type == OTX2_CPT_DES3_CBC) {
 140                rctx = skcipher_request_ctx(sreq);
 141                req_info = &rctx->cpt_req;
 142                ivsize = crypto_skcipher_ivsize(stfm);
 143                start = sreq->cryptlen - ivsize;
 144
 145                if (req_info->is_enc) {
 146                        scatterwalk_map_and_copy(sreq->iv, sreq->dst, start,
 147                                                 ivsize, 0);
 148                } else {
 149                        if (sreq->src != sreq->dst) {
 150                                scatterwalk_map_and_copy(sreq->iv, sreq->src,
 151                                                         start, ivsize, 0);
 152                        } else {
 153                                memcpy(sreq->iv, req_info->iv_out, ivsize);
 154                                kfree(req_info->iv_out);
 155                        }
 156                }
 157        }
 158}
 159
 160static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2)
 161{
 162        struct otx2_cpt_inst_info *inst_info = arg2;
 163        struct crypto_async_request *areq = arg1;
 164        struct pci_dev *pdev;
 165
 166        if (areq) {
 167                if (!status)
 168                        output_iv_copyback(areq);
 169                if (inst_info) {
 170                        pdev = inst_info->pdev;
 171                        otx2_cpt_info_destroy(pdev, inst_info);
 172                }
 173                areq->complete(areq, status);
 174        }
 175}
 176
 177static inline void update_input_data(struct otx2_cpt_req_info *req_info,
 178                                     struct scatterlist *inp_sg,
 179                                     u32 nbytes, u32 *argcnt)
 180{
 181        req_info->req.dlen += nbytes;
 182
 183        while (nbytes) {
 184                u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length;
 185                u8 *ptr = sg_virt(inp_sg);
 186
 187                req_info->in[*argcnt].vptr = (void *)ptr;
 188                req_info->in[*argcnt].size = len;
 189                nbytes -= len;
 190                ++(*argcnt);
 191                inp_sg = sg_next(inp_sg);
 192        }
 193}
 194
 195static inline void update_output_data(struct otx2_cpt_req_info *req_info,
 196                                      struct scatterlist *outp_sg,
 197                                      u32 offset, u32 nbytes, u32 *argcnt)
 198{
 199        u32 len, sg_len;
 200        u8 *ptr;
 201
 202        req_info->rlen += nbytes;
 203
 204        while (nbytes) {
 205                sg_len = outp_sg->length - offset;
 206                len = (nbytes < sg_len) ? nbytes : sg_len;
 207                ptr = sg_virt(outp_sg);
 208
 209                req_info->out[*argcnt].vptr = (void *) (ptr + offset);
 210                req_info->out[*argcnt].size = len;
 211                nbytes -= len;
 212                ++(*argcnt);
 213                offset = 0;
 214                outp_sg = sg_next(outp_sg);
 215        }
 216}
 217
 218static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
 219                                 u32 *argcnt)
 220{
 221        struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
 222        struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
 223        struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
 224        struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
 225        struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
 226        int ivsize = crypto_skcipher_ivsize(stfm);
 227        u32 start = req->cryptlen - ivsize;
 228        gfp_t flags;
 229
 230        flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
 231                        GFP_KERNEL : GFP_ATOMIC;
 232        req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
 233        req_info->ctrl.s.se_req = 1;
 234
 235        req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
 236                                DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
 237        if (enc) {
 238                req_info->req.opcode.s.minor = 2;
 239        } else {
 240                req_info->req.opcode.s.minor = 3;
 241                if ((ctx->cipher_type == OTX2_CPT_AES_CBC ||
 242                    ctx->cipher_type == OTX2_CPT_DES3_CBC) &&
 243                    req->src == req->dst) {
 244                        req_info->iv_out = kmalloc(ivsize, flags);
 245                        if (!req_info->iv_out)
 246                                return -ENOMEM;
 247
 248                        scatterwalk_map_and_copy(req_info->iv_out, req->src,
 249                                                 start, ivsize, 0);
 250                }
 251        }
 252        /* Encryption data length */
 253        req_info->req.param1 = req->cryptlen;
 254        /* Authentication data length */
 255        req_info->req.param2 = 0;
 256
 257        fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
 258        fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
 259        fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
 260
 261        if (ctx->cipher_type == OTX2_CPT_AES_XTS)
 262                memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
 263        else
 264                memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
 265
 266        memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));
 267
 268        cpu_to_be64s(&fctx->enc.enc_ctrl.u);
 269
 270        /*
 271         * Storing  Packet Data Information in offset
 272         * Control Word First 8 bytes
 273         */
 274        req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
 275        req_info->in[*argcnt].size = CONTROL_WORD_LEN;
 276        req_info->req.dlen += CONTROL_WORD_LEN;
 277        ++(*argcnt);
 278
 279        req_info->in[*argcnt].vptr = (u8 *)fctx;
 280        req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
 281        req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
 282
 283        ++(*argcnt);
 284
 285        return 0;
 286}
 287
 288static inline int create_input_list(struct skcipher_request *req, u32 enc,
 289                                    u32 enc_iv_len)
 290{
 291        struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
 292        struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
 293        u32 argcnt =  0;
 294        int ret;
 295
 296        ret = create_ctx_hdr(req, enc, &argcnt);
 297        if (ret)
 298                return ret;
 299
 300        update_input_data(req_info, req->src, req->cryptlen, &argcnt);
 301        req_info->in_cnt = argcnt;
 302
 303        return 0;
 304}
 305
 306static inline void create_output_list(struct skcipher_request *req,
 307                                      u32 enc_iv_len)
 308{
 309        struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
 310        struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
 311        u32 argcnt = 0;
 312
 313        /*
 314         * OUTPUT Buffer Processing
 315         * AES encryption/decryption output would be
 316         * received in the following format
 317         *
 318         * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
 319         * [ 16 Bytes/     [   Request Enc/Dec/ DATA Len AES CBC ]
 320         */
 321        update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt);
 322        req_info->out_cnt = argcnt;
 323}
 324
 325static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
 326{
 327        struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
 328        struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
 329        struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
 330        int ret;
 331
 332        if (ctx->fbk_cipher) {
 333                skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher);
 334                skcipher_request_set_callback(&rctx->sk_fbk_req,
 335                                              req->base.flags,
 336                                              req->base.complete,
 337                                              req->base.data);
 338                skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src,
 339                                           req->dst, req->cryptlen, req->iv);
 340                ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) :
 341                               crypto_skcipher_decrypt(&rctx->sk_fbk_req);
 342        } else {
 343                ret = -EINVAL;
 344        }
 345        return ret;
 346}
 347
 348static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
 349{
 350        struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
 351        struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
 352        struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
 353        struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
 354        u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
 355        struct pci_dev *pdev;
 356        int status, cpu_num;
 357
 358        if (req->cryptlen == 0)
 359                return 0;
 360
 361        if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len))
 362                return -EINVAL;
 363
 364        if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE)
 365                return skcipher_do_fallback(req, enc);
 366
 367        /* Clear control words */
 368        rctx->ctrl_word.flags = 0;
 369        rctx->fctx.enc.enc_ctrl.u = 0;
 370
 371        status = create_input_list(req, enc, enc_iv_len);
 372        if (status)
 373                return status;
 374        create_output_list(req, enc_iv_len);
 375
 376        status = get_se_device(&pdev, &cpu_num);
 377        if (status)
 378                return status;
 379
 380        req_info->callback = otx2_cpt_skcipher_callback;
 381        req_info->areq = &req->base;
 382        req_info->req_type = OTX2_CPT_ENC_DEC_REQ;
 383        req_info->is_enc = enc;
 384        req_info->is_trunc_hmac = false;
 385        req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
 386
 387        /*
 388         * We perform an asynchronous send and once
 389         * the request is completed the driver would
 390         * intimate through registered call back functions
 391         */
 392        status = otx2_cpt_do_request(pdev, req_info, cpu_num);
 393
 394        return status;
 395}
 396
 397static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req)
 398{
 399        return cpt_enc_dec(req, true);
 400}
 401
 402static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req)
 403{
 404        return cpt_enc_dec(req, false);
 405}
 406
 407static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,
 408                                       const u8 *key, u32 keylen)
 409{
 410        struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
 411        const u8 *key2 = key + (keylen / 2);
 412        const u8 *key1 = key;
 413        int ret;
 414
 415        ret = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen);
 416        if (ret)
 417                return ret;
 418        ctx->key_len = keylen;
 419        ctx->enc_align_len = 1;
 420        memcpy(ctx->enc_key, key1, keylen / 2);
 421        memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
 422        ctx->cipher_type = OTX2_CPT_AES_XTS;
 423        switch (ctx->key_len) {
 424        case 2 * AES_KEYSIZE_128:
 425                ctx->key_type = OTX2_CPT_AES_128_BIT;
 426                break;
 427        case 2 * AES_KEYSIZE_192:
 428                ctx->key_type = OTX2_CPT_AES_192_BIT;
 429                break;
 430        case 2 * AES_KEYSIZE_256:
 431                ctx->key_type = OTX2_CPT_AES_256_BIT;
 432                break;
 433        default:
 434                return -EINVAL;
 435        }
 436        return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
 437}
 438
 439static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
 440                          u32 keylen, u8 cipher_type)
 441{
 442        struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
 443
 444        if (keylen != DES3_EDE_KEY_SIZE)
 445                return -EINVAL;
 446
 447        ctx->key_len = keylen;
 448        ctx->cipher_type = cipher_type;
 449        ctx->enc_align_len = 8;
 450
 451        memcpy(ctx->enc_key, key, keylen);
 452
 453        return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
 454}
 455
 456static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
 457                          u32 keylen, u8 cipher_type)
 458{
 459        struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
 460
 461        switch (keylen) {
 462        case AES_KEYSIZE_128:
 463                ctx->key_type = OTX2_CPT_AES_128_BIT;
 464                break;
 465        case AES_KEYSIZE_192:
 466                ctx->key_type = OTX2_CPT_AES_192_BIT;
 467                break;
 468        case AES_KEYSIZE_256:
 469                ctx->key_type = OTX2_CPT_AES_256_BIT;
 470                break;
 471        default:
 472                return -EINVAL;
 473        }
 474        if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB)
 475                ctx->enc_align_len = 16;
 476        else
 477                ctx->enc_align_len = 1;
 478
 479        ctx->key_len = keylen;
 480        ctx->cipher_type = cipher_type;
 481
 482        memcpy(ctx->enc_key, key, keylen);
 483
 484        return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
 485}
 486
 487static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,
 488                                            const u8 *key, u32 keylen)
 489{
 490        return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC);
 491}
 492
 493static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,
 494                                            const u8 *key, u32 keylen)
 495{
 496        return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB);
 497}
 498
 499static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,
 500                                             const u8 *key, u32 keylen)
 501{
 502        return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC);
 503}
 504
 505static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,
 506                                             const u8 *key, u32 keylen)
 507{
 508        return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB);
 509}
 510
 511static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx,
 512                                      struct crypto_alg *alg)
 513{
 514        if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
 515                ctx->fbk_cipher =
 516                                crypto_alloc_skcipher(alg->cra_name, 0,
 517                                                      CRYPTO_ALG_ASYNC |
 518                                                      CRYPTO_ALG_NEED_FALLBACK);
 519                if (IS_ERR(ctx->fbk_cipher)) {
 520                        pr_err("%s() failed to allocate fallback for %s\n",
 521                                __func__, alg->cra_name);
 522                        return PTR_ERR(ctx->fbk_cipher);
 523                }
 524        }
 525        return 0;
 526}
 527
 528static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
 529{
 530        struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
 531        struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
 532        struct crypto_alg *alg = tfm->__crt_alg;
 533
 534        memset(ctx, 0, sizeof(*ctx));
 535        /*
 536         * Additional memory for skcipher_request is
 537         * allocated since the cryptd daemon uses
 538         * this memory for request_ctx information
 539         */
 540        crypto_skcipher_set_reqsize(stfm, sizeof(struct otx2_cpt_req_ctx) +
 541                                        sizeof(struct skcipher_request));
 542
 543        return cpt_skcipher_fallback_init(ctx, alg);
 544}
 545
 546static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm)
 547{
 548        struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
 549
 550        if (ctx->fbk_cipher) {
 551                crypto_free_skcipher(ctx->fbk_cipher);
 552                ctx->fbk_cipher = NULL;
 553        }
 554}
 555
 556static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
 557                                  struct crypto_alg *alg)
 558{
 559        if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
 560                ctx->fbk_cipher =
 561                            crypto_alloc_aead(alg->cra_name, 0,
 562                                              CRYPTO_ALG_ASYNC |
 563                                              CRYPTO_ALG_NEED_FALLBACK);
 564                if (IS_ERR(ctx->fbk_cipher)) {
 565                        pr_err("%s() failed to allocate fallback for %s\n",
 566                                __func__, alg->cra_name);
 567                        return PTR_ERR(ctx->fbk_cipher);
 568                }
 569        }
 570        return 0;
 571}
 572
 573static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
 574{
 575        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(atfm);
 576        struct crypto_tfm *tfm = crypto_aead_tfm(atfm);
 577        struct crypto_alg *alg = tfm->__crt_alg;
 578
 579        ctx->cipher_type = cipher_type;
 580        ctx->mac_type = mac_type;
 581
 582        /*
 583         * When selected cipher is NULL we use HMAC opcode instead of
 584         * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms
 585         * for calculating ipad and opad
 586         */
 587        if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL) {
 588                switch (ctx->mac_type) {
 589                case OTX2_CPT_SHA1:
 590                        ctx->hashalg = crypto_alloc_shash("sha1", 0,
 591                                                          CRYPTO_ALG_ASYNC);
 592                        if (IS_ERR(ctx->hashalg))
 593                                return PTR_ERR(ctx->hashalg);
 594                        break;
 595
 596                case OTX2_CPT_SHA256:
 597                        ctx->hashalg = crypto_alloc_shash("sha256", 0,
 598                                                          CRYPTO_ALG_ASYNC);
 599                        if (IS_ERR(ctx->hashalg))
 600                                return PTR_ERR(ctx->hashalg);
 601                        break;
 602
 603                case OTX2_CPT_SHA384:
 604                        ctx->hashalg = crypto_alloc_shash("sha384", 0,
 605                                                          CRYPTO_ALG_ASYNC);
 606                        if (IS_ERR(ctx->hashalg))
 607                                return PTR_ERR(ctx->hashalg);
 608                        break;
 609
 610                case OTX2_CPT_SHA512:
 611                        ctx->hashalg = crypto_alloc_shash("sha512", 0,
 612                                                          CRYPTO_ALG_ASYNC);
 613                        if (IS_ERR(ctx->hashalg))
 614                                return PTR_ERR(ctx->hashalg);
 615                        break;
 616                }
 617        }
 618        switch (ctx->cipher_type) {
 619        case OTX2_CPT_AES_CBC:
 620        case OTX2_CPT_AES_ECB:
 621                ctx->enc_align_len = 16;
 622                break;
 623        case OTX2_CPT_DES3_CBC:
 624        case OTX2_CPT_DES3_ECB:
 625                ctx->enc_align_len = 8;
 626                break;
 627        case OTX2_CPT_AES_GCM:
 628        case OTX2_CPT_CIPHER_NULL:
 629                ctx->enc_align_len = 1;
 630                break;
 631        }
 632        crypto_aead_set_reqsize(atfm, sizeof(struct otx2_cpt_req_ctx));
 633
 634        return cpt_aead_fallback_init(ctx, alg);
 635}
 636
 637static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)
 638{
 639        return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1);
 640}
 641
 642static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)
 643{
 644        return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256);
 645}
 646
 647static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)
 648{
 649        return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384);
 650}
 651
 652static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)
 653{
 654        return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512);
 655}
 656
 657static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)
 658{
 659        return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1);
 660}
 661
 662static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)
 663{
 664        return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256);
 665}
 666
 667static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)
 668{
 669        return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384);
 670}
 671
 672static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)
 673{
 674        return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512);
 675}
 676
 677static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
 678{
 679        return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL);
 680}
 681
 682static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
 683{
 684        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
 685
 686        kfree(ctx->ipad);
 687        kfree(ctx->opad);
 688        if (ctx->hashalg)
 689                crypto_free_shash(ctx->hashalg);
 690        kfree(ctx->sdesc);
 691
 692        if (ctx->fbk_cipher) {
 693                crypto_free_aead(ctx->fbk_cipher);
 694                ctx->fbk_cipher = NULL;
 695        }
 696}
 697
 698static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
 699                                          unsigned int authsize)
 700{
 701        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
 702
 703        if (crypto_rfc4106_check_authsize(authsize))
 704                return -EINVAL;
 705
 706        tfm->authsize = authsize;
 707        /* Set authsize for fallback case */
 708        if (ctx->fbk_cipher)
 709                ctx->fbk_cipher->authsize = authsize;
 710
 711        return 0;
 712}
 713
 714static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,
 715                                      unsigned int authsize)
 716{
 717        tfm->authsize = authsize;
 718
 719        return 0;
 720}
 721
 722static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm,
 723                                           unsigned int authsize)
 724{
 725        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
 726
 727        ctx->is_trunc_hmac = true;
 728        tfm->authsize = authsize;
 729
 730        return 0;
 731}
 732
 733static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)
 734{
 735        struct otx2_cpt_sdesc *sdesc;
 736        int size;
 737
 738        size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);
 739        sdesc = kmalloc(size, GFP_KERNEL);
 740        if (!sdesc)
 741                return NULL;
 742
 743        sdesc->shash.tfm = alg;
 744
 745        return sdesc;
 746}
 747
 748static inline void swap_data32(void *buf, u32 len)
 749{
 750        cpu_to_be32_array(buf, buf, len / 4);
 751}
 752
 753static inline void swap_data64(void *buf, u32 len)
 754{
 755        u64 *src = buf;
 756        int i = 0;
 757
 758        for (i = 0 ; i < len / 8; i++, src++)
 759                cpu_to_be64s(src);
 760}
 761
 762static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)
 763{
 764        struct sha512_state *sha512;
 765        struct sha256_state *sha256;
 766        struct sha1_state *sha1;
 767
 768        switch (mac_type) {
 769        case OTX2_CPT_SHA1:
 770                sha1 = (struct sha1_state *) in_pad;
 771                swap_data32(sha1->state, SHA1_DIGEST_SIZE);
 772                memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE);
 773                break;
 774
 775        case OTX2_CPT_SHA256:
 776                sha256 = (struct sha256_state *) in_pad;
 777                swap_data32(sha256->state, SHA256_DIGEST_SIZE);
 778                memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE);
 779                break;
 780
 781        case OTX2_CPT_SHA384:
 782        case OTX2_CPT_SHA512:
 783                sha512 = (struct sha512_state *) in_pad;
 784                swap_data64(sha512->state, SHA512_DIGEST_SIZE);
 785                memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE);
 786                break;
 787
 788        default:
 789                return -EINVAL;
 790        }
 791
 792        return 0;
 793}
 794
 795static int aead_hmac_init(struct crypto_aead *cipher)
 796{
 797        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
 798        int state_size = crypto_shash_statesize(ctx->hashalg);
 799        int ds = crypto_shash_digestsize(ctx->hashalg);
 800        int bs = crypto_shash_blocksize(ctx->hashalg);
 801        int authkeylen = ctx->auth_key_len;
 802        u8 *ipad = NULL, *opad = NULL;
 803        int ret = 0, icount = 0;
 804
 805        ctx->sdesc = alloc_sdesc(ctx->hashalg);
 806        if (!ctx->sdesc)
 807                return -ENOMEM;
 808
 809        ctx->ipad = kzalloc(bs, GFP_KERNEL);
 810        if (!ctx->ipad) {
 811                ret = -ENOMEM;
 812                goto calc_fail;
 813        }
 814
 815        ctx->opad = kzalloc(bs, GFP_KERNEL);
 816        if (!ctx->opad) {
 817                ret = -ENOMEM;
 818                goto calc_fail;
 819        }
 820
 821        ipad = kzalloc(state_size, GFP_KERNEL);
 822        if (!ipad) {
 823                ret = -ENOMEM;
 824                goto calc_fail;
 825        }
 826
 827        opad = kzalloc(state_size, GFP_KERNEL);
 828        if (!opad) {
 829                ret = -ENOMEM;
 830                goto calc_fail;
 831        }
 832
 833        if (authkeylen > bs) {
 834                ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key,
 835                                          authkeylen, ipad);
 836                if (ret)
 837                        goto calc_fail;
 838
 839                authkeylen = ds;
 840        } else {
 841                memcpy(ipad, ctx->key, authkeylen);
 842        }
 843
 844        memset(ipad + authkeylen, 0, bs - authkeylen);
 845        memcpy(opad, ipad, bs);
 846
 847        for (icount = 0; icount < bs; icount++) {
 848                ipad[icount] ^= 0x36;
 849                opad[icount] ^= 0x5c;
 850        }
 851
 852        /*
 853         * Partial Hash calculated from the software
 854         * algorithm is retrieved for IPAD & OPAD
 855         */
 856
 857        /* IPAD Calculation */
 858        crypto_shash_init(&ctx->sdesc->shash);
 859        crypto_shash_update(&ctx->sdesc->shash, ipad, bs);
 860        crypto_shash_export(&ctx->sdesc->shash, ipad);
 861        ret = copy_pad(ctx->mac_type, ctx->ipad, ipad);
 862        if (ret)
 863                goto calc_fail;
 864
 865        /* OPAD Calculation */
 866        crypto_shash_init(&ctx->sdesc->shash);
 867        crypto_shash_update(&ctx->sdesc->shash, opad, bs);
 868        crypto_shash_export(&ctx->sdesc->shash, opad);
 869        ret = copy_pad(ctx->mac_type, ctx->opad, opad);
 870        if (ret)
 871                goto calc_fail;
 872
 873        kfree(ipad);
 874        kfree(opad);
 875
 876        return 0;
 877
 878calc_fail:
 879        kfree(ctx->ipad);
 880        ctx->ipad = NULL;
 881        kfree(ctx->opad);
 882        ctx->opad = NULL;
 883        kfree(ipad);
 884        kfree(opad);
 885        kfree(ctx->sdesc);
 886        ctx->sdesc = NULL;
 887
 888        return ret;
 889}
 890
 891static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
 892                                            const unsigned char *key,
 893                                            unsigned int keylen)
 894{
 895        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
 896        struct crypto_authenc_key_param *param;
 897        int enckeylen = 0, authkeylen = 0;
 898        struct rtattr *rta = (void *)key;
 899        int status;
 900
 901        if (!RTA_OK(rta, keylen))
 902                return -EINVAL;
 903
 904        if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
 905                return -EINVAL;
 906
 907        if (RTA_PAYLOAD(rta) < sizeof(*param))
 908                return -EINVAL;
 909
 910        param = RTA_DATA(rta);
 911        enckeylen = be32_to_cpu(param->enckeylen);
 912        key += RTA_ALIGN(rta->rta_len);
 913        keylen -= RTA_ALIGN(rta->rta_len);
 914        if (keylen < enckeylen)
 915                return -EINVAL;
 916
 917        if (keylen > OTX2_CPT_MAX_KEY_SIZE)
 918                return -EINVAL;
 919
 920        authkeylen = keylen - enckeylen;
 921        memcpy(ctx->key, key, keylen);
 922
 923        switch (enckeylen) {
 924        case AES_KEYSIZE_128:
 925                ctx->key_type = OTX2_CPT_AES_128_BIT;
 926                break;
 927        case AES_KEYSIZE_192:
 928                ctx->key_type = OTX2_CPT_AES_192_BIT;
 929                break;
 930        case AES_KEYSIZE_256:
 931                ctx->key_type = OTX2_CPT_AES_256_BIT;
 932                break;
 933        default:
 934                /* Invalid key length */
 935                return -EINVAL;
 936        }
 937
 938        ctx->enc_key_len = enckeylen;
 939        ctx->auth_key_len = authkeylen;
 940
 941        status = aead_hmac_init(cipher);
 942        if (status)
 943                return status;
 944
 945        return 0;
 946}
 947
 948static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
 949                                             const unsigned char *key,
 950                                             unsigned int keylen)
 951{
 952        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
 953        struct crypto_authenc_key_param *param;
 954        struct rtattr *rta = (void *)key;
 955        int enckeylen = 0;
 956
 957        if (!RTA_OK(rta, keylen))
 958                return -EINVAL;
 959
 960        if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
 961                return -EINVAL;
 962
 963        if (RTA_PAYLOAD(rta) < sizeof(*param))
 964                return -EINVAL;
 965
 966        param = RTA_DATA(rta);
 967        enckeylen = be32_to_cpu(param->enckeylen);
 968        key += RTA_ALIGN(rta->rta_len);
 969        keylen -= RTA_ALIGN(rta->rta_len);
 970        if (enckeylen != 0)
 971                return -EINVAL;
 972
 973        if (keylen > OTX2_CPT_MAX_KEY_SIZE)
 974                return -EINVAL;
 975
 976        memcpy(ctx->key, key, keylen);
 977        ctx->enc_key_len = enckeylen;
 978        ctx->auth_key_len = keylen;
 979
 980        return 0;
 981}
 982
 983static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
 984                                        const unsigned char *key,
 985                                        unsigned int keylen)
 986{
 987        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
 988
 989        /*
 990         * For aes gcm we expect to get encryption key (16, 24, 32 bytes)
 991         * and salt (4 bytes)
 992         */
 993        switch (keylen) {
 994        case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:
 995                ctx->key_type = OTX2_CPT_AES_128_BIT;
 996                ctx->enc_key_len = AES_KEYSIZE_128;
 997                break;
 998        case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:
 999                ctx->key_type = OTX2_CPT_AES_192_BIT;
1000                ctx->enc_key_len = AES_KEYSIZE_192;
1001                break;
1002        case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:
1003                ctx->key_type = OTX2_CPT_AES_256_BIT;
1004                ctx->enc_key_len = AES_KEYSIZE_256;
1005                break;
1006        default:
1007                /* Invalid key and salt length */
1008                return -EINVAL;
1009        }
1010
1011        /* Store encryption key and salt */
1012        memcpy(ctx->key, key, keylen);
1013
1014        return crypto_aead_setkey(ctx->fbk_cipher, key, keylen);
1015}
1016
1017static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
1018                                      u32 *argcnt)
1019{
1020        struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1021        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1022        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1023        struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1024        struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
1025        int mac_len = crypto_aead_authsize(tfm);
1026        int ds;
1027
1028        rctx->ctrl_word.e.enc_data_offset = req->assoclen;
1029
1030        switch (ctx->cipher_type) {
1031        case OTX2_CPT_AES_CBC:
1032                if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8))
1033                        return -EINVAL;
1034
1035                fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
1036                /* Copy encryption key to context */
1037                memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,
1038                       ctx->enc_key_len);
1039                /* Copy IV to context */
1040                memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));
1041
1042                ds = crypto_shash_digestsize(ctx->hashalg);
1043                if (ctx->mac_type == OTX2_CPT_SHA384)
1044                        ds = SHA512_DIGEST_SIZE;
1045                if (ctx->ipad)
1046                        memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);
1047                if (ctx->opad)
1048                        memcpy(fctx->hmac.e.opad, ctx->opad, ds);
1049                break;
1050
1051        case OTX2_CPT_AES_GCM:
1052                if (crypto_ipsec_check_assoclen(req->assoclen))
1053                        return -EINVAL;
1054
1055                fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR;
1056                /* Copy encryption key to context */
1057                memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);
1058                /* Copy salt to context */
1059                memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,
1060                       AES_GCM_SALT_SIZE);
1061
1062                rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;
1063                break;
1064
1065        default:
1066                /* Unknown cipher type */
1067                return -EINVAL;
1068        }
1069        cpu_to_be64s(&rctx->ctrl_word.flags);
1070
1071        req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1072        req_info->ctrl.s.se_req = 1;
1073        req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
1074                                 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1075        if (enc) {
1076                req_info->req.opcode.s.minor = 2;
1077                req_info->req.param1 = req->cryptlen;
1078                req_info->req.param2 = req->cryptlen + req->assoclen;
1079        } else {
1080                req_info->req.opcode.s.minor = 3;
1081                req_info->req.param1 = req->cryptlen - mac_len;
1082                req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;
1083        }
1084
1085        fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
1086        fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
1087        fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;
1088        fctx->enc.enc_ctrl.e.mac_len = mac_len;
1089        cpu_to_be64s(&fctx->enc.enc_ctrl.u);
1090
1091        /*
1092         * Storing Packet Data Information in offset
1093         * Control Word First 8 bytes
1094         */
1095        req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
1096        req_info->in[*argcnt].size = CONTROL_WORD_LEN;
1097        req_info->req.dlen += CONTROL_WORD_LEN;
1098        ++(*argcnt);
1099
1100        req_info->in[*argcnt].vptr = (u8 *)fctx;
1101        req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
1102        req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
1103        ++(*argcnt);
1104
1105        return 0;
1106}
1107
1108static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
1109                                      u32 enc)
1110{
1111        struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1112        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1113        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1114        struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1115
1116        req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1117        req_info->ctrl.s.se_req = 1;
1118        req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC |
1119                                 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1120        req_info->is_trunc_hmac = ctx->is_trunc_hmac;
1121
1122        req_info->req.opcode.s.minor = 0;
1123        req_info->req.param1 = ctx->auth_key_len;
1124        req_info->req.param2 = ctx->mac_type << 8;
1125
1126        /* Add authentication key */
1127        req_info->in[*argcnt].vptr = ctx->key;
1128        req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);
1129        req_info->req.dlen += round_up(ctx->auth_key_len, 8);
1130        ++(*argcnt);
1131}
1132
1133static inline int create_aead_input_list(struct aead_request *req, u32 enc)
1134{
1135        struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1136        struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1137        u32 inputlen =  req->cryptlen + req->assoclen;
1138        u32 status, argcnt = 0;
1139
1140        status = create_aead_ctx_hdr(req, enc, &argcnt);
1141        if (status)
1142                return status;
1143        update_input_data(req_info, req->src, inputlen, &argcnt);
1144        req_info->in_cnt = argcnt;
1145
1146        return 0;
1147}
1148
1149static inline void create_aead_output_list(struct aead_request *req, u32 enc,
1150                                           u32 mac_len)
1151{
1152        struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1153        struct otx2_cpt_req_info *req_info =  &rctx->cpt_req;
1154        u32 argcnt = 0, outputlen = 0;
1155
1156        if (enc)
1157                outputlen = req->cryptlen +  req->assoclen + mac_len;
1158        else
1159                outputlen = req->cryptlen + req->assoclen - mac_len;
1160
1161        update_output_data(req_info, req->dst, 0, outputlen, &argcnt);
1162        req_info->out_cnt = argcnt;
1163}
1164
1165static inline void create_aead_null_input_list(struct aead_request *req,
1166                                               u32 enc, u32 mac_len)
1167{
1168        struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1169        struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1170        u32 inputlen, argcnt = 0;
1171
1172        if (enc)
1173                inputlen =  req->cryptlen + req->assoclen;
1174        else
1175                inputlen =  req->cryptlen + req->assoclen - mac_len;
1176
1177        create_hmac_ctx_hdr(req, &argcnt, enc);
1178        update_input_data(req_info, req->src, inputlen, &argcnt);
1179        req_info->in_cnt = argcnt;
1180}
1181
1182static inline int create_aead_null_output_list(struct aead_request *req,
1183                                               u32 enc, u32 mac_len)
1184{
1185        struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1186        struct otx2_cpt_req_info *req_info =  &rctx->cpt_req;
1187        struct scatterlist *dst;
1188        u8 *ptr = NULL;
1189        int argcnt = 0, status, offset;
1190        u32 inputlen;
1191
1192        if (enc)
1193                inputlen =  req->cryptlen + req->assoclen;
1194        else
1195                inputlen =  req->cryptlen + req->assoclen - mac_len;
1196
1197        /*
1198         * If source and destination are different
1199         * then copy payload to destination
1200         */
1201        if (req->src != req->dst) {
1202
1203                ptr = kmalloc(inputlen, (req_info->areq->flags &
1204                                         CRYPTO_TFM_REQ_MAY_SLEEP) ?
1205                                         GFP_KERNEL : GFP_ATOMIC);
1206                if (!ptr)
1207                        return -ENOMEM;
1208
1209                status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr,
1210                                           inputlen);
1211                if (status != inputlen) {
1212                        status = -EINVAL;
1213                        goto error_free;
1214                }
1215                status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr,
1216                                             inputlen);
1217                if (status != inputlen) {
1218                        status = -EINVAL;
1219                        goto error_free;
1220                }
1221                kfree(ptr);
1222        }
1223
1224        if (enc) {
1225                /*
1226                 * In an encryption scenario hmac needs
1227                 * to be appended after payload
1228                 */
1229                dst = req->dst;
1230                offset = inputlen;
1231                while (offset >= dst->length) {
1232                        offset -= dst->length;
1233                        dst = sg_next(dst);
1234                        if (!dst)
1235                                return -ENOENT;
1236                }
1237
1238                update_output_data(req_info, dst, offset, mac_len, &argcnt);
1239        } else {
1240                /*
1241                 * In a decryption scenario calculated hmac for received
1242                 * payload needs to be compare with hmac received
1243                 */
1244                status = sg_copy_buffer(req->src, sg_nents(req->src),
1245                                        rctx->fctx.hmac.s.hmac_recv, mac_len,
1246                                        inputlen, true);
1247                if (status != mac_len)
1248                        return -EINVAL;
1249
1250                req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;
1251                req_info->out[argcnt].size = mac_len;
1252                argcnt++;
1253        }
1254
1255        req_info->out_cnt = argcnt;
1256        return 0;
1257
1258error_free:
1259        kfree(ptr);
1260        return status;
1261}
1262
1263static int aead_do_fallback(struct aead_request *req, bool is_enc)
1264{
1265        struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1266        struct crypto_aead *aead = crypto_aead_reqtfm(req);
1267        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(aead);
1268        int ret;
1269
1270        if (ctx->fbk_cipher) {
1271                /* Store the cipher tfm and then use the fallback tfm */
1272                aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher);
1273                aead_request_set_callback(&rctx->fbk_req, req->base.flags,
1274                                          req->base.complete, req->base.data);
1275                aead_request_set_crypt(&rctx->fbk_req, req->src,
1276                                       req->dst, req->cryptlen, req->iv);
1277                ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) :
1278                               crypto_aead_decrypt(&rctx->fbk_req);
1279        } else {
1280                ret = -EINVAL;
1281        }
1282
1283        return ret;
1284}
1285
1286static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
1287{
1288        struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1289        struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1290        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1291        struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1292        struct pci_dev *pdev;
1293        int status, cpu_num;
1294
1295        /* Clear control words */
1296        rctx->ctrl_word.flags = 0;
1297        rctx->fctx.enc.enc_ctrl.u = 0;
1298
1299        req_info->callback = otx2_cpt_aead_callback;
1300        req_info->areq = &req->base;
1301        req_info->req_type = reg_type;
1302        req_info->is_enc = enc;
1303        req_info->is_trunc_hmac = false;
1304
1305        switch (reg_type) {
1306        case OTX2_CPT_AEAD_ENC_DEC_REQ:
1307                status = create_aead_input_list(req, enc);
1308                if (status)
1309                        return status;
1310                create_aead_output_list(req, enc, crypto_aead_authsize(tfm));
1311                break;
1312
1313        case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ:
1314                create_aead_null_input_list(req, enc,
1315                                            crypto_aead_authsize(tfm));
1316                status = create_aead_null_output_list(req, enc,
1317                                                crypto_aead_authsize(tfm));
1318                if (status)
1319                        return status;
1320                break;
1321
1322        default:
1323                return -EINVAL;
1324        }
1325        if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len))
1326                return -EINVAL;
1327
1328        if (!req_info->req.param2 ||
1329            (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) ||
1330            (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE))
1331                return aead_do_fallback(req, enc);
1332
1333        status = get_se_device(&pdev, &cpu_num);
1334        if (status)
1335                return status;
1336
1337        req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
1338
1339        /*
1340         * We perform an asynchronous send and once
1341         * the request is completed the driver would
1342         * intimate through registered call back functions
1343         */
1344        return otx2_cpt_do_request(pdev, req_info, cpu_num);
1345}
1346
1347static int otx2_cpt_aead_encrypt(struct aead_request *req)
1348{
1349        return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true);
1350}
1351
1352static int otx2_cpt_aead_decrypt(struct aead_request *req)
1353{
1354        return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false);
1355}
1356
1357static int otx2_cpt_aead_null_encrypt(struct aead_request *req)
1358{
1359        return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true);
1360}
1361
1362static int otx2_cpt_aead_null_decrypt(struct aead_request *req)
1363{
1364        return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false);
1365}
1366
1367static struct skcipher_alg otx2_cpt_skciphers[] = { {
1368        .base.cra_name = "xts(aes)",
1369        .base.cra_driver_name = "cpt_xts_aes",
1370        .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1371        .base.cra_blocksize = AES_BLOCK_SIZE,
1372        .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1373        .base.cra_alignmask = 7,
1374        .base.cra_priority = 4001,
1375        .base.cra_module = THIS_MODULE,
1376
1377        .init = otx2_cpt_enc_dec_init,
1378        .exit = otx2_cpt_skcipher_exit,
1379        .ivsize = AES_BLOCK_SIZE,
1380        .min_keysize = 2 * AES_MIN_KEY_SIZE,
1381        .max_keysize = 2 * AES_MAX_KEY_SIZE,
1382        .setkey = otx2_cpt_skcipher_xts_setkey,
1383        .encrypt = otx2_cpt_skcipher_encrypt,
1384        .decrypt = otx2_cpt_skcipher_decrypt,
1385}, {
1386        .base.cra_name = "cbc(aes)",
1387        .base.cra_driver_name = "cpt_cbc_aes",
1388        .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1389        .base.cra_blocksize = AES_BLOCK_SIZE,
1390        .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1391        .base.cra_alignmask = 7,
1392        .base.cra_priority = 4001,
1393        .base.cra_module = THIS_MODULE,
1394
1395        .init = otx2_cpt_enc_dec_init,
1396        .exit = otx2_cpt_skcipher_exit,
1397        .ivsize = AES_BLOCK_SIZE,
1398        .min_keysize = AES_MIN_KEY_SIZE,
1399        .max_keysize = AES_MAX_KEY_SIZE,
1400        .setkey = otx2_cpt_skcipher_cbc_aes_setkey,
1401        .encrypt = otx2_cpt_skcipher_encrypt,
1402        .decrypt = otx2_cpt_skcipher_decrypt,
1403}, {
1404        .base.cra_name = "ecb(aes)",
1405        .base.cra_driver_name = "cpt_ecb_aes",
1406        .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1407        .base.cra_blocksize = AES_BLOCK_SIZE,
1408        .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1409        .base.cra_alignmask = 7,
1410        .base.cra_priority = 4001,
1411        .base.cra_module = THIS_MODULE,
1412
1413        .init = otx2_cpt_enc_dec_init,
1414        .exit = otx2_cpt_skcipher_exit,
1415        .ivsize = 0,
1416        .min_keysize = AES_MIN_KEY_SIZE,
1417        .max_keysize = AES_MAX_KEY_SIZE,
1418        .setkey = otx2_cpt_skcipher_ecb_aes_setkey,
1419        .encrypt = otx2_cpt_skcipher_encrypt,
1420        .decrypt = otx2_cpt_skcipher_decrypt,
1421}, {
1422        .base.cra_name = "cbc(des3_ede)",
1423        .base.cra_driver_name = "cpt_cbc_des3_ede",
1424        .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1425        .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1426        .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1427        .base.cra_alignmask = 7,
1428        .base.cra_priority = 4001,
1429        .base.cra_module = THIS_MODULE,
1430
1431        .init = otx2_cpt_enc_dec_init,
1432        .exit = otx2_cpt_skcipher_exit,
1433        .min_keysize = DES3_EDE_KEY_SIZE,
1434        .max_keysize = DES3_EDE_KEY_SIZE,
1435        .ivsize = DES_BLOCK_SIZE,
1436        .setkey = otx2_cpt_skcipher_cbc_des3_setkey,
1437        .encrypt = otx2_cpt_skcipher_encrypt,
1438        .decrypt = otx2_cpt_skcipher_decrypt,
1439}, {
1440        .base.cra_name = "ecb(des3_ede)",
1441        .base.cra_driver_name = "cpt_ecb_des3_ede",
1442        .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1443        .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1444        .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1445        .base.cra_alignmask = 7,
1446        .base.cra_priority = 4001,
1447        .base.cra_module = THIS_MODULE,
1448
1449        .init = otx2_cpt_enc_dec_init,
1450        .exit = otx2_cpt_skcipher_exit,
1451        .min_keysize = DES3_EDE_KEY_SIZE,
1452        .max_keysize = DES3_EDE_KEY_SIZE,
1453        .ivsize = 0,
1454        .setkey = otx2_cpt_skcipher_ecb_des3_setkey,
1455        .encrypt = otx2_cpt_skcipher_encrypt,
1456        .decrypt = otx2_cpt_skcipher_decrypt,
1457} };
1458
1459static struct aead_alg otx2_cpt_aeads[] = { {
1460        .base = {
1461                .cra_name = "authenc(hmac(sha1),cbc(aes))",
1462                .cra_driver_name = "cpt_hmac_sha1_cbc_aes",
1463                .cra_blocksize = AES_BLOCK_SIZE,
1464                .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1465                .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1466                .cra_priority = 4001,
1467                .cra_alignmask = 0,
1468                .cra_module = THIS_MODULE,
1469        },
1470        .init = otx2_cpt_aead_cbc_aes_sha1_init,
1471        .exit = otx2_cpt_aead_exit,
1472        .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1473        .setauthsize = otx2_cpt_aead_set_authsize,
1474        .encrypt = otx2_cpt_aead_encrypt,
1475        .decrypt = otx2_cpt_aead_decrypt,
1476        .ivsize = AES_BLOCK_SIZE,
1477        .maxauthsize = SHA1_DIGEST_SIZE,
1478}, {
1479        .base = {
1480                .cra_name = "authenc(hmac(sha256),cbc(aes))",
1481                .cra_driver_name = "cpt_hmac_sha256_cbc_aes",
1482                .cra_blocksize = AES_BLOCK_SIZE,
1483                .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1484                .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1485                .cra_priority = 4001,
1486                .cra_alignmask = 0,
1487                .cra_module = THIS_MODULE,
1488        },
1489        .init = otx2_cpt_aead_cbc_aes_sha256_init,
1490        .exit = otx2_cpt_aead_exit,
1491        .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1492        .setauthsize = otx2_cpt_aead_set_authsize,
1493        .encrypt = otx2_cpt_aead_encrypt,
1494        .decrypt = otx2_cpt_aead_decrypt,
1495        .ivsize = AES_BLOCK_SIZE,
1496        .maxauthsize = SHA256_DIGEST_SIZE,
1497}, {
1498        .base = {
1499                .cra_name = "authenc(hmac(sha384),cbc(aes))",
1500                .cra_driver_name = "cpt_hmac_sha384_cbc_aes",
1501                .cra_blocksize = AES_BLOCK_SIZE,
1502                .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1503                .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1504                .cra_priority = 4001,
1505                .cra_alignmask = 0,
1506                .cra_module = THIS_MODULE,
1507        },
1508        .init = otx2_cpt_aead_cbc_aes_sha384_init,
1509        .exit = otx2_cpt_aead_exit,
1510        .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1511        .setauthsize = otx2_cpt_aead_set_authsize,
1512        .encrypt = otx2_cpt_aead_encrypt,
1513        .decrypt = otx2_cpt_aead_decrypt,
1514        .ivsize = AES_BLOCK_SIZE,
1515        .maxauthsize = SHA384_DIGEST_SIZE,
1516}, {
1517        .base = {
1518                .cra_name = "authenc(hmac(sha512),cbc(aes))",
1519                .cra_driver_name = "cpt_hmac_sha512_cbc_aes",
1520                .cra_blocksize = AES_BLOCK_SIZE,
1521                .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1522                .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1523                .cra_priority = 4001,
1524                .cra_alignmask = 0,
1525                .cra_module = THIS_MODULE,
1526        },
1527        .init = otx2_cpt_aead_cbc_aes_sha512_init,
1528        .exit = otx2_cpt_aead_exit,
1529        .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1530        .setauthsize = otx2_cpt_aead_set_authsize,
1531        .encrypt = otx2_cpt_aead_encrypt,
1532        .decrypt = otx2_cpt_aead_decrypt,
1533        .ivsize = AES_BLOCK_SIZE,
1534        .maxauthsize = SHA512_DIGEST_SIZE,
1535}, {
1536        .base = {
1537                .cra_name = "authenc(hmac(sha1),ecb(cipher_null))",
1538                .cra_driver_name = "cpt_hmac_sha1_ecb_null",
1539                .cra_blocksize = 1,
1540                .cra_flags = CRYPTO_ALG_ASYNC,
1541                .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1542                .cra_priority = 4001,
1543                .cra_alignmask = 0,
1544                .cra_module = THIS_MODULE,
1545        },
1546        .init = otx2_cpt_aead_ecb_null_sha1_init,
1547        .exit = otx2_cpt_aead_exit,
1548        .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1549        .setauthsize = otx2_cpt_aead_null_set_authsize,
1550        .encrypt = otx2_cpt_aead_null_encrypt,
1551        .decrypt = otx2_cpt_aead_null_decrypt,
1552        .ivsize = 0,
1553        .maxauthsize = SHA1_DIGEST_SIZE,
1554}, {
1555        .base = {
1556                .cra_name = "authenc(hmac(sha256),ecb(cipher_null))",
1557                .cra_driver_name = "cpt_hmac_sha256_ecb_null",
1558                .cra_blocksize = 1,
1559                .cra_flags = CRYPTO_ALG_ASYNC,
1560                .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1561                .cra_priority = 4001,
1562                .cra_alignmask = 0,
1563                .cra_module = THIS_MODULE,
1564        },
1565        .init = otx2_cpt_aead_ecb_null_sha256_init,
1566        .exit = otx2_cpt_aead_exit,
1567        .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1568        .setauthsize = otx2_cpt_aead_null_set_authsize,
1569        .encrypt = otx2_cpt_aead_null_encrypt,
1570        .decrypt = otx2_cpt_aead_null_decrypt,
1571        .ivsize = 0,
1572        .maxauthsize = SHA256_DIGEST_SIZE,
1573}, {
1574        .base = {
1575                .cra_name = "authenc(hmac(sha384),ecb(cipher_null))",
1576                .cra_driver_name = "cpt_hmac_sha384_ecb_null",
1577                .cra_blocksize = 1,
1578                .cra_flags = CRYPTO_ALG_ASYNC,
1579                .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1580                .cra_priority = 4001,
1581                .cra_alignmask = 0,
1582                .cra_module = THIS_MODULE,
1583        },
1584        .init = otx2_cpt_aead_ecb_null_sha384_init,
1585        .exit = otx2_cpt_aead_exit,
1586        .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1587        .setauthsize = otx2_cpt_aead_null_set_authsize,
1588        .encrypt = otx2_cpt_aead_null_encrypt,
1589        .decrypt = otx2_cpt_aead_null_decrypt,
1590        .ivsize = 0,
1591        .maxauthsize = SHA384_DIGEST_SIZE,
1592}, {
1593        .base = {
1594                .cra_name = "authenc(hmac(sha512),ecb(cipher_null))",
1595                .cra_driver_name = "cpt_hmac_sha512_ecb_null",
1596                .cra_blocksize = 1,
1597                .cra_flags = CRYPTO_ALG_ASYNC,
1598                .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1599                .cra_priority = 4001,
1600                .cra_alignmask = 0,
1601                .cra_module = THIS_MODULE,
1602        },
1603        .init = otx2_cpt_aead_ecb_null_sha512_init,
1604        .exit = otx2_cpt_aead_exit,
1605        .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1606        .setauthsize = otx2_cpt_aead_null_set_authsize,
1607        .encrypt = otx2_cpt_aead_null_encrypt,
1608        .decrypt = otx2_cpt_aead_null_decrypt,
1609        .ivsize = 0,
1610        .maxauthsize = SHA512_DIGEST_SIZE,
1611}, {
1612        .base = {
1613                .cra_name = "rfc4106(gcm(aes))",
1614                .cra_driver_name = "cpt_rfc4106_gcm_aes",
1615                .cra_blocksize = 1,
1616                .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1617                .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1618                .cra_priority = 4001,
1619                .cra_alignmask = 0,
1620                .cra_module = THIS_MODULE,
1621        },
1622        .init = otx2_cpt_aead_gcm_aes_init,
1623        .exit = otx2_cpt_aead_exit,
1624        .setkey = otx2_cpt_aead_gcm_aes_setkey,
1625        .setauthsize = otx2_cpt_aead_gcm_set_authsize,
1626        .encrypt = otx2_cpt_aead_encrypt,
1627        .decrypt = otx2_cpt_aead_decrypt,
1628        .ivsize = AES_GCM_IV_SIZE,
1629        .maxauthsize = AES_GCM_ICV_SIZE,
1630} };
1631
1632static inline int cpt_register_algs(void)
1633{
1634        int i, err = 0;
1635
1636        if (!IS_ENABLED(CONFIG_DM_CRYPT)) {
1637                for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++)
1638                        otx2_cpt_skciphers[i].base.cra_flags &=
1639                                                        ~CRYPTO_ALG_DEAD;
1640
1641                err = crypto_register_skciphers(otx2_cpt_skciphers,
1642                                                ARRAY_SIZE(otx2_cpt_skciphers));
1643                if (err)
1644                        return err;
1645        }
1646
1647        for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++)
1648                otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1649
1650        err = crypto_register_aeads(otx2_cpt_aeads,
1651                                    ARRAY_SIZE(otx2_cpt_aeads));
1652        if (err) {
1653                crypto_unregister_skciphers(otx2_cpt_skciphers,
1654                                            ARRAY_SIZE(otx2_cpt_skciphers));
1655                return err;
1656        }
1657
1658        return 0;
1659}
1660
1661static inline void cpt_unregister_algs(void)
1662{
1663        crypto_unregister_skciphers(otx2_cpt_skciphers,
1664                                    ARRAY_SIZE(otx2_cpt_skciphers));
1665        crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads));
1666}
1667
1668static int compare_func(const void *lptr, const void *rptr)
1669{
1670        const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;
1671        const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;
1672
1673        if (ldesc->dev->devfn < rdesc->dev->devfn)
1674                return -1;
1675        if (ldesc->dev->devfn > rdesc->dev->devfn)
1676                return 1;
1677        return 0;
1678}
1679
1680static void swap_func(void *lptr, void *rptr, int size)
1681{
1682        struct cpt_device_desc *ldesc = lptr;
1683        struct cpt_device_desc *rdesc = rptr;
1684        struct cpt_device_desc desc;
1685
1686        desc = *ldesc;
1687        *ldesc = *rdesc;
1688        *rdesc = desc;
1689}
1690
1691int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
1692                         int num_queues, int num_devices)
1693{
1694        int ret = 0;
1695        int count;
1696
1697        mutex_lock(&mutex);
1698        count = atomic_read(&se_devices.count);
1699        if (count >= OTX2_CPT_MAX_LFS_NUM) {
1700                dev_err(&pdev->dev, "No space to add a new device\n");
1701                ret = -ENOSPC;
1702                goto unlock;
1703        }
1704        se_devices.desc[count].num_queues = num_queues;
1705        se_devices.desc[count++].dev = pdev;
1706        atomic_inc(&se_devices.count);
1707
1708        if (atomic_read(&se_devices.count) == num_devices &&
1709            is_crypto_registered == false) {
1710                if (cpt_register_algs()) {
1711                        dev_err(&pdev->dev,
1712                                "Error in registering crypto algorithms\n");
1713                        ret =  -EINVAL;
1714                        goto unlock;
1715                }
1716                try_module_get(mod);
1717                is_crypto_registered = true;
1718        }
1719        sort(se_devices.desc, count, sizeof(struct cpt_device_desc),
1720             compare_func, swap_func);
1721
1722unlock:
1723        mutex_unlock(&mutex);
1724        return ret;
1725}
1726
1727void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod)
1728{
1729        struct cpt_device_table *dev_tbl;
1730        bool dev_found = false;
1731        int i, j, count;
1732
1733        mutex_lock(&mutex);
1734
1735        dev_tbl = &se_devices;
1736        count = atomic_read(&dev_tbl->count);
1737        for (i = 0; i < count; i++) {
1738                if (pdev == dev_tbl->desc[i].dev) {
1739                        for (j = i; j < count-1; j++)
1740                                dev_tbl->desc[j] = dev_tbl->desc[j+1];
1741                        dev_found = true;
1742                        break;
1743                }
1744        }
1745
1746        if (!dev_found) {
1747                dev_err(&pdev->dev, "%s device not found\n", __func__);
1748                goto unlock;
1749        }
1750        if (atomic_dec_and_test(&se_devices.count)) {
1751                cpt_unregister_algs();
1752                module_put(mod);
1753                is_crypto_registered = false;
1754        }
1755
1756unlock:
1757        mutex_unlock(&mutex);
1758}
1759
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.