linux/drivers/crypto/hisilicon/hpre/hpre_crypto.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2019 HiSilicon Limited. */
   3#include <crypto/akcipher.h>
   4#include <crypto/curve25519.h>
   5#include <crypto/dh.h>
   6#include <crypto/ecc_curve.h>
   7#include <crypto/ecdh.h>
   8#include <crypto/internal/akcipher.h>
   9#include <crypto/internal/kpp.h>
  10#include <crypto/internal/rsa.h>
  11#include <crypto/kpp.h>
  12#include <crypto/scatterwalk.h>
  13#include <linux/dma-mapping.h>
  14#include <linux/fips.h>
  15#include <linux/module.h>
  16#include <linux/time.h>
  17#include "hpre.h"
  18
  19struct hpre_ctx;
  20
  21#define HPRE_CRYPTO_ALG_PRI     1000
  22#define HPRE_ALIGN_SZ           64
  23#define HPRE_BITS_2_BYTES_SHIFT 3
  24#define HPRE_RSA_512BITS_KSZ    64
  25#define HPRE_RSA_1536BITS_KSZ   192
  26#define HPRE_CRT_PRMS           5
  27#define HPRE_CRT_Q              2
  28#define HPRE_CRT_P              3
  29#define HPRE_CRT_INV            4
  30#define HPRE_DH_G_FLAG          0x02
  31#define HPRE_TRY_SEND_TIMES     100
  32#define HPRE_INVLD_REQ_ID               (-1)
  33#define HPRE_DEV(ctx)           (&((ctx)->qp->qm->pdev->dev))
  34
  35#define HPRE_SQE_ALG_BITS       5
  36#define HPRE_SQE_DONE_SHIFT     30
  37#define HPRE_DH_MAX_P_SZ        512
  38
  39#define HPRE_DFX_SEC_TO_US      1000000
  40#define HPRE_DFX_US_TO_NS       1000
  41
  42/* size in bytes of the n prime */
  43#define HPRE_ECC_NIST_P192_N_SIZE       24
  44#define HPRE_ECC_NIST_P256_N_SIZE       32
  45
  46/* size in bytes */
  47#define HPRE_ECC_HW256_KSZ_B    32
  48
  49typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
  50
  51struct hpre_rsa_ctx {
  52        /* low address: e--->n */
  53        char *pubkey;
  54        dma_addr_t dma_pubkey;
  55
  56        /* low address: d--->n */
  57        char *prikey;
  58        dma_addr_t dma_prikey;
  59
  60        /* low address: dq->dp->q->p->qinv */
  61        char *crt_prikey;
  62        dma_addr_t dma_crt_prikey;
  63
  64        struct crypto_akcipher *soft_tfm;
  65};
  66
  67struct hpre_dh_ctx {
  68        /*
  69         * If base is g we compute the public key
  70         *      ya = g^xa mod p; [RFC2631 sec 2.1.1]
  71         * else if base if the counterpart public key we
  72         * compute the shared secret
  73         *      ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
  74         * low address: d--->n, please refer to Hisilicon HPRE UM
  75         */
  76        char *xa_p;
  77        dma_addr_t dma_xa_p;
  78
  79        char *g; /* m */
  80        dma_addr_t dma_g;
  81};
  82
  83struct hpre_ecdh_ctx {
  84        /* low address: p->a->k->b */
  85        unsigned char *p;
  86        dma_addr_t dma_p;
  87
  88        /* low address: x->y */
  89        unsigned char *g;
  90        dma_addr_t dma_g;
  91};
  92
  93struct hpre_curve25519_ctx {
  94        /* low address: p->a->k */
  95        unsigned char *p;
  96        dma_addr_t dma_p;
  97
  98        /* gx coordinate */
  99        unsigned char *g;
 100        dma_addr_t dma_g;
 101};
 102
 103struct hpre_ctx {
 104        struct hisi_qp *qp;
 105        struct hpre_asym_request **req_list;
 106        struct hpre *hpre;
 107        spinlock_t req_lock;
 108        unsigned int key_sz;
 109        bool crt_g2_mode;
 110        struct idr req_idr;
 111        union {
 112                struct hpre_rsa_ctx rsa;
 113                struct hpre_dh_ctx dh;
 114                struct hpre_ecdh_ctx ecdh;
 115                struct hpre_curve25519_ctx curve25519;
 116        };
 117        /* for ecc algorithms */
 118        unsigned int curve_id;
 119};
 120
 121struct hpre_asym_request {
 122        char *src;
 123        char *dst;
 124        struct hpre_sqe req;
 125        struct hpre_ctx *ctx;
 126        union {
 127                struct akcipher_request *rsa;
 128                struct kpp_request *dh;
 129                struct kpp_request *ecdh;
 130                struct kpp_request *curve25519;
 131        } areq;
 132        int err;
 133        int req_id;
 134        hpre_cb cb;
 135        struct timespec64 req_time;
 136};
 137
 138static int hpre_alloc_req_id(struct hpre_ctx *ctx)
 139{
 140        unsigned long flags;
 141        int id;
 142
 143        spin_lock_irqsave(&ctx->req_lock, flags);
 144        id = idr_alloc(&ctx->req_idr, NULL, 0, QM_Q_DEPTH, GFP_ATOMIC);
 145        spin_unlock_irqrestore(&ctx->req_lock, flags);
 146
 147        return id;
 148}
 149
 150static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)
 151{
 152        unsigned long flags;
 153
 154        spin_lock_irqsave(&ctx->req_lock, flags);
 155        idr_remove(&ctx->req_idr, req_id);
 156        spin_unlock_irqrestore(&ctx->req_lock, flags);
 157}
 158
 159static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
 160{
 161        struct hpre_ctx *ctx;
 162        struct hpre_dfx *dfx;
 163        int id;
 164
 165        ctx = hpre_req->ctx;
 166        id = hpre_alloc_req_id(ctx);
 167        if (unlikely(id < 0))
 168                return -EINVAL;
 169
 170        ctx->req_list[id] = hpre_req;
 171        hpre_req->req_id = id;
 172
 173        dfx = ctx->hpre->debug.dfx;
 174        if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value))
 175                ktime_get_ts64(&hpre_req->req_time);
 176
 177        return id;
 178}
 179
 180static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)
 181{
 182        struct hpre_ctx *ctx = hpre_req->ctx;
 183        int id = hpre_req->req_id;
 184
 185        if (hpre_req->req_id >= 0) {
 186                hpre_req->req_id = HPRE_INVLD_REQ_ID;
 187                ctx->req_list[id] = NULL;
 188                hpre_free_req_id(ctx, id);
 189        }
 190}
 191
 192static struct hisi_qp *hpre_get_qp_and_start(u8 type)
 193{
 194        struct hisi_qp *qp;
 195        int ret;
 196
 197        qp = hpre_create_qp(type);
 198        if (!qp) {
 199                pr_err("Can not create hpre qp!\n");
 200                return ERR_PTR(-ENODEV);
 201        }
 202
 203        ret = hisi_qm_start_qp(qp, 0);
 204        if (ret < 0) {
 205                hisi_qm_free_qps(&qp, 1);
 206                pci_err(qp->qm->pdev, "Can not start qp!\n");
 207                return ERR_PTR(-EINVAL);
 208        }
 209
 210        return qp;
 211}
 212
 213static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req,
 214                                  struct scatterlist *data, unsigned int len,
 215                                  int is_src, dma_addr_t *tmp)
 216{
 217        struct hpre_ctx *ctx = hpre_req->ctx;
 218        struct device *dev = HPRE_DEV(ctx);
 219        enum dma_data_direction dma_dir;
 220
 221        if (is_src) {
 222                hpre_req->src = NULL;
 223                dma_dir = DMA_TO_DEVICE;
 224        } else {
 225                hpre_req->dst = NULL;
 226                dma_dir = DMA_FROM_DEVICE;
 227        }
 228        *tmp = dma_map_single(dev, sg_virt(data), len, dma_dir);
 229        if (unlikely(dma_mapping_error(dev, *tmp))) {
 230                dev_err(dev, "dma map data err!\n");
 231                return -ENOMEM;
 232        }
 233
 234        return 0;
 235}
 236
 237static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req,
 238                                struct scatterlist *data, unsigned int len,
 239                                int is_src, dma_addr_t *tmp)
 240{
 241        struct hpre_ctx *ctx = hpre_req->ctx;
 242        struct device *dev = HPRE_DEV(ctx);
 243        void *ptr;
 244        int shift;
 245
 246        shift = ctx->key_sz - len;
 247        if (unlikely(shift < 0))
 248                return -EINVAL;
 249
 250        ptr = dma_alloc_coherent(dev, ctx->key_sz, tmp, GFP_KERNEL);
 251        if (unlikely(!ptr))
 252                return -ENOMEM;
 253
 254        if (is_src) {
 255                scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0);
 256                hpre_req->src = ptr;
 257        } else {
 258                hpre_req->dst = ptr;
 259        }
 260
 261        return 0;
 262}
 263
 264static int hpre_hw_data_init(struct hpre_asym_request *hpre_req,
 265                             struct scatterlist *data, unsigned int len,
 266                             int is_src, int is_dh)
 267{
 268        struct hpre_sqe *msg = &hpre_req->req;
 269        struct hpre_ctx *ctx = hpre_req->ctx;
 270        dma_addr_t tmp = 0;
 271        int ret;
 272
 273        /* when the data is dh's source, we should format it */
 274        if ((sg_is_last(data) && len == ctx->key_sz) &&
 275            ((is_dh && !is_src) || !is_dh))
 276                ret = hpre_get_data_dma_addr(hpre_req, data, len, is_src, &tmp);
 277        else
 278                ret = hpre_prepare_dma_buf(hpre_req, data, len, is_src, &tmp);
 279
 280        if (unlikely(ret))
 281                return ret;
 282
 283        if (is_src)
 284                msg->in = cpu_to_le64(tmp);
 285        else
 286                msg->out = cpu_to_le64(tmp);
 287
 288        return 0;
 289}
 290
 291static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
 292                                 struct hpre_asym_request *req,
 293                                 struct scatterlist *dst,
 294                                 struct scatterlist *src)
 295{
 296        struct device *dev = HPRE_DEV(ctx);
 297        struct hpre_sqe *sqe = &req->req;
 298        dma_addr_t tmp;
 299
 300        tmp = le64_to_cpu(sqe->in);
 301        if (unlikely(dma_mapping_error(dev, tmp)))
 302                return;
 303
 304        if (src) {
 305                if (req->src)
 306                        dma_free_coherent(dev, ctx->key_sz, req->src, tmp);
 307                else
 308                        dma_unmap_single(dev, tmp, ctx->key_sz, DMA_TO_DEVICE);
 309        }
 310
 311        tmp = le64_to_cpu(sqe->out);
 312        if (unlikely(dma_mapping_error(dev, tmp)))
 313                return;
 314
 315        if (req->dst) {
 316                if (dst)
 317                        scatterwalk_map_and_copy(req->dst, dst, 0,
 318                                                 ctx->key_sz, 1);
 319                dma_free_coherent(dev, ctx->key_sz, req->dst, tmp);
 320        } else {
 321                dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE);
 322        }
 323}
 324
 325static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
 326                                void **kreq)
 327{
 328        struct device *dev = HPRE_DEV(ctx);
 329        struct hpre_asym_request *req;
 330        unsigned int err, done, alg;
 331        int id;
 332
 333#define HPRE_NO_HW_ERR          0
 334#define HPRE_HW_TASK_DONE       3
 335#define HREE_HW_ERR_MASK        0x7ff
 336#define HREE_SQE_DONE_MASK      0x3
 337#define HREE_ALG_TYPE_MASK      0x1f
 338        id = (int)le16_to_cpu(sqe->tag);
 339        req = ctx->req_list[id];
 340        hpre_rm_req_from_ctx(req);
 341        *kreq = req;
 342
 343        err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) &
 344                HREE_HW_ERR_MASK;
 345
 346        done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) &
 347                HREE_SQE_DONE_MASK;
 348
 349        if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE))
 350                return 0;
 351
 352        alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK;
 353        dev_err_ratelimited(dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",
 354                alg, done, err);
 355
 356        return -EINVAL;
 357}
 358
 359static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
 360{
 361        struct hpre *hpre;
 362
 363        if (!ctx || !qp || qlen < 0)
 364                return -EINVAL;
 365
 366        spin_lock_init(&ctx->req_lock);
 367        ctx->qp = qp;
 368
 369        hpre = container_of(ctx->qp->qm, struct hpre, qm);
 370        ctx->hpre = hpre;
 371        ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);
 372        if (!ctx->req_list)
 373                return -ENOMEM;
 374        ctx->key_sz = 0;
 375        ctx->crt_g2_mode = false;
 376        idr_init(&ctx->req_idr);
 377
 378        return 0;
 379}
 380
 381static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all)
 382{
 383        if (is_clear_all) {
 384                idr_destroy(&ctx->req_idr);
 385                kfree(ctx->req_list);
 386                hisi_qm_free_qps(&ctx->qp, 1);
 387        }
 388
 389        ctx->crt_g2_mode = false;
 390        ctx->key_sz = 0;
 391}
 392
 393static bool hpre_is_bd_timeout(struct hpre_asym_request *req,
 394                               u64 overtime_thrhld)
 395{
 396        struct timespec64 reply_time;
 397        u64 time_use_us;
 398
 399        ktime_get_ts64(&reply_time);
 400        time_use_us = (reply_time.tv_sec - req->req_time.tv_sec) *
 401                HPRE_DFX_SEC_TO_US +
 402                (reply_time.tv_nsec - req->req_time.tv_nsec) /
 403                HPRE_DFX_US_TO_NS;
 404
 405        if (time_use_us <= overtime_thrhld)
 406                return false;
 407
 408        return true;
 409}
 410
 411static void hpre_dh_cb(struct hpre_ctx *ctx, void *resp)
 412{
 413        struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
 414        struct hpre_asym_request *req;
 415        struct kpp_request *areq;
 416        u64 overtime_thrhld;
 417        int ret;
 418
 419        ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
 420        areq = req->areq.dh;
 421        areq->dst_len = ctx->key_sz;
 422
 423        overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
 424        if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
 425                atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
 426
 427        hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
 428        kpp_request_complete(areq, ret);
 429        atomic64_inc(&dfx[HPRE_RECV_CNT].value);
 430}
 431
 432static void hpre_rsa_cb(struct hpre_ctx *ctx, void *resp)
 433{
 434        struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
 435        struct hpre_asym_request *req;
 436        struct akcipher_request *areq;
 437        u64 overtime_thrhld;
 438        int ret;
 439
 440        ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
 441
 442        overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
 443        if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
 444                atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
 445
 446        areq = req->areq.rsa;
 447        areq->dst_len = ctx->key_sz;
 448        hpre_hw_data_clr_all(ctx, req, areq->dst, areq->src);
 449        akcipher_request_complete(areq, ret);
 450        atomic64_inc(&dfx[HPRE_RECV_CNT].value);
 451}
 452
 453static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
 454{
 455        struct hpre_ctx *ctx = qp->qp_ctx;
 456        struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
 457        struct hpre_sqe *sqe = resp;
 458        struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)];
 459
 460        if (unlikely(!req)) {
 461                atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value);
 462                return;
 463        }
 464
 465        req->cb(ctx, resp);
 466}
 467
 468static void hpre_stop_qp_and_put(struct hisi_qp *qp)
 469{
 470        hisi_qm_stop_qp(qp);
 471        hisi_qm_free_qps(&qp, 1);
 472}
 473
 474static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)
 475{
 476        struct hisi_qp *qp;
 477        int ret;
 478
 479        qp = hpre_get_qp_and_start(type);
 480        if (IS_ERR(qp))
 481                return PTR_ERR(qp);
 482
 483        qp->qp_ctx = ctx;
 484        qp->req_cb = hpre_alg_cb;
 485
 486        ret = hpre_ctx_set(ctx, qp, QM_Q_DEPTH);
 487        if (ret)
 488                hpre_stop_qp_and_put(qp);
 489
 490        return ret;
 491}
 492
 493static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
 494{
 495        struct hpre_asym_request *h_req;
 496        struct hpre_sqe *msg;
 497        int req_id;
 498        void *tmp;
 499
 500        if (is_rsa) {
 501                struct akcipher_request *akreq = req;
 502
 503                if (akreq->dst_len < ctx->key_sz) {
 504                        akreq->dst_len = ctx->key_sz;
 505                        return -EOVERFLOW;
 506                }
 507
 508                tmp = akcipher_request_ctx(akreq);
 509                h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
 510                h_req->cb = hpre_rsa_cb;
 511                h_req->areq.rsa = akreq;
 512                msg = &h_req->req;
 513                memset(msg, 0, sizeof(*msg));
 514        } else {
 515                struct kpp_request *kreq = req;
 516
 517                if (kreq->dst_len < ctx->key_sz) {
 518                        kreq->dst_len = ctx->key_sz;
 519                        return -EOVERFLOW;
 520                }
 521
 522                tmp = kpp_request_ctx(kreq);
 523                h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
 524                h_req->cb = hpre_dh_cb;
 525                h_req->areq.dh = kreq;
 526                msg = &h_req->req;
 527                memset(msg, 0, sizeof(*msg));
 528                msg->key = cpu_to_le64(ctx->dh.dma_xa_p);
 529        }
 530
 531        msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
 532        msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
 533        msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT);
 534        msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
 535        h_req->ctx = ctx;
 536
 537        req_id = hpre_add_req_to_ctx(h_req);
 538        if (req_id < 0)
 539                return -EBUSY;
 540
 541        msg->tag = cpu_to_le16((u16)req_id);
 542
 543        return 0;
 544}
 545
 546static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
 547{
 548        struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
 549        int ctr = 0;
 550        int ret;
 551
 552        do {
 553                atomic64_inc(&dfx[HPRE_SEND_CNT].value);
 554                ret = hisi_qp_send(ctx->qp, msg);
 555                if (ret != -EBUSY)
 556                        break;
 557                atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value);
 558        } while (ctr++ < HPRE_TRY_SEND_TIMES);
 559
 560        if (likely(!ret))
 561                return ret;
 562
 563        if (ret != -EBUSY)
 564                atomic64_inc(&dfx[HPRE_SEND_FAIL_CNT].value);
 565
 566        return ret;
 567}
 568
 569static int hpre_dh_compute_value(struct kpp_request *req)
 570{
 571        struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
 572        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
 573        void *tmp = kpp_request_ctx(req);
 574        struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
 575        struct hpre_sqe *msg = &hpre_req->req;
 576        int ret;
 577
 578        ret = hpre_msg_request_set(ctx, req, false);
 579        if (unlikely(ret))
 580                return ret;
 581
 582        if (req->src) {
 583                ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 1);
 584                if (unlikely(ret))
 585                        goto clear_all;
 586        } else {
 587                msg->in = cpu_to_le64(ctx->dh.dma_g);
 588        }
 589
 590        ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 1);
 591        if (unlikely(ret))
 592                goto clear_all;
 593
 594        if (ctx->crt_g2_mode && !req->src)
 595                msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH_G2);
 596        else
 597                msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_DH);
 598
 599        /* success */
 600        ret = hpre_send(ctx, msg);
 601        if (likely(!ret))
 602                return -EINPROGRESS;
 603
 604clear_all:
 605        hpre_rm_req_from_ctx(hpre_req);
 606        hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
 607
 608        return ret;
 609}
 610
 611static int hpre_is_dh_params_length_valid(unsigned int key_sz)
 612{
 613#define _HPRE_DH_GRP1           768
 614#define _HPRE_DH_GRP2           1024
 615#define _HPRE_DH_GRP5           1536
 616#define _HPRE_DH_GRP14          2048
 617#define _HPRE_DH_GRP15          3072
 618#define _HPRE_DH_GRP16          4096
 619        switch (key_sz) {
 620        case _HPRE_DH_GRP1:
 621        case _HPRE_DH_GRP2:
 622        case _HPRE_DH_GRP5:
 623        case _HPRE_DH_GRP14:
 624        case _HPRE_DH_GRP15:
 625        case _HPRE_DH_GRP16:
 626                return 0;
 627        }
 628
 629        return -EINVAL;
 630}
 631
 632static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
 633{
 634        struct device *dev = HPRE_DEV(ctx);
 635        unsigned int sz;
 636
 637        if (params->p_size > HPRE_DH_MAX_P_SZ)
 638                return -EINVAL;
 639
 640        if (hpre_is_dh_params_length_valid(params->p_size <<
 641                                           HPRE_BITS_2_BYTES_SHIFT))
 642                return -EINVAL;
 643
 644        sz = ctx->key_sz = params->p_size;
 645        ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1,
 646                                          &ctx->dh.dma_xa_p, GFP_KERNEL);
 647        if (!ctx->dh.xa_p)
 648                return -ENOMEM;
 649
 650        memcpy(ctx->dh.xa_p + sz, params->p, sz);
 651
 652        /* If g equals 2 don't copy it */
 653        if (params->g_size == 1 && *(char *)params->g == HPRE_DH_G_FLAG) {
 654                ctx->crt_g2_mode = true;
 655                return 0;
 656        }
 657
 658        ctx->dh.g = dma_alloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL);
 659        if (!ctx->dh.g) {
 660                dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
 661                                  ctx->dh.dma_xa_p);
 662                ctx->dh.xa_p = NULL;
 663                return -ENOMEM;
 664        }
 665
 666        memcpy(ctx->dh.g + (sz - params->g_size), params->g, params->g_size);
 667
 668        return 0;
 669}
 670
 671static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
 672{
 673        struct device *dev = HPRE_DEV(ctx);
 674        unsigned int sz = ctx->key_sz;
 675
 676        if (is_clear_all)
 677                hisi_qm_stop_qp(ctx->qp);
 678
 679        if (ctx->dh.g) {
 680                dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g);
 681                ctx->dh.g = NULL;
 682        }
 683
 684        if (ctx->dh.xa_p) {
 685                memzero_explicit(ctx->dh.xa_p, sz);
 686                dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
 687                                  ctx->dh.dma_xa_p);
 688                ctx->dh.xa_p = NULL;
 689        }
 690
 691        hpre_ctx_clear(ctx, is_clear_all);
 692}
 693
 694static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf,
 695                              unsigned int len)
 696{
 697        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
 698        struct dh params;
 699        int ret;
 700
 701        if (crypto_dh_decode_key(buf, len, &params) < 0)
 702                return -EINVAL;
 703
 704        /* Free old secret if any */
 705        hpre_dh_clear_ctx(ctx, false);
 706
 707        ret = hpre_dh_set_params(ctx, &params);
 708        if (ret < 0)
 709                goto err_clear_ctx;
 710
 711        memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key,
 712               params.key_size);
 713
 714        return 0;
 715
 716err_clear_ctx:
 717        hpre_dh_clear_ctx(ctx, false);
 718        return ret;
 719}
 720
 721static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm)
 722{
 723        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
 724
 725        return ctx->key_sz;
 726}
 727
 728static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
 729{
 730        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
 731
 732        return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
 733}
 734
 735static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
 736{
 737        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
 738
 739        hpre_dh_clear_ctx(ctx, true);
 740}
 741
 742static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)
 743{
 744        while (!**ptr && *len) {
 745                (*ptr)++;
 746                (*len)--;
 747        }
 748}
 749
 750static bool hpre_rsa_key_size_is_support(unsigned int len)
 751{
 752        unsigned int bits = len << HPRE_BITS_2_BYTES_SHIFT;
 753
 754#define _RSA_1024BITS_KEY_WDTH          1024
 755#define _RSA_2048BITS_KEY_WDTH          2048
 756#define _RSA_3072BITS_KEY_WDTH          3072
 757#define _RSA_4096BITS_KEY_WDTH          4096
 758
 759        switch (bits) {
 760        case _RSA_1024BITS_KEY_WDTH:
 761        case _RSA_2048BITS_KEY_WDTH:
 762        case _RSA_3072BITS_KEY_WDTH:
 763        case _RSA_4096BITS_KEY_WDTH:
 764                return true;
 765        default:
 766                return false;
 767        }
 768}
 769
 770static int hpre_rsa_enc(struct akcipher_request *req)
 771{
 772        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
 773        struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
 774        void *tmp = akcipher_request_ctx(req);
 775        struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
 776        struct hpre_sqe *msg = &hpre_req->req;
 777        int ret;
 778
 779        /* For 512 and 1536 bits key size, use soft tfm instead */
 780        if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
 781            ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
 782                akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
 783                ret = crypto_akcipher_encrypt(req);
 784                akcipher_request_set_tfm(req, tfm);
 785                return ret;
 786        }
 787
 788        if (unlikely(!ctx->rsa.pubkey))
 789                return -EINVAL;
 790
 791        ret = hpre_msg_request_set(ctx, req, true);
 792        if (unlikely(ret))
 793                return ret;
 794
 795        msg->dw0 |= cpu_to_le32(HPRE_ALG_NC_NCRT);
 796        msg->key = cpu_to_le64(ctx->rsa.dma_pubkey);
 797
 798        ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
 799        if (unlikely(ret))
 800                goto clear_all;
 801
 802        ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
 803        if (unlikely(ret))
 804                goto clear_all;
 805
 806        /* success */
 807        ret = hpre_send(ctx, msg);
 808        if (likely(!ret))
 809                return -EINPROGRESS;
 810
 811clear_all:
 812        hpre_rm_req_from_ctx(hpre_req);
 813        hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
 814
 815        return ret;
 816}
 817
 818static int hpre_rsa_dec(struct akcipher_request *req)
 819{
 820        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
 821        struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
 822        void *tmp = akcipher_request_ctx(req);
 823        struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
 824        struct hpre_sqe *msg = &hpre_req->req;
 825        int ret;
 826
 827        /* For 512 and 1536 bits key size, use soft tfm instead */
 828        if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
 829            ctx->key_sz == HPRE_RSA_1536BITS_KSZ) {
 830                akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
 831                ret = crypto_akcipher_decrypt(req);
 832                akcipher_request_set_tfm(req, tfm);
 833                return ret;
 834        }
 835
 836        if (unlikely(!ctx->rsa.prikey))
 837                return -EINVAL;
 838
 839        ret = hpre_msg_request_set(ctx, req, true);
 840        if (unlikely(ret))
 841                return ret;
 842
 843        if (ctx->crt_g2_mode) {
 844                msg->key = cpu_to_le64(ctx->rsa.dma_crt_prikey);
 845                msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
 846                                       HPRE_ALG_NC_CRT);
 847        } else {
 848                msg->key = cpu_to_le64(ctx->rsa.dma_prikey);
 849                msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) |
 850                                       HPRE_ALG_NC_NCRT);
 851        }
 852
 853        ret = hpre_hw_data_init(hpre_req, req->src, req->src_len, 1, 0);
 854        if (unlikely(ret))
 855                goto clear_all;
 856
 857        ret = hpre_hw_data_init(hpre_req, req->dst, req->dst_len, 0, 0);
 858        if (unlikely(ret))
 859                goto clear_all;
 860
 861        /* success */
 862        ret = hpre_send(ctx, msg);
 863        if (likely(!ret))
 864                return -EINPROGRESS;
 865
 866clear_all:
 867        hpre_rm_req_from_ctx(hpre_req);
 868        hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
 869
 870        return ret;
 871}
 872
 873static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,
 874                          size_t vlen, bool private)
 875{
 876        const char *ptr = value;
 877
 878        hpre_rsa_drop_leading_zeros(&ptr, &vlen);
 879
 880        ctx->key_sz = vlen;
 881
 882        /* if invalid key size provided, we use software tfm */
 883        if (!hpre_rsa_key_size_is_support(ctx->key_sz))
 884                return 0;
 885
 886        ctx->rsa.pubkey = dma_alloc_coherent(HPRE_DEV(ctx), vlen << 1,
 887                                             &ctx->rsa.dma_pubkey,
 888                                             GFP_KERNEL);
 889        if (!ctx->rsa.pubkey)
 890                return -ENOMEM;
 891
 892        if (private) {
 893                ctx->rsa.prikey = dma_alloc_coherent(HPRE_DEV(ctx), vlen << 1,
 894                                                     &ctx->rsa.dma_prikey,
 895                                                     GFP_KERNEL);
 896                if (!ctx->rsa.prikey) {
 897                        dma_free_coherent(HPRE_DEV(ctx), vlen << 1,
 898                                          ctx->rsa.pubkey,
 899                                          ctx->rsa.dma_pubkey);
 900                        ctx->rsa.pubkey = NULL;
 901                        return -ENOMEM;
 902                }
 903                memcpy(ctx->rsa.prikey + vlen, ptr, vlen);
 904        }
 905        memcpy(ctx->rsa.pubkey + vlen, ptr, vlen);
 906
 907        /* Using hardware HPRE to do RSA */
 908        return 1;
 909}
 910
 911static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value,
 912                          size_t vlen)
 913{
 914        const char *ptr = value;
 915
 916        hpre_rsa_drop_leading_zeros(&ptr, &vlen);
 917
 918        if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
 919                return -EINVAL;
 920
 921        memcpy(ctx->rsa.pubkey + ctx->key_sz - vlen, ptr, vlen);
 922
 923        return 0;
 924}
 925
 926static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value,
 927                          size_t vlen)
 928{
 929        const char *ptr = value;
 930
 931        hpre_rsa_drop_leading_zeros(&ptr, &vlen);
 932
 933        if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
 934                return -EINVAL;
 935
 936        memcpy(ctx->rsa.prikey + ctx->key_sz - vlen, ptr, vlen);
 937
 938        return 0;
 939}
 940
 941static int hpre_crt_para_get(char *para, size_t para_sz,
 942                             const char *raw, size_t raw_sz)
 943{
 944        const char *ptr = raw;
 945        size_t len = raw_sz;
 946
 947        hpre_rsa_drop_leading_zeros(&ptr, &len);
 948        if (!len || len > para_sz)
 949                return -EINVAL;
 950
 951        memcpy(para + para_sz - len, ptr, len);
 952
 953        return 0;
 954}
 955
 956static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
 957{
 958        unsigned int hlf_ksz = ctx->key_sz >> 1;
 959        struct device *dev = HPRE_DEV(ctx);
 960        u64 offset;
 961        int ret;
 962
 963        ctx->rsa.crt_prikey = dma_alloc_coherent(dev, hlf_ksz * HPRE_CRT_PRMS,
 964                                        &ctx->rsa.dma_crt_prikey,
 965                                        GFP_KERNEL);
 966        if (!ctx->rsa.crt_prikey)
 967                return -ENOMEM;
 968
 969        ret = hpre_crt_para_get(ctx->rsa.crt_prikey, hlf_ksz,
 970                                rsa_key->dq, rsa_key->dq_sz);
 971        if (ret)
 972                goto free_key;
 973
 974        offset = hlf_ksz;
 975        ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
 976                                rsa_key->dp, rsa_key->dp_sz);
 977        if (ret)
 978                goto free_key;
 979
 980        offset = hlf_ksz * HPRE_CRT_Q;
 981        ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
 982                                rsa_key->q, rsa_key->q_sz);
 983        if (ret)
 984                goto free_key;
 985
 986        offset = hlf_ksz * HPRE_CRT_P;
 987        ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
 988                                rsa_key->p, rsa_key->p_sz);
 989        if (ret)
 990                goto free_key;
 991
 992        offset = hlf_ksz * HPRE_CRT_INV;
 993        ret = hpre_crt_para_get(ctx->rsa.crt_prikey + offset, hlf_ksz,
 994                                rsa_key->qinv, rsa_key->qinv_sz);
 995        if (ret)
 996                goto free_key;
 997
 998        ctx->crt_g2_mode = true;
 999
1000        return 0;
1001
1002free_key:
1003        offset = hlf_ksz * HPRE_CRT_PRMS;
1004        memzero_explicit(ctx->rsa.crt_prikey, offset);
1005        dma_free_coherent(dev, hlf_ksz * HPRE_CRT_PRMS, ctx->rsa.crt_prikey,
1006                          ctx->rsa.dma_crt_prikey);
1007        ctx->rsa.crt_prikey = NULL;
1008        ctx->crt_g2_mode = false;
1009
1010        return ret;
1011}
1012
1013/* If it is clear all, all the resources of the QP will be cleaned. */
1014static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all)
1015{
1016        unsigned int half_key_sz = ctx->key_sz >> 1;
1017        struct device *dev = HPRE_DEV(ctx);
1018
1019        if (is_clear_all)
1020                hisi_qm_stop_qp(ctx->qp);
1021
1022        if (ctx->rsa.pubkey) {
1023                dma_free_coherent(dev, ctx->key_sz << 1,
1024                                  ctx->rsa.pubkey, ctx->rsa.dma_pubkey);
1025                ctx->rsa.pubkey = NULL;
1026        }
1027
1028        if (ctx->rsa.crt_prikey) {
1029                memzero_explicit(ctx->rsa.crt_prikey,
1030                                 half_key_sz * HPRE_CRT_PRMS);
1031                dma_free_coherent(dev, half_key_sz * HPRE_CRT_PRMS,
1032                                  ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey);
1033                ctx->rsa.crt_prikey = NULL;
1034        }
1035
1036        if (ctx->rsa.prikey) {
1037                memzero_explicit(ctx->rsa.prikey, ctx->key_sz);
1038                dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey,
1039                                  ctx->rsa.dma_prikey);
1040                ctx->rsa.prikey = NULL;
1041        }
1042
1043        hpre_ctx_clear(ctx, is_clear_all);
1044}
1045
1046/*
1047 * we should judge if it is CRT or not,
1048 * CRT: return true,  N-CRT: return false .
1049 */
1050static bool hpre_is_crt_key(struct rsa_key *key)
1051{
1052        u16 len = key->p_sz + key->q_sz + key->dp_sz + key->dq_sz +
1053                  key->qinv_sz;
1054
1055#define LEN_OF_NCRT_PARA        5
1056
1057        /* N-CRT less than 5 parameters */
1058        return len > LEN_OF_NCRT_PARA;
1059}
1060
1061static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key,
1062                           unsigned int keylen, bool private)
1063{
1064        struct rsa_key rsa_key;
1065        int ret;
1066
1067        hpre_rsa_clear_ctx(ctx, false);
1068
1069        if (private)
1070                ret = rsa_parse_priv_key(&rsa_key, key, keylen);
1071        else
1072                ret = rsa_parse_pub_key(&rsa_key, key, keylen);
1073        if (ret < 0)
1074                return ret;
1075
1076        ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private);
1077        if (ret <= 0)
1078                return ret;
1079
1080        if (private) {
1081                ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);
1082                if (ret < 0)
1083                        goto free;
1084
1085                if (hpre_is_crt_key(&rsa_key)) {
1086                        ret = hpre_rsa_setkey_crt(ctx, &rsa_key);
1087                        if (ret < 0)
1088                                goto free;
1089                }
1090        }
1091
1092        ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);
1093        if (ret < 0)
1094                goto free;
1095
1096        if ((private && !ctx->rsa.prikey) || !ctx->rsa.pubkey) {
1097                ret = -EINVAL;
1098                goto free;
1099        }
1100
1101        return 0;
1102
1103free:
1104        hpre_rsa_clear_ctx(ctx, false);
1105        return ret;
1106}
1107
1108static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
1109                              unsigned int keylen)
1110{
1111        struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1112        int ret;
1113
1114        ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen);
1115        if (ret)
1116                return ret;
1117
1118        return hpre_rsa_setkey(ctx, key, keylen, false);
1119}
1120
1121static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
1122                               unsigned int keylen)
1123{
1124        struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1125        int ret;
1126
1127        ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen);
1128        if (ret)
1129                return ret;
1130
1131        return hpre_rsa_setkey(ctx, key, keylen, true);
1132}
1133
1134static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm)
1135{
1136        struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1137
1138        /* For 512 and 1536 bits key size, use soft tfm instead */
1139        if (ctx->key_sz == HPRE_RSA_512BITS_KSZ ||
1140            ctx->key_sz == HPRE_RSA_1536BITS_KSZ)
1141                return crypto_akcipher_maxsize(ctx->rsa.soft_tfm);
1142
1143        return ctx->key_sz;
1144}
1145
1146static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
1147{
1148        struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1149        int ret;
1150
1151        ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0);
1152        if (IS_ERR(ctx->rsa.soft_tfm)) {
1153                pr_err("Can not alloc_akcipher!\n");
1154                return PTR_ERR(ctx->rsa.soft_tfm);
1155        }
1156
1157        ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
1158        if (ret)
1159                crypto_free_akcipher(ctx->rsa.soft_tfm);
1160
1161        return ret;
1162}
1163
1164static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
1165{
1166        struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
1167
1168        hpre_rsa_clear_ctx(ctx, true);
1169        crypto_free_akcipher(ctx->rsa.soft_tfm);
1170}
1171
1172static void hpre_key_to_big_end(u8 *data, int len)
1173{
1174        int i, j;
1175        u8 tmp;
1176
1177        for (i = 0; i < len / 2; i++) {
1178                j = len - i - 1;
1179                tmp = data[j];
1180                data[j] = data[i];
1181                data[i] = tmp;
1182        }
1183}
1184
1185static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all,
1186                               bool is_ecdh)
1187{
1188        struct device *dev = HPRE_DEV(ctx);
1189        unsigned int sz = ctx->key_sz;
1190        unsigned int shift = sz << 1;
1191
1192        if (is_clear_all)
1193                hisi_qm_stop_qp(ctx->qp);
1194
1195        if (is_ecdh && ctx->ecdh.p) {
1196                /* ecdh: p->a->k->b */
1197                memzero_explicit(ctx->ecdh.p + shift, sz);
1198                dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1199                ctx->ecdh.p = NULL;
1200        } else if (!is_ecdh && ctx->curve25519.p) {
1201                /* curve25519: p->a->k */
1202                memzero_explicit(ctx->curve25519.p + shift, sz);
1203                dma_free_coherent(dev, sz << 2, ctx->curve25519.p,
1204                                  ctx->curve25519.dma_p);
1205                ctx->curve25519.p = NULL;
1206        }
1207
1208        hpre_ctx_clear(ctx, is_clear_all);
1209}
1210
1211static unsigned int hpre_ecdh_supported_curve(unsigned short id)
1212{
1213        switch (id) {
1214        case ECC_CURVE_NIST_P192:
1215        case ECC_CURVE_NIST_P256:
1216                return HPRE_ECC_HW256_KSZ_B;
1217        default:
1218                break;
1219        }
1220
1221        return 0;
1222}
1223
1224static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits)
1225{
1226        unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64);
1227        u8 i = 0;
1228
1229        while (i < ndigits - 1) {
1230                memcpy(addr + sizeof(u64) * i, &param[i], sizeof(u64));
1231                i++;
1232        }
1233
1234        memcpy(addr + sizeof(u64) * i, &param[ndigits - 1], sz);
1235        hpre_key_to_big_end((u8 *)addr, cur_sz);
1236}
1237
1238static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params,
1239                                unsigned int cur_sz)
1240{
1241        unsigned int shifta = ctx->key_sz << 1;
1242        unsigned int shiftb = ctx->key_sz << 2;
1243        void *p = ctx->ecdh.p + ctx->key_sz - cur_sz;
1244        void *a = ctx->ecdh.p + shifta - cur_sz;
1245        void *b = ctx->ecdh.p + shiftb - cur_sz;
1246        void *x = ctx->ecdh.g + ctx->key_sz - cur_sz;
1247        void *y = ctx->ecdh.g + shifta - cur_sz;
1248        const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id);
1249        char *n;
1250
1251        if (unlikely(!curve))
1252                return -EINVAL;
1253
1254        n = kzalloc(ctx->key_sz, GFP_KERNEL);
1255        if (!n)
1256                return -ENOMEM;
1257
1258        fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits);
1259        fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits);
1260        fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits);
1261        fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits);
1262        fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits);
1263        fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits);
1264
1265        if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) {
1266                kfree(n);
1267                return -EINVAL;
1268        }
1269
1270        kfree(n);
1271        return 0;
1272}
1273
1274static unsigned int hpre_ecdh_get_curvesz(unsigned short id)
1275{
1276        switch (id) {
1277        case ECC_CURVE_NIST_P192:
1278                return HPRE_ECC_NIST_P192_N_SIZE;
1279        case ECC_CURVE_NIST_P256:
1280                return HPRE_ECC_NIST_P256_N_SIZE;
1281        default:
1282                break;
1283        }
1284
1285        return 0;
1286}
1287
1288static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params)
1289{
1290        struct device *dev = HPRE_DEV(ctx);
1291        unsigned int sz, shift, curve_sz;
1292        int ret;
1293
1294        ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id);
1295        if (!ctx->key_sz)
1296                return -EINVAL;
1297
1298        curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1299        if (!curve_sz || params->key_size > curve_sz)
1300                return -EINVAL;
1301
1302        sz = ctx->key_sz;
1303
1304        if (!ctx->ecdh.p) {
1305                ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p,
1306                                                 GFP_KERNEL);
1307                if (!ctx->ecdh.p)
1308                        return -ENOMEM;
1309        }
1310
1311        shift = sz << 2;
1312        ctx->ecdh.g = ctx->ecdh.p + shift;
1313        ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift;
1314
1315        ret = hpre_ecdh_fill_curve(ctx, params, curve_sz);
1316        if (ret) {
1317                dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret);
1318                dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
1319                ctx->ecdh.p = NULL;
1320                return ret;
1321        }
1322
1323        return 0;
1324}
1325
1326static bool hpre_key_is_zero(char *key, unsigned short key_sz)
1327{
1328        int i;
1329
1330        for (i = 0; i < key_sz; i++)
1331                if (key[i])
1332                        return false;
1333
1334        return true;
1335}
1336
1337static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
1338                                unsigned int len)
1339{
1340        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1341        struct device *dev = HPRE_DEV(ctx);
1342        unsigned int sz, sz_shift;
1343        struct ecdh params;
1344        int ret;
1345
1346        if (crypto_ecdh_decode_key(buf, len, &params) < 0) {
1347                dev_err(dev, "failed to decode ecdh key!\n");
1348                return -EINVAL;
1349        }
1350
1351        if (hpre_key_is_zero(params.key, params.key_size)) {
1352                dev_err(dev, "Invalid hpre key!\n");
1353                return -EINVAL;
1354        }
1355
1356        hpre_ecc_clear_ctx(ctx, false, true);
1357
1358        ret = hpre_ecdh_set_param(ctx, &params);
1359        if (ret < 0) {
1360                dev_err(dev, "failed to set hpre param, ret = %d!\n", ret);
1361                return ret;
1362        }
1363
1364        sz = ctx->key_sz;
1365        sz_shift = (sz << 1) + sz - params.key_size;
1366        memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size);
1367
1368        return 0;
1369}
1370
1371static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
1372                                      struct hpre_asym_request *req,
1373                                      struct scatterlist *dst,
1374                                      struct scatterlist *src)
1375{
1376        struct device *dev = HPRE_DEV(ctx);
1377        struct hpre_sqe *sqe = &req->req;
1378        dma_addr_t dma;
1379
1380        dma = le64_to_cpu(sqe->in);
1381        if (unlikely(dma_mapping_error(dev, dma)))
1382                return;
1383
1384        if (src && req->src)
1385                dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);
1386
1387        dma = le64_to_cpu(sqe->out);
1388        if (unlikely(dma_mapping_error(dev, dma)))
1389                return;
1390
1391        if (req->dst)
1392                dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
1393        if (dst)
1394                dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
1395}
1396
1397static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)
1398{
1399        unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
1400        struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1401        struct hpre_asym_request *req = NULL;
1402        struct kpp_request *areq;
1403        u64 overtime_thrhld;
1404        char *p;
1405        int ret;
1406
1407        ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1408        areq = req->areq.ecdh;
1409        areq->dst_len = ctx->key_sz << 1;
1410
1411        overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1412        if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1413                atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1414
1415        p = sg_virt(areq->dst);
1416        memmove(p, p + ctx->key_sz - curve_sz, curve_sz);
1417        memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);
1418
1419        hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1420        kpp_request_complete(areq, ret);
1421
1422        atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1423}
1424
1425static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
1426                                     struct kpp_request *req)
1427{
1428        struct hpre_asym_request *h_req;
1429        struct hpre_sqe *msg;
1430        int req_id;
1431        void *tmp;
1432
1433        if (req->dst_len < ctx->key_sz << 1) {
1434                req->dst_len = ctx->key_sz << 1;
1435                return -EINVAL;
1436        }
1437
1438        tmp = kpp_request_ctx(req);
1439        h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
1440        h_req->cb = hpre_ecdh_cb;
1441        h_req->areq.ecdh = req;
1442        msg = &h_req->req;
1443        memset(msg, 0, sizeof(*msg));
1444        msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1445        msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1446        msg->key = cpu_to_le64(ctx->ecdh.dma_p);
1447
1448        msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1449        msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1450        h_req->ctx = ctx;
1451
1452        req_id = hpre_add_req_to_ctx(h_req);
1453        if (req_id < 0)
1454                return -EBUSY;
1455
1456        msg->tag = cpu_to_le16((u16)req_id);
1457        return 0;
1458}
1459
1460static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
1461                                   struct scatterlist *data, unsigned int len)
1462{
1463        struct hpre_sqe *msg = &hpre_req->req;
1464        struct hpre_ctx *ctx = hpre_req->ctx;
1465        struct device *dev = HPRE_DEV(ctx);
1466        unsigned int tmpshift;
1467        dma_addr_t dma = 0;
1468        void *ptr;
1469        int shift;
1470
1471        /* Src_data include gx and gy. */
1472        shift = ctx->key_sz - (len >> 1);
1473        if (unlikely(shift < 0))
1474                return -EINVAL;
1475
1476        ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL);
1477        if (unlikely(!ptr))
1478                return -ENOMEM;
1479
1480        tmpshift = ctx->key_sz << 1;
1481        scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0);
1482        memcpy(ptr + shift, ptr + tmpshift, len >> 1);
1483        memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1);
1484
1485        hpre_req->src = ptr;
1486        msg->in = cpu_to_le64(dma);
1487        return 0;
1488}
1489
1490static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,
1491                                   struct scatterlist *data, unsigned int len)
1492{
1493        struct hpre_sqe *msg = &hpre_req->req;
1494        struct hpre_ctx *ctx = hpre_req->ctx;
1495        struct device *dev = HPRE_DEV(ctx);
1496        dma_addr_t dma = 0;
1497
1498        if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {
1499                dev_err(dev, "data or data length is illegal!\n");
1500                return -EINVAL;
1501        }
1502
1503        hpre_req->dst = NULL;
1504        dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1505        if (unlikely(dma_mapping_error(dev, dma))) {
1506                dev_err(dev, "dma map data err!\n");
1507                return -ENOMEM;
1508        }
1509
1510        msg->out = cpu_to_le64(dma);
1511        return 0;
1512}
1513
1514static int hpre_ecdh_compute_value(struct kpp_request *req)
1515{
1516        struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1517        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1518        struct device *dev = HPRE_DEV(ctx);
1519        void *tmp = kpp_request_ctx(req);
1520        struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
1521        struct hpre_sqe *msg = &hpre_req->req;
1522        int ret;
1523
1524        ret = hpre_ecdh_msg_request_set(ctx, req);
1525        if (unlikely(ret)) {
1526                dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret);
1527                return ret;
1528        }
1529
1530        if (req->src) {
1531                ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len);
1532                if (unlikely(ret)) {
1533                        dev_err(dev, "failed to init src data, ret = %d!\n", ret);
1534                        goto clear_all;
1535                }
1536        } else {
1537                msg->in = cpu_to_le64(ctx->ecdh.dma_g);
1538        }
1539
1540        ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len);
1541        if (unlikely(ret)) {
1542                dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1543                goto clear_all;
1544        }
1545
1546        msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);
1547        ret = hpre_send(ctx, msg);
1548        if (likely(!ret))
1549                return -EINPROGRESS;
1550
1551clear_all:
1552        hpre_rm_req_from_ctx(hpre_req);
1553        hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1554        return ret;
1555}
1556
1557static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm)
1558{
1559        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1560
1561        /* max size is the pub_key_size, include x and y */
1562        return ctx->key_sz << 1;
1563}
1564
1565static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm)
1566{
1567        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1568
1569        ctx->curve_id = ECC_CURVE_NIST_P192;
1570
1571        return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1572}
1573
1574static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
1575{
1576        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1577
1578        ctx->curve_id = ECC_CURVE_NIST_P256;
1579
1580        return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1581}
1582
1583static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)
1584{
1585        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1586
1587        hpre_ecc_clear_ctx(ctx, true, true);
1588}
1589
1590static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf,
1591                                       unsigned int len)
1592{
1593        u8 secret[CURVE25519_KEY_SIZE] = { 0 };
1594        unsigned int sz = ctx->key_sz;
1595        const struct ecc_curve *curve;
1596        unsigned int shift = sz << 1;
1597        void *p;
1598
1599        /*
1600         * The key from 'buf' is in little-endian, we should preprocess it as
1601         * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64",
1602         * then convert it to big endian. Only in this way, the result can be
1603         * the same as the software curve-25519 that exists in crypto.
1604         */
1605        memcpy(secret, buf, len);
1606        curve25519_clamp_secret(secret);
1607        hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE);
1608
1609        p = ctx->curve25519.p + sz - len;
1610
1611        curve = ecc_get_curve25519();
1612
1613        /* fill curve parameters */
1614        fill_curve_param(p, curve->p, len, curve->g.ndigits);
1615        fill_curve_param(p + sz, curve->a, len, curve->g.ndigits);
1616        memcpy(p + shift, secret, len);
1617        fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits);
1618        memzero_explicit(secret, CURVE25519_KEY_SIZE);
1619}
1620
1621static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf,
1622                                     unsigned int len)
1623{
1624        struct device *dev = HPRE_DEV(ctx);
1625        unsigned int sz = ctx->key_sz;
1626        unsigned int shift = sz << 1;
1627
1628        /* p->a->k->gx */
1629        if (!ctx->curve25519.p) {
1630                ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2,
1631                                                       &ctx->curve25519.dma_p,
1632                                                       GFP_KERNEL);
1633                if (!ctx->curve25519.p)
1634                        return -ENOMEM;
1635        }
1636
1637        ctx->curve25519.g = ctx->curve25519.p + shift + sz;
1638        ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz;
1639
1640        hpre_curve25519_fill_curve(ctx, buf, len);
1641
1642        return 0;
1643}
1644
1645static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
1646                                      unsigned int len)
1647{
1648        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1649        struct device *dev = HPRE_DEV(ctx);
1650        int ret = -EINVAL;
1651
1652        if (len != CURVE25519_KEY_SIZE ||
1653            !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1654                dev_err(dev, "key is null or key len is not 32bytes!\n");
1655                return ret;
1656        }
1657
1658        /* Free old secret if any */
1659        hpre_ecc_clear_ctx(ctx, false, false);
1660
1661        ctx->key_sz = CURVE25519_KEY_SIZE;
1662        ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE);
1663        if (ret) {
1664                dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret);
1665                hpre_ecc_clear_ctx(ctx, false, false);
1666                return ret;
1667        }
1668
1669        return 0;
1670}
1671
1672static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx,
1673                                            struct hpre_asym_request *req,
1674                                            struct scatterlist *dst,
1675                                            struct scatterlist *src)
1676{
1677        struct device *dev = HPRE_DEV(ctx);
1678        struct hpre_sqe *sqe = &req->req;
1679        dma_addr_t dma;
1680
1681        dma = le64_to_cpu(sqe->in);
1682        if (unlikely(dma_mapping_error(dev, dma)))
1683                return;
1684
1685        if (src && req->src)
1686                dma_free_coherent(dev, ctx->key_sz, req->src, dma);
1687
1688        dma = le64_to_cpu(sqe->out);
1689        if (unlikely(dma_mapping_error(dev, dma)))
1690                return;
1691
1692        if (req->dst)
1693                dma_free_coherent(dev, ctx->key_sz, req->dst, dma);
1694        if (dst)
1695                dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE);
1696}
1697
1698static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp)
1699{
1700        struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
1701        struct hpre_asym_request *req = NULL;
1702        struct kpp_request *areq;
1703        u64 overtime_thrhld;
1704        int ret;
1705
1706        ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
1707        areq = req->areq.curve25519;
1708        areq->dst_len = ctx->key_sz;
1709
1710        overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
1711        if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1712                atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1713
1714        hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE);
1715
1716        hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1717        kpp_request_complete(areq, ret);
1718
1719        atomic64_inc(&dfx[HPRE_RECV_CNT].value);
1720}
1721
1722static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx,
1723                                           struct kpp_request *req)
1724{
1725        struct hpre_asym_request *h_req;
1726        struct hpre_sqe *msg;
1727        int req_id;
1728        void *tmp;
1729
1730        if (unlikely(req->dst_len < ctx->key_sz)) {
1731                req->dst_len = ctx->key_sz;
1732                return -EINVAL;
1733        }
1734
1735        tmp = kpp_request_ctx(req);
1736        h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
1737        h_req->cb = hpre_curve25519_cb;
1738        h_req->areq.curve25519 = req;
1739        msg = &h_req->req;
1740        memset(msg, 0, sizeof(*msg));
1741        msg->in = cpu_to_le64(DMA_MAPPING_ERROR);
1742        msg->out = cpu_to_le64(DMA_MAPPING_ERROR);
1743        msg->key = cpu_to_le64(ctx->curve25519.dma_p);
1744
1745        msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
1746        msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
1747        h_req->ctx = ctx;
1748
1749        req_id = hpre_add_req_to_ctx(h_req);
1750        if (req_id < 0)
1751                return -EBUSY;
1752
1753        msg->tag = cpu_to_le16((u16)req_id);
1754        return 0;
1755}
1756
1757static void hpre_curve25519_src_modulo_p(u8 *ptr)
1758{
1759        int i;
1760
1761        for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++)
1762                ptr[i] = 0;
1763
1764        /* The modulus is ptr's last byte minus '0xed'(last byte of p) */
1765        ptr[i] -= 0xed;
1766}
1767
1768static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
1769                                    struct scatterlist *data, unsigned int len)
1770{
1771        struct hpre_sqe *msg = &hpre_req->req;
1772        struct hpre_ctx *ctx = hpre_req->ctx;
1773        struct device *dev = HPRE_DEV(ctx);
1774        u8 p[CURVE25519_KEY_SIZE] = { 0 };
1775        const struct ecc_curve *curve;
1776        dma_addr_t dma = 0;
1777        u8 *ptr;
1778
1779        if (len != CURVE25519_KEY_SIZE) {
1780                dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len);
1781                return -EINVAL;
1782        }
1783
1784        ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL);
1785        if (unlikely(!ptr))
1786                return -ENOMEM;
1787
1788        scatterwalk_map_and_copy(ptr, data, 0, len, 0);
1789
1790        if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) {
1791                dev_err(dev, "gx is null!\n");
1792                goto err;
1793        }
1794
1795        /*
1796         * Src_data(gx) is in little-endian order, MSB in the final byte should
1797         * be masked as described in RFC7748, then transform it to big-endian
1798         * form, then hisi_hpre can use the data.
1799         */
1800        ptr[31] &= 0x7f;
1801        hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE);
1802
1803        curve = ecc_get_curve25519();
1804
1805        fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits);
1806
1807        /*
1808         * When src_data equals (2^255 - 19) ~  (2^255 - 1), it is out of p,
1809         * we get its modulus to p, and then use it.
1810         */
1811        if (memcmp(ptr, p, ctx->key_sz) >= 0)
1812                hpre_curve25519_src_modulo_p(ptr);
1813
1814        hpre_req->src = ptr;
1815        msg->in = cpu_to_le64(dma);
1816        return 0;
1817
1818err:
1819        dma_free_coherent(dev, ctx->key_sz, ptr, dma);
1820        return -EINVAL;
1821}
1822
1823static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req,
1824                                    struct scatterlist *data, unsigned int len)
1825{
1826        struct hpre_sqe *msg = &hpre_req->req;
1827        struct hpre_ctx *ctx = hpre_req->ctx;
1828        struct device *dev = HPRE_DEV(ctx);
1829        dma_addr_t dma = 0;
1830
1831        if (!data || !sg_is_last(data) || len != ctx->key_sz) {
1832                dev_err(dev, "data or data length is illegal!\n");
1833                return -EINVAL;
1834        }
1835
1836        hpre_req->dst = NULL;
1837        dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
1838        if (unlikely(dma_mapping_error(dev, dma))) {
1839                dev_err(dev, "dma map data err!\n");
1840                return -ENOMEM;
1841        }
1842
1843        msg->out = cpu_to_le64(dma);
1844        return 0;
1845}
1846
1847static int hpre_curve25519_compute_value(struct kpp_request *req)
1848{
1849        struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
1850        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1851        struct device *dev = HPRE_DEV(ctx);
1852        void *tmp = kpp_request_ctx(req);
1853        struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
1854        struct hpre_sqe *msg = &hpre_req->req;
1855        int ret;
1856
1857        ret = hpre_curve25519_msg_request_set(ctx, req);
1858        if (unlikely(ret)) {
1859                dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret);
1860                return ret;
1861        }
1862
1863        if (req->src) {
1864                ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len);
1865                if (unlikely(ret)) {
1866                        dev_err(dev, "failed to init src data, ret = %d!\n",
1867                                ret);
1868                        goto clear_all;
1869                }
1870        } else {
1871                msg->in = cpu_to_le64(ctx->curve25519.dma_g);
1872        }
1873
1874        ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len);
1875        if (unlikely(ret)) {
1876                dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
1877                goto clear_all;
1878        }
1879
1880        msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL);
1881        ret = hpre_send(ctx, msg);
1882        if (likely(!ret))
1883                return -EINPROGRESS;
1884
1885clear_all:
1886        hpre_rm_req_from_ctx(hpre_req);
1887        hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
1888        return ret;
1889}
1890
1891static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm)
1892{
1893        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1894
1895        return ctx->key_sz;
1896}
1897
1898static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm)
1899{
1900        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1901
1902        return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
1903}
1904
1905static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm)
1906{
1907        struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
1908
1909        hpre_ecc_clear_ctx(ctx, true, false);
1910}
1911
1912static struct akcipher_alg rsa = {
1913        .sign = hpre_rsa_dec,
1914        .verify = hpre_rsa_enc,
1915        .encrypt = hpre_rsa_enc,
1916        .decrypt = hpre_rsa_dec,
1917        .set_pub_key = hpre_rsa_setpubkey,
1918        .set_priv_key = hpre_rsa_setprivkey,
1919        .max_size = hpre_rsa_max_size,
1920        .init = hpre_rsa_init_tfm,
1921        .exit = hpre_rsa_exit_tfm,
1922        .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
1923        .base = {
1924                .cra_ctxsize = sizeof(struct hpre_ctx),
1925                .cra_priority = HPRE_CRYPTO_ALG_PRI,
1926                .cra_name = "rsa",
1927                .cra_driver_name = "hpre-rsa",
1928                .cra_module = THIS_MODULE,
1929        },
1930};
1931
1932static struct kpp_alg dh = {
1933        .set_secret = hpre_dh_set_secret,
1934        .generate_public_key = hpre_dh_compute_value,
1935        .compute_shared_secret = hpre_dh_compute_value,
1936        .max_size = hpre_dh_max_size,
1937        .init = hpre_dh_init_tfm,
1938        .exit = hpre_dh_exit_tfm,
1939        .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
1940        .base = {
1941                .cra_ctxsize = sizeof(struct hpre_ctx),
1942                .cra_priority = HPRE_CRYPTO_ALG_PRI,
1943                .cra_name = "dh",
1944                .cra_driver_name = "hpre-dh",
1945                .cra_module = THIS_MODULE,
1946        },
1947};
1948
1949static struct kpp_alg ecdh_nist_p192 = {
1950        .set_secret = hpre_ecdh_set_secret,
1951        .generate_public_key = hpre_ecdh_compute_value,
1952        .compute_shared_secret = hpre_ecdh_compute_value,
1953        .max_size = hpre_ecdh_max_size,
1954        .init = hpre_ecdh_nist_p192_init_tfm,
1955        .exit = hpre_ecdh_exit_tfm,
1956        .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
1957        .base = {
1958                .cra_ctxsize = sizeof(struct hpre_ctx),
1959                .cra_priority = HPRE_CRYPTO_ALG_PRI,
1960                .cra_name = "ecdh-nist-p192",
1961                .cra_driver_name = "hpre-ecdh",
1962                .cra_module = THIS_MODULE,
1963        },
1964};
1965
1966static struct kpp_alg ecdh_nist_p256 = {
1967        .set_secret = hpre_ecdh_set_secret,
1968        .generate_public_key = hpre_ecdh_compute_value,
1969        .compute_shared_secret = hpre_ecdh_compute_value,
1970        .max_size = hpre_ecdh_max_size,
1971        .init = hpre_ecdh_nist_p256_init_tfm,
1972        .exit = hpre_ecdh_exit_tfm,
1973        .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
1974        .base = {
1975                .cra_ctxsize = sizeof(struct hpre_ctx),
1976                .cra_priority = HPRE_CRYPTO_ALG_PRI,
1977                .cra_name = "ecdh-nist-p256",
1978                .cra_driver_name = "hpre-ecdh",
1979                .cra_module = THIS_MODULE,
1980        },
1981};
1982
1983static struct kpp_alg curve25519_alg = {
1984        .set_secret = hpre_curve25519_set_secret,
1985        .generate_public_key = hpre_curve25519_compute_value,
1986        .compute_shared_secret = hpre_curve25519_compute_value,
1987        .max_size = hpre_curve25519_max_size,
1988        .init = hpre_curve25519_init_tfm,
1989        .exit = hpre_curve25519_exit_tfm,
1990        .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
1991        .base = {
1992                .cra_ctxsize = sizeof(struct hpre_ctx),
1993                .cra_priority = HPRE_CRYPTO_ALG_PRI,
1994                .cra_name = "curve25519",
1995                .cra_driver_name = "hpre-curve25519",
1996                .cra_module = THIS_MODULE,
1997        },
1998};
1999
2000
2001static int hpre_register_ecdh(void)
2002{
2003        int ret;
2004
2005        ret = crypto_register_kpp(&ecdh_nist_p192);
2006        if (ret)
2007                return ret;
2008
2009        ret = crypto_register_kpp(&ecdh_nist_p256);
2010        if (ret) {
2011                crypto_unregister_kpp(&ecdh_nist_p192);
2012                return ret;
2013        }
2014
2015        return 0;
2016}
2017
2018static void hpre_unregister_ecdh(void)
2019{
2020        crypto_unregister_kpp(&ecdh_nist_p256);
2021        crypto_unregister_kpp(&ecdh_nist_p192);
2022}
2023
2024int hpre_algs_register(struct hisi_qm *qm)
2025{
2026        int ret;
2027
2028        rsa.base.cra_flags = 0;
2029        ret = crypto_register_akcipher(&rsa);
2030        if (ret)
2031                return ret;
2032
2033        ret = crypto_register_kpp(&dh);
2034        if (ret)
2035                goto unreg_rsa;
2036
2037        if (qm->ver >= QM_HW_V3) {
2038                ret = hpre_register_ecdh();
2039                if (ret)
2040                        goto unreg_dh;
2041                ret = crypto_register_kpp(&curve25519_alg);
2042                if (ret)
2043                        goto unreg_ecdh;
2044        }
2045        return 0;
2046
2047unreg_ecdh:
2048        hpre_unregister_ecdh();
2049unreg_dh:
2050        crypto_unregister_kpp(&dh);
2051unreg_rsa:
2052        crypto_unregister_akcipher(&rsa);
2053        return ret;
2054}
2055
2056void hpre_algs_unregister(struct hisi_qm *qm)
2057{
2058        if (qm->ver >= QM_HW_V3) {
2059                crypto_unregister_kpp(&curve25519_alg);
2060                hpre_unregister_ecdh();
2061        }
2062
2063        crypto_unregister_kpp(&dh);
2064        crypto_unregister_akcipher(&rsa);
2065}
2066