linux/crypto/simd.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Shared crypto simd helpers
   4 *
   5 * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   6 * Copyright (c) 2016 Herbert Xu <herbert@gondor.apana.org.au>
   7 * Copyright (c) 2019 Google LLC
   8 *
   9 * Based on aesni-intel_glue.c by:
  10 *  Copyright (C) 2008, Intel Corp.
  11 *    Author: Huang Ying <ying.huang@intel.com>
  12 */
  13
  14/*
  15 * Shared crypto SIMD helpers.  These functions dynamically create and register
  16 * an skcipher or AEAD algorithm that wraps another, internal algorithm.  The
  17 * wrapper ensures that the internal algorithm is only executed in a context
  18 * where SIMD instructions are usable, i.e. where may_use_simd() returns true.
  19 * If SIMD is already usable, the wrapper directly calls the internal algorithm.
  20 * Otherwise it defers execution to a workqueue via cryptd.
  21 *
  22 * This is an alternative to the internal algorithm implementing a fallback for
  23 * the !may_use_simd() case itself.
  24 *
  25 * Note that the wrapper algorithm is asynchronous, i.e. it has the
  26 * CRYPTO_ALG_ASYNC flag set.  Therefore it won't be found by users who
  27 * explicitly allocate a synchronous algorithm.
  28 */
  29
  30#include <crypto/cryptd.h>
  31#include <crypto/internal/aead.h>
  32#include <crypto/internal/simd.h>
  33#include <crypto/internal/skcipher.h>
  34#include <linux/kernel.h>
  35#include <linux/module.h>
  36#include <linux/preempt.h>
  37#include <asm/simd.h>
  38
  39/* skcipher support */
  40
  41struct simd_skcipher_alg {
  42        const char *ialg_name;
  43        struct skcipher_alg alg;
  44};
  45
  46struct simd_skcipher_ctx {
  47        struct cryptd_skcipher *cryptd_tfm;
  48};
  49
  50static int simd_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
  51                                unsigned int key_len)
  52{
  53        struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
  54        struct crypto_skcipher *child = &ctx->cryptd_tfm->base;
  55
  56        crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
  57        crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(tfm) &
  58                                         CRYPTO_TFM_REQ_MASK);
  59        return crypto_skcipher_setkey(child, key, key_len);
  60}
  61
  62static int simd_skcipher_encrypt(struct skcipher_request *req)
  63{
  64        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  65        struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
  66        struct skcipher_request *subreq;
  67        struct crypto_skcipher *child;
  68
  69        subreq = skcipher_request_ctx(req);
  70        *subreq = *req;
  71
  72        if (!crypto_simd_usable() ||
  73            (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm)))
  74                child = &ctx->cryptd_tfm->base;
  75        else
  76                child = cryptd_skcipher_child(ctx->cryptd_tfm);
  77
  78        skcipher_request_set_tfm(subreq, child);
  79
  80        return crypto_skcipher_encrypt(subreq);
  81}
  82
  83static int simd_skcipher_decrypt(struct skcipher_request *req)
  84{
  85        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
  86        struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
  87        struct skcipher_request *subreq;
  88        struct crypto_skcipher *child;
  89
  90        subreq = skcipher_request_ctx(req);
  91        *subreq = *req;
  92
  93        if (!crypto_simd_usable() ||
  94            (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm)))
  95                child = &ctx->cryptd_tfm->base;
  96        else
  97                child = cryptd_skcipher_child(ctx->cryptd_tfm);
  98
  99        skcipher_request_set_tfm(subreq, child);
 100
 101        return crypto_skcipher_decrypt(subreq);
 102}
 103
 104static void simd_skcipher_exit(struct crypto_skcipher *tfm)
 105{
 106        struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
 107
 108        cryptd_free_skcipher(ctx->cryptd_tfm);
 109}
 110
 111static int simd_skcipher_init(struct crypto_skcipher *tfm)
 112{
 113        struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
 114        struct cryptd_skcipher *cryptd_tfm;
 115        struct simd_skcipher_alg *salg;
 116        struct skcipher_alg *alg;
 117        unsigned reqsize;
 118
 119        alg = crypto_skcipher_alg(tfm);
 120        salg = container_of(alg, struct simd_skcipher_alg, alg);
 121
 122        cryptd_tfm = cryptd_alloc_skcipher(salg->ialg_name,
 123                                           CRYPTO_ALG_INTERNAL,
 124                                           CRYPTO_ALG_INTERNAL);
 125        if (IS_ERR(cryptd_tfm))
 126                return PTR_ERR(cryptd_tfm);
 127
 128        ctx->cryptd_tfm = cryptd_tfm;
 129
 130        reqsize = crypto_skcipher_reqsize(cryptd_skcipher_child(cryptd_tfm));
 131        reqsize = max(reqsize, crypto_skcipher_reqsize(&cryptd_tfm->base));
 132        reqsize += sizeof(struct skcipher_request);
 133
 134        crypto_skcipher_set_reqsize(tfm, reqsize);
 135
 136        return 0;
 137}
 138
 139struct simd_skcipher_alg *simd_skcipher_create_compat(const char *algname,
 140                                                      const char *drvname,
 141                                                      const char *basename)
 142{
 143        struct simd_skcipher_alg *salg;
 144        struct crypto_skcipher *tfm;
 145        struct skcipher_alg *ialg;
 146        struct skcipher_alg *alg;
 147        int err;
 148
 149        tfm = crypto_alloc_skcipher(basename, CRYPTO_ALG_INTERNAL,
 150                                    CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC);
 151        if (IS_ERR(tfm))
 152                return ERR_CAST(tfm);
 153
 154        ialg = crypto_skcipher_alg(tfm);
 155
 156        salg = kzalloc(sizeof(*salg), GFP_KERNEL);
 157        if (!salg) {
 158                salg = ERR_PTR(-ENOMEM);
 159                goto out_put_tfm;
 160        }
 161
 162        salg->ialg_name = basename;
 163        alg = &salg->alg;
 164
 165        err = -ENAMETOOLONG;
 166        if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >=
 167            CRYPTO_MAX_ALG_NAME)
 168                goto out_free_salg;
 169
 170        if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
 171                     drvname) >= CRYPTO_MAX_ALG_NAME)
 172                goto out_free_salg;
 173
 174        alg->base.cra_flags = CRYPTO_ALG_ASYNC |
 175                (ialg->base.cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
 176        alg->base.cra_priority = ialg->base.cra_priority;
 177        alg->base.cra_blocksize = ialg->base.cra_blocksize;
 178        alg->base.cra_alignmask = ialg->base.cra_alignmask;
 179        alg->base.cra_module = ialg->base.cra_module;
 180        alg->base.cra_ctxsize = sizeof(struct simd_skcipher_ctx);
 181
 182        alg->ivsize = ialg->ivsize;
 183        alg->chunksize = ialg->chunksize;
 184        alg->min_keysize = ialg->min_keysize;
 185        alg->max_keysize = ialg->max_keysize;
 186
 187        alg->init = simd_skcipher_init;
 188        alg->exit = simd_skcipher_exit;
 189
 190        alg->setkey = simd_skcipher_setkey;
 191        alg->encrypt = simd_skcipher_encrypt;
 192        alg->decrypt = simd_skcipher_decrypt;
 193
 194        err = crypto_register_skcipher(alg);
 195        if (err)
 196                goto out_free_salg;
 197
 198out_put_tfm:
 199        crypto_free_skcipher(tfm);
 200        return salg;
 201
 202out_free_salg:
 203        kfree(salg);
 204        salg = ERR_PTR(err);
 205        goto out_put_tfm;
 206}
 207EXPORT_SYMBOL_GPL(simd_skcipher_create_compat);
 208
 209struct simd_skcipher_alg *simd_skcipher_create(const char *algname,
 210                                               const char *basename)
 211{
 212        char drvname[CRYPTO_MAX_ALG_NAME];
 213
 214        if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >=
 215            CRYPTO_MAX_ALG_NAME)
 216                return ERR_PTR(-ENAMETOOLONG);
 217
 218        return simd_skcipher_create_compat(algname, drvname, basename);
 219}
 220EXPORT_SYMBOL_GPL(simd_skcipher_create);
 221
 222void simd_skcipher_free(struct simd_skcipher_alg *salg)
 223{
 224        crypto_unregister_skcipher(&salg->alg);
 225        kfree(salg);
 226}
 227EXPORT_SYMBOL_GPL(simd_skcipher_free);
 228
 229int simd_register_skciphers_compat(struct skcipher_alg *algs, int count,
 230                                   struct simd_skcipher_alg **simd_algs)
 231{
 232        int err;
 233        int i;
 234        const char *algname;
 235        const char *drvname;
 236        const char *basename;
 237        struct simd_skcipher_alg *simd;
 238
 239        err = crypto_register_skciphers(algs, count);
 240        if (err)
 241                return err;
 242
 243        for (i = 0; i < count; i++) {
 244                WARN_ON(strncmp(algs[i].base.cra_name, "__", 2));
 245                WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2));
 246                algname = algs[i].base.cra_name + 2;
 247                drvname = algs[i].base.cra_driver_name + 2;
 248                basename = algs[i].base.cra_driver_name;
 249                simd = simd_skcipher_create_compat(algname, drvname, basename);
 250                err = PTR_ERR(simd);
 251                if (IS_ERR(simd))
 252                        goto err_unregister;
 253                simd_algs[i] = simd;
 254        }
 255        return 0;
 256
 257err_unregister:
 258        simd_unregister_skciphers(algs, count, simd_algs);
 259        return err;
 260}
 261EXPORT_SYMBOL_GPL(simd_register_skciphers_compat);
 262
 263void simd_unregister_skciphers(struct skcipher_alg *algs, int count,
 264                               struct simd_skcipher_alg **simd_algs)
 265{
 266        int i;
 267
 268        crypto_unregister_skciphers(algs, count);
 269
 270        for (i = 0; i < count; i++) {
 271                if (simd_algs[i]) {
 272                        simd_skcipher_free(simd_algs[i]);
 273                        simd_algs[i] = NULL;
 274                }
 275        }
 276}
 277EXPORT_SYMBOL_GPL(simd_unregister_skciphers);
 278
 279/* AEAD support */
 280
 281struct simd_aead_alg {
 282        const char *ialg_name;
 283        struct aead_alg alg;
 284};
 285
 286struct simd_aead_ctx {
 287        struct cryptd_aead *cryptd_tfm;
 288};
 289
 290static int simd_aead_setkey(struct crypto_aead *tfm, const u8 *key,
 291                                unsigned int key_len)
 292{
 293        struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
 294        struct crypto_aead *child = &ctx->cryptd_tfm->base;
 295
 296        crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 297        crypto_aead_set_flags(child, crypto_aead_get_flags(tfm) &
 298                                     CRYPTO_TFM_REQ_MASK);
 299        return crypto_aead_setkey(child, key, key_len);
 300}
 301
 302static int simd_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
 303{
 304        struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
 305        struct crypto_aead *child = &ctx->cryptd_tfm->base;
 306
 307        return crypto_aead_setauthsize(child, authsize);
 308}
 309
 310static int simd_aead_encrypt(struct aead_request *req)
 311{
 312        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 313        struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
 314        struct aead_request *subreq;
 315        struct crypto_aead *child;
 316
 317        subreq = aead_request_ctx(req);
 318        *subreq = *req;
 319
 320        if (!crypto_simd_usable() ||
 321            (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm)))
 322                child = &ctx->cryptd_tfm->base;
 323        else
 324                child = cryptd_aead_child(ctx->cryptd_tfm);
 325
 326        aead_request_set_tfm(subreq, child);
 327
 328        return crypto_aead_encrypt(subreq);
 329}
 330
 331static int simd_aead_decrypt(struct aead_request *req)
 332{
 333        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 334        struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
 335        struct aead_request *subreq;
 336        struct crypto_aead *child;
 337
 338        subreq = aead_request_ctx(req);
 339        *subreq = *req;
 340
 341        if (!crypto_simd_usable() ||
 342            (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm)))
 343                child = &ctx->cryptd_tfm->base;
 344        else
 345                child = cryptd_aead_child(ctx->cryptd_tfm);
 346
 347        aead_request_set_tfm(subreq, child);
 348
 349        return crypto_aead_decrypt(subreq);
 350}
 351
 352static void simd_aead_exit(struct crypto_aead *tfm)
 353{
 354        struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
 355
 356        cryptd_free_aead(ctx->cryptd_tfm);
 357}
 358
 359static int simd_aead_init(struct crypto_aead *tfm)
 360{
 361        struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm);
 362        struct cryptd_aead *cryptd_tfm;
 363        struct simd_aead_alg *salg;
 364        struct aead_alg *alg;
 365        unsigned reqsize;
 366
 367        alg = crypto_aead_alg(tfm);
 368        salg = container_of(alg, struct simd_aead_alg, alg);
 369
 370        cryptd_tfm = cryptd_alloc_aead(salg->ialg_name, CRYPTO_ALG_INTERNAL,
 371                                       CRYPTO_ALG_INTERNAL);
 372        if (IS_ERR(cryptd_tfm))
 373                return PTR_ERR(cryptd_tfm);
 374
 375        ctx->cryptd_tfm = cryptd_tfm;
 376
 377        reqsize = crypto_aead_reqsize(cryptd_aead_child(cryptd_tfm));
 378        reqsize = max(reqsize, crypto_aead_reqsize(&cryptd_tfm->base));
 379        reqsize += sizeof(struct aead_request);
 380
 381        crypto_aead_set_reqsize(tfm, reqsize);
 382
 383        return 0;
 384}
 385
 386struct simd_aead_alg *simd_aead_create_compat(const char *algname,
 387                                              const char *drvname,
 388                                              const char *basename)
 389{
 390        struct simd_aead_alg *salg;
 391        struct crypto_aead *tfm;
 392        struct aead_alg *ialg;
 393        struct aead_alg *alg;
 394        int err;
 395
 396        tfm = crypto_alloc_aead(basename, CRYPTO_ALG_INTERNAL,
 397                                CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC);
 398        if (IS_ERR(tfm))
 399                return ERR_CAST(tfm);
 400
 401        ialg = crypto_aead_alg(tfm);
 402
 403        salg = kzalloc(sizeof(*salg), GFP_KERNEL);
 404        if (!salg) {
 405                salg = ERR_PTR(-ENOMEM);
 406                goto out_put_tfm;
 407        }
 408
 409        salg->ialg_name = basename;
 410        alg = &salg->alg;
 411
 412        err = -ENAMETOOLONG;
 413        if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >=
 414            CRYPTO_MAX_ALG_NAME)
 415                goto out_free_salg;
 416
 417        if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
 418                     drvname) >= CRYPTO_MAX_ALG_NAME)
 419                goto out_free_salg;
 420
 421        alg->base.cra_flags = CRYPTO_ALG_ASYNC |
 422                (ialg->base.cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
 423        alg->base.cra_priority = ialg->base.cra_priority;
 424        alg->base.cra_blocksize = ialg->base.cra_blocksize;
 425        alg->base.cra_alignmask = ialg->base.cra_alignmask;
 426        alg->base.cra_module = ialg->base.cra_module;
 427        alg->base.cra_ctxsize = sizeof(struct simd_aead_ctx);
 428
 429        alg->ivsize = ialg->ivsize;
 430        alg->maxauthsize = ialg->maxauthsize;
 431        alg->chunksize = ialg->chunksize;
 432
 433        alg->init = simd_aead_init;
 434        alg->exit = simd_aead_exit;
 435
 436        alg->setkey = simd_aead_setkey;
 437        alg->setauthsize = simd_aead_setauthsize;
 438        alg->encrypt = simd_aead_encrypt;
 439        alg->decrypt = simd_aead_decrypt;
 440
 441        err = crypto_register_aead(alg);
 442        if (err)
 443                goto out_free_salg;
 444
 445out_put_tfm:
 446        crypto_free_aead(tfm);
 447        return salg;
 448
 449out_free_salg:
 450        kfree(salg);
 451        salg = ERR_PTR(err);
 452        goto out_put_tfm;
 453}
 454EXPORT_SYMBOL_GPL(simd_aead_create_compat);
 455
 456struct simd_aead_alg *simd_aead_create(const char *algname,
 457                                       const char *basename)
 458{
 459        char drvname[CRYPTO_MAX_ALG_NAME];
 460
 461        if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >=
 462            CRYPTO_MAX_ALG_NAME)
 463                return ERR_PTR(-ENAMETOOLONG);
 464
 465        return simd_aead_create_compat(algname, drvname, basename);
 466}
 467EXPORT_SYMBOL_GPL(simd_aead_create);
 468
 469void simd_aead_free(struct simd_aead_alg *salg)
 470{
 471        crypto_unregister_aead(&salg->alg);
 472        kfree(salg);
 473}
 474EXPORT_SYMBOL_GPL(simd_aead_free);
 475
 476int simd_register_aeads_compat(struct aead_alg *algs, int count,
 477                               struct simd_aead_alg **simd_algs)
 478{
 479        int err;
 480        int i;
 481        const char *algname;
 482        const char *drvname;
 483        const char *basename;
 484        struct simd_aead_alg *simd;
 485
 486        err = crypto_register_aeads(algs, count);
 487        if (err)
 488                return err;
 489
 490        for (i = 0; i < count; i++) {
 491                WARN_ON(strncmp(algs[i].base.cra_name, "__", 2));
 492                WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2));
 493                algname = algs[i].base.cra_name + 2;
 494                drvname = algs[i].base.cra_driver_name + 2;
 495                basename = algs[i].base.cra_driver_name;
 496                simd = simd_aead_create_compat(algname, drvname, basename);
 497                err = PTR_ERR(simd);
 498                if (IS_ERR(simd))
 499                        goto err_unregister;
 500                simd_algs[i] = simd;
 501        }
 502        return 0;
 503
 504err_unregister:
 505        simd_unregister_aeads(algs, count, simd_algs);
 506        return err;
 507}
 508EXPORT_SYMBOL_GPL(simd_register_aeads_compat);
 509
 510void simd_unregister_aeads(struct aead_alg *algs, int count,
 511                           struct simd_aead_alg **simd_algs)
 512{
 513        int i;
 514
 515        crypto_unregister_aeads(algs, count);
 516
 517        for (i = 0; i < count; i++) {
 518                if (simd_algs[i]) {
 519                        simd_aead_free(simd_algs[i]);
 520                        simd_algs[i] = NULL;
 521                }
 522        }
 523}
 524EXPORT_SYMBOL_GPL(simd_unregister_aeads);
 525
 526MODULE_LICENSE("GPL");
 527
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.