linux/arch/arm/crypto/chacha-glue.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * ARM NEON accelerated ChaCha and XChaCha stream ciphers,
   4 * including ChaCha20 (RFC7539)
   5 *
   6 * Copyright (C) 2016-2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
   7 * Copyright (C) 2015 Martin Willi
   8 */
   9
  10#include <crypto/algapi.h>
  11#include <crypto/internal/chacha.h>
  12#include <crypto/internal/simd.h>
  13#include <crypto/internal/skcipher.h>
  14#include <linux/jump_label.h>
  15#include <linux/kernel.h>
  16#include <linux/module.h>
  17
  18#include <asm/cputype.h>
  19#include <asm/hwcap.h>
  20#include <asm/neon.h>
  21#include <asm/simd.h>
  22
  23asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
  24                                      int nrounds);
  25asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
  26                                       int nrounds, unsigned int nbytes);
  27asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds);
  28asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
  29
  30asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,
  31                             const u32 *state, int nrounds);
  32
  33static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon);
  34
  35static inline bool neon_usable(void)
  36{
  37        return static_branch_likely(&use_neon) && crypto_simd_usable();
  38}
  39
  40static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
  41                          unsigned int bytes, int nrounds)
  42{
  43        u8 buf[CHACHA_BLOCK_SIZE];
  44
  45        while (bytes > CHACHA_BLOCK_SIZE) {
  46                unsigned int l = min(bytes, CHACHA_BLOCK_SIZE * 4U);
  47
  48                chacha_4block_xor_neon(state, dst, src, nrounds, l);
  49                bytes -= l;
  50                src += l;
  51                dst += l;
  52                state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE);
  53        }
  54        if (bytes) {
  55                const u8 *s = src;
  56                u8 *d = dst;
  57
  58                if (bytes != CHACHA_BLOCK_SIZE)
  59                        s = d = memcpy(buf, src, bytes);
  60                chacha_block_xor_neon(state, d, s, nrounds);
  61                if (d != dst)
  62                        memcpy(dst, buf, bytes);
  63                state[12]++;
  64        }
  65}
  66
  67void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
  68{
  69        if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {
  70                hchacha_block_arm(state, stream, nrounds);
  71        } else {
  72                kernel_neon_begin();
  73                hchacha_block_neon(state, stream, nrounds);
  74                kernel_neon_end();
  75        }
  76}
  77EXPORT_SYMBOL(hchacha_block_arch);
  78
  79void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
  80{
  81        chacha_init_generic(state, key, iv);
  82}
  83EXPORT_SYMBOL(chacha_init_arch);
  84
  85void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
  86                       int nrounds)
  87{
  88        if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() ||
  89            bytes <= CHACHA_BLOCK_SIZE) {
  90                chacha_doarm(dst, src, bytes, state, nrounds);
  91                state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
  92                return;
  93        }
  94
  95        do {
  96                unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
  97
  98                kernel_neon_begin();
  99                chacha_doneon(state, dst, src, todo, nrounds);
 100                kernel_neon_end();
 101
 102                bytes -= todo;
 103                src += todo;
 104                dst += todo;
 105        } while (bytes);
 106}
 107EXPORT_SYMBOL(chacha_crypt_arch);
 108
 109static int chacha_stream_xor(struct skcipher_request *req,
 110                             const struct chacha_ctx *ctx, const u8 *iv,
 111                             bool neon)
 112{
 113        struct skcipher_walk walk;
 114        u32 state[16];
 115        int err;
 116
 117        err = skcipher_walk_virt(&walk, req, false);
 118
 119        chacha_init_generic(state, ctx->key, iv);
 120
 121        while (walk.nbytes > 0) {
 122                unsigned int nbytes = walk.nbytes;
 123
 124                if (nbytes < walk.total)
 125                        nbytes = round_down(nbytes, walk.stride);
 126
 127                if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
 128                        chacha_doarm(walk.dst.virt.addr, walk.src.virt.addr,
 129                                     nbytes, state, ctx->nrounds);
 130                        state[12] += DIV_ROUND_UP(nbytes, CHACHA_BLOCK_SIZE);
 131                } else {
 132                        kernel_neon_begin();
 133                        chacha_doneon(state, walk.dst.virt.addr,
 134                                      walk.src.virt.addr, nbytes, ctx->nrounds);
 135                        kernel_neon_end();
 136                }
 137                err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 138        }
 139
 140        return err;
 141}
 142
 143static int do_chacha(struct skcipher_request *req, bool neon)
 144{
 145        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 146        struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 147
 148        return chacha_stream_xor(req, ctx, req->iv, neon);
 149}
 150
 151static int chacha_arm(struct skcipher_request *req)
 152{
 153        return do_chacha(req, false);
 154}
 155
 156static int chacha_neon(struct skcipher_request *req)
 157{
 158        return do_chacha(req, neon_usable());
 159}
 160
 161static int do_xchacha(struct skcipher_request *req, bool neon)
 162{
 163        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 164        struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
 165        struct chacha_ctx subctx;
 166        u32 state[16];
 167        u8 real_iv[16];
 168
 169        chacha_init_generic(state, ctx->key, req->iv);
 170
 171        if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
 172                hchacha_block_arm(state, subctx.key, ctx->nrounds);
 173        } else {
 174                kernel_neon_begin();
 175                hchacha_block_neon(state, subctx.key, ctx->nrounds);
 176                kernel_neon_end();
 177        }
 178        subctx.nrounds = ctx->nrounds;
 179
 180        memcpy(&real_iv[0], req->iv + 24, 8);
 181        memcpy(&real_iv[8], req->iv + 16, 8);
 182        return chacha_stream_xor(req, &subctx, real_iv, neon);
 183}
 184
 185static int xchacha_arm(struct skcipher_request *req)
 186{
 187        return do_xchacha(req, false);
 188}
 189
 190static int xchacha_neon(struct skcipher_request *req)
 191{
 192        return do_xchacha(req, neon_usable());
 193}
 194
 195static struct skcipher_alg arm_algs[] = {
 196        {
 197                .base.cra_name          = "chacha20",
 198                .base.cra_driver_name   = "chacha20-arm",
 199                .base.cra_priority      = 200,
 200                .base.cra_blocksize     = 1,
 201                .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 202                .base.cra_module        = THIS_MODULE,
 203
 204                .min_keysize            = CHACHA_KEY_SIZE,
 205                .max_keysize            = CHACHA_KEY_SIZE,
 206                .ivsize                 = CHACHA_IV_SIZE,
 207                .chunksize              = CHACHA_BLOCK_SIZE,
 208                .setkey                 = chacha20_setkey,
 209                .encrypt                = chacha_arm,
 210                .decrypt                = chacha_arm,
 211        }, {
 212                .base.cra_name          = "xchacha20",
 213                .base.cra_driver_name   = "xchacha20-arm",
 214                .base.cra_priority      = 200,
 215                .base.cra_blocksize     = 1,
 216                .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 217                .base.cra_module        = THIS_MODULE,
 218
 219                .min_keysize            = CHACHA_KEY_SIZE,
 220                .max_keysize            = CHACHA_KEY_SIZE,
 221                .ivsize                 = XCHACHA_IV_SIZE,
 222                .chunksize              = CHACHA_BLOCK_SIZE,
 223                .setkey                 = chacha20_setkey,
 224                .encrypt                = xchacha_arm,
 225                .decrypt                = xchacha_arm,
 226        }, {
 227                .base.cra_name          = "xchacha12",
 228                .base.cra_driver_name   = "xchacha12-arm",
 229                .base.cra_priority      = 200,
 230                .base.cra_blocksize     = 1,
 231                .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 232                .base.cra_module        = THIS_MODULE,
 233
 234                .min_keysize            = CHACHA_KEY_SIZE,
 235                .max_keysize            = CHACHA_KEY_SIZE,
 236                .ivsize                 = XCHACHA_IV_SIZE,
 237                .chunksize              = CHACHA_BLOCK_SIZE,
 238                .setkey                 = chacha12_setkey,
 239                .encrypt                = xchacha_arm,
 240                .decrypt                = xchacha_arm,
 241        },
 242};
 243
 244static struct skcipher_alg neon_algs[] = {
 245        {
 246                .base.cra_name          = "chacha20",
 247                .base.cra_driver_name   = "chacha20-neon",
 248                .base.cra_priority      = 300,
 249                .base.cra_blocksize     = 1,
 250                .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 251                .base.cra_module        = THIS_MODULE,
 252
 253                .min_keysize            = CHACHA_KEY_SIZE,
 254                .max_keysize            = CHACHA_KEY_SIZE,
 255                .ivsize                 = CHACHA_IV_SIZE,
 256                .chunksize              = CHACHA_BLOCK_SIZE,
 257                .walksize               = 4 * CHACHA_BLOCK_SIZE,
 258                .setkey                 = chacha20_setkey,
 259                .encrypt                = chacha_neon,
 260                .decrypt                = chacha_neon,
 261        }, {
 262                .base.cra_name          = "xchacha20",
 263                .base.cra_driver_name   = "xchacha20-neon",
 264                .base.cra_priority      = 300,
 265                .base.cra_blocksize     = 1,
 266                .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 267                .base.cra_module        = THIS_MODULE,
 268
 269                .min_keysize            = CHACHA_KEY_SIZE,
 270                .max_keysize            = CHACHA_KEY_SIZE,
 271                .ivsize                 = XCHACHA_IV_SIZE,
 272                .chunksize              = CHACHA_BLOCK_SIZE,
 273                .walksize               = 4 * CHACHA_BLOCK_SIZE,
 274                .setkey                 = chacha20_setkey,
 275                .encrypt                = xchacha_neon,
 276                .decrypt                = xchacha_neon,
 277        }, {
 278                .base.cra_name          = "xchacha12",
 279                .base.cra_driver_name   = "xchacha12-neon",
 280                .base.cra_priority      = 300,
 281                .base.cra_blocksize     = 1,
 282                .base.cra_ctxsize       = sizeof(struct chacha_ctx),
 283                .base.cra_module        = THIS_MODULE,
 284
 285                .min_keysize            = CHACHA_KEY_SIZE,
 286                .max_keysize            = CHACHA_KEY_SIZE,
 287                .ivsize                 = XCHACHA_IV_SIZE,
 288                .chunksize              = CHACHA_BLOCK_SIZE,
 289                .walksize               = 4 * CHACHA_BLOCK_SIZE,
 290                .setkey                 = chacha12_setkey,
 291                .encrypt                = xchacha_neon,
 292                .decrypt                = xchacha_neon,
 293        }
 294};
 295
 296static int __init chacha_simd_mod_init(void)
 297{
 298        int err = 0;
 299
 300        if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
 301                err = crypto_register_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
 302                if (err)
 303                        return err;
 304        }
 305
 306        if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) {
 307                int i;
 308
 309                switch (read_cpuid_part()) {
 310                case ARM_CPU_PART_CORTEX_A7:
 311                case ARM_CPU_PART_CORTEX_A5:
 312                        /*
 313                         * The Cortex-A7 and Cortex-A5 do not perform well with
 314                         * the NEON implementation but do incredibly with the
 315                         * scalar one and use less power.
 316                         */
 317                        for (i = 0; i < ARRAY_SIZE(neon_algs); i++)
 318                                neon_algs[i].base.cra_priority = 0;
 319                        break;
 320                default:
 321                        static_branch_enable(&use_neon);
 322                }
 323
 324                if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
 325                        err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
 326                        if (err)
 327                                crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
 328                }
 329        }
 330        return err;
 331}
 332
 333static void __exit chacha_simd_mod_fini(void)
 334{
 335        if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
 336                crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
 337                if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON))
 338                        crypto_unregister_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
 339        }
 340}
 341
 342module_init(chacha_simd_mod_init);
 343module_exit(chacha_simd_mod_fini);
 344
 345MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)");
 346MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 347MODULE_LICENSE("GPL v2");
 348MODULE_ALIAS_CRYPTO("chacha20");
 349MODULE_ALIAS_CRYPTO("chacha20-arm");
 350MODULE_ALIAS_CRYPTO("xchacha20");
 351MODULE_ALIAS_CRYPTO("xchacha20-arm");
 352MODULE_ALIAS_CRYPTO("xchacha12");
 353MODULE_ALIAS_CRYPTO("xchacha12-arm");
 354#ifdef CONFIG_KERNEL_MODE_NEON
 355MODULE_ALIAS_CRYPTO("chacha20-neon");
 356MODULE_ALIAS_CRYPTO("xchacha20-neon");
 357MODULE_ALIAS_CRYPTO("xchacha12-neon");
 358#endif
 359
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.