linux/lib/crypto/chacha20poly1305.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0 OR MIT
   2/*
   3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4 *
   5 * This is an implementation of the ChaCha20Poly1305 AEAD construction.
   6 *
   7 * Information: https://tools.ietf.org/html/rfc8439
   8 */
   9
  10#include <crypto/algapi.h>
  11#include <crypto/chacha20poly1305.h>
  12#include <crypto/chacha.h>
  13#include <crypto/poly1305.h>
  14#include <crypto/scatterwalk.h>
  15
  16#include <asm/unaligned.h>
  17#include <linux/kernel.h>
  18#include <linux/init.h>
  19#include <linux/mm.h>
  20#include <linux/module.h>
  21
  22#define CHACHA_KEY_WORDS        (CHACHA_KEY_SIZE / sizeof(u32))
  23
  24static void chacha_load_key(u32 *k, const u8 *in)
  25{
  26        k[0] = get_unaligned_le32(in);
  27        k[1] = get_unaligned_le32(in + 4);
  28        k[2] = get_unaligned_le32(in + 8);
  29        k[3] = get_unaligned_le32(in + 12);
  30        k[4] = get_unaligned_le32(in + 16);
  31        k[5] = get_unaligned_le32(in + 20);
  32        k[6] = get_unaligned_le32(in + 24);
  33        k[7] = get_unaligned_le32(in + 28);
  34}
  35
  36static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
  37{
  38        u32 k[CHACHA_KEY_WORDS];
  39        u8 iv[CHACHA_IV_SIZE];
  40
  41        memset(iv, 0, 8);
  42        memcpy(iv + 8, nonce + 16, 8);
  43
  44        chacha_load_key(k, key);
  45
  46        /* Compute the subkey given the original key and first 128 nonce bits */
  47        chacha_init(chacha_state, k, nonce);
  48        hchacha_block(chacha_state, k, 20);
  49
  50        chacha_init(chacha_state, k, iv);
  51
  52        memzero_explicit(k, sizeof(k));
  53        memzero_explicit(iv, sizeof(iv));
  54}
  55
  56static void
  57__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
  58                           const u8 *ad, const size_t ad_len, u32 *chacha_state)
  59{
  60        const u8 *pad0 = page_address(ZERO_PAGE(0));
  61        struct poly1305_desc_ctx poly1305_state;
  62        union {
  63                u8 block0[POLY1305_KEY_SIZE];
  64                __le64 lens[2];
  65        } b;
  66
  67        chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
  68        poly1305_init(&poly1305_state, b.block0);
  69
  70        poly1305_update(&poly1305_state, ad, ad_len);
  71        if (ad_len & 0xf)
  72                poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
  73
  74        chacha20_crypt(chacha_state, dst, src, src_len);
  75
  76        poly1305_update(&poly1305_state, dst, src_len);
  77        if (src_len & 0xf)
  78                poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
  79
  80        b.lens[0] = cpu_to_le64(ad_len);
  81        b.lens[1] = cpu_to_le64(src_len);
  82        poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
  83
  84        poly1305_final(&poly1305_state, dst + src_len);
  85
  86        memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
  87        memzero_explicit(&b, sizeof(b));
  88}
  89
  90void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
  91                              const u8 *ad, const size_t ad_len,
  92                              const u64 nonce,
  93                              const u8 key[CHACHA20POLY1305_KEY_SIZE])
  94{
  95        u32 chacha_state[CHACHA_STATE_WORDS];
  96        u32 k[CHACHA_KEY_WORDS];
  97        __le64 iv[2];
  98
  99        chacha_load_key(k, key);
 100
 101        iv[0] = 0;
 102        iv[1] = cpu_to_le64(nonce);
 103
 104        chacha_init(chacha_state, k, (u8 *)iv);
 105        __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
 106
 107        memzero_explicit(iv, sizeof(iv));
 108        memzero_explicit(k, sizeof(k));
 109}
 110EXPORT_SYMBOL(chacha20poly1305_encrypt);
 111
 112void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
 113                               const u8 *ad, const size_t ad_len,
 114                               const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
 115                               const u8 key[CHACHA20POLY1305_KEY_SIZE])
 116{
 117        u32 chacha_state[CHACHA_STATE_WORDS];
 118
 119        xchacha_init(chacha_state, key, nonce);
 120        __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
 121}
 122EXPORT_SYMBOL(xchacha20poly1305_encrypt);
 123
 124static bool
 125__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 126                           const u8 *ad, const size_t ad_len, u32 *chacha_state)
 127{
 128        const u8 *pad0 = page_address(ZERO_PAGE(0));
 129        struct poly1305_desc_ctx poly1305_state;
 130        size_t dst_len;
 131        int ret;
 132        union {
 133                u8 block0[POLY1305_KEY_SIZE];
 134                u8 mac[POLY1305_DIGEST_SIZE];
 135                __le64 lens[2];
 136        } b;
 137
 138        if (unlikely(src_len < POLY1305_DIGEST_SIZE))
 139                return false;
 140
 141        chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
 142        poly1305_init(&poly1305_state, b.block0);
 143
 144        poly1305_update(&poly1305_state, ad, ad_len);
 145        if (ad_len & 0xf)
 146                poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
 147
 148        dst_len = src_len - POLY1305_DIGEST_SIZE;
 149        poly1305_update(&poly1305_state, src, dst_len);
 150        if (dst_len & 0xf)
 151                poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
 152
 153        b.lens[0] = cpu_to_le64(ad_len);
 154        b.lens[1] = cpu_to_le64(dst_len);
 155        poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
 156
 157        poly1305_final(&poly1305_state, b.mac);
 158
 159        ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
 160        if (likely(!ret))
 161                chacha20_crypt(chacha_state, dst, src, dst_len);
 162
 163        memzero_explicit(&b, sizeof(b));
 164
 165        return !ret;
 166}
 167
 168bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 169                              const u8 *ad, const size_t ad_len,
 170                              const u64 nonce,
 171                              const u8 key[CHACHA20POLY1305_KEY_SIZE])
 172{
 173        u32 chacha_state[CHACHA_STATE_WORDS];
 174        u32 k[CHACHA_KEY_WORDS];
 175        __le64 iv[2];
 176        bool ret;
 177
 178        chacha_load_key(k, key);
 179
 180        iv[0] = 0;
 181        iv[1] = cpu_to_le64(nonce);
 182
 183        chacha_init(chacha_state, k, (u8 *)iv);
 184        ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
 185                                         chacha_state);
 186
 187        memzero_explicit(chacha_state, sizeof(chacha_state));
 188        memzero_explicit(iv, sizeof(iv));
 189        memzero_explicit(k, sizeof(k));
 190        return ret;
 191}
 192EXPORT_SYMBOL(chacha20poly1305_decrypt);
 193
 194bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
 195                               const u8 *ad, const size_t ad_len,
 196                               const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
 197                               const u8 key[CHACHA20POLY1305_KEY_SIZE])
 198{
 199        u32 chacha_state[CHACHA_STATE_WORDS];
 200
 201        xchacha_init(chacha_state, key, nonce);
 202        return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
 203                                          chacha_state);
 204}
 205EXPORT_SYMBOL(xchacha20poly1305_decrypt);
 206
 207static
 208bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
 209                                       const size_t src_len,
 210                                       const u8 *ad, const size_t ad_len,
 211                                       const u64 nonce,
 212                                       const u8 key[CHACHA20POLY1305_KEY_SIZE],
 213                                       int encrypt)
 214{
 215        const u8 *pad0 = page_address(ZERO_PAGE(0));
 216        struct poly1305_desc_ctx poly1305_state;
 217        u32 chacha_state[CHACHA_STATE_WORDS];
 218        struct sg_mapping_iter miter;
 219        size_t partial = 0;
 220        unsigned int flags;
 221        bool ret = true;
 222        int sl;
 223        union {
 224                struct {
 225                        u32 k[CHACHA_KEY_WORDS];
 226                        __le64 iv[2];
 227                };
 228                u8 block0[POLY1305_KEY_SIZE];
 229                u8 chacha_stream[CHACHA_BLOCK_SIZE];
 230                struct {
 231                        u8 mac[2][POLY1305_DIGEST_SIZE];
 232                };
 233                __le64 lens[2];
 234        } b __aligned(16);
 235
 236        if (WARN_ON(src_len > INT_MAX))
 237                return false;
 238
 239        chacha_load_key(b.k, key);
 240
 241        b.iv[0] = 0;
 242        b.iv[1] = cpu_to_le64(nonce);
 243
 244        chacha_init(chacha_state, b.k, (u8 *)b.iv);
 245        chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
 246        poly1305_init(&poly1305_state, b.block0);
 247
 248        if (unlikely(ad_len)) {
 249                poly1305_update(&poly1305_state, ad, ad_len);
 250                if (ad_len & 0xf)
 251                        poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
 252        }
 253
 254        flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;
 255
 256        sg_miter_start(&miter, src, sg_nents(src), flags);
 257
 258        for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
 259                u8 *addr = miter.addr;
 260                size_t length = min_t(size_t, sl, miter.length);
 261
 262                if (!encrypt)
 263                        poly1305_update(&poly1305_state, addr, length);
 264
 265                if (unlikely(partial)) {
 266                        size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
 267
 268                        crypto_xor(addr, b.chacha_stream + partial, l);
 269                        partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
 270
 271                        addr += l;
 272                        length -= l;
 273                }
 274
 275                if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
 276                        size_t l = length;
 277
 278                        if (unlikely(length < sl))
 279                                l &= ~(CHACHA_BLOCK_SIZE - 1);
 280                        chacha20_crypt(chacha_state, addr, addr, l);
 281                        addr += l;
 282                        length -= l;
 283                }
 284
 285                if (unlikely(length > 0)) {
 286                        chacha20_crypt(chacha_state, b.chacha_stream, pad0,
 287                                       CHACHA_BLOCK_SIZE);
 288                        crypto_xor(addr, b.chacha_stream, length);
 289                        partial = length;
 290                }
 291
 292                if (encrypt)
 293                        poly1305_update(&poly1305_state, miter.addr,
 294                                        min_t(size_t, sl, miter.length));
 295        }
 296
 297        if (src_len & 0xf)
 298                poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
 299
 300        b.lens[0] = cpu_to_le64(ad_len);
 301        b.lens[1] = cpu_to_le64(src_len);
 302        poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
 303
 304        if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
 305                if (encrypt) {
 306                        poly1305_final(&poly1305_state,
 307                                       miter.addr + miter.length + sl);
 308                        ret = true;
 309                } else {
 310                        poly1305_final(&poly1305_state, b.mac[0]);
 311                        ret = !crypto_memneq(b.mac[0],
 312                                             miter.addr + miter.length + sl,
 313                                             POLY1305_DIGEST_SIZE);
 314                }
 315        }
 316
 317        sg_miter_stop(&miter);
 318
 319        if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
 320                poly1305_final(&poly1305_state, b.mac[1]);
 321                scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
 322                                         sizeof(b.mac[1]), encrypt);
 323                ret = encrypt ||
 324                      !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
 325        }
 326
 327        memzero_explicit(chacha_state, sizeof(chacha_state));
 328        memzero_explicit(&b, sizeof(b));
 329
 330        return ret;
 331}
 332
 333bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
 334                                         const u8 *ad, const size_t ad_len,
 335                                         const u64 nonce,
 336                                         const u8 key[CHACHA20POLY1305_KEY_SIZE])
 337{
 338        return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
 339                                                 nonce, key, 1);
 340}
 341EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
 342
 343bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
 344                                         const u8 *ad, const size_t ad_len,
 345                                         const u64 nonce,
 346                                         const u8 key[CHACHA20POLY1305_KEY_SIZE])
 347{
 348        if (unlikely(src_len < POLY1305_DIGEST_SIZE))
 349                return false;
 350
 351        return chacha20poly1305_crypt_sg_inplace(src,
 352                                                 src_len - POLY1305_DIGEST_SIZE,
 353                                                 ad, ad_len, nonce, key, 0);
 354}
 355EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
 356
 357static int __init mod_init(void)
 358{
 359        if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
 360            WARN_ON(!chacha20poly1305_selftest()))
 361                return -ENODEV;
 362        return 0;
 363}
 364
 365static void __exit mod_exit(void)
 366{
 367}
 368
 369module_init(mod_init);
 370module_exit(mod_exit);
 371MODULE_LICENSE("GPL v2");
 372MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
 373MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
 374
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.