diff --git a/cipher/cipher-ccm.c b/cipher/cipher-ccm.c index e71c6f15..fd284caa 100644 --- a/cipher/cipher-ccm.c +++ b/cipher/cipher-ccm.c @@ -1,367 +1,406 @@ /* cipher-ccm.c - CTR mode with CBC-MAC mode implementation * Copyright (C) 2013 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser general Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . */ #include #include #include #include #include #include "g10lib.h" #include "cipher.h" #include "bufhelp.h" #include "./cipher-internal.h" #define set_burn(burn, nburn) do { \ unsigned int __nburn = (nburn); \ (burn) = (burn) > __nburn ? (burn) : __nburn; } while (0) static unsigned int do_cbc_mac (gcry_cipher_hd_t c, const unsigned char *inbuf, size_t inlen, int do_padding) { const unsigned int blocksize = 16; gcry_cipher_encrypt_t enc_fn = c->spec->encrypt; unsigned char tmp[blocksize]; unsigned int burn = 0; unsigned int unused = c->u_mode.ccm.mac_unused; size_t nblocks; if (inlen == 0 && (unused == 0 || !do_padding)) return 0; do { if (inlen + unused < blocksize || unused > 0) { for (; inlen && unused < blocksize; inlen--) c->u_mode.ccm.macbuf[unused++] = *inbuf++; } if (!inlen) { if (!do_padding) break; while (unused < blocksize) c->u_mode.ccm.macbuf[unused++] = 0; } if (unused > 0) { /* Process one block from macbuf. */ cipher_block_xor(c->u_iv.iv, c->u_iv.iv, c->u_mode.ccm.macbuf, blocksize); set_burn (burn, enc_fn ( &c->context.c, c->u_iv.iv, c->u_iv.iv )); unused = 0; } if (c->bulk.cbc_enc) { nblocks = inlen / blocksize; c->bulk.cbc_enc (&c->context.c, c->u_iv.iv, tmp, inbuf, nblocks, 1); inbuf += nblocks * blocksize; inlen -= nblocks * blocksize; wipememory (tmp, sizeof(tmp)); } else { while (inlen >= blocksize) { cipher_block_xor(c->u_iv.iv, c->u_iv.iv, inbuf, blocksize); set_burn (burn, enc_fn ( &c->context.c, c->u_iv.iv, c->u_iv.iv )); inlen -= blocksize; inbuf += blocksize; } } } while (inlen > 0); c->u_mode.ccm.mac_unused = unused; if (burn) burn += 4 * sizeof(void *); return burn; } gcry_err_code_t _gcry_cipher_ccm_set_nonce (gcry_cipher_hd_t c, const unsigned char *nonce, size_t noncelen) { unsigned int marks_key; size_t L = 15 - noncelen; size_t L_; L_ = L - 1; if (!nonce) return GPG_ERR_INV_ARG; /* Length field must be 2, 3, ..., or 8. */ if (L < 2 || L > 8) return GPG_ERR_INV_LENGTH; /* Reset state */ marks_key = c->marks.key; memset (&c->u_mode, 0, sizeof(c->u_mode)); memset (&c->marks, 0, sizeof(c->marks)); memset (&c->u_iv, 0, sizeof(c->u_iv)); memset (&c->u_ctr, 0, sizeof(c->u_ctr)); memset (c->lastiv, 0, sizeof(c->lastiv)); c->unused = 0; c->marks.key = marks_key; /* Setup CTR */ c->u_ctr.ctr[0] = L_; memcpy (&c->u_ctr.ctr[1], nonce, noncelen); memset (&c->u_ctr.ctr[1 + noncelen], 0, L); /* Setup IV */ c->u_iv.iv[0] = L_; memcpy (&c->u_iv.iv[1], nonce, noncelen); /* Add (8 * M_ + 64 * flags) to iv[0] and set iv[noncelen + 1 ... 15] later in set_aad. */ memset (&c->u_iv.iv[1 + noncelen], 0, L); c->u_mode.ccm.nonce = 1; return GPG_ERR_NO_ERROR; } gcry_err_code_t _gcry_cipher_ccm_set_lengths (gcry_cipher_hd_t c, u64 encryptlen, u64 aadlen, u64 taglen) { unsigned int burn = 0; unsigned char b0[16]; size_t noncelen = 15 - (c->u_iv.iv[0] + 1); u64 M = taglen; u64 M_; int i; M_ = (M - 2) / 2; /* Authentication field must be 4, 6, 8, 10, 12, 14 or 16. */ if ((M_ * 2 + 2) != M || M < 4 || M > 16) return GPG_ERR_INV_LENGTH; if (!c->u_mode.ccm.nonce || c->marks.tag) return GPG_ERR_INV_STATE; if (c->u_mode.ccm.lengths) return GPG_ERR_INV_STATE; c->u_mode.ccm.authlen = taglen; c->u_mode.ccm.encryptlen = encryptlen; c->u_mode.ccm.aadlen = aadlen; /* Complete IV setup. */ c->u_iv.iv[0] += (aadlen > 0) * 64 + M_ * 8; for (i = 16 - 1; i >= 1 + noncelen; i--) { c->u_iv.iv[i] = encryptlen & 0xff; encryptlen >>= 8; } memcpy (b0, c->u_iv.iv, 16); memset (c->u_iv.iv, 0, 16); set_burn (burn, do_cbc_mac (c, b0, 16, 0)); if (aadlen == 0) { /* Do nothing. */ } else if (aadlen > 0 && aadlen <= (unsigned int)0xfeff) { b0[0] = (aadlen >> 8) & 0xff; b0[1] = aadlen & 0xff; set_burn (burn, do_cbc_mac (c, b0, 2, 0)); } else if (aadlen > 0xfeff && aadlen <= (unsigned int)0xffffffff) { b0[0] = 0xff; b0[1] = 0xfe; buf_put_be32(&b0[2], aadlen); set_burn (burn, do_cbc_mac (c, b0, 6, 0)); } else if (aadlen > (unsigned int)0xffffffff) { b0[0] = 0xff; b0[1] = 0xff; buf_put_be64(&b0[2], aadlen); set_burn (burn, do_cbc_mac (c, b0, 10, 0)); } /* Generate S_0 and increase counter. */ set_burn (burn, c->spec->encrypt ( &c->context.c, c->u_mode.ccm.s0, c->u_ctr.ctr )); c->u_ctr.ctr[15]++; if (burn) _gcry_burn_stack (burn + sizeof(void *) * 5); c->u_mode.ccm.lengths = 1; return GPG_ERR_NO_ERROR; } gcry_err_code_t _gcry_cipher_ccm_authenticate (gcry_cipher_hd_t c, const unsigned char *abuf, size_t abuflen) { unsigned int burn; if (abuflen > 0 && !abuf) return GPG_ERR_INV_ARG; if (!c->u_mode.ccm.nonce || !c->u_mode.ccm.lengths || c->marks.tag) return GPG_ERR_INV_STATE; if (abuflen > c->u_mode.ccm.aadlen) return GPG_ERR_INV_LENGTH; c->u_mode.ccm.aadlen -= abuflen; burn = do_cbc_mac (c, abuf, abuflen, c->u_mode.ccm.aadlen == 0); if (burn) _gcry_burn_stack (burn + sizeof(void *) * 5); return GPG_ERR_NO_ERROR; } gcry_err_code_t _gcry_cipher_ccm_tag (gcry_cipher_hd_t c, unsigned char *outbuf, size_t outbuflen, int check) { unsigned int burn; if (!outbuf || outbuflen == 0) return GPG_ERR_INV_ARG; /* Tag length must be same as initial authlen. */ if (c->u_mode.ccm.authlen != outbuflen) return GPG_ERR_INV_LENGTH; if (!c->u_mode.ccm.nonce || !c->u_mode.ccm.lengths || c->u_mode.ccm.aadlen > 0) return GPG_ERR_INV_STATE; /* Initial encrypt length must match with length of actual data processed. */ if (c->u_mode.ccm.encryptlen > 0) return GPG_ERR_UNFINISHED; if (!c->marks.tag) { burn = do_cbc_mac (c, NULL, 0, 1); /* Perform final padding. */ /* Add S_0 */ cipher_block_xor (c->u_iv.iv, c->u_iv.iv, c->u_mode.ccm.s0, 16); wipememory (c->u_ctr.ctr, 16); wipememory (c->u_mode.ccm.s0, 16); wipememory (c->u_mode.ccm.macbuf, 16); if (burn) _gcry_burn_stack (burn + sizeof(void *) * 5); c->marks.tag = 1; } if (!check) { memcpy (outbuf, c->u_iv.iv, outbuflen); return GPG_ERR_NO_ERROR; } else { return buf_eq_const(outbuf, c->u_iv.iv, outbuflen) ? GPG_ERR_NO_ERROR : GPG_ERR_CHECKSUM; } } gcry_err_code_t _gcry_cipher_ccm_get_tag (gcry_cipher_hd_t c, unsigned char *outtag, size_t taglen) { return _gcry_cipher_ccm_tag (c, outtag, taglen, 0); } gcry_err_code_t _gcry_cipher_ccm_check_tag (gcry_cipher_hd_t c, const unsigned char *intag, size_t taglen) { return _gcry_cipher_ccm_tag (c, (unsigned char *)intag, taglen, 1); } gcry_err_code_t _gcry_cipher_ccm_encrypt (gcry_cipher_hd_t c, unsigned char *outbuf, size_t outbuflen, const unsigned char *inbuf, size_t inbuflen) { - unsigned int burn; + gcry_err_code_t err = 0; + unsigned int burn = 0; + unsigned int nburn; if (outbuflen < inbuflen) return GPG_ERR_BUFFER_TOO_SHORT; if (!c->u_mode.ccm.nonce || c->marks.tag || !c->u_mode.ccm.lengths || c->u_mode.ccm.aadlen > 0) return GPG_ERR_INV_STATE; if (inbuflen > c->u_mode.ccm.encryptlen) return GPG_ERR_INV_LENGTH; - c->u_mode.ccm.encryptlen -= inbuflen; - burn = do_cbc_mac (c, inbuf, inbuflen, 0); + while (inbuflen) + { + size_t currlen = inbuflen; + + /* Since checksumming is done before encryption, process input in 24KiB + * chunks to keep data loaded in L1 cache for encryption. */ + if (currlen > 24 * 1024) + currlen = 24 * 1024; + + c->u_mode.ccm.encryptlen -= currlen; + nburn = do_cbc_mac (c, inbuf, currlen, 0); + burn = nburn > burn ? nburn : burn; + + err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, currlen); + if (err) + break; + + outbuf += currlen; + inbuf += currlen; + outbuflen -= currlen; + inbuflen -= currlen; + } + if (burn) _gcry_burn_stack (burn + sizeof(void *) * 5); - - return _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, inbuflen); + return err; } gcry_err_code_t _gcry_cipher_ccm_decrypt (gcry_cipher_hd_t c, unsigned char *outbuf, size_t outbuflen, const unsigned char *inbuf, size_t inbuflen) { - gcry_err_code_t err; - unsigned int burn; + gcry_err_code_t err = 0; + unsigned int burn = 0; + unsigned int nburn; if (outbuflen < inbuflen) return GPG_ERR_BUFFER_TOO_SHORT; if (!c->u_mode.ccm.nonce || c->marks.tag || !c->u_mode.ccm.lengths || c->u_mode.ccm.aadlen > 0) return GPG_ERR_INV_STATE; if (inbuflen > c->u_mode.ccm.encryptlen) return GPG_ERR_INV_LENGTH; - err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, inbuflen); - if (err) - return err; + while (inbuflen) + { + size_t currlen = inbuflen; + + /* Since checksumming is done after decryption, process input in 24KiB + * chunks to keep data loaded in L1 cache for checksumming. */ + if (currlen > 24 * 1024) + currlen = 24 * 1024; + + err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, currlen); + if (err) + break; + + c->u_mode.ccm.encryptlen -= currlen; + nburn = do_cbc_mac (c, outbuf, currlen, 0); + burn = nburn > burn ? nburn : burn; + + outbuf += currlen; + inbuf += currlen; + outbuflen -= currlen; + inbuflen -= currlen; + } - c->u_mode.ccm.encryptlen -= inbuflen; - burn = do_cbc_mac (c, outbuf, inbuflen, 0); if (burn) _gcry_burn_stack (burn + sizeof(void *) * 5); - return err; } diff --git a/cipher/cipher-eax.c b/cipher/cipher-eax.c index 3b17bb64..08f815a9 100644 --- a/cipher/cipher-eax.c +++ b/cipher/cipher-eax.c @@ -1,249 +1,289 @@ /* cipher-eax.c - EAX implementation * Copyright (C) 2018 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser general Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . */ #include #include #include #include #include #include "g10lib.h" #include "cipher.h" #include "bufhelp.h" #include "./cipher-internal.h" gcry_err_code_t _gcry_cipher_eax_encrypt (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen, const byte *inbuf, size_t inbuflen) { gcry_err_code_t err; if (outbuflen < inbuflen) return GPG_ERR_BUFFER_TOO_SHORT; if (c->marks.tag) return GPG_ERR_INV_STATE; if (!c->marks.iv) { err = _gcry_cipher_eax_set_nonce (c, NULL, 0); if (err != 0) return err; } - err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, inbuflen); - if (err != 0) - return err; + while (inbuflen) + { + size_t currlen = inbuflen; + + /* Since checksumming is done after encryption, process input in 24KiB + * chunks to keep data loaded in L1 cache for checksumming. */ + if (currlen > 24 * 1024) + currlen = 24 * 1024; + + err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, currlen); + if (err != 0) + return err; - return _gcry_cmac_write (c, &c->u_mode.eax.cmac_ciphertext, outbuf, inbuflen); + err = _gcry_cmac_write (c, &c->u_mode.eax.cmac_ciphertext, outbuf, + currlen); + if (err != 0) + return err; + + outbuf += currlen; + inbuf += currlen; + outbuflen -= currlen; + inbuflen -= currlen; + } + + return 0; } gcry_err_code_t _gcry_cipher_eax_decrypt (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen, const byte *inbuf, size_t inbuflen) { gcry_err_code_t err; if (outbuflen < inbuflen) return GPG_ERR_BUFFER_TOO_SHORT; if (c->marks.tag) return GPG_ERR_INV_STATE; if (!c->marks.iv) { err = _gcry_cipher_eax_set_nonce (c, NULL, 0); if (err != 0) return err; } - err = _gcry_cmac_write (c, &c->u_mode.eax.cmac_ciphertext, inbuf, inbuflen); - if (err != 0) - return err; + while (inbuflen) + { + size_t currlen = inbuflen; + + /* Since checksumming is done before decryption, process input in 24KiB + * chunks to keep data loaded in L1 cache for decryption. */ + if (currlen > 24 * 1024) + currlen = 24 * 1024; + + err = _gcry_cmac_write (c, &c->u_mode.eax.cmac_ciphertext, inbuf, + currlen); + if (err != 0) + return err; - return _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, inbuflen); + err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, currlen); + if (err != 0) + return err; + + outbuf += currlen; + inbuf += currlen; + outbuflen -= currlen; + inbuflen -= currlen; + } + + return 0; } gcry_err_code_t _gcry_cipher_eax_authenticate (gcry_cipher_hd_t c, const byte * aadbuf, size_t aadbuflen) { gcry_err_code_t err; if (c->marks.tag) return GPG_ERR_INV_STATE; if (!c->marks.iv) { err = _gcry_cipher_eax_set_nonce (c, NULL, 0); if (err != 0) return err; } return _gcry_cmac_write (c, &c->u_mode.eax.cmac_header, aadbuf, aadbuflen); } gcry_err_code_t _gcry_cipher_eax_setkey (gcry_cipher_hd_t c) { gcry_err_code_t err; err = _gcry_cmac_generate_subkeys (c, &c->u_mode.eax.cmac_header); if (err != 0) return err; buf_cpy (c->u_mode.eax.cmac_ciphertext.subkeys, c->u_mode.eax.cmac_header.subkeys, sizeof(c->u_mode.eax.cmac_header.subkeys)); return 0; } gcry_err_code_t _gcry_cipher_eax_set_nonce (gcry_cipher_hd_t c, const byte *nonce, size_t noncelen) { gcry_cmac_context_t nonce_cmac; unsigned char initbuf[MAX_BLOCKSIZE]; gcry_err_code_t err; c->marks.iv = 0; c->marks.tag = 0; _gcry_cmac_reset (&c->u_mode.eax.cmac_header); _gcry_cmac_reset (&c->u_mode.eax.cmac_ciphertext); /* Calculate nonce CMAC */ memset(&nonce_cmac, 0, sizeof(nonce_cmac)); memset(&initbuf, 0, sizeof(initbuf)); buf_cpy (&nonce_cmac.subkeys, c->u_mode.eax.cmac_header.subkeys, sizeof(c->u_mode.eax.cmac_header.subkeys)); err = _gcry_cmac_write (c, &nonce_cmac, initbuf, c->spec->blocksize); if (err != 0) return err; if (noncelen != 0) { err = _gcry_cmac_write (c, &nonce_cmac, nonce, noncelen); if (err != 0) return err; } err = _gcry_cmac_final (c, &nonce_cmac); if (err != 0) return err; cipher_block_cpy (c->u_iv.iv, nonce_cmac.u_iv.iv, MAX_BLOCKSIZE); cipher_block_cpy (c->u_ctr.ctr, nonce_cmac.u_iv.iv, MAX_BLOCKSIZE); wipememory (&nonce_cmac, sizeof(nonce_cmac)); /* Prepare header CMAC */ initbuf[c->spec->blocksize - 1] = 1; err = _gcry_cmac_write (c, &c->u_mode.eax.cmac_header, initbuf, c->spec->blocksize); if (err != 0) return err; /* Prepare ciphertext CMAC */ initbuf[c->spec->blocksize - 1] = 2; err = _gcry_cmac_write (c, &c->u_mode.eax.cmac_ciphertext, initbuf, c->spec->blocksize); if (err != 0) return err; c->marks.iv = 1; c->marks.tag = 0; return 0; } static gcry_err_code_t _gcry_cipher_eax_tag (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen, int check) { gcry_err_code_t err; if (!c->marks.tag) { err = _gcry_cmac_final (c, &c->u_mode.eax.cmac_header); if (err != 0) return err; err = _gcry_cmac_final (c, &c->u_mode.eax.cmac_ciphertext); if (err != 0) return err; cipher_block_xor_1 (c->u_iv.iv, c->u_mode.eax.cmac_header.u_iv.iv, MAX_BLOCKSIZE); cipher_block_xor_1 (c->u_iv.iv, c->u_mode.eax.cmac_ciphertext.u_iv.iv, MAX_BLOCKSIZE); _gcry_cmac_reset (&c->u_mode.eax.cmac_header); _gcry_cmac_reset (&c->u_mode.eax.cmac_ciphertext); c->marks.tag = 1; } if (!check) { if (outbuflen > c->spec->blocksize) outbuflen = c->spec->blocksize; /* NB: We already checked that OUTBUF is large enough to hold * the result or has valid truncated length. */ memcpy (outbuf, c->u_iv.iv, outbuflen); } else { /* OUTBUFLEN gives the length of the user supplied tag in OUTBUF * and thus we need to compare its length first. */ if (!(outbuflen <= c->spec->blocksize) || !buf_eq_const (outbuf, c->u_iv.iv, outbuflen)) return GPG_ERR_CHECKSUM; } return 0; } gcry_err_code_t _gcry_cipher_eax_get_tag (gcry_cipher_hd_t c, unsigned char *outtag, size_t taglen) { return _gcry_cipher_eax_tag (c, outtag, taglen, 0); } gcry_err_code_t _gcry_cipher_eax_check_tag (gcry_cipher_hd_t c, const unsigned char *intag, size_t taglen) { return _gcry_cipher_eax_tag (c, (unsigned char *) intag, taglen, 1); } diff --git a/cipher/cipher-gcm.c b/cipher/cipher-gcm.c index 32ec9fa0..f9ddbc56 100644 --- a/cipher/cipher-gcm.c +++ b/cipher/cipher-gcm.c @@ -1,980 +1,1015 @@ /* cipher-gcm.c - Generic Galois Counter Mode implementation * Copyright (C) 2013 Dmitry Eremin-Solenikov * Copyright (C) 2013, 2018 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser general Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . */ #include #include #include #include #include #include "g10lib.h" #include "cipher.h" #include "bufhelp.h" #include "./cipher-internal.h" #ifdef GCM_USE_INTEL_PCLMUL extern void _gcry_ghash_setup_intel_pclmul (gcry_cipher_hd_t c); extern unsigned int _gcry_ghash_intel_pclmul (gcry_cipher_hd_t c, byte *result, const byte *buf, size_t nblocks); #endif #ifdef GCM_USE_ARM_PMULL extern void _gcry_ghash_setup_armv8_ce_pmull (void *gcm_key, void *gcm_table); extern unsigned int _gcry_ghash_armv8_ce_pmull (void *gcm_key, byte *result, const byte *buf, size_t nblocks, void *gcm_table); static void ghash_setup_armv8_ce_pmull (gcry_cipher_hd_t c) { _gcry_ghash_setup_armv8_ce_pmull(c->u_mode.gcm.u_ghash_key.key, c->u_mode.gcm.gcm_table); } static unsigned int ghash_armv8_ce_pmull (gcry_cipher_hd_t c, byte *result, const byte *buf, size_t nblocks) { return _gcry_ghash_armv8_ce_pmull(c->u_mode.gcm.u_ghash_key.key, result, buf, nblocks, c->u_mode.gcm.gcm_table); } #endif #ifdef GCM_USE_TABLES static const u16 gcmR[256] = { 0x0000, 0x01c2, 0x0384, 0x0246, 0x0708, 0x06ca, 0x048c, 0x054e, 0x0e10, 0x0fd2, 0x0d94, 0x0c56, 0x0918, 0x08da, 0x0a9c, 0x0b5e, 0x1c20, 0x1de2, 0x1fa4, 0x1e66, 0x1b28, 0x1aea, 0x18ac, 0x196e, 0x1230, 0x13f2, 0x11b4, 0x1076, 0x1538, 0x14fa, 0x16bc, 0x177e, 0x3840, 0x3982, 0x3bc4, 0x3a06, 0x3f48, 0x3e8a, 0x3ccc, 0x3d0e, 0x3650, 0x3792, 0x35d4, 0x3416, 0x3158, 0x309a, 0x32dc, 0x331e, 0x2460, 0x25a2, 0x27e4, 0x2626, 0x2368, 0x22aa, 0x20ec, 0x212e, 0x2a70, 0x2bb2, 0x29f4, 0x2836, 0x2d78, 0x2cba, 0x2efc, 0x2f3e, 0x7080, 0x7142, 0x7304, 0x72c6, 0x7788, 0x764a, 0x740c, 0x75ce, 0x7e90, 0x7f52, 0x7d14, 0x7cd6, 0x7998, 0x785a, 0x7a1c, 0x7bde, 0x6ca0, 0x6d62, 0x6f24, 0x6ee6, 0x6ba8, 0x6a6a, 0x682c, 0x69ee, 0x62b0, 0x6372, 0x6134, 0x60f6, 0x65b8, 0x647a, 0x663c, 0x67fe, 0x48c0, 0x4902, 0x4b44, 0x4a86, 0x4fc8, 0x4e0a, 0x4c4c, 0x4d8e, 0x46d0, 0x4712, 0x4554, 0x4496, 0x41d8, 0x401a, 0x425c, 0x439e, 0x54e0, 0x5522, 0x5764, 0x56a6, 0x53e8, 0x522a, 0x506c, 0x51ae, 0x5af0, 0x5b32, 0x5974, 0x58b6, 0x5df8, 0x5c3a, 0x5e7c, 0x5fbe, 0xe100, 0xe0c2, 0xe284, 0xe346, 0xe608, 0xe7ca, 0xe58c, 0xe44e, 0xef10, 0xeed2, 0xec94, 0xed56, 0xe818, 0xe9da, 0xeb9c, 0xea5e, 0xfd20, 0xfce2, 0xfea4, 0xff66, 0xfa28, 0xfbea, 0xf9ac, 0xf86e, 0xf330, 0xf2f2, 0xf0b4, 0xf176, 0xf438, 0xf5fa, 0xf7bc, 0xf67e, 0xd940, 0xd882, 0xdac4, 0xdb06, 0xde48, 0xdf8a, 0xddcc, 0xdc0e, 0xd750, 0xd692, 0xd4d4, 0xd516, 0xd058, 0xd19a, 0xd3dc, 0xd21e, 0xc560, 0xc4a2, 0xc6e4, 0xc726, 0xc268, 0xc3aa, 0xc1ec, 0xc02e, 0xcb70, 0xcab2, 0xc8f4, 0xc936, 0xcc78, 0xcdba, 0xcffc, 0xce3e, 0x9180, 0x9042, 0x9204, 0x93c6, 0x9688, 0x974a, 0x950c, 0x94ce, 0x9f90, 0x9e52, 0x9c14, 0x9dd6, 0x9898, 0x995a, 0x9b1c, 0x9ade, 0x8da0, 0x8c62, 0x8e24, 0x8fe6, 0x8aa8, 0x8b6a, 0x892c, 0x88ee, 0x83b0, 0x8272, 0x8034, 0x81f6, 0x84b8, 0x857a, 0x873c, 0x86fe, 0xa9c0, 0xa802, 0xaa44, 0xab86, 0xaec8, 0xaf0a, 0xad4c, 0xac8e, 0xa7d0, 0xa612, 0xa454, 0xa596, 0xa0d8, 0xa11a, 0xa35c, 0xa29e, 0xb5e0, 0xb422, 0xb664, 0xb7a6, 0xb2e8, 0xb32a, 0xb16c, 0xb0ae, 0xbbf0, 0xba32, 0xb874, 0xb9b6, 0xbcf8, 0xbd3a, 0xbf7c, 0xbebe, }; #ifdef GCM_TABLES_USE_U64 static void bshift (u64 * b0, u64 * b1) { u64 t[2], mask; t[0] = *b0; t[1] = *b1; mask = t[1] & 1 ? 0xe1 : 0; mask <<= 56; *b1 = (t[1] >> 1) ^ (t[0] << 63); *b0 = (t[0] >> 1) ^ mask; } static void do_fillM (unsigned char *h, u64 *M) { int i, j; M[0 + 0] = 0; M[0 + 16] = 0; M[8 + 0] = buf_get_be64 (h + 0); M[8 + 16] = buf_get_be64 (h + 8); for (i = 4; i > 0; i /= 2) { M[i + 0] = M[2 * i + 0]; M[i + 16] = M[2 * i + 16]; bshift (&M[i], &M[i + 16]); } for (i = 2; i < 16; i *= 2) for (j = 1; j < i; j++) { M[(i + j) + 0] = M[i + 0] ^ M[j + 0]; M[(i + j) + 16] = M[i + 16] ^ M[j + 16]; } } static inline unsigned int do_ghash (unsigned char *result, const unsigned char *buf, const u64 *gcmM) { u64 V[2]; u64 tmp[2]; const u64 *M; u64 T; u32 A; int i; cipher_block_xor (V, result, buf, 16); V[0] = be_bswap64 (V[0]); V[1] = be_bswap64 (V[1]); /* First round can be manually tweaked based on fact that 'tmp' is zero. */ i = 15; M = &gcmM[(V[1] & 0xf)]; V[1] >>= 4; tmp[0] = (M[0] >> 4) ^ ((u64) gcmR[(M[16] & 0xf) << 4] << 48); tmp[1] = (M[16] >> 4) ^ (M[0] << 60); tmp[0] ^= gcmM[(V[1] & 0xf) + 0]; tmp[1] ^= gcmM[(V[1] & 0xf) + 16]; V[1] >>= 4; --i; while (1) { M = &gcmM[(V[1] & 0xf)]; V[1] >>= 4; A = tmp[1] & 0xff; T = tmp[0]; tmp[0] = (T >> 8) ^ ((u64) gcmR[A] << 48) ^ gcmM[(V[1] & 0xf) + 0]; tmp[1] = (T << 56) ^ (tmp[1] >> 8) ^ gcmM[(V[1] & 0xf) + 16]; tmp[0] ^= (M[0] >> 4) ^ ((u64) gcmR[(M[16] & 0xf) << 4] << 48); tmp[1] ^= (M[16] >> 4) ^ (M[0] << 60); if (i == 0) break; else if (i == 8) V[1] = V[0]; else V[1] >>= 4; --i; } buf_put_be64 (result + 0, tmp[0]); buf_put_be64 (result + 8, tmp[1]); return (sizeof(V) + sizeof(T) + sizeof(tmp) + sizeof(int)*2 + sizeof(void*)*5); } #else /*!GCM_TABLES_USE_U64*/ static void bshift (u32 * M, int i) { u32 t[4], mask; t[0] = M[i * 4 + 0]; t[1] = M[i * 4 + 1]; t[2] = M[i * 4 + 2]; t[3] = M[i * 4 + 3]; mask = t[3] & 1 ? 0xe1 : 0; M[i * 4 + 3] = (t[3] >> 1) ^ (t[2] << 31); M[i * 4 + 2] = (t[2] >> 1) ^ (t[1] << 31); M[i * 4 + 1] = (t[1] >> 1) ^ (t[0] << 31); M[i * 4 + 0] = (t[0] >> 1) ^ (mask << 24); } static void do_fillM (unsigned char *h, u32 *M) { int i, j; M[0 * 4 + 0] = 0; M[0 * 4 + 1] = 0; M[0 * 4 + 2] = 0; M[0 * 4 + 3] = 0; M[8 * 4 + 0] = buf_get_be32 (h + 0); M[8 * 4 + 1] = buf_get_be32 (h + 4); M[8 * 4 + 2] = buf_get_be32 (h + 8); M[8 * 4 + 3] = buf_get_be32 (h + 12); for (i = 4; i > 0; i /= 2) { M[i * 4 + 0] = M[2 * i * 4 + 0]; M[i * 4 + 1] = M[2 * i * 4 + 1]; M[i * 4 + 2] = M[2 * i * 4 + 2]; M[i * 4 + 3] = M[2 * i * 4 + 3]; bshift (M, i); } for (i = 2; i < 16; i *= 2) for (j = 1; j < i; j++) { M[(i + j) * 4 + 0] = M[i * 4 + 0] ^ M[j * 4 + 0]; M[(i + j) * 4 + 1] = M[i * 4 + 1] ^ M[j * 4 + 1]; M[(i + j) * 4 + 2] = M[i * 4 + 2] ^ M[j * 4 + 2]; M[(i + j) * 4 + 3] = M[i * 4 + 3] ^ M[j * 4 + 3]; } } static inline unsigned int do_ghash (unsigned char *result, const unsigned char *buf, const u32 *gcmM) { byte V[16]; u32 tmp[4]; u32 v; const u32 *M, *m; u32 T[3]; int i; cipher_block_xor (V, result, buf, 16); /* V is big-endian */ /* First round can be manually tweaked based on fact that 'tmp' is zero. */ i = 15; v = V[i]; M = &gcmM[(v & 0xf) * 4]; v = (v & 0xf0) >> 4; m = &gcmM[v * 4]; v = V[--i]; tmp[0] = (M[0] >> 4) ^ ((u64) gcmR[(M[3] << 4) & 0xf0] << 16) ^ m[0]; tmp[1] = (M[1] >> 4) ^ (M[0] << 28) ^ m[1]; tmp[2] = (M[2] >> 4) ^ (M[1] << 28) ^ m[2]; tmp[3] = (M[3] >> 4) ^ (M[2] << 28) ^ m[3]; while (1) { M = &gcmM[(v & 0xf) * 4]; v = (v & 0xf0) >> 4; m = &gcmM[v * 4]; T[0] = tmp[0]; T[1] = tmp[1]; T[2] = tmp[2]; tmp[0] = (T[0] >> 8) ^ ((u32) gcmR[tmp[3] & 0xff] << 16) ^ m[0]; tmp[1] = (T[0] << 24) ^ (tmp[1] >> 8) ^ m[1]; tmp[2] = (T[1] << 24) ^ (tmp[2] >> 8) ^ m[2]; tmp[3] = (T[2] << 24) ^ (tmp[3] >> 8) ^ m[3]; tmp[0] ^= (M[0] >> 4) ^ ((u64) gcmR[(M[3] << 4) & 0xf0] << 16); tmp[1] ^= (M[1] >> 4) ^ (M[0] << 28); tmp[2] ^= (M[2] >> 4) ^ (M[1] << 28); tmp[3] ^= (M[3] >> 4) ^ (M[2] << 28); if (i == 0) break; v = V[--i]; } buf_put_be32 (result + 0, tmp[0]); buf_put_be32 (result + 4, tmp[1]); buf_put_be32 (result + 8, tmp[2]); buf_put_be32 (result + 12, tmp[3]); return (sizeof(V) + sizeof(T) + sizeof(tmp) + sizeof(int)*2 + sizeof(void*)*6); } #endif /*!GCM_TABLES_USE_U64*/ #define fillM(c) \ do_fillM (c->u_mode.gcm.u_ghash_key.key, c->u_mode.gcm.gcm_table) #define GHASH(c, result, buf) do_ghash (result, buf, c->u_mode.gcm.gcm_table) #else static unsigned long bshift (unsigned long *b) { unsigned long c; int i; c = b[3] & 1; for (i = 3; i > 0; i--) { b[i] = (b[i] >> 1) | (b[i - 1] << 31); } b[i] >>= 1; return c; } static unsigned int do_ghash (unsigned char *hsub, unsigned char *result, const unsigned char *buf) { unsigned long V[4]; int i, j; byte *p; #ifdef WORDS_BIGENDIAN p = result; #else unsigned long T[4]; cipher_block_xor (V, result, buf, 16); for (i = 0; i < 4; i++) { V[i] = (V[i] & 0x00ff00ff) << 8 | (V[i] & 0xff00ff00) >> 8; V[i] = (V[i] & 0x0000ffff) << 16 | (V[i] & 0xffff0000) >> 16; } p = (byte *) T; #endif memset (p, 0, 16); for (i = 0; i < 16; i++) { for (j = 0x80; j; j >>= 1) { if (hsub[i] & j) cipher_block_xor (p, p, V, 16); if (bshift (V)) V[0] ^= 0xe1000000; } } #ifndef WORDS_BIGENDIAN for (i = 0, p = (byte *) T; i < 16; i += 4, p += 4) { result[i + 0] = p[3]; result[i + 1] = p[2]; result[i + 2] = p[1]; result[i + 3] = p[0]; } #endif return (sizeof(V) + sizeof(T) + sizeof(int)*2 + sizeof(void*)*5); } #define fillM(c) do { } while (0) #define GHASH(c, result, buf) do_ghash (c->u_mode.gcm.u_ghash_key.key, result, buf) #endif /* !GCM_USE_TABLES */ static unsigned int ghash_internal (gcry_cipher_hd_t c, byte *result, const byte *buf, size_t nblocks) { const unsigned int blocksize = GCRY_GCM_BLOCK_LEN; unsigned int burn = 0; while (nblocks) { burn = GHASH (c, result, buf); buf += blocksize; nblocks--; } return burn + (burn ? 5*sizeof(void*) : 0); } static void setupM (gcry_cipher_hd_t c) { #if defined(GCM_USE_INTEL_PCLMUL) || defined(GCM_USE_ARM_PMULL) unsigned int features = _gcry_get_hw_features (); #endif if (0) ; #ifdef GCM_USE_INTEL_PCLMUL else if (features & HWF_INTEL_PCLMUL) { c->u_mode.gcm.ghash_fn = _gcry_ghash_intel_pclmul; _gcry_ghash_setup_intel_pclmul (c); } #endif #ifdef GCM_USE_ARM_PMULL else if (features & HWF_ARM_PMULL) { c->u_mode.gcm.ghash_fn = ghash_armv8_ce_pmull; ghash_setup_armv8_ce_pmull (c); } #endif else { c->u_mode.gcm.ghash_fn = ghash_internal; fillM (c); } } static inline void gcm_bytecounter_add (u32 ctr[2], size_t add) { if (sizeof(add) > sizeof(u32)) { u32 high_add = ((add >> 31) >> 1) & 0xffffffff; ctr[1] += high_add; } ctr[0] += add; if (ctr[0] >= add) return; ++ctr[1]; } static inline u32 gcm_add32_be128 (byte *ctr, unsigned int add) { /* 'ctr' must be aligned to four bytes. */ const unsigned int blocksize = GCRY_GCM_BLOCK_LEN; u32 *pval = (u32 *)(void *)(ctr + blocksize - sizeof(u32)); u32 val; val = be_bswap32(*pval) + add; *pval = be_bswap32(val); return val; /* return result as host-endian value */ } static inline int gcm_check_datalen (u32 ctr[2]) { /* len(plaintext) <= 2^39-256 bits == 2^36-32 bytes == 2^32-2 blocks */ if (ctr[1] > 0xfU) return 0; if (ctr[1] < 0xfU) return 1; if (ctr[0] <= 0xffffffe0U) return 1; return 0; } static inline int gcm_check_aadlen_or_ivlen (u32 ctr[2]) { /* len(aad/iv) <= 2^64-1 bits ~= 2^61-1 bytes */ if (ctr[1] > 0x1fffffffU) return 0; if (ctr[1] < 0x1fffffffU) return 1; if (ctr[0] <= 0xffffffffU) return 1; return 0; } static void do_ghash_buf(gcry_cipher_hd_t c, byte *hash, const byte *buf, size_t buflen, int do_padding) { unsigned int blocksize = GCRY_GCM_BLOCK_LEN; unsigned int unused = c->u_mode.gcm.mac_unused; ghash_fn_t ghash_fn = c->u_mode.gcm.ghash_fn; size_t nblocks, n; unsigned int burn = 0; if (buflen == 0 && (unused == 0 || !do_padding)) return; do { if (buflen > 0 && (buflen + unused < blocksize || unused > 0)) { n = blocksize - unused; n = n < buflen ? n : buflen; buf_cpy (&c->u_mode.gcm.macbuf[unused], buf, n); unused += n; buf += n; buflen -= n; } if (!buflen) { if (!do_padding) break; while (unused < blocksize) c->u_mode.gcm.macbuf[unused++] = 0; } if (unused > 0) { gcry_assert (unused == blocksize); /* Process one block from macbuf. */ burn = ghash_fn (c, hash, c->u_mode.gcm.macbuf, 1); unused = 0; } nblocks = buflen / blocksize; if (nblocks) { burn = ghash_fn (c, hash, buf, nblocks); buf += blocksize * nblocks; buflen -= blocksize * nblocks; } } while (buflen > 0); c->u_mode.gcm.mac_unused = unused; if (burn) _gcry_burn_stack (burn); } static gcry_err_code_t gcm_ctr_encrypt (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen, const byte *inbuf, size_t inbuflen) { gcry_err_code_t err = 0; while (inbuflen) { u32 nblocks_to_overflow; u32 num_ctr_increments; u32 curr_ctr_low; size_t currlen = inbuflen; byte ctr_copy[GCRY_GCM_BLOCK_LEN]; int fix_ctr = 0; /* GCM CTR increments only least significant 32-bits, without carry * to upper 96-bits of counter. Using generic CTR implementation * directly would carry 32-bit overflow to upper 96-bit. Detect * if input length is long enough to cause overflow, and limit * input length so that CTR overflow happen but updated CTR value is * not used to encrypt further input. After overflow, upper 96 bits * of CTR are restored to cancel out modification done by generic CTR * encryption. */ if (inbuflen > c->unused) { curr_ctr_low = gcm_add32_be128 (c->u_ctr.ctr, 0); /* Number of CTR increments this inbuflen would cause. */ num_ctr_increments = (inbuflen - c->unused) / GCRY_GCM_BLOCK_LEN + !!((inbuflen - c->unused) % GCRY_GCM_BLOCK_LEN); if ((u32)(num_ctr_increments + curr_ctr_low) < curr_ctr_low) { nblocks_to_overflow = 0xffffffffU - curr_ctr_low + 1; currlen = nblocks_to_overflow * GCRY_GCM_BLOCK_LEN + c->unused; if (currlen > inbuflen) { currlen = inbuflen; } fix_ctr = 1; cipher_block_cpy(ctr_copy, c->u_ctr.ctr, GCRY_GCM_BLOCK_LEN); } } err = _gcry_cipher_ctr_encrypt(c, outbuf, outbuflen, inbuf, currlen); if (err != 0) return err; if (fix_ctr) { /* Lower 32-bits of CTR should now be zero. */ gcry_assert(gcm_add32_be128 (c->u_ctr.ctr, 0) == 0); /* Restore upper part of CTR. */ buf_cpy(c->u_ctr.ctr, ctr_copy, GCRY_GCM_BLOCK_LEN - sizeof(u32)); wipememory(ctr_copy, sizeof(ctr_copy)); } inbuflen -= currlen; inbuf += currlen; outbuflen -= currlen; outbuf += currlen; } return err; } gcry_err_code_t _gcry_cipher_gcm_encrypt (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen, const byte *inbuf, size_t inbuflen) { static const unsigned char zerobuf[MAX_BLOCKSIZE]; gcry_err_code_t err; if (c->spec->blocksize != GCRY_GCM_BLOCK_LEN) return GPG_ERR_CIPHER_ALGO; if (outbuflen < inbuflen) return GPG_ERR_BUFFER_TOO_SHORT; if (c->u_mode.gcm.datalen_over_limits) return GPG_ERR_INV_LENGTH; if (c->marks.tag || c->u_mode.gcm.ghash_data_finalized || !c->u_mode.gcm.ghash_fn) return GPG_ERR_INV_STATE; if (!c->marks.iv) _gcry_cipher_gcm_setiv (c, zerobuf, GCRY_GCM_BLOCK_LEN); if (c->u_mode.gcm.disallow_encryption_because_of_setiv_in_fips_mode) return GPG_ERR_INV_STATE; if (!c->u_mode.gcm.ghash_aad_finalized) { /* Start of encryption marks end of AAD stream. */ do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, NULL, 0, 1); c->u_mode.gcm.ghash_aad_finalized = 1; } gcm_bytecounter_add(c->u_mode.gcm.datalen, inbuflen); if (!gcm_check_datalen(c->u_mode.gcm.datalen)) { c->u_mode.gcm.datalen_over_limits = 1; return GPG_ERR_INV_LENGTH; } - err = gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, inbuflen); - if (err != 0) - return err; + while (inbuflen) + { + size_t currlen = inbuflen; - do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, outbuf, inbuflen, 0); + /* Since checksumming is done after encryption, process input in 24KiB + * chunks to keep data loaded in L1 cache for checksumming. */ + if (currlen > 24 * 1024) + currlen = 24 * 1024; + + err = gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, currlen); + if (err != 0) + return err; + + do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, outbuf, currlen, 0); + + outbuf += currlen; + inbuf += currlen; + outbuflen -= currlen; + inbuflen -= currlen; + } return 0; } gcry_err_code_t _gcry_cipher_gcm_decrypt (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen, const byte *inbuf, size_t inbuflen) { static const unsigned char zerobuf[MAX_BLOCKSIZE]; + gcry_err_code_t err; if (c->spec->blocksize != GCRY_GCM_BLOCK_LEN) return GPG_ERR_CIPHER_ALGO; if (outbuflen < inbuflen) return GPG_ERR_BUFFER_TOO_SHORT; if (c->u_mode.gcm.datalen_over_limits) return GPG_ERR_INV_LENGTH; if (c->marks.tag || c->u_mode.gcm.ghash_data_finalized || !c->u_mode.gcm.ghash_fn) return GPG_ERR_INV_STATE; if (!c->marks.iv) _gcry_cipher_gcm_setiv (c, zerobuf, GCRY_GCM_BLOCK_LEN); if (!c->u_mode.gcm.ghash_aad_finalized) { /* Start of decryption marks end of AAD stream. */ do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, NULL, 0, 1); c->u_mode.gcm.ghash_aad_finalized = 1; } gcm_bytecounter_add(c->u_mode.gcm.datalen, inbuflen); if (!gcm_check_datalen(c->u_mode.gcm.datalen)) { c->u_mode.gcm.datalen_over_limits = 1; return GPG_ERR_INV_LENGTH; } - do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, inbuf, inbuflen, 0); + while (inbuflen) + { + size_t currlen = inbuflen; + + /* Since checksumming is done before decryption, process input in + * 24KiB chunks to keep data loaded in L1 cache for decryption. */ + if (currlen > 24 * 1024) + currlen = 24 * 1024; - return gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, inbuflen); + do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, inbuf, currlen, 0); + + err = gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, currlen); + if (err) + return err; + + outbuf += currlen; + inbuf += currlen; + outbuflen -= currlen; + inbuflen -= currlen; + } + + return 0; } gcry_err_code_t _gcry_cipher_gcm_authenticate (gcry_cipher_hd_t c, const byte * aadbuf, size_t aadbuflen) { static const unsigned char zerobuf[MAX_BLOCKSIZE]; if (c->spec->blocksize != GCRY_GCM_BLOCK_LEN) return GPG_ERR_CIPHER_ALGO; if (c->u_mode.gcm.datalen_over_limits) return GPG_ERR_INV_LENGTH; if (c->marks.tag || c->u_mode.gcm.ghash_aad_finalized || c->u_mode.gcm.ghash_data_finalized || !c->u_mode.gcm.ghash_fn) return GPG_ERR_INV_STATE; if (!c->marks.iv) _gcry_cipher_gcm_setiv (c, zerobuf, GCRY_GCM_BLOCK_LEN); gcm_bytecounter_add(c->u_mode.gcm.aadlen, aadbuflen); if (!gcm_check_aadlen_or_ivlen(c->u_mode.gcm.aadlen)) { c->u_mode.gcm.datalen_over_limits = 1; return GPG_ERR_INV_LENGTH; } do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, aadbuf, aadbuflen, 0); return 0; } void _gcry_cipher_gcm_setkey (gcry_cipher_hd_t c) { memset (c->u_mode.gcm.u_ghash_key.key, 0, GCRY_GCM_BLOCK_LEN); c->spec->encrypt (&c->context.c, c->u_mode.gcm.u_ghash_key.key, c->u_mode.gcm.u_ghash_key.key); setupM (c); } static gcry_err_code_t _gcry_cipher_gcm_initiv (gcry_cipher_hd_t c, const byte *iv, size_t ivlen) { memset (c->u_mode.gcm.aadlen, 0, sizeof(c->u_mode.gcm.aadlen)); memset (c->u_mode.gcm.datalen, 0, sizeof(c->u_mode.gcm.datalen)); memset (c->u_mode.gcm.u_tag.tag, 0, GCRY_GCM_BLOCK_LEN); c->u_mode.gcm.datalen_over_limits = 0; c->u_mode.gcm.ghash_data_finalized = 0; c->u_mode.gcm.ghash_aad_finalized = 0; if (ivlen == 0) return GPG_ERR_INV_LENGTH; if (ivlen != GCRY_GCM_BLOCK_LEN - 4) { u32 iv_bytes[2] = {0, 0}; u32 bitlengths[2][2]; if (!c->u_mode.gcm.ghash_fn) return GPG_ERR_INV_STATE; memset(c->u_ctr.ctr, 0, GCRY_GCM_BLOCK_LEN); gcm_bytecounter_add(iv_bytes, ivlen); if (!gcm_check_aadlen_or_ivlen(iv_bytes)) { c->u_mode.gcm.datalen_over_limits = 1; return GPG_ERR_INV_LENGTH; } do_ghash_buf(c, c->u_ctr.ctr, iv, ivlen, 1); /* iv length, 64-bit */ bitlengths[1][1] = be_bswap32(iv_bytes[0] << 3); bitlengths[1][0] = be_bswap32((iv_bytes[0] >> 29) | (iv_bytes[1] << 3)); /* zeros, 64-bit */ bitlengths[0][1] = 0; bitlengths[0][0] = 0; do_ghash_buf(c, c->u_ctr.ctr, (byte*)bitlengths, GCRY_GCM_BLOCK_LEN, 1); wipememory (iv_bytes, sizeof iv_bytes); wipememory (bitlengths, sizeof bitlengths); } else { /* 96-bit IV is handled differently. */ memcpy (c->u_ctr.ctr, iv, ivlen); c->u_ctr.ctr[12] = c->u_ctr.ctr[13] = c->u_ctr.ctr[14] = 0; c->u_ctr.ctr[15] = 1; } c->spec->encrypt (&c->context.c, c->u_mode.gcm.tagiv, c->u_ctr.ctr); gcm_add32_be128 (c->u_ctr.ctr, 1); c->unused = 0; c->marks.iv = 1; c->marks.tag = 0; return 0; } gcry_err_code_t _gcry_cipher_gcm_setiv (gcry_cipher_hd_t c, const byte *iv, size_t ivlen) { c->marks.iv = 0; c->marks.tag = 0; c->u_mode.gcm.disallow_encryption_because_of_setiv_in_fips_mode = 0; if (fips_mode ()) { /* Direct invocation of GCM setiv in FIPS mode disables encryption. */ c->u_mode.gcm.disallow_encryption_because_of_setiv_in_fips_mode = 1; } return _gcry_cipher_gcm_initiv (c, iv, ivlen); } #if 0 && TODO void _gcry_cipher_gcm_geniv (gcry_cipher_hd_t c, byte *ivout, size_t ivoutlen, const byte *nonce, size_t noncelen) { /* nonce: user provided part (might be null) */ /* noncelen: check if proper length (if nonce not null) */ /* ivout: iv used to initialize gcm, output to user */ /* ivoutlen: check correct size */ byte iv[IVLEN]; if (!ivout) return GPG_ERR_INV_ARG; if (ivoutlen != IVLEN) return GPG_ERR_INV_LENGTH; if (nonce != NULL && !is_nonce_ok_len(noncelen)) return GPG_ERR_INV_ARG; gcm_generate_iv(iv, nonce, noncelen); c->marks.iv = 0; c->marks.tag = 0; c->u_mode.gcm.disallow_encryption_because_of_setiv_in_fips_mode = 0; _gcry_cipher_gcm_initiv (c, iv, IVLEN); buf_cpy(ivout, iv, IVLEN); wipememory(iv, sizeof(iv)); } #endif static int is_tag_length_valid(size_t taglen) { switch (taglen) { /* Allowed tag lengths from NIST SP 800-38D. */ case 128 / 8: /* GCRY_GCM_BLOCK_LEN */ case 120 / 8: case 112 / 8: case 104 / 8: case 96 / 8: case 64 / 8: case 32 / 8: return 1; default: return 0; } } static gcry_err_code_t _gcry_cipher_gcm_tag (gcry_cipher_hd_t c, byte * outbuf, size_t outbuflen, int check) { if (!(is_tag_length_valid (outbuflen) || outbuflen >= GCRY_GCM_BLOCK_LEN)) return GPG_ERR_INV_LENGTH; if (c->u_mode.gcm.datalen_over_limits) return GPG_ERR_INV_LENGTH; if (!c->marks.tag) { u32 bitlengths[2][2]; if (!c->u_mode.gcm.ghash_fn) return GPG_ERR_INV_STATE; /* aad length */ bitlengths[0][1] = be_bswap32(c->u_mode.gcm.aadlen[0] << 3); bitlengths[0][0] = be_bswap32((c->u_mode.gcm.aadlen[0] >> 29) | (c->u_mode.gcm.aadlen[1] << 3)); /* data length */ bitlengths[1][1] = be_bswap32(c->u_mode.gcm.datalen[0] << 3); bitlengths[1][0] = be_bswap32((c->u_mode.gcm.datalen[0] >> 29) | (c->u_mode.gcm.datalen[1] << 3)); /* Finalize data-stream. */ do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, NULL, 0, 1); c->u_mode.gcm.ghash_aad_finalized = 1; c->u_mode.gcm.ghash_data_finalized = 1; /* Add bitlengths to tag. */ do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, (byte*)bitlengths, GCRY_GCM_BLOCK_LEN, 1); cipher_block_xor (c->u_mode.gcm.u_tag.tag, c->u_mode.gcm.tagiv, c->u_mode.gcm.u_tag.tag, GCRY_GCM_BLOCK_LEN); c->marks.tag = 1; wipememory (bitlengths, sizeof (bitlengths)); wipememory (c->u_mode.gcm.macbuf, GCRY_GCM_BLOCK_LEN); wipememory (c->u_mode.gcm.tagiv, GCRY_GCM_BLOCK_LEN); wipememory (c->u_mode.gcm.aadlen, sizeof (c->u_mode.gcm.aadlen)); wipememory (c->u_mode.gcm.datalen, sizeof (c->u_mode.gcm.datalen)); } if (!check) { if (outbuflen > GCRY_GCM_BLOCK_LEN) outbuflen = GCRY_GCM_BLOCK_LEN; /* NB: We already checked that OUTBUF is large enough to hold * the result or has valid truncated length. */ memcpy (outbuf, c->u_mode.gcm.u_tag.tag, outbuflen); } else { /* OUTBUFLEN gives the length of the user supplied tag in OUTBUF * and thus we need to compare its length first. */ if (!is_tag_length_valid (outbuflen) || !buf_eq_const (outbuf, c->u_mode.gcm.u_tag.tag, outbuflen)) return GPG_ERR_CHECKSUM; } return 0; } gcry_err_code_t _gcry_cipher_gcm_get_tag (gcry_cipher_hd_t c, unsigned char *outtag, size_t taglen) { /* Outputting authentication tag is part of encryption. */ if (c->u_mode.gcm.disallow_encryption_because_of_setiv_in_fips_mode) return GPG_ERR_INV_STATE; return _gcry_cipher_gcm_tag (c, outtag, taglen, 0); } gcry_err_code_t _gcry_cipher_gcm_check_tag (gcry_cipher_hd_t c, const unsigned char *intag, size_t taglen) { return _gcry_cipher_gcm_tag (c, (unsigned char *) intag, taglen, 1); } diff --git a/cipher/cipher-poly1305.c b/cipher/cipher-poly1305.c index 82537aa4..607586b5 100644 --- a/cipher/cipher-poly1305.c +++ b/cipher/cipher-poly1305.c @@ -1,334 +1,365 @@ /* cipher-poly1305.c - Poly1305 based AEAD cipher mode, RFC-8439 * Copyright (C) 2014 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser general Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . */ #include #include #include #include #include #include "g10lib.h" #include "cipher.h" #include "bufhelp.h" #include "./cipher-internal.h" #include "./poly1305-internal.h" static inline int poly1305_bytecounter_add (u32 ctr[2], size_t add) { int overflow = 0; if (sizeof(add) > sizeof(u32)) { u32 high_add = ((add >> 31) >> 1) & 0xffffffff; ctr[1] += high_add; if (ctr[1] < high_add) overflow = 1; } ctr[0] += add; if (ctr[0] >= add) return overflow; ctr[1] += 1; return (ctr[1] < 1) || overflow; } static void poly1305_fill_bytecounts (gcry_cipher_hd_t c) { u32 lenbuf[4]; lenbuf[0] = le_bswap32(c->u_mode.poly1305.aadcount[0]); lenbuf[1] = le_bswap32(c->u_mode.poly1305.aadcount[1]); lenbuf[2] = le_bswap32(c->u_mode.poly1305.datacount[0]); lenbuf[3] = le_bswap32(c->u_mode.poly1305.datacount[1]); _gcry_poly1305_update (&c->u_mode.poly1305.ctx, (byte*)lenbuf, sizeof(lenbuf)); wipememory(lenbuf, sizeof(lenbuf)); } static void poly1305_do_padding (gcry_cipher_hd_t c, u32 ctr[2]) { static const byte zero_padding_buf[15] = {}; u32 padding_count; /* Padding to 16 byte boundary. */ if (ctr[0] % 16 > 0) { padding_count = 16 - ctr[0] % 16; _gcry_poly1305_update (&c->u_mode.poly1305.ctx, zero_padding_buf, padding_count); } } static void poly1305_aad_finish (gcry_cipher_hd_t c) { /* After AAD, feed padding bytes so we get 16 byte alignment. */ poly1305_do_padding (c, c->u_mode.poly1305.aadcount); /* Start of encryption marks end of AAD stream. */ c->u_mode.poly1305.aad_finalized = 1; c->u_mode.poly1305.datacount[0] = 0; c->u_mode.poly1305.datacount[1] = 0; } static gcry_err_code_t poly1305_set_zeroiv (gcry_cipher_hd_t c) { byte zero[8] = { 0, }; return _gcry_cipher_poly1305_setiv (c, zero, sizeof(zero)); } gcry_err_code_t _gcry_cipher_poly1305_authenticate (gcry_cipher_hd_t c, const byte * aadbuf, size_t aadbuflen) { if (c->u_mode.poly1305.bytecount_over_limits) return GPG_ERR_INV_LENGTH; if (c->u_mode.poly1305.aad_finalized) return GPG_ERR_INV_STATE; if (c->marks.tag) return GPG_ERR_INV_STATE; if (!c->marks.iv) poly1305_set_zeroiv(c); if (poly1305_bytecounter_add(c->u_mode.poly1305.aadcount, aadbuflen)) { c->u_mode.poly1305.bytecount_over_limits = 1; return GPG_ERR_INV_LENGTH; } _gcry_poly1305_update (&c->u_mode.poly1305.ctx, aadbuf, aadbuflen); return 0; } gcry_err_code_t _gcry_cipher_poly1305_encrypt (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen, const byte *inbuf, size_t inbuflen) { gcry_err_code_t err; if (outbuflen < inbuflen) return GPG_ERR_BUFFER_TOO_SHORT; if (c->marks.tag) return GPG_ERR_INV_STATE; if (c->u_mode.poly1305.bytecount_over_limits) return GPG_ERR_INV_LENGTH; if (!c->marks.iv) { err = poly1305_set_zeroiv(c); if (err) return err; } if (!c->u_mode.poly1305.aad_finalized) poly1305_aad_finish(c); if (poly1305_bytecounter_add(c->u_mode.poly1305.datacount, inbuflen)) { c->u_mode.poly1305.bytecount_over_limits = 1; return GPG_ERR_INV_LENGTH; } - c->spec->stencrypt(&c->context.c, outbuf, (byte*)inbuf, inbuflen); + while (inbuflen) + { + size_t currlen = inbuflen; + + /* Since checksumming is done after encryption, process input in 24KiB + * chunks to keep data loaded in L1 cache for checksumming. */ + if (currlen > 24 * 1024) + currlen = 24 * 1024; + + c->spec->stencrypt(&c->context.c, outbuf, (byte*)inbuf, currlen); - _gcry_poly1305_update (&c->u_mode.poly1305.ctx, outbuf, inbuflen); + _gcry_poly1305_update (&c->u_mode.poly1305.ctx, outbuf, currlen); + + outbuf += currlen; + inbuf += currlen; + outbuflen -= currlen; + inbuflen -= currlen; + } return 0; } gcry_err_code_t _gcry_cipher_poly1305_decrypt (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen, const byte *inbuf, size_t inbuflen) { gcry_err_code_t err; if (outbuflen < inbuflen) return GPG_ERR_BUFFER_TOO_SHORT; if (c->marks.tag) return GPG_ERR_INV_STATE; if (c->u_mode.poly1305.bytecount_over_limits) return GPG_ERR_INV_LENGTH; if (!c->marks.iv) { err = poly1305_set_zeroiv(c); if (err) return err; } if (!c->u_mode.poly1305.aad_finalized) poly1305_aad_finish(c); if (poly1305_bytecounter_add(c->u_mode.poly1305.datacount, inbuflen)) { c->u_mode.poly1305.bytecount_over_limits = 1; return GPG_ERR_INV_LENGTH; } - _gcry_poly1305_update (&c->u_mode.poly1305.ctx, inbuf, inbuflen); + while (inbuflen) + { + size_t currlen = inbuflen; + + /* Since checksumming is done before decryption, process input in 24KiB + * chunks to keep data loaded in L1 cache for decryption. */ + if (currlen > 24 * 1024) + currlen = 24 * 1024; + + _gcry_poly1305_update (&c->u_mode.poly1305.ctx, inbuf, currlen); + + c->spec->stdecrypt(&c->context.c, outbuf, (byte*)inbuf, currlen); + + outbuf += currlen; + inbuf += currlen; + outbuflen -= currlen; + inbuflen -= currlen; + } - c->spec->stdecrypt(&c->context.c, outbuf, (byte*)inbuf, inbuflen); return 0; } static gcry_err_code_t _gcry_cipher_poly1305_tag (gcry_cipher_hd_t c, byte * outbuf, size_t outbuflen, int check) { gcry_err_code_t err; if (outbuflen < POLY1305_TAGLEN) return GPG_ERR_BUFFER_TOO_SHORT; if (c->u_mode.poly1305.bytecount_over_limits) return GPG_ERR_INV_LENGTH; if (!c->marks.iv) { err = poly1305_set_zeroiv(c); if (err) return err; } if (!c->u_mode.poly1305.aad_finalized) poly1305_aad_finish(c); if (!c->marks.tag) { /* After data, feed padding bytes so we get 16 byte alignment. */ poly1305_do_padding (c, c->u_mode.poly1305.datacount); /* Write byte counts to poly1305. */ poly1305_fill_bytecounts(c); _gcry_poly1305_finish(&c->u_mode.poly1305.ctx, c->u_iv.iv); c->marks.tag = 1; } if (!check) { memcpy (outbuf, c->u_iv.iv, POLY1305_TAGLEN); } else { /* OUTBUFLEN gives the length of the user supplied tag in OUTBUF * and thus we need to compare its length first. */ if (outbuflen != POLY1305_TAGLEN || !buf_eq_const (outbuf, c->u_iv.iv, POLY1305_TAGLEN)) return GPG_ERR_CHECKSUM; } return 0; } gcry_err_code_t _gcry_cipher_poly1305_get_tag (gcry_cipher_hd_t c, unsigned char *outtag, size_t taglen) { return _gcry_cipher_poly1305_tag (c, outtag, taglen, 0); } gcry_err_code_t _gcry_cipher_poly1305_check_tag (gcry_cipher_hd_t c, const unsigned char *intag, size_t taglen) { return _gcry_cipher_poly1305_tag (c, (unsigned char *) intag, taglen, 1); } void _gcry_cipher_poly1305_setkey (gcry_cipher_hd_t c) { c->u_mode.poly1305.aadcount[0] = 0; c->u_mode.poly1305.aadcount[1] = 0; c->u_mode.poly1305.datacount[0] = 0; c->u_mode.poly1305.datacount[1] = 0; c->u_mode.poly1305.bytecount_over_limits = 0; c->u_mode.poly1305.aad_finalized = 0; c->marks.tag = 0; c->marks.iv = 0; } gcry_err_code_t _gcry_cipher_poly1305_setiv (gcry_cipher_hd_t c, const byte *iv, size_t ivlen) { byte tmpbuf[64]; /* size of ChaCha20 block */ gcry_err_code_t err; /* IV must be 96-bits */ if (!iv && ivlen != (96 / 8)) return GPG_ERR_INV_ARG; memset(&c->u_mode.poly1305.ctx, 0, sizeof(c->u_mode.poly1305.ctx)); c->u_mode.poly1305.aadcount[0] = 0; c->u_mode.poly1305.aadcount[1] = 0; c->u_mode.poly1305.datacount[0] = 0; c->u_mode.poly1305.datacount[1] = 0; c->u_mode.poly1305.bytecount_over_limits = 0; c->u_mode.poly1305.aad_finalized = 0; c->marks.tag = 0; c->marks.iv = 0; /* Set up IV for stream cipher. */ c->spec->setiv (&c->context.c, iv, ivlen); /* Get the first block from ChaCha20. */ memset(tmpbuf, 0, sizeof(tmpbuf)); c->spec->stencrypt(&c->context.c, tmpbuf, tmpbuf, sizeof(tmpbuf)); /* Use the first 32-bytes as Poly1305 key. */ err = _gcry_poly1305_init (&c->u_mode.poly1305.ctx, tmpbuf, POLY1305_KEYLEN); wipememory(tmpbuf, sizeof(tmpbuf)); if (err) return err; c->marks.iv = 1; return 0; }