Index: cipher/rijndael-ppc8.S =================================================================== --- cipher/rijndael-ppc8.S +++ cipher/rijndael-ppc8.S @@ -486,7 +486,6 @@ neg 11,3 lvsr 10,0,6 - lwz 9,480(6) lvsr 6,0,11 lvx 5,0,3 @@ -1222,7 +1221,6 @@ neg 11,3 lvsr 10,0,6 - lwz 9,480(6) lvsr 6,0,11 lvx 5,0,3 Index: cipher/rijndael-ppc8.pl =================================================================== --- cipher/rijndael-ppc8.pl +++ cipher/rijndael-ppc8.pl @@ -526,7 +526,6 @@ neg r11,$inp ?lvsl $keyperm,0,$key # prepare for unaligned key - lwz $rounds,480($key) lvsr $inpperm,0,r11 # prepare for unaligned load lvx $inptail,0,$inp @@ -1287,7 +1286,6 @@ neg r11,$inp ?lvsl $keyperm,0,$key # prepare for unaligned key - lwz $rounds,480($key) lvsr $inpperm,0,r11 # prepare for unaligned load lvx $inptail,0,$inp Index: cipher/rijndael-ppc832.S =================================================================== --- cipher/rijndael-ppc832.S +++ cipher/rijndael-ppc832.S @@ -486,7 +486,6 @@ neg 11,3 lvsl 10,0,6 - lwz 9,480(6) lvsr 6,0,11 lvx 5,0,3 Index: cipher/rijndael-ppc8be.S =================================================================== --- cipher/rijndael-ppc8be.S +++ cipher/rijndael-ppc8be.S @@ -486,7 +486,6 @@ neg 11,3 lvsl 10,0,6 - lwz 9,480(6) lvsr 6,0,11 lvx 5,0,3 Index: cipher/rijndael.c =================================================================== --- cipher/rijndael.c +++ cipher/rijndael.c @@ -199,11 +199,6 @@ size_t nblocks, int encrypt); #endif /*USE_ARM_ASM*/ -/* forward declaration */ -static int _gcry_aes_generic_cbc_enc (const void *context, unsigned char *iv, - void *outbuf_arg, const void *inbuf_arg, - size_t nblocks, - int cbc_mac); #ifdef USE_PPC_ASM /* POWER 8 AES extensions */ extern void aes_p8_encrypt (const unsigned char *in, @@ -233,6 +228,80 @@ extern int aes_p8_set_decrypt_key (const unsigned char *userKey, const int bits, /* this is the decryption key part of context */ const unsigned (*)[15][4]); +extern void aes_p8_cbc_encrypt (const unsigned char *in, unsigned char *out, + size_t length, const void *key, + unsigned char *ivec, const int enc, int rounds); +static void _gcry_aes_ppc8_cbc_dec (void *context, unsigned char *iv, + void *outbuf_arg, const void *inbuf_arg, + size_t nblocks) { + const RIJNDAEL_context *ctx = context; + aes_p8_cbc_encrypt (inbuf_arg, outbuf_arg, nblocks * 16, &ctx->u2, iv, 0, ctx->rounds); + return; +} +/* forward declaration */ +static int _gcry_aes_generic_cbc_enc (const void *context, unsigned char *iv, + void *outbuf_arg, const void *inbuf_arg, + size_t nblocks, + int cbc_mac); +static void _gcry_aes_ppc8_cbc_enc (void *context, unsigned char *iv, + void *outbuf_arg, const void *inbuf_arg, + size_t nblocks, int cbc_mac) { + const RIJNDAEL_context *ctx = context; +#ifdef __builtin_expect + __builtin_expect (cbc_mac, 0); +#endif + if (cbc_mac) { + _gcry_aes_generic_cbc_enc (context, iv, outbuf_arg, inbuf_arg, nblocks, cbc_mac); + return; + } + aes_p8_cbc_encrypt (inbuf_arg, outbuf_arg, nblocks * 16, &ctx->u1, iv, 1, ctx->rounds); + return; +} +extern void aes_p8_xts_encrypt(const unsigned char *inp, unsigned char *out, + size_t len, const void *key1, + const void *key2, const void *iv); +extern void aes_p8_xts_decrypt(const unsigned char *inp, unsigned char *out, + size_t len, const void *key1, + const void *key2, const void *iv); +void _gcry_aes_ppc8_xts_crypt (void *context, unsigned char *tweak, + void *outbuf_arg, + const void *inbuf_arg, + size_t nblocks, int encrypt) { + const RIJNDAEL_context *ctx = context; + if (encrypt) + aes_p8_xts_encrypt (inbuf_arg, outbuf_arg, nblocks * 16, &ctx->u1, NULL, tweak); + else + aes_p8_xts_decrypt (inbuf_arg, outbuf_arg, nblocks * 16, &ctx->u2, NULL, tweak); +} +extern void aes_p8_ctr32_encrypt_blocks (const unsigned char *in, unsigned char *out, + size_t len, const void *key, + const void *ivec, int unused, int rounds); +static void _gcry_aes_ppc8_ctr_enc (void *context, unsigned char *ctr, + void *outbuf_arg, const void *inbuf_arg, + size_t nblocks) { + const RIJNDAEL_context *ctx = context; + u64 s[2]; + s[0] = buf_get_be64(ctr + 8); + s[1] = buf_get_be64(ctr + 0); + s[0] += nblocks; + s[1] += (s[0] < nblocks); +#ifdef __builtin_expect + __builtin_expect(s[0] < nblocks, 0); +#endif + if (s[0] < nblocks) { + aes_p8_ctr32_encrypt_blocks (inbuf_arg, outbuf_arg, nblocks - s[0], &ctx->u1, ctr, /*unused*/0, ctx->rounds); + buf_put_be64(ctr + 8, 0); + buf_put_be64(ctr + 0, s[1]); + inbuf_arg += (nblocks - s[0]) * BLOCKSIZE; + outbuf_arg += (nblocks - s[0]) * BLOCKSIZE; + aes_p8_ctr32_encrypt_blocks (inbuf_arg, outbuf_arg, s[0], &ctx->u1, ctr, /*unused*/0, ctx->rounds); + } else { + aes_p8_ctr32_encrypt_blocks (inbuf_arg, outbuf_arg, nblocks, &ctx->u1, ctr, /*unused*/0, ctx->rounds); + buf_put_be64(ctr + 0, s[1]); + } + buf_put_be64(ctr + 8, s[0]); + return; +} #endif /*USE_PPC_ASM*/ static unsigned int do_encrypt (const RIJNDAEL_context *ctx, unsigned char *bx, @@ -448,6 +517,12 @@ ctx->prefetch_enc_fn = NULL; ctx->prefetch_dec_fn = NULL; ctx->use_ppc_asm = 1; + if (hd) { + hd->bulk.cbc_dec = _gcry_aes_ppc8_cbc_dec; + hd->bulk.cbc_enc = _gcry_aes_ppc8_cbc_enc; + hd->bulk.xts_crypt = _gcry_aes_ppc8_xts_crypt; + hd->bulk.ctr_enc = _gcry_aes_ppc8_ctr_enc; + } } #endif else @@ -478,8 +553,7 @@ #endif #ifdef USE_PPC_ASM else if (ctx->use_ppc_asm) { - /* These are both done here to avoid having to store the key. - * These S-boxes are generated on-the-fly. */ + /* These are both done here to avoid having to store the key. */ aes_p8_set_encrypt_key (key, keylen * 8, ctx); aes_p8_set_decrypt_key (key, keylen * 8, &ctx->keyschdec32); } @@ -983,6 +1057,13 @@ _gcry_aes_armv8_ce_cbc_enc (ctx, iv, outbuf, inbuf, nblocks, cbc_mac); return; } +#endif /*USE_ARM_CE*/ +#ifdef USE_PPC_ASM + else if (ctx->use_ppc_asm) + { + _gcry_aes_ppc8_cbc_enc (ctx, iv, outbuf, inbuf, nblocks, cbc_mac); + return; + } #endif /*USE_ARM_CE*/ else { @@ -1033,6 +1114,13 @@ return; } #endif /*USE_ARM_CE*/ +#ifdef USE_PPC_ASM + else if (ctx->use_ppc_asm) + { + _gcry_aes_ppc8_ctr_enc (ctx, ctr, outbuf, inbuf, nblocks); + return; + } +#endif /*USE_PPC_ASM*/ else { union { unsigned char x1[16] ATTR_ALIGNED_16; u32 x32[4]; } tmp; @@ -1298,6 +1386,40 @@ _gcry_burn_stack (burn_depth + 4 * sizeof(void *)); } +static void +_gcry_aes_generic_cbc_dec (void *context, unsigned char *iv, + void *outbuf_arg, const void *inbuf_arg, + size_t nblocks) +{ + RIJNDAEL_context *ctx = context; + unsigned char *outbuf = outbuf_arg; + const unsigned char *inbuf = inbuf_arg; + unsigned char savebuf[BLOCKSIZE] ATTR_ALIGNED_16; + unsigned burn_depth = 0; + rijndael_cryptfn_t decrypt_fn = ctx->decrypt_fn; + + check_decryption_preparation (ctx); + + if (ctx->prefetch_dec_fn) + ctx->prefetch_dec_fn(); + + for ( ;nblocks; nblocks-- ) + { + /* INBUF is needed later and it may be identical to OUTBUF, so store + the intermediate result to SAVEBUF. */ + + burn_depth = decrypt_fn (ctx, savebuf, inbuf); + + cipher_block_xor_n_copy_2(outbuf, savebuf, iv, inbuf, BLOCKSIZE); + inbuf += BLOCKSIZE; + outbuf += BLOCKSIZE; + } + + wipememory(savebuf, sizeof(savebuf)); + + if (burn_depth) + _gcry_burn_stack (burn_depth + 4 * sizeof(void *)); +} /* Bulk decryption of complete blocks in CBC mode. Caller needs to make sure that IV is aligned on an unsigned long boundary. This @@ -1311,7 +1433,6 @@ RIJNDAEL_context *ctx = context; unsigned char *outbuf = outbuf_arg; const unsigned char *inbuf = inbuf_arg; - unsigned int burn_depth = 0; if (0) ; @@ -1338,31 +1459,9 @@ #endif /*USE_ARM_CE*/ else { - unsigned char savebuf[BLOCKSIZE] ATTR_ALIGNED_16; - rijndael_cryptfn_t decrypt_fn = ctx->decrypt_fn; - - check_decryption_preparation (ctx); - - if (ctx->prefetch_dec_fn) - ctx->prefetch_dec_fn(); - - for ( ;nblocks; nblocks-- ) - { - /* INBUF is needed later and it may be identical to OUTBUF, so store - the intermediate result to SAVEBUF. */ - - burn_depth = decrypt_fn (ctx, savebuf, inbuf); - - cipher_block_xor_n_copy_2(outbuf, savebuf, iv, inbuf, BLOCKSIZE); - inbuf += BLOCKSIZE; - outbuf += BLOCKSIZE; - } - - wipememory(savebuf, sizeof(savebuf)); + _gcry_aes_generic_cbc_dec (ctx, iv, outbuf, inbuf, nblocks); + return; } - - if (burn_depth) - _gcry_burn_stack (burn_depth + 4 * sizeof(void *)); } @@ -1554,6 +1653,13 @@ return; } #endif /*USE_ARM_CE*/ +#ifdef USE_PPC_ASM + else if (ctx->use_ppc_asm) + { + _gcry_aes_ppc8_xts_crypt (ctx, tweak, outbuf, inbuf, nblocks, encrypt); + return; + } +#endif /*USE_PPC_ASM*/ else { if (encrypt)