diff --git a/cipher/asm-common-amd64.h b/cipher/asm-common-amd64.h index cd93abc3..d9bbc01b 100644 --- a/cipher/asm-common-amd64.h +++ b/cipher/asm-common-amd64.h @@ -1,205 +1,207 @@ /* asm-common-amd64.h - Common macros for AMD64 assembly * * Copyright (C) 2018 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . */ #ifndef GCRY_ASM_COMMON_AMD64_H #define GCRY_ASM_COMMON_AMD64_H #include #ifdef HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS # define ELF(...) __VA_ARGS__ #else # define ELF(...) /*_*/ #endif #ifdef __PIC__ # define rRIP (%rip) #else # define rRIP #endif #ifdef __PIC__ # define RIP %rip #else # define RIP #endif #ifdef __PIC__ # define ADD_RIP +rip #else # define ADD_RIP #endif #if defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS) || !defined(__PIC__) # define GET_EXTERN_POINTER(name, reg) movabsq $name, reg #else # ifdef __code_model_large__ # define GET_EXTERN_POINTER(name, reg) \ pushq %r15; \ pushq %r14; \ 1: leaq 1b(%rip), reg; \ movabsq $_GLOBAL_OFFSET_TABLE_-1b, %r14; \ movabsq $name@GOT, %r15; \ addq %r14, reg; \ popq %r14; \ movq (reg, %r15), reg; \ popq %r15; # else # define GET_EXTERN_POINTER(name, reg) movq name@GOTPCREL(%rip), reg # endif #endif #ifdef HAVE_GCC_ASM_CFI_DIRECTIVES /* CFI directives to emit DWARF stack unwinding information. */ # define CFI_STARTPROC() .cfi_startproc # define CFI_ENDPROC() .cfi_endproc # define CFI_REMEMBER_STATE() .cfi_remember_state # define CFI_RESTORE_STATE() .cfi_restore_state # define CFI_ADJUST_CFA_OFFSET(off) .cfi_adjust_cfa_offset off # define CFI_REL_OFFSET(reg,off) .cfi_rel_offset reg, off # define CFI_DEF_CFA_REGISTER(reg) .cfi_def_cfa_register reg # define CFI_REGISTER(ro,rn) .cfi_register ro, rn # define CFI_RESTORE(reg) .cfi_restore reg # define CFI_PUSH(reg) \ CFI_ADJUST_CFA_OFFSET(8); CFI_REL_OFFSET(reg, 0) # define CFI_POP(reg) \ CFI_ADJUST_CFA_OFFSET(-8); CFI_RESTORE(reg) # define CFI_POP_TMP_REG() \ CFI_ADJUST_CFA_OFFSET(-8); # define CFI_LEAVE() \ CFI_ADJUST_CFA_OFFSET(-8); CFI_DEF_CFA_REGISTER(%rsp) /* CFA expressions are used for pointing CFA and registers to * %rsp relative offsets. */ # define DW_REGNO_rax 0 # define DW_REGNO_rdx 1 # define DW_REGNO_rcx 2 # define DW_REGNO_rbx 3 # define DW_REGNO_rsi 4 # define DW_REGNO_rdi 5 # define DW_REGNO_rbp 6 # define DW_REGNO_rsp 7 # define DW_REGNO_r8 8 # define DW_REGNO_r9 9 # define DW_REGNO_r10 10 # define DW_REGNO_r11 11 # define DW_REGNO_r12 12 # define DW_REGNO_r13 13 # define DW_REGNO_r14 14 # define DW_REGNO_r15 15 # define DW_REGNO(reg) DW_REGNO_ ## reg /* Fixed length encoding used for integers for now. */ # define DW_SLEB128_7BIT(value) \ 0x00|((value) & 0x7f) # define DW_SLEB128_28BIT(value) \ 0x80|((value)&0x7f), \ 0x80|(((value)>>7)&0x7f), \ 0x80|(((value)>>14)&0x7f), \ 0x00|(((value)>>21)&0x7f) # define CFI_CFA_ON_STACK(rsp_offs,cfa_depth) \ .cfi_escape \ 0x0f, /* DW_CFA_def_cfa_expression */ \ DW_SLEB128_7BIT(11), /* length */ \ 0x77, /* DW_OP_breg7, rsp + constant */ \ DW_SLEB128_28BIT(rsp_offs), \ 0x06, /* DW_OP_deref */ \ 0x23, /* DW_OP_plus_constu */ \ DW_SLEB128_28BIT((cfa_depth)+8) # define CFI_REG_ON_STACK(reg,rsp_offs) \ .cfi_escape \ 0x10, /* DW_CFA_expression */ \ DW_SLEB128_7BIT(DW_REGNO(reg)), \ DW_SLEB128_7BIT(5), /* length */ \ 0x77, /* DW_OP_breg7, rsp + constant */ \ DW_SLEB128_28BIT(rsp_offs) #else # define CFI_STARTPROC() # define CFI_ENDPROC() # define CFI_REMEMBER_STATE() # define CFI_RESTORE_STATE() # define CFI_ADJUST_CFA_OFFSET(off) # define CFI_REL_OFFSET(reg,off) # define CFI_DEF_CFA_REGISTER(reg) # define CFI_REGISTER(ro,rn) # define CFI_RESTORE(reg) # define CFI_PUSH(reg) # define CFI_POP(reg) # define CFI_POP_TMP_REG() # define CFI_LEAVE() # define CFI_CFA_ON_STACK(rsp_offs,cfa_depth) # define CFI_REG_ON_STACK(reg,rsp_offs) #endif #ifdef HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS # define ENTER_SYSV_FUNC_PARAMS_0_4 \ pushq %rdi; \ CFI_PUSH(%rdi); \ pushq %rsi; \ CFI_PUSH(%rsi); \ movq %rcx, %rdi; \ movq %rdx, %rsi; \ movq %r8, %rdx; \ movq %r9, %rcx; \ # define ENTER_SYSV_FUNC_PARAMS_5 \ ENTER_SYSV_FUNC_PARAMS_0_4; \ movq 0x38(%rsp), %r8; # define ENTER_SYSV_FUNC_PARAMS_6 \ ENTER_SYSV_FUNC_PARAMS_5; \ movq 0x40(%rsp), %r9; # define EXIT_SYSV_FUNC \ popq %rsi; \ CFI_POP(%rsi); \ popq %rdi; \ CFI_POP(%rdi); #else # define ENTER_SYSV_FUNC_PARAMS_0_4 # define ENTER_SYSV_FUNC_PARAMS_5 # define ENTER_SYSV_FUNC_PARAMS_6 # define EXIT_SYSV_FUNC #endif /* 'ret' instruction replacement for straight-line speculation mitigation. */ #define ret_spec_stop \ ret; int3; /* This prevents speculative execution on old AVX512 CPUs, to prevent * speculative execution to AVX512 code. The vpopcntb instruction is * available on newer CPUs that do not suffer from significant frequency * drop when 512-bit vectors are utilized. */ #define spec_stop_avx512 \ - vpxord %xmm16, %xmm16, %xmm16; \ - vpopcntb %xmm16, %xmm16; /* Supported only by newer AVX512 CPUs. */ + vpxord %ymm16, %ymm16, %ymm16; \ + vpopcntb %xmm16, %xmm16; /* Supported only by newer AVX512 CPUs. */ \ + vpxord %ymm16, %ymm16, %ymm16; #define spec_stop_avx512_intel_syntax \ - vpxord xmm16, xmm16, xmm16; \ - vpopcntb xmm16, xmm16; /* Supported only by newer AVX512 CPUs. */ + vpxord ymm16, ymm16, ymm16; \ + vpopcntb xmm16, xmm16; /* Supported only by newer AVX512 CPUs. */ \ + vpxord ymm16, ymm16, ymm16; #endif /* GCRY_ASM_COMMON_AMD64_H */ diff --git a/cipher/camellia-gfni-avx512-amd64.S b/cipher/camellia-gfni-avx512-amd64.S index bddad804..14725b4a 100644 --- a/cipher/camellia-gfni-avx512-amd64.S +++ b/cipher/camellia-gfni-avx512-amd64.S @@ -1,1572 +1,1572 @@ /* camellia-gfni-avx512-amd64.S - GFNI/AVX512 implementation of Camellia * * Copyright (C) 2022 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . */ #include #ifdef __x86_64 #if (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \ defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) && \ defined(ENABLE_GFNI_SUPPORT) && defined(ENABLE_AVX512_SUPPORT) #include "asm-common-amd64.h" #define CAMELLIA_TABLE_BYTE_LEN 272 /* struct CAMELLIA_context: */ #define key_table 0 #define key_bitlength CAMELLIA_TABLE_BYTE_LEN /* register macros */ #define CTX %rdi #define RIO %r8 /********************************************************************** helper macros **********************************************************************/ #define zmm0_x xmm0 #define zmm1_x xmm1 #define zmm2_x xmm2 #define zmm3_x xmm3 #define zmm4_x xmm4 #define zmm5_x xmm5 #define zmm6_x xmm6 #define zmm7_x xmm7 #define zmm8_x xmm8 #define zmm9_x xmm9 #define zmm10_x xmm10 #define zmm11_x xmm11 #define zmm12_x xmm12 #define zmm13_x xmm13 #define zmm14_x xmm14 #define zmm15_x xmm15 #define zmm0_y ymm0 #define zmm1_y ymm1 #define zmm2_y ymm2 #define zmm3_y ymm3 #define zmm4_y ymm4 #define zmm5_y ymm5 #define zmm6_y ymm6 #define zmm7_y ymm7 #define zmm8_y ymm8 #define zmm9_y ymm9 #define zmm10_y ymm10 #define zmm11_y ymm11 #define zmm12_y ymm12 #define zmm13_y ymm13 #define zmm14_y ymm14 #define zmm15_y ymm15 #define mem_ab_0 %zmm16 #define mem_ab_1 %zmm17 #define mem_ab_2 %zmm31 #define mem_ab_3 %zmm18 #define mem_ab_4 %zmm19 #define mem_ab_5 %zmm20 #define mem_ab_6 %zmm21 #define mem_ab_7 %zmm22 #define mem_cd_0 %zmm23 #define mem_cd_1 %zmm24 #define mem_cd_2 %zmm30 #define mem_cd_3 %zmm25 #define mem_cd_4 %zmm26 #define mem_cd_5 %zmm27 #define mem_cd_6 %zmm28 #define mem_cd_7 %zmm29 #define clear_vec4(v0,v1,v2,v3) \ vpxord v0, v0, v0; \ vpxord v1, v1, v1; \ vpxord v2, v2, v2; \ vpxord v3, v3, v3 #define clear_zmm16_zmm31() \ - clear_vec4(%xmm16, %xmm20, %xmm24, %xmm28); \ - clear_vec4(%xmm17, %xmm21, %xmm25, %xmm29); \ - clear_vec4(%xmm18, %xmm22, %xmm26, %xmm30); \ - clear_vec4(%xmm19, %xmm23, %xmm27, %xmm31) + clear_vec4(%ymm16, %ymm20, %ymm24, %ymm28); \ + clear_vec4(%ymm17, %ymm21, %ymm25, %ymm29); \ + clear_vec4(%ymm18, %ymm22, %ymm26, %ymm30); \ + clear_vec4(%ymm19, %ymm23, %ymm27, %ymm31) #define clear_regs() \ kxorq %k1, %k1, %k1; \ vzeroall; \ clear_zmm16_zmm31() /********************************************************************** GFNI helper macros and constants **********************************************************************/ #define BV8(a0,a1,a2,a3,a4,a5,a6,a7) \ ( (((a0) & 1) << 0) | \ (((a1) & 1) << 1) | \ (((a2) & 1) << 2) | \ (((a3) & 1) << 3) | \ (((a4) & 1) << 4) | \ (((a5) & 1) << 5) | \ (((a6) & 1) << 6) | \ (((a7) & 1) << 7) ) #define BM8X8(l0,l1,l2,l3,l4,l5,l6,l7) \ ( ((l7) << (0 * 8)) | \ ((l6) << (1 * 8)) | \ ((l5) << (2 * 8)) | \ ((l4) << (3 * 8)) | \ ((l3) << (4 * 8)) | \ ((l2) << (5 * 8)) | \ ((l1) << (6 * 8)) | \ ((l0) << (7 * 8)) ) /* Pre-filters and post-filters constants for Camellia sboxes s1, s2, s3 and s4. * See http://urn.fi/URN:NBN:fi:oulu-201305311409, pages 43-48. * * Pre-filters are directly from above source, "θ₁"/"θ₄". Post-filters are * combination of function "A" (AES SubBytes affine transformation) and * "ψ₁"/"ψ₂"/"ψ₃". */ /* Constant from "θ₁(x)" and "θ₄(x)" functions. */ #define pre_filter_constant_s1234 BV8(1, 0, 1, 0, 0, 0, 1, 0) /* Constant from "ψ₁(A(x))" function: */ #define post_filter_constant_s14 BV8(0, 1, 1, 1, 0, 1, 1, 0) /* Constant from "ψ₂(A(x))" function: */ #define post_filter_constant_s2 BV8(0, 0, 1, 1, 1, 0, 1, 1) /* Constant from "ψ₃(A(x))" function: */ #define post_filter_constant_s3 BV8(1, 1, 1, 0, 1, 1, 0, 0) /********************************************************************** 64-way parallel camellia **********************************************************************/ /* roundsm64 (GFNI/AVX512 version) * IN: * x0..x7: byte-sliced AB state * mem_cd: register pointer storing CD state * key: index for key material * OUT: * x0..x7: new byte-sliced CD state */ #define roundsm64(x0, x1, x2, x3, x4, x5, x6, x7, t0, t1, t2, t3, t4, t5, \ t6, t7, mem_cd, key) \ /* \ * S-function with AES subbytes \ */ \ vpbroadcastq .Lpre_filter_bitmatrix_s123 rRIP, t5; \ vpbroadcastq .Lpre_filter_bitmatrix_s4 rRIP, t2; \ vpbroadcastq .Lpost_filter_bitmatrix_s14 rRIP, t4; \ vpbroadcastq .Lpost_filter_bitmatrix_s2 rRIP, t3; \ vpbroadcastq .Lpost_filter_bitmatrix_s3 rRIP, t6; \ vpxor t7##_x, t7##_x, t7##_x; \ vpbroadcastq key, t0; /* higher 64-bit duplicate ignored */ \ \ /* prefilter sboxes */ \ vgf2p8affineqb $(pre_filter_constant_s1234), t5, x0, x0; \ vgf2p8affineqb $(pre_filter_constant_s1234), t5, x7, x7; \ vgf2p8affineqb $(pre_filter_constant_s1234), t2, x3, x3; \ vgf2p8affineqb $(pre_filter_constant_s1234), t2, x6, x6; \ vgf2p8affineqb $(pre_filter_constant_s1234), t5, x2, x2; \ vgf2p8affineqb $(pre_filter_constant_s1234), t5, x5, x5; \ vgf2p8affineqb $(pre_filter_constant_s1234), t5, x1, x1; \ vgf2p8affineqb $(pre_filter_constant_s1234), t5, x4, x4; \ \ /* sbox GF8 inverse + postfilter sboxes 1 and 4 */ \ vgf2p8affineinvqb $(post_filter_constant_s14), t4, x0, x0; \ vgf2p8affineinvqb $(post_filter_constant_s14), t4, x7, x7; \ vgf2p8affineinvqb $(post_filter_constant_s14), t4, x3, x3; \ vgf2p8affineinvqb $(post_filter_constant_s14), t4, x6, x6; \ \ /* sbox GF8 inverse + postfilter sbox 3 */ \ vgf2p8affineinvqb $(post_filter_constant_s3), t6, x2, x2; \ vgf2p8affineinvqb $(post_filter_constant_s3), t6, x5, x5; \ \ /* sbox GF8 inverse + postfilter sbox 2 */ \ vgf2p8affineinvqb $(post_filter_constant_s2), t3, x1, x1; \ vgf2p8affineinvqb $(post_filter_constant_s2), t3, x4, x4; \ \ vpsrldq $1, t0, t1; \ vpsrldq $2, t0, t2; \ vpshufb t7, t1, t1; \ vpsrldq $3, t0, t3; \ \ /* P-function */ \ vpxorq x5, x0, x0; \ vpxorq x6, x1, x1; \ vpxorq x7, x2, x2; \ vpxorq x4, x3, x3; \ \ vpshufb t7, t2, t2; \ vpsrldq $4, t0, t4; \ vpshufb t7, t3, t3; \ vpsrldq $5, t0, t5; \ vpshufb t7, t4, t4; \ \ vpxorq x2, x4, x4; \ vpxorq x3, x5, x5; \ vpxorq x0, x6, x6; \ vpxorq x1, x7, x7; \ \ vpsrldq $6, t0, t6; \ vpshufb t7, t5, t5; \ vpshufb t7, t6, t6; \ \ vpxorq x7, x0, x0; \ vpxorq x4, x1, x1; \ vpxorq x5, x2, x2; \ vpxorq x6, x3, x3; \ \ vpxorq x3, x4, x4; \ vpxorq x0, x5, x5; \ vpxorq x1, x6, x6; \ vpxorq x2, x7, x7; /* note: high and low parts swapped */ \ \ /* Add key material and result to CD (x becomes new CD) */ \ \ vpternlogq $0x96, mem_cd##_5, t6, x1; \ \ vpsrldq $7, t0, t6; \ vpshufb t7, t0, t0; \ vpshufb t7, t6, t7; \ \ vpternlogq $0x96, mem_cd##_4, t7, x0; \ vpternlogq $0x96, mem_cd##_6, t5, x2; \ vpternlogq $0x96, mem_cd##_7, t4, x3; \ vpternlogq $0x96, mem_cd##_0, t3, x4; \ vpternlogq $0x96, mem_cd##_1, t2, x5; \ vpternlogq $0x96, mem_cd##_2, t1, x6; \ vpternlogq $0x96, mem_cd##_3, t0, x7; /* * IN/OUT: * x0..x7: byte-sliced AB state preloaded * mem_ab: byte-sliced AB state in memory * mem_cb: byte-sliced CD state in memory */ #define two_roundsm64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, mem_cd, i, dir, store_ab) \ roundsm64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_cd, (key_table + (i) * 8)(CTX)); \ \ vmovdqu64 x0, mem_cd##_4; \ vmovdqu64 x1, mem_cd##_5; \ vmovdqu64 x2, mem_cd##_6; \ vmovdqu64 x3, mem_cd##_7; \ vmovdqu64 x4, mem_cd##_0; \ vmovdqu64 x5, mem_cd##_1; \ vmovdqu64 x6, mem_cd##_2; \ vmovdqu64 x7, mem_cd##_3; \ \ roundsm64(x4, x5, x6, x7, x0, x1, x2, x3, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, (key_table + ((i) + (dir)) * 8)(CTX)); \ \ store_ab(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab); #define dummy_store(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) /* do nothing */ #define store_ab_state(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) \ /* Store new AB state */ \ vmovdqu64 x4, mem_ab##_4; \ vmovdqu64 x5, mem_ab##_5; \ vmovdqu64 x6, mem_ab##_6; \ vmovdqu64 x7, mem_ab##_7; \ vmovdqu64 x0, mem_ab##_0; \ vmovdqu64 x1, mem_ab##_1; \ vmovdqu64 x2, mem_ab##_2; \ vmovdqu64 x3, mem_ab##_3; #define enc_rounds64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, mem_cd, i) \ two_roundsm64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, mem_cd, (i) + 2, 1, store_ab_state); \ two_roundsm64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, mem_cd, (i) + 4, 1, store_ab_state); \ two_roundsm64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, mem_cd, (i) + 6, 1, dummy_store); #define dec_rounds64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, mem_cd, i) \ two_roundsm64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, mem_cd, (i) + 7, -1, store_ab_state); \ two_roundsm64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, mem_cd, (i) + 5, -1, store_ab_state); \ two_roundsm64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, mem_cd, (i) + 3, -1, dummy_store); /* * IN: * v0..3: byte-sliced 32-bit integers * OUT: * v0..3: (IN << 1) * t0, t1, t2, zero: (IN >> 7) */ #define rol32_1_64(v0, v1, v2, v3, t0, t1, t2, zero, one) \ vpcmpltb zero, v0, %k1; \ vpaddb v0, v0, v0; \ vpaddb one, zero, t0{%k1}{z}; \ \ vpcmpltb zero, v1, %k1; \ vpaddb v1, v1, v1; \ vpaddb one, zero, t1{%k1}{z}; \ \ vpcmpltb zero, v2, %k1; \ vpaddb v2, v2, v2; \ vpaddb one, zero, t2{%k1}{z}; \ \ vpcmpltb zero, v3, %k1; \ vpaddb v3, v3, v3; \ vpaddb one, zero, zero{%k1}{z}; /* * IN: * r: byte-sliced AB state in memory * l: byte-sliced CD state in memory * OUT: * x0..x7: new byte-sliced CD state */ #define fls64(l, l0, l1, l2, l3, l4, l5, l6, l7, r, t0, t1, t2, t3, tt0, \ tt1, tt2, tt3, kll, klr, krl, krr, tmp) \ /* \ * t0 = kll; \ * t0 &= ll; \ * lr ^= rol32(t0, 1); \ */ \ vpbroadcastd kll, t0; /* only lowest 32-bit used */ \ vpbroadcastq .Lbyte_ones rRIP, tmp; \ vpxor tt3##_x, tt3##_x, tt3##_x; \ vpshufb tt3, t0, t3; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t2; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t1; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t0; \ \ vpandq l0, t0, t0; \ vpandq l1, t1, t1; \ vpandq l2, t2, t2; \ vpandq l3, t3, t3; \ \ rol32_1_64(t3, t2, t1, t0, tt0, tt1, tt2, tt3, tmp); \ \ vpternlogq $0x96, tt2, t0, l4; \ vpbroadcastd krr, t0; /* only lowest 32-bit used */ \ vmovdqu64 l4, l##_4; \ vpternlogq $0x96, tt1, t1, l5; \ vmovdqu64 l5, l##_5; \ vpternlogq $0x96, tt0, t2, l6; \ vmovdqu64 l6, l##_6; \ vpternlogq $0x96, tt3, t3, l7; \ vmovdqu64 l7, l##_7; \ vpxor tt3##_x, tt3##_x, tt3##_x; \ \ /* \ * t2 = krr; \ * t2 |= rr; \ * rl ^= t2; \ */ \ \ vpshufb tt3, t0, t3; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t2; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t1; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t0; \ \ vpternlogq $0x1e, r##_4, t0, r##_0; \ vpbroadcastd krl, t0; /* only lowest 32-bit used */ \ vpternlogq $0x1e, r##_5, t1, r##_1; \ vpternlogq $0x1e, r##_6, t2, r##_2; \ vpternlogq $0x1e, r##_7, t3, r##_3; \ \ /* \ * t2 = krl; \ * t2 &= rl; \ * rr ^= rol32(t2, 1); \ */ \ vpshufb tt3, t0, t3; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t2; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t1; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t0; \ \ vpandq r##_0, t0, t0; \ vpandq r##_1, t1, t1; \ vpandq r##_2, t2, t2; \ vpandq r##_3, t3, t3; \ \ rol32_1_64(t3, t2, t1, t0, tt0, tt1, tt2, tt3, tmp); \ \ vpternlogq $0x96, tt2, t0, r##_4; \ vpbroadcastd klr, t0; /* only lowest 32-bit used */ \ vpternlogq $0x96, tt1, t1, r##_5; \ vpternlogq $0x96, tt0, t2, r##_6; \ vpternlogq $0x96, tt3, t3, r##_7; \ vpxor tt3##_x, tt3##_x, tt3##_x; \ \ /* \ * t0 = klr; \ * t0 |= lr; \ * ll ^= t0; \ */ \ \ vpshufb tt3, t0, t3; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t2; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t1; \ vpsrldq $1, t0, t0; \ vpshufb tt3, t0, t0; \ \ vpternlogq $0x1e, l4, t0, l0; \ vmovdqu64 l0, l##_0; \ vpternlogq $0x1e, l5, t1, l1; \ vmovdqu64 l1, l##_1; \ vpternlogq $0x1e, l6, t2, l2; \ vmovdqu64 l2, l##_2; \ vpternlogq $0x1e, l7, t3, l3; \ vmovdqu64 l3, l##_3; #define transpose_4x4(x0, x1, x2, x3, t1, t2) \ vpunpckhdq x1, x0, t2; \ vpunpckldq x1, x0, x0; \ \ vpunpckldq x3, x2, t1; \ vpunpckhdq x3, x2, x2; \ \ vpunpckhqdq t1, x0, x1; \ vpunpcklqdq t1, x0, x0; \ \ vpunpckhqdq x2, t2, x3; \ vpunpcklqdq x2, t2, x2; #define byteslice_16x16b_fast(a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, \ a3, b3, c3, d3, st0, st1) \ transpose_4x4(a0, a1, a2, a3, st0, st1); \ transpose_4x4(b0, b1, b2, b3, st0, st1); \ \ transpose_4x4(c0, c1, c2, c3, st0, st1); \ transpose_4x4(d0, d1, d2, d3, st0, st1); \ \ vbroadcasti64x2 .Lshufb_16x16b rRIP, st0; \ vpshufb st0, a0, a0; \ vpshufb st0, a1, a1; \ vpshufb st0, a2, a2; \ vpshufb st0, a3, a3; \ vpshufb st0, b0, b0; \ vpshufb st0, b1, b1; \ vpshufb st0, b2, b2; \ vpshufb st0, b3, b3; \ vpshufb st0, c0, c0; \ vpshufb st0, c1, c1; \ vpshufb st0, c2, c2; \ vpshufb st0, c3, c3; \ vpshufb st0, d0, d0; \ vpshufb st0, d1, d1; \ vpshufb st0, d2, d2; \ vpshufb st0, d3, d3; \ \ transpose_4x4(a0, b0, c0, d0, st0, st1); \ transpose_4x4(a1, b1, c1, d1, st0, st1); \ \ transpose_4x4(a2, b2, c2, d2, st0, st1); \ transpose_4x4(a3, b3, c3, d3, st0, st1); \ /* does not adjust output bytes inside vectors */ /* load blocks to registers and apply pre-whitening */ #define inpack64_pre(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, rio, key) \ vpbroadcastq key, x0; \ vpshufb .Lpack_bswap rRIP, x0, x0; \ \ vpxorq 0 * 64(rio), x0, y7; \ vpxorq 1 * 64(rio), x0, y6; \ vpxorq 2 * 64(rio), x0, y5; \ vpxorq 3 * 64(rio), x0, y4; \ vpxorq 4 * 64(rio), x0, y3; \ vpxorq 5 * 64(rio), x0, y2; \ vpxorq 6 * 64(rio), x0, y1; \ vpxorq 7 * 64(rio), x0, y0; \ vpxorq 8 * 64(rio), x0, x7; \ vpxorq 9 * 64(rio), x0, x6; \ vpxorq 10 * 64(rio), x0, x5; \ vpxorq 11 * 64(rio), x0, x4; \ vpxorq 12 * 64(rio), x0, x3; \ vpxorq 13 * 64(rio), x0, x2; \ vpxorq 14 * 64(rio), x0, x1; \ vpxorq 15 * 64(rio), x0, x0; /* byteslice pre-whitened blocks and store to temporary memory */ #define inpack64_post(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, mem_ab, mem_cd, tmp0, tmp1) \ byteslice_16x16b_fast(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, \ y4, y5, y6, y7, tmp0, tmp1); \ \ vmovdqu64 x0, mem_ab##_0; \ vmovdqu64 x1, mem_ab##_1; \ vmovdqu64 x2, mem_ab##_2; \ vmovdqu64 x3, mem_ab##_3; \ vmovdqu64 x4, mem_ab##_4; \ vmovdqu64 x5, mem_ab##_5; \ vmovdqu64 x6, mem_ab##_6; \ vmovdqu64 x7, mem_ab##_7; \ vmovdqu64 y0, mem_cd##_0; \ vmovdqu64 y1, mem_cd##_1; \ vmovdqu64 y2, mem_cd##_2; \ vmovdqu64 y3, mem_cd##_3; \ vmovdqu64 y4, mem_cd##_4; \ vmovdqu64 y5, mem_cd##_5; \ vmovdqu64 y6, mem_cd##_6; \ vmovdqu64 y7, mem_cd##_7; /* de-byteslice, apply post-whitening and store blocks */ #define outunpack64(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, \ y5, y6, y7, key, tmp0, tmp1) \ byteslice_16x16b_fast(y0, y4, x0, x4, y1, y5, x1, x5, y2, y6, x2, x6, \ y3, y7, x3, x7, tmp0, tmp1); \ \ vpbroadcastq key, tmp0; \ vpshufb .Lpack_bswap rRIP, tmp0, tmp0; \ \ vpxorq tmp0, y7, y7; \ vpxorq tmp0, y6, y6; \ vpxorq tmp0, y5, y5; \ vpxorq tmp0, y4, y4; \ vpxorq tmp0, y3, y3; \ vpxorq tmp0, y2, y2; \ vpxorq tmp0, y1, y1; \ vpxorq tmp0, y0, y0; \ vpxorq tmp0, x7, x7; \ vpxorq tmp0, x6, x6; \ vpxorq tmp0, x5, x5; \ vpxorq tmp0, x4, x4; \ vpxorq tmp0, x3, x3; \ vpxorq tmp0, x2, x2; \ vpxorq tmp0, x1, x1; \ vpxorq tmp0, x0, x0; #define write_output(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ y6, y7, rio) \ vmovdqu64 x0, 0 * 64(rio); \ vmovdqu64 x1, 1 * 64(rio); \ vmovdqu64 x2, 2 * 64(rio); \ vmovdqu64 x3, 3 * 64(rio); \ vmovdqu64 x4, 4 * 64(rio); \ vmovdqu64 x5, 5 * 64(rio); \ vmovdqu64 x6, 6 * 64(rio); \ vmovdqu64 x7, 7 * 64(rio); \ vmovdqu64 y0, 8 * 64(rio); \ vmovdqu64 y1, 9 * 64(rio); \ vmovdqu64 y2, 10 * 64(rio); \ vmovdqu64 y3, 11 * 64(rio); \ vmovdqu64 y4, 12 * 64(rio); \ vmovdqu64 y5, 13 * 64(rio); \ vmovdqu64 y6, 14 * 64(rio); \ vmovdqu64 y7, 15 * 64(rio); .text #define SHUFB_BYTES(idx) \ 0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx) _gcry_camellia_gfni_avx512__constants: ELF(.type _gcry_camellia_gfni_avx512__constants,@object;) .align 64 .Lpack_bswap: .long 0x00010203, 0x04050607, 0x80808080, 0x80808080 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080 .Lcounter0123_lo: .quad 0, 0 .quad 1, 0 .quad 2, 0 .quad 3, 0 .align 16 .Lcounter4444_lo: .quad 4, 0 .Lcounter8888_lo: .quad 8, 0 .Lcounter16161616_lo: .quad 16, 0 .Lcounter1111_hi: .quad 0, 1 .Lshufb_16x16b: .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3) /* For CTR-mode IV byteswap */ .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 .Lbyte_ones: .byte 1, 1, 1, 1, 1, 1, 1, 1 /* Pre-filters and post-filters bit-matrixes for Camellia sboxes s1, s2, s3 * and s4. * See http://urn.fi/URN:NBN:fi:oulu-201305311409, pages 43-48. * * Pre-filters are directly from above source, "θ₁"/"θ₄". Post-filters are * combination of function "A" (AES SubBytes affine transformation) and * "ψ₁"/"ψ₂"/"ψ₃". */ /* Bit-matrix from "θ₁(x)" function: */ .Lpre_filter_bitmatrix_s123: .quad BM8X8(BV8(1, 1, 1, 0, 1, 1, 0, 1), BV8(0, 0, 1, 1, 0, 0, 1, 0), BV8(1, 1, 0, 1, 0, 0, 0, 0), BV8(1, 0, 1, 1, 0, 0, 1, 1), BV8(0, 0, 0, 0, 1, 1, 0, 0), BV8(1, 0, 1, 0, 0, 1, 0, 0), BV8(0, 0, 1, 0, 1, 1, 0, 0), BV8(1, 0, 0, 0, 0, 1, 1, 0)) /* Bit-matrix from "θ₄(x)" function: */ .Lpre_filter_bitmatrix_s4: .quad BM8X8(BV8(1, 1, 0, 1, 1, 0, 1, 1), BV8(0, 1, 1, 0, 0, 1, 0, 0), BV8(1, 0, 1, 0, 0, 0, 0, 1), BV8(0, 1, 1, 0, 0, 1, 1, 1), BV8(0, 0, 0, 1, 1, 0, 0, 0), BV8(0, 1, 0, 0, 1, 0, 0, 1), BV8(0, 1, 0, 1, 1, 0, 0, 0), BV8(0, 0, 0, 0, 1, 1, 0, 1)) /* Bit-matrix from "ψ₁(A(x))" function: */ .Lpost_filter_bitmatrix_s14: .quad BM8X8(BV8(0, 0, 0, 0, 0, 0, 0, 1), BV8(0, 1, 1, 0, 0, 1, 1, 0), BV8(1, 0, 1, 1, 1, 1, 1, 0), BV8(0, 0, 0, 1, 1, 0, 1, 1), BV8(1, 0, 0, 0, 1, 1, 1, 0), BV8(0, 1, 0, 1, 1, 1, 1, 0), BV8(0, 1, 1, 1, 1, 1, 1, 1), BV8(0, 0, 0, 1, 1, 1, 0, 0)) /* Bit-matrix from "ψ₂(A(x))" function: */ .Lpost_filter_bitmatrix_s2: .quad BM8X8(BV8(0, 0, 0, 1, 1, 1, 0, 0), BV8(0, 0, 0, 0, 0, 0, 0, 1), BV8(0, 1, 1, 0, 0, 1, 1, 0), BV8(1, 0, 1, 1, 1, 1, 1, 0), BV8(0, 0, 0, 1, 1, 0, 1, 1), BV8(1, 0, 0, 0, 1, 1, 1, 0), BV8(0, 1, 0, 1, 1, 1, 1, 0), BV8(0, 1, 1, 1, 1, 1, 1, 1)) /* Bit-matrix from "ψ₃(A(x))" function: */ .Lpost_filter_bitmatrix_s3: .quad BM8X8(BV8(0, 1, 1, 0, 0, 1, 1, 0), BV8(1, 0, 1, 1, 1, 1, 1, 0), BV8(0, 0, 0, 1, 1, 0, 1, 1), BV8(1, 0, 0, 0, 1, 1, 1, 0), BV8(0, 1, 0, 1, 1, 1, 1, 0), BV8(0, 1, 1, 1, 1, 1, 1, 1), BV8(0, 0, 0, 1, 1, 1, 0, 0), BV8(0, 0, 0, 0, 0, 0, 0, 1)) ELF(.size _gcry_camellia_gfni_avx512__constants,.-_gcry_camellia_gfni_avx512__constants;) .align 8 ELF(.type __camellia_gfni_avx512_enc_blk64,@function;) __camellia_gfni_avx512_enc_blk64: /* input: * %rdi: ctx, CTX * %r8d: 24 for 16 byte key, 32 for larger * %zmm0..%zmm15: 64 plaintext blocks * output: * %zmm0..%zmm15: 64 encrypted blocks, order swapped: * 7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 */ CFI_STARTPROC(); leaq (-8 * 8)(CTX, %r8, 8), %r8; inpack64_post(%zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7, %zmm8, %zmm9, %zmm10, %zmm11, %zmm12, %zmm13, %zmm14, %zmm15, mem_ab, mem_cd, %zmm30, %zmm31); .align 8 .Lenc_loop: enc_rounds64(%zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7, %zmm8, %zmm9, %zmm10, %zmm11, %zmm12, %zmm13, %zmm14, %zmm15, mem_ab, mem_cd, 0); cmpq %r8, CTX; je .Lenc_done; leaq (8 * 8)(CTX), CTX; fls64(mem_ab, %zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7, mem_cd, %zmm8, %zmm9, %zmm10, %zmm11, %zmm12, %zmm13, %zmm14, %zmm15, ((key_table) + 0)(CTX), ((key_table) + 4)(CTX), ((key_table) + 8)(CTX), ((key_table) + 12)(CTX), %zmm31); jmp .Lenc_loop; .align 8 .Lenc_done: /* load CD for output */ vmovdqu64 mem_cd_0, %zmm8; vmovdqu64 mem_cd_1, %zmm9; vmovdqu64 mem_cd_2, %zmm10; vmovdqu64 mem_cd_3, %zmm11; vmovdqu64 mem_cd_4, %zmm12; vmovdqu64 mem_cd_5, %zmm13; vmovdqu64 mem_cd_6, %zmm14; vmovdqu64 mem_cd_7, %zmm15; outunpack64(%zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7, %zmm8, %zmm9, %zmm10, %zmm11, %zmm12, %zmm13, %zmm14, %zmm15, ((key_table) + 8 * 8)(%r8), %zmm30, %zmm31); ret_spec_stop; CFI_ENDPROC(); ELF(.size __camellia_gfni_avx512_enc_blk64,.-__camellia_gfni_avx512_enc_blk64;) .align 8 ELF(.type __camellia_gfni_avx512_dec_blk64,@function;) __camellia_gfni_avx512_dec_blk64: /* input: * %rdi: ctx, CTX * %r8d: 24 for 16 byte key, 32 for larger * %zmm0..%zmm15: 64 encrypted blocks * output: * %zmm0..%zmm15: 64 plaintext blocks, order swapped: * 7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 */ CFI_STARTPROC(); movq %r8, %rcx; movq CTX, %r8 leaq (-8 * 8)(CTX, %rcx, 8), CTX; inpack64_post(%zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7, %zmm8, %zmm9, %zmm10, %zmm11, %zmm12, %zmm13, %zmm14, %zmm15, mem_ab, mem_cd, %zmm30, %zmm31); .align 8 .Ldec_loop: dec_rounds64(%zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7, %zmm8, %zmm9, %zmm10, %zmm11, %zmm12, %zmm13, %zmm14, %zmm15, mem_ab, mem_cd, 0); cmpq %r8, CTX; je .Ldec_done; fls64(mem_ab, %zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7, mem_cd, %zmm8, %zmm9, %zmm10, %zmm11, %zmm12, %zmm13, %zmm14, %zmm15, ((key_table) + 8)(CTX), ((key_table) + 12)(CTX), ((key_table) + 0)(CTX), ((key_table) + 4)(CTX), %zmm31); leaq (-8 * 8)(CTX), CTX; jmp .Ldec_loop; .align 8 .Ldec_done: /* load CD for output */ vmovdqu64 mem_cd_0, %zmm8; vmovdqu64 mem_cd_1, %zmm9; vmovdqu64 mem_cd_2, %zmm10; vmovdqu64 mem_cd_3, %zmm11; vmovdqu64 mem_cd_4, %zmm12; vmovdqu64 mem_cd_5, %zmm13; vmovdqu64 mem_cd_6, %zmm14; vmovdqu64 mem_cd_7, %zmm15; outunpack64(%zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7, %zmm8, %zmm9, %zmm10, %zmm11, %zmm12, %zmm13, %zmm14, %zmm15, (key_table)(CTX), %zmm30, %zmm31); ret_spec_stop; CFI_ENDPROC(); ELF(.size __camellia_gfni_avx512_dec_blk64,.-__camellia_gfni_avx512_dec_blk64;) #define add_le128(out, in, lo_counter, hi_counter1) \ vpaddq lo_counter, in, out; \ vpcmpuq $1, lo_counter, out, %k1; \ kaddb %k1, %k1, %k1; \ vpaddq hi_counter1, out, out{%k1}; .align 8 .globl _gcry_camellia_gfni_avx512_ctr_enc ELF(.type _gcry_camellia_gfni_avx512_ctr_enc,@function;) _gcry_camellia_gfni_avx512_ctr_enc: /* input: * %rdi: ctx, CTX * %rsi: dst (64 blocks) * %rdx: src (64 blocks) * %rcx: iv (big endian, 128bit) */ CFI_STARTPROC(); spec_stop_avx512; vbroadcasti64x2 .Lbswap128_mask rRIP, %zmm19; vmovdqa64 .Lcounter0123_lo rRIP, %zmm21; vbroadcasti64x2 .Lcounter4444_lo rRIP, %zmm22; vbroadcasti64x2 .Lcounter8888_lo rRIP, %zmm23; vbroadcasti64x2 .Lcounter16161616_lo rRIP, %zmm24; vbroadcasti64x2 .Lcounter1111_hi rRIP, %zmm25; /* load IV and byteswap */ movq 8(%rcx), %r11; movq (%rcx), %r10; bswapq %r11; bswapq %r10; vbroadcasti64x2 (%rcx), %zmm0; vpshufb %zmm19, %zmm0, %zmm0; cmpl $128, key_bitlength(CTX); movl $32, %r8d; movl $24, %eax; cmovel %eax, %r8d; /* max */ /* check need for handling 64-bit overflow and carry */ cmpq $(0xffffffffffffffff - 64), %r11; ja .Lload_ctr_carry; /* construct IVs */ vpaddq %zmm21, %zmm0, %zmm15; /* +0:+1:+2:+3 */ vpaddq %zmm22, %zmm15, %zmm14; /* +4:+5:+6:+7 */ vpaddq %zmm23, %zmm15, %zmm13; /* +8:+9:+10:+11 */ vpaddq %zmm23, %zmm14, %zmm12; /* +12:+13:+14:+15 */ vpaddq %zmm24, %zmm15, %zmm11; /* +16... */ vpaddq %zmm24, %zmm14, %zmm10; /* +20... */ vpaddq %zmm24, %zmm13, %zmm9; /* +24... */ vpaddq %zmm24, %zmm12, %zmm8; /* +28... */ vpaddq %zmm24, %zmm11, %zmm7; /* +32... */ vpaddq %zmm24, %zmm10, %zmm6; /* +36... */ vpaddq %zmm24, %zmm9, %zmm5; /* +40... */ vpaddq %zmm24, %zmm8, %zmm4; /* +44... */ vpaddq %zmm24, %zmm7, %zmm3; /* +48... */ vpaddq %zmm24, %zmm6, %zmm2; /* +52... */ vpaddq %zmm24, %zmm5, %zmm1; /* +56... */ vpaddq %zmm24, %zmm4, %zmm0; /* +60... */ jmp .Lload_ctr_done; .align 4 .Lload_ctr_carry: /* construct IVs */ add_le128(%zmm15, %zmm0, %zmm21, %zmm25); /* +0:+1:+2:+3 */ add_le128(%zmm14, %zmm15, %zmm22, %zmm25); /* +4:+5:+6:+7 */ add_le128(%zmm13, %zmm15, %zmm23, %zmm25); /* +8:+9:+10:+11 */ add_le128(%zmm12, %zmm14, %zmm23, %zmm25); /* +12:+13:+14:+15 */ add_le128(%zmm11, %zmm15, %zmm24, %zmm25); /* +16... */ add_le128(%zmm10, %zmm14, %zmm24, %zmm25); /* +20... */ add_le128(%zmm9, %zmm13, %zmm24, %zmm25); /* +24... */ add_le128(%zmm8, %zmm12, %zmm24, %zmm25); /* +28... */ add_le128(%zmm7, %zmm11, %zmm24, %zmm25); /* +32... */ add_le128(%zmm6, %zmm10, %zmm24, %zmm25); /* +36... */ add_le128(%zmm5, %zmm9, %zmm24, %zmm25); /* +40... */ add_le128(%zmm4, %zmm8, %zmm24, %zmm25); /* +44... */ add_le128(%zmm3, %zmm7, %zmm24, %zmm25); /* +48... */ add_le128(%zmm2, %zmm6, %zmm24, %zmm25); /* +52... */ add_le128(%zmm1, %zmm5, %zmm24, %zmm25); /* +56... */ add_le128(%zmm0, %zmm4, %zmm24, %zmm25); /* +60... */ .align 4 .Lload_ctr_done: vpbroadcastq (key_table)(CTX), %zmm16; vpshufb .Lpack_bswap rRIP, %zmm16, %zmm16; /* Byte-swap IVs and update counter. */ addq $64, %r11; adcq $0, %r10; vpshufb %zmm19, %zmm15, %zmm15; vpshufb %zmm19, %zmm14, %zmm14; vpshufb %zmm19, %zmm13, %zmm13; vpshufb %zmm19, %zmm12, %zmm12; vpshufb %zmm19, %zmm11, %zmm11; vpshufb %zmm19, %zmm10, %zmm10; vpshufb %zmm19, %zmm9, %zmm9; vpshufb %zmm19, %zmm8, %zmm8; bswapq %r11; bswapq %r10; vpshufb %zmm19, %zmm7, %zmm7; vpshufb %zmm19, %zmm6, %zmm6; vpshufb %zmm19, %zmm5, %zmm5; vpshufb %zmm19, %zmm4, %zmm4; vpshufb %zmm19, %zmm3, %zmm3; vpshufb %zmm19, %zmm2, %zmm2; vpshufb %zmm19, %zmm1, %zmm1; vpshufb %zmm19, %zmm0, %zmm0; movq %r11, 8(%rcx); movq %r10, (%rcx); /* inpack64_pre: */ vpxorq %zmm0, %zmm16, %zmm0; vpxorq %zmm1, %zmm16, %zmm1; vpxorq %zmm2, %zmm16, %zmm2; vpxorq %zmm3, %zmm16, %zmm3; vpxorq %zmm4, %zmm16, %zmm4; vpxorq %zmm5, %zmm16, %zmm5; vpxorq %zmm6, %zmm16, %zmm6; vpxorq %zmm7, %zmm16, %zmm7; vpxorq %zmm8, %zmm16, %zmm8; vpxorq %zmm9, %zmm16, %zmm9; vpxorq %zmm10, %zmm16, %zmm10; vpxorq %zmm11, %zmm16, %zmm11; vpxorq %zmm12, %zmm16, %zmm12; vpxorq %zmm13, %zmm16, %zmm13; vpxorq %zmm14, %zmm16, %zmm14; vpxorq %zmm15, %zmm16, %zmm15; call __camellia_gfni_avx512_enc_blk64; vpxorq 0 * 64(%rdx), %zmm7, %zmm7; vpxorq 1 * 64(%rdx), %zmm6, %zmm6; vpxorq 2 * 64(%rdx), %zmm5, %zmm5; vpxorq 3 * 64(%rdx), %zmm4, %zmm4; vpxorq 4 * 64(%rdx), %zmm3, %zmm3; vpxorq 5 * 64(%rdx), %zmm2, %zmm2; vpxorq 6 * 64(%rdx), %zmm1, %zmm1; vpxorq 7 * 64(%rdx), %zmm0, %zmm0; vpxorq 8 * 64(%rdx), %zmm15, %zmm15; vpxorq 9 * 64(%rdx), %zmm14, %zmm14; vpxorq 10 * 64(%rdx), %zmm13, %zmm13; vpxorq 11 * 64(%rdx), %zmm12, %zmm12; vpxorq 12 * 64(%rdx), %zmm11, %zmm11; vpxorq 13 * 64(%rdx), %zmm10, %zmm10; vpxorq 14 * 64(%rdx), %zmm9, %zmm9; vpxorq 15 * 64(%rdx), %zmm8, %zmm8; write_output(%zmm7, %zmm6, %zmm5, %zmm4, %zmm3, %zmm2, %zmm1, %zmm0, %zmm15, %zmm14, %zmm13, %zmm12, %zmm11, %zmm10, %zmm9, %zmm8, %rsi); clear_regs(); ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_camellia_gfni_avx512_ctr_enc,.-_gcry_camellia_gfni_avx512_ctr_enc;) .align 8 .globl _gcry_camellia_gfni_avx512_cbc_dec ELF(.type _gcry_camellia_gfni_avx512_cbc_dec,@function;) _gcry_camellia_gfni_avx512_cbc_dec: /* input: * %rdi: ctx, CTX * %rsi: dst (64 blocks) * %rdx: src (64 blocks) * %rcx: iv */ CFI_STARTPROC(); spec_stop_avx512; movq %rcx, %r9; cmpl $128, key_bitlength(CTX); movl $32, %r8d; movl $24, %eax; cmovel %eax, %r8d; /* max */ inpack64_pre(%zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7, %zmm8, %zmm9, %zmm10, %zmm11, %zmm12, %zmm13, %zmm14, %zmm15, %rdx, (key_table)(CTX, %r8, 8)); call __camellia_gfni_avx512_dec_blk64; /* XOR output with IV */ vmovdqu64 (%r9), %xmm16; vinserti64x2 $1, (0 * 16)(%rdx), %ymm16, %ymm16; vinserti64x4 $1, (1 * 16)(%rdx), %zmm16, %zmm16; vpxorq %zmm16, %zmm7, %zmm7; vpxorq (0 * 64 + 48)(%rdx), %zmm6, %zmm6; vpxorq (1 * 64 + 48)(%rdx), %zmm5, %zmm5; vpxorq (2 * 64 + 48)(%rdx), %zmm4, %zmm4; vpxorq (3 * 64 + 48)(%rdx), %zmm3, %zmm3; vpxorq (4 * 64 + 48)(%rdx), %zmm2, %zmm2; vpxorq (5 * 64 + 48)(%rdx), %zmm1, %zmm1; vpxorq (6 * 64 + 48)(%rdx), %zmm0, %zmm0; vpxorq (7 * 64 + 48)(%rdx), %zmm15, %zmm15; vpxorq (8 * 64 + 48)(%rdx), %zmm14, %zmm14; vpxorq (9 * 64 + 48)(%rdx), %zmm13, %zmm13; vpxorq (10 * 64 + 48)(%rdx), %zmm12, %zmm12; vpxorq (11 * 64 + 48)(%rdx), %zmm11, %zmm11; vpxorq (12 * 64 + 48)(%rdx), %zmm10, %zmm10; vpxorq (13 * 64 + 48)(%rdx), %zmm9, %zmm9; vpxorq (14 * 64 + 48)(%rdx), %zmm8, %zmm8; vmovdqu64 (15 * 64 + 48)(%rdx), %xmm16; write_output(%zmm7, %zmm6, %zmm5, %zmm4, %zmm3, %zmm2, %zmm1, %zmm0, %zmm15, %zmm14, %zmm13, %zmm12, %zmm11, %zmm10, %zmm9, %zmm8, %rsi); /* store new IV */ vmovdqu64 %xmm16, (0)(%r9); clear_regs(); ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_camellia_gfni_avx512_cbc_dec,.-_gcry_camellia_gfni_avx512_cbc_dec;) .align 8 .globl _gcry_camellia_gfni_avx512_cfb_dec ELF(.type _gcry_camellia_gfni_avx512_cfb_dec,@function;) _gcry_camellia_gfni_avx512_cfb_dec: /* input: * %rdi: ctx, CTX * %rsi: dst (32 blocks) * %rdx: src (32 blocks) * %rcx: iv */ CFI_STARTPROC(); spec_stop_avx512; cmpl $128, key_bitlength(CTX); movl $32, %r8d; movl $24, %eax; cmovel %eax, %r8d; /* max */ /* inpack64_pre: */ vpbroadcastq (key_table)(CTX), %zmm0; vpshufb .Lpack_bswap rRIP, %zmm0, %zmm0; vmovdqu64 (%rcx), %xmm15; vinserti64x2 $1, (%rdx), %ymm15, %ymm15; vinserti64x4 $1, 16(%rdx), %zmm15, %zmm15; vpxorq %zmm15, %zmm0, %zmm15; vpxorq (0 * 64 + 48)(%rdx), %zmm0, %zmm14; vpxorq (1 * 64 + 48)(%rdx), %zmm0, %zmm13; vpxorq (2 * 64 + 48)(%rdx), %zmm0, %zmm12; vpxorq (3 * 64 + 48)(%rdx), %zmm0, %zmm11; vpxorq (4 * 64 + 48)(%rdx), %zmm0, %zmm10; vpxorq (5 * 64 + 48)(%rdx), %zmm0, %zmm9; vpxorq (6 * 64 + 48)(%rdx), %zmm0, %zmm8; vpxorq (7 * 64 + 48)(%rdx), %zmm0, %zmm7; vpxorq (8 * 64 + 48)(%rdx), %zmm0, %zmm6; vpxorq (9 * 64 + 48)(%rdx), %zmm0, %zmm5; vpxorq (10 * 64 + 48)(%rdx), %zmm0, %zmm4; vpxorq (11 * 64 + 48)(%rdx), %zmm0, %zmm3; vpxorq (12 * 64 + 48)(%rdx), %zmm0, %zmm2; vpxorq (13 * 64 + 48)(%rdx), %zmm0, %zmm1; vpxorq (14 * 64 + 48)(%rdx), %zmm0, %zmm0; vmovdqu64 (15 * 64 + 48)(%rdx), %xmm16; vmovdqu64 %xmm16, (%rcx); /* store new IV */ call __camellia_gfni_avx512_enc_blk64; vpxorq 0 * 64(%rdx), %zmm7, %zmm7; vpxorq 1 * 64(%rdx), %zmm6, %zmm6; vpxorq 2 * 64(%rdx), %zmm5, %zmm5; vpxorq 3 * 64(%rdx), %zmm4, %zmm4; vpxorq 4 * 64(%rdx), %zmm3, %zmm3; vpxorq 5 * 64(%rdx), %zmm2, %zmm2; vpxorq 6 * 64(%rdx), %zmm1, %zmm1; vpxorq 7 * 64(%rdx), %zmm0, %zmm0; vpxorq 8 * 64(%rdx), %zmm15, %zmm15; vpxorq 9 * 64(%rdx), %zmm14, %zmm14; vpxorq 10 * 64(%rdx), %zmm13, %zmm13; vpxorq 11 * 64(%rdx), %zmm12, %zmm12; vpxorq 12 * 64(%rdx), %zmm11, %zmm11; vpxorq 13 * 64(%rdx), %zmm10, %zmm10; vpxorq 14 * 64(%rdx), %zmm9, %zmm9; vpxorq 15 * 64(%rdx), %zmm8, %zmm8; write_output(%zmm7, %zmm6, %zmm5, %zmm4, %zmm3, %zmm2, %zmm1, %zmm0, %zmm15, %zmm14, %zmm13, %zmm12, %zmm11, %zmm10, %zmm9, %zmm8, %rsi); clear_regs(); ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_camellia_gfni_avx512_cfb_dec,.-_gcry_camellia_gfni_avx512_cfb_dec;) .align 8 .globl _gcry_camellia_gfni_avx512_ocb_enc ELF(.type _gcry_camellia_gfni_avx512_ocb_enc,@function;) _gcry_camellia_gfni_avx512_ocb_enc: /* input: * %rdi: ctx, CTX * %rsi: dst (64 blocks) * %rdx: src (64 blocks) * %rcx: offset * %r8 : checksum * %r9 : L pointers (void *L[64]) */ CFI_STARTPROC(); spec_stop_avx512; pushq %r12; CFI_PUSH(%r12); pushq %r13; CFI_PUSH(%r13); pushq %r14; CFI_PUSH(%r14); pushq %r15; CFI_PUSH(%r15); pushq %rbx; CFI_PUSH(%rbx); vmovdqu64 (%rcx), %xmm30; /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* Checksum_i = Checksum_{i-1} xor P_i */ /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */ #define OCB_INPUT(n, l0reg, l1reg, l2reg, l3reg, zreg, zplain) \ vmovdqu64 (n * 64)(%rdx), zplain; \ vpxorq (l0reg), %xmm30, %xmm16; \ vpxorq (l1reg), %xmm16, %xmm30; \ vinserti64x2 $1, %xmm30, %ymm16, %ymm16; \ vpxorq (l2reg), %xmm30, %xmm30; \ vinserti64x2 $2, %xmm30, %zmm16, %zmm16; \ vpxorq (l3reg), %xmm30, %xmm30; \ vinserti64x2 $3, %xmm30, %zmm16, %zmm16; \ vpxorq zplain, %zmm16, zreg; \ vmovdqu64 %zmm16, (n * 64)(%rsi); #define OCB_LOAD_PTRS(n) \ movq ((n * 4 * 8) + (0 * 8))(%r9), %r10; \ movq ((n * 4 * 8) + (1 * 8))(%r9), %r11; \ movq ((n * 4 * 8) + (2 * 8))(%r9), %r12; \ movq ((n * 4 * 8) + (3 * 8))(%r9), %r13; \ movq ((n * 4 * 8) + (4 * 8))(%r9), %r14; \ movq ((n * 4 * 8) + (5 * 8))(%r9), %r15; \ movq ((n * 4 * 8) + (6 * 8))(%r9), %rax; \ movq ((n * 4 * 8) + (7 * 8))(%r9), %rbx; OCB_LOAD_PTRS(0); OCB_INPUT(0, %r10, %r11, %r12, %r13, %zmm15, %zmm20); OCB_INPUT(1, %r14, %r15, %rax, %rbx, %zmm14, %zmm21); OCB_LOAD_PTRS(2); OCB_INPUT(2, %r10, %r11, %r12, %r13, %zmm13, %zmm22); vpternlogq $0x96, %zmm20, %zmm21, %zmm22; OCB_INPUT(3, %r14, %r15, %rax, %rbx, %zmm12, %zmm23); OCB_LOAD_PTRS(4); OCB_INPUT(4, %r10, %r11, %r12, %r13, %zmm11, %zmm24); OCB_INPUT(5, %r14, %r15, %rax, %rbx, %zmm10, %zmm25); vpternlogq $0x96, %zmm23, %zmm24, %zmm25; OCB_LOAD_PTRS(6); OCB_INPUT(6, %r10, %r11, %r12, %r13, %zmm9, %zmm20); OCB_INPUT(7, %r14, %r15, %rax, %rbx, %zmm8, %zmm21); OCB_LOAD_PTRS(8); OCB_INPUT(8, %r10, %r11, %r12, %r13, %zmm7, %zmm26); vpternlogq $0x96, %zmm20, %zmm21, %zmm26; OCB_INPUT(9, %r14, %r15, %rax, %rbx, %zmm6, %zmm23); OCB_LOAD_PTRS(10); OCB_INPUT(10, %r10, %r11, %r12, %r13, %zmm5, %zmm24); OCB_INPUT(11, %r14, %r15, %rax, %rbx, %zmm4, %zmm27); vpternlogq $0x96, %zmm23, %zmm24, %zmm27; OCB_LOAD_PTRS(12); OCB_INPUT(12, %r10, %r11, %r12, %r13, %zmm3, %zmm20); OCB_INPUT(13, %r14, %r15, %rax, %rbx, %zmm2, %zmm21); OCB_LOAD_PTRS(14); OCB_INPUT(14, %r10, %r11, %r12, %r13, %zmm1, %zmm23); vpternlogq $0x96, %zmm20, %zmm21, %zmm23; OCB_INPUT(15, %r14, %r15, %rax, %rbx, %zmm0, %zmm24); #undef OCB_LOAD_PTRS #undef OCB_INPUT vpbroadcastq (key_table)(CTX), %zmm16; vpshufb .Lpack_bswap rRIP, %zmm16, %zmm16; vpternlogq $0x96, %zmm24, %zmm22, %zmm25; vpternlogq $0x96, %zmm26, %zmm27, %zmm23; vpxorq %zmm25, %zmm23, %zmm20; vextracti64x4 $1, %zmm20, %ymm21; vpxorq %ymm21, %ymm20, %ymm20; vextracti64x2 $1, %ymm20, %xmm21; vpternlogq $0x96, (%r8), %xmm21, %xmm20; vmovdqu64 %xmm30, (%rcx); vmovdqu64 %xmm20, (%r8); cmpl $128, key_bitlength(CTX); movl $32, %r8d; movl $24, %eax; cmovel %eax, %r8d; /* max */ /* inpack64_pre: */ vpxorq %zmm0, %zmm16, %zmm0; vpxorq %zmm1, %zmm16, %zmm1; vpxorq %zmm2, %zmm16, %zmm2; vpxorq %zmm3, %zmm16, %zmm3; vpxorq %zmm4, %zmm16, %zmm4; vpxorq %zmm5, %zmm16, %zmm5; vpxorq %zmm6, %zmm16, %zmm6; vpxorq %zmm7, %zmm16, %zmm7; vpxorq %zmm8, %zmm16, %zmm8; vpxorq %zmm9, %zmm16, %zmm9; vpxorq %zmm10, %zmm16, %zmm10; vpxorq %zmm11, %zmm16, %zmm11; vpxorq %zmm12, %zmm16, %zmm12; vpxorq %zmm13, %zmm16, %zmm13; vpxorq %zmm14, %zmm16, %zmm14; vpxorq %zmm15, %zmm16, %zmm15; call __camellia_gfni_avx512_enc_blk64; vpxorq 0 * 64(%rsi), %zmm7, %zmm7; vpxorq 1 * 64(%rsi), %zmm6, %zmm6; vpxorq 2 * 64(%rsi), %zmm5, %zmm5; vpxorq 3 * 64(%rsi), %zmm4, %zmm4; vpxorq 4 * 64(%rsi), %zmm3, %zmm3; vpxorq 5 * 64(%rsi), %zmm2, %zmm2; vpxorq 6 * 64(%rsi), %zmm1, %zmm1; vpxorq 7 * 64(%rsi), %zmm0, %zmm0; vpxorq 8 * 64(%rsi), %zmm15, %zmm15; vpxorq 9 * 64(%rsi), %zmm14, %zmm14; vpxorq 10 * 64(%rsi), %zmm13, %zmm13; vpxorq 11 * 64(%rsi), %zmm12, %zmm12; vpxorq 12 * 64(%rsi), %zmm11, %zmm11; vpxorq 13 * 64(%rsi), %zmm10, %zmm10; vpxorq 14 * 64(%rsi), %zmm9, %zmm9; vpxorq 15 * 64(%rsi), %zmm8, %zmm8; write_output(%zmm7, %zmm6, %zmm5, %zmm4, %zmm3, %zmm2, %zmm1, %zmm0, %zmm15, %zmm14, %zmm13, %zmm12, %zmm11, %zmm10, %zmm9, %zmm8, %rsi); clear_regs(); popq %rbx; CFI_RESTORE(%rbx); popq %r15; CFI_RESTORE(%r15); popq %r14; CFI_RESTORE(%r14); popq %r13; CFI_RESTORE(%r12); popq %r12; CFI_RESTORE(%r13); ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_camellia_gfni_avx512_ocb_enc,.-_gcry_camellia_gfni_avx512_ocb_enc;) .align 8 .globl _gcry_camellia_gfni_avx512_ocb_dec ELF(.type _gcry_camellia_gfni_avx512_ocb_dec,@function;) _gcry_camellia_gfni_avx512_ocb_dec: /* input: * %rdi: ctx, CTX * %rsi: dst (64 blocks) * %rdx: src (64 blocks) * %rcx: offset * %r8 : checksum * %r9 : L pointers (void *L[64]) */ CFI_STARTPROC(); spec_stop_avx512; pushq %r12; CFI_PUSH(%r12); pushq %r13; CFI_PUSH(%r13); pushq %r14; CFI_PUSH(%r14); pushq %r15; CFI_PUSH(%r15); pushq %rbx; CFI_PUSH(%rbx); pushq %r8; CFI_PUSH(%r8); vmovdqu64 (%rcx), %xmm30; /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* C_i = Offset_i xor DECIPHER(K, P_i xor Offset_i) */ #define OCB_INPUT(n, l0reg, l1reg, l2reg, l3reg, zreg) \ vpxorq (l0reg), %xmm30, %xmm16; \ vpxorq (l1reg), %xmm16, %xmm30; \ vinserti64x2 $1, %xmm30, %ymm16, %ymm16; \ vpxorq (l2reg), %xmm30, %xmm30; \ vinserti64x2 $2, %xmm30, %zmm16, %zmm16; \ vpxorq (l3reg), %xmm30, %xmm30; \ vinserti64x2 $3, %xmm30, %zmm16, %zmm16; \ vpxorq (n * 64)(%rdx), %zmm16, zreg; \ vmovdqu64 %zmm16, (n * 64)(%rsi); #define OCB_LOAD_PTRS(n) \ movq ((n * 4 * 8) + (0 * 8))(%r9), %r10; \ movq ((n * 4 * 8) + (1 * 8))(%r9), %r11; \ movq ((n * 4 * 8) + (2 * 8))(%r9), %r12; \ movq ((n * 4 * 8) + (3 * 8))(%r9), %r13; \ movq ((n * 4 * 8) + (4 * 8))(%r9), %r14; \ movq ((n * 4 * 8) + (5 * 8))(%r9), %r15; \ movq ((n * 4 * 8) + (6 * 8))(%r9), %rax; \ movq ((n * 4 * 8) + (7 * 8))(%r9), %rbx; OCB_LOAD_PTRS(0); OCB_INPUT(0, %r10, %r11, %r12, %r13, %zmm15); OCB_INPUT(1, %r14, %r15, %rax, %rbx, %zmm14); OCB_LOAD_PTRS(2); OCB_INPUT(2, %r10, %r11, %r12, %r13, %zmm13); OCB_INPUT(3, %r14, %r15, %rax, %rbx, %zmm12); OCB_LOAD_PTRS(4); OCB_INPUT(4, %r10, %r11, %r12, %r13, %zmm11); OCB_INPUT(5, %r14, %r15, %rax, %rbx, %zmm10); OCB_LOAD_PTRS(6); OCB_INPUT(6, %r10, %r11, %r12, %r13, %zmm9); OCB_INPUT(7, %r14, %r15, %rax, %rbx, %zmm8); OCB_LOAD_PTRS(8); OCB_INPUT(8, %r10, %r11, %r12, %r13, %zmm7); OCB_INPUT(9, %r14, %r15, %rax, %rbx, %zmm6); OCB_LOAD_PTRS(10); OCB_INPUT(10, %r10, %r11, %r12, %r13, %zmm5); OCB_INPUT(11, %r14, %r15, %rax, %rbx, %zmm4); OCB_LOAD_PTRS(12); OCB_INPUT(12, %r10, %r11, %r12, %r13, %zmm3); OCB_INPUT(13, %r14, %r15, %rax, %rbx, %zmm2); OCB_LOAD_PTRS(14); OCB_INPUT(14, %r10, %r11, %r12, %r13, %zmm1); OCB_INPUT(15, %r14, %r15, %rax, %rbx, %zmm0); #undef OCB_LOAD_PTRS #undef OCB_INPUT vmovdqu64 %xmm30, (%rcx); cmpl $128, key_bitlength(CTX); movl $32, %r8d; movl $24, %eax; cmovel %eax, %r8d; /* max */ vpbroadcastq (key_table)(CTX, %r8, 8), %zmm16; vpshufb .Lpack_bswap rRIP, %zmm16, %zmm16; /* inpack64_pre: */ vpxorq %zmm0, %zmm16, %zmm0; vpxorq %zmm1, %zmm16, %zmm1; vpxorq %zmm2, %zmm16, %zmm2; vpxorq %zmm3, %zmm16, %zmm3; vpxorq %zmm4, %zmm16, %zmm4; vpxorq %zmm5, %zmm16, %zmm5; vpxorq %zmm6, %zmm16, %zmm6; vpxorq %zmm7, %zmm16, %zmm7; vpxorq %zmm8, %zmm16, %zmm8; vpxorq %zmm9, %zmm16, %zmm9; vpxorq %zmm10, %zmm16, %zmm10; vpxorq %zmm11, %zmm16, %zmm11; vpxorq %zmm12, %zmm16, %zmm12; vpxorq %zmm13, %zmm16, %zmm13; vpxorq %zmm14, %zmm16, %zmm14; vpxorq %zmm15, %zmm16, %zmm15; call __camellia_gfni_avx512_dec_blk64; vpxorq 0 * 64(%rsi), %zmm7, %zmm7; vpxorq 1 * 64(%rsi), %zmm6, %zmm6; vpxorq 2 * 64(%rsi), %zmm5, %zmm5; vpxorq 3 * 64(%rsi), %zmm4, %zmm4; vpxorq 4 * 64(%rsi), %zmm3, %zmm3; vpxorq 5 * 64(%rsi), %zmm2, %zmm2; vpxorq 6 * 64(%rsi), %zmm1, %zmm1; vpxorq 7 * 64(%rsi), %zmm0, %zmm0; vpxorq 8 * 64(%rsi), %zmm15, %zmm15; vpxorq 9 * 64(%rsi), %zmm14, %zmm14; vpxorq 10 * 64(%rsi), %zmm13, %zmm13; vpxorq 11 * 64(%rsi), %zmm12, %zmm12; vpxorq 12 * 64(%rsi), %zmm11, %zmm11; vpxorq 13 * 64(%rsi), %zmm10, %zmm10; vpxorq 14 * 64(%rsi), %zmm9, %zmm9; vpxorq 15 * 64(%rsi), %zmm8, %zmm8; write_output(%zmm7, %zmm6, %zmm5, %zmm4, %zmm3, %zmm2, %zmm1, %zmm0, %zmm15, %zmm14, %zmm13, %zmm12, %zmm11, %zmm10, %zmm9, %zmm8, %rsi); popq %r8; CFI_RESTORE(%r8); /* Checksum_i = Checksum_{i-1} xor C_i */ vpternlogq $0x96, %zmm7, %zmm6, %zmm5; vpternlogq $0x96, %zmm4, %zmm3, %zmm2; vpternlogq $0x96, %zmm1, %zmm0, %zmm15; vpternlogq $0x96, %zmm14, %zmm13, %zmm12; vpternlogq $0x96, %zmm11, %zmm10, %zmm9; vpternlogq $0x96, %zmm5, %zmm2, %zmm15; vpternlogq $0x96, %zmm12, %zmm9, %zmm8; vpxorq %zmm15, %zmm8, %zmm8; vextracti64x4 $1, %zmm8, %ymm0; vpxor %ymm0, %ymm8, %ymm8; vextracti128 $1, %ymm8, %xmm0; vpternlogq $0x96, (%r8), %xmm0, %xmm8; vmovdqu64 %xmm8, (%r8); clear_regs(); popq %rbx; CFI_RESTORE(%rbx); popq %r15; CFI_RESTORE(%r15); popq %r14; CFI_RESTORE(%r14); popq %r13; CFI_RESTORE(%r12); popq %r12; CFI_RESTORE(%r13); ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_camellia_gfni_avx512_ocb_dec,.-_gcry_camellia_gfni_avx512_ocb_dec;) .align 8 .globl _gcry_camellia_gfni_avx512_enc_blk64 ELF(.type _gcry_camellia_gfni_avx512_enc_blk64,@function;) _gcry_camellia_gfni_avx512_enc_blk64: /* input: * %rdi: ctx, CTX * %rsi: dst (64 blocks) * %rdx: src (64 blocks) */ CFI_STARTPROC(); spec_stop_avx512; cmpl $128, key_bitlength(CTX); movl $32, %r8d; movl $24, %eax; cmovel %eax, %r8d; /* max */ xorl %eax, %eax; vpbroadcastq (key_table)(CTX), %zmm0; vpshufb .Lpack_bswap rRIP, %zmm0, %zmm0; vpxorq (0) * 64(%rdx), %zmm0, %zmm15; vpxorq (1) * 64(%rdx), %zmm0, %zmm14; vpxorq (2) * 64(%rdx), %zmm0, %zmm13; vpxorq (3) * 64(%rdx), %zmm0, %zmm12; vpxorq (4) * 64(%rdx), %zmm0, %zmm11; vpxorq (5) * 64(%rdx), %zmm0, %zmm10; vpxorq (6) * 64(%rdx), %zmm0, %zmm9; vpxorq (7) * 64(%rdx), %zmm0, %zmm8; vpxorq (8) * 64(%rdx), %zmm0, %zmm7; vpxorq (9) * 64(%rdx), %zmm0, %zmm6; vpxorq (10) * 64(%rdx), %zmm0, %zmm5; vpxorq (11) * 64(%rdx), %zmm0, %zmm4; vpxorq (12) * 64(%rdx), %zmm0, %zmm3; vpxorq (13) * 64(%rdx), %zmm0, %zmm2; vpxorq (14) * 64(%rdx), %zmm0, %zmm1; vpxorq (15) * 64(%rdx), %zmm0, %zmm0; call __camellia_gfni_avx512_enc_blk64; vmovdqu64 %zmm7, (0) * 64(%rsi); vmovdqu64 %zmm6, (1) * 64(%rsi); vmovdqu64 %zmm5, (2) * 64(%rsi); vmovdqu64 %zmm4, (3) * 64(%rsi); vmovdqu64 %zmm3, (4) * 64(%rsi); vmovdqu64 %zmm2, (5) * 64(%rsi); vmovdqu64 %zmm1, (6) * 64(%rsi); vmovdqu64 %zmm0, (7) * 64(%rsi); vmovdqu64 %zmm15, (8) * 64(%rsi); vmovdqu64 %zmm14, (9) * 64(%rsi); vmovdqu64 %zmm13, (10) * 64(%rsi); vmovdqu64 %zmm12, (11) * 64(%rsi); vmovdqu64 %zmm11, (12) * 64(%rsi); vmovdqu64 %zmm10, (13) * 64(%rsi); vmovdqu64 %zmm9, (14) * 64(%rsi); vmovdqu64 %zmm8, (15) * 64(%rsi); clear_regs(); ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_camellia_gfni_avx512_enc_blk64,.-_gcry_camellia_gfni_avx512_enc_blk64;) .align 8 .globl _gcry_camellia_gfni_avx512_dec_blk64 ELF(.type _gcry_camellia_gfni_avx512_dec_blk64,@function;) _gcry_camellia_gfni_avx512_dec_blk64: /* input: * %rdi: ctx, CTX * %rsi: dst (64 blocks) * %rdx: src (64 blocks) */ CFI_STARTPROC(); spec_stop_avx512; cmpl $128, key_bitlength(CTX); movl $32, %r8d; movl $24, %eax; cmovel %eax, %r8d; /* max */ xorl %eax, %eax; vpbroadcastq (key_table)(CTX, %r8, 8), %zmm0; vpshufb .Lpack_bswap rRIP, %zmm0, %zmm0; vpxorq (0) * 64(%rdx), %zmm0, %zmm15; vpxorq (1) * 64(%rdx), %zmm0, %zmm14; vpxorq (2) * 64(%rdx), %zmm0, %zmm13; vpxorq (3) * 64(%rdx), %zmm0, %zmm12; vpxorq (4) * 64(%rdx), %zmm0, %zmm11; vpxorq (5) * 64(%rdx), %zmm0, %zmm10; vpxorq (6) * 64(%rdx), %zmm0, %zmm9; vpxorq (7) * 64(%rdx), %zmm0, %zmm8; vpxorq (8) * 64(%rdx), %zmm0, %zmm7; vpxorq (9) * 64(%rdx), %zmm0, %zmm6; vpxorq (10) * 64(%rdx), %zmm0, %zmm5; vpxorq (11) * 64(%rdx), %zmm0, %zmm4; vpxorq (12) * 64(%rdx), %zmm0, %zmm3; vpxorq (13) * 64(%rdx), %zmm0, %zmm2; vpxorq (14) * 64(%rdx), %zmm0, %zmm1; vpxorq (15) * 64(%rdx), %zmm0, %zmm0; call __camellia_gfni_avx512_dec_blk64; vmovdqu64 %zmm7, (0) * 64(%rsi); vmovdqu64 %zmm6, (1) * 64(%rsi); vmovdqu64 %zmm5, (2) * 64(%rsi); vmovdqu64 %zmm4, (3) * 64(%rsi); vmovdqu64 %zmm3, (4) * 64(%rsi); vmovdqu64 %zmm2, (5) * 64(%rsi); vmovdqu64 %zmm1, (6) * 64(%rsi); vmovdqu64 %zmm0, (7) * 64(%rsi); vmovdqu64 %zmm15, (8) * 64(%rsi); vmovdqu64 %zmm14, (9) * 64(%rsi); vmovdqu64 %zmm13, (10) * 64(%rsi); vmovdqu64 %zmm12, (11) * 64(%rsi); vmovdqu64 %zmm11, (12) * 64(%rsi); vmovdqu64 %zmm10, (13) * 64(%rsi); vmovdqu64 %zmm9, (14) * 64(%rsi); vmovdqu64 %zmm8, (15) * 64(%rsi); clear_regs(); ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_camellia_gfni_avx512_dec_blk64,.-_gcry_camellia_gfni_avx512_dec_blk64;) #endif /* defined(ENABLE_GFNI_SUPPORT) && defined(ENABLE_AVX512_SUPPORT) */ #endif /* __x86_64 */ diff --git a/cipher/chacha20-amd64-avx512.S b/cipher/chacha20-amd64-avx512.S index 544e7cdc..4b183528 100644 --- a/cipher/chacha20-amd64-avx512.S +++ b/cipher/chacha20-amd64-avx512.S @@ -1,734 +1,734 @@ /* chacha20-amd64-avx512.S - AVX512 implementation of ChaCha20 cipher * * Copyright (C) 2022 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . */ /* * Based on D. J. Bernstein reference implementation at * http://cr.yp.to/chacha.html: * * chacha-regs.c version 20080118 * D. J. Bernstein * Public domain. */ #ifdef __x86_64 #include #if defined(HAVE_GCC_INLINE_ASM_AVX512) && \ (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \ defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) .text #include "asm-common-amd64.h" /* register macros */ #define INPUT %rdi #define DST %rsi #define SRC %rdx #define NBLKS %rcx #define ROUND %eax /* vector registers */ #define X0 %zmm0 #define X1 %zmm1 #define X2 %zmm2 #define X3 %zmm3 #define X4 %zmm4 #define X5 %zmm5 #define X6 %zmm6 #define X7 %zmm7 #define X8 %zmm8 #define X9 %zmm9 #define X10 %zmm10 #define X11 %zmm11 #define X12 %zmm12 #define X13 %zmm13 #define X14 %zmm14 #define X15 %zmm15 #define X0y %ymm0 #define X1y %ymm1 #define X2y %ymm2 #define X3y %ymm3 #define X4y %ymm4 #define X5y %ymm5 #define X6y %ymm6 #define X7y %ymm7 #define X8y %ymm8 #define X9y %ymm9 #define X10y %ymm10 #define X11y %ymm11 #define X12y %ymm12 #define X13y %ymm13 #define X14y %ymm14 #define X15y %ymm15 #define X0x %xmm0 #define X1x %xmm1 #define X2x %xmm2 #define X3x %xmm3 #define X4x %xmm4 #define X5x %xmm5 #define X6x %xmm6 #define X7x %xmm7 #define X8x %xmm8 #define X9x %xmm9 #define X10x %xmm10 #define X11x %xmm11 #define X12x %xmm12 #define X13x %xmm13 #define X14x %xmm14 #define X15x %xmm15 #define TMP0 %zmm16 #define TMP1 %zmm17 #define TMP0y %ymm16 #define TMP1y %ymm17 #define TMP0x %xmm16 #define TMP1x %xmm17 #define COUNTER_ADD %zmm18 #define COUNTER_ADDy %ymm18 #define COUNTER_ADDx %xmm18 #define X12_SAVE %zmm19 #define X12_SAVEy %ymm19 #define X12_SAVEx %xmm19 #define X13_SAVE %zmm20 #define X13_SAVEy %ymm20 #define X13_SAVEx %xmm20 #define S0 %zmm21 #define S1 %zmm22 #define S2 %zmm23 #define S3 %zmm24 #define S4 %zmm25 #define S5 %zmm26 #define S6 %zmm27 #define S7 %zmm28 #define S8 %zmm29 #define S14 %zmm30 #define S15 %zmm31 #define S0y %ymm21 #define S1y %ymm22 #define S2y %ymm23 #define S3y %ymm24 #define S4y %ymm25 #define S5y %ymm26 #define S6y %ymm27 #define S7y %ymm28 #define S8y %ymm29 #define S14y %ymm30 #define S15y %ymm31 #define S0x %xmm21 #define S1x %xmm22 #define S2x %xmm23 #define S3x %xmm24 #define S4x %xmm25 #define S5x %xmm26 #define S6x %xmm27 #define S7x %xmm28 #define S8x %xmm29 #define S14x %xmm30 #define S15x %xmm31 /********************************************************************** helper macros **********************************************************************/ /* 4x4 32-bit integer matrix transpose */ #define transpose_4x4(x0,x1,x2,x3,t1,t2) \ vpunpckhdq x1, x0, t2; \ vpunpckldq x1, x0, x0; \ \ vpunpckldq x3, x2, t1; \ vpunpckhdq x3, x2, x2; \ \ vpunpckhqdq t1, x0, x1; \ vpunpcklqdq t1, x0, x0; \ \ vpunpckhqdq x2, t2, x3; \ vpunpcklqdq x2, t2, x2; /* 4x4 128-bit matrix transpose */ #define transpose_16byte_4x4(x0,x1,x2,x3,t1,t2) \ vshufi32x4 $0xee, x1, x0, t2; \ vshufi32x4 $0x44, x1, x0, x0; \ \ vshufi32x4 $0x44, x3, x2, t1; \ vshufi32x4 $0xee, x3, x2, x2; \ \ vshufi32x4 $0xdd, t1, x0, x1; \ vshufi32x4 $0x88, t1, x0, x0; \ \ vshufi32x4 $0xdd, x2, t2, x3; \ vshufi32x4 $0x88, x2, t2, x2; /* 2x2 128-bit matrix transpose */ #define transpose_16byte_2x2(x0,x1,t1) \ vmovdqa32 x0, t1; \ vshufi32x4 $0x0, x1, x0, x0; \ vshufi32x4 $0x3, x1, t1, x1; #define xor_src_dst_4x4(dst, src, offset, add, x0, x4, x8, x12) \ vpxord (offset + 0 * (add))(src), x0, x0; \ vpxord (offset + 1 * (add))(src), x4, x4; \ vpxord (offset + 2 * (add))(src), x8, x8; \ vpxord (offset + 3 * (add))(src), x12, x12; \ vmovdqu32 x0, (offset + 0 * (add))(dst); \ vmovdqu32 x4, (offset + 1 * (add))(dst); \ vmovdqu32 x8, (offset + 2 * (add))(dst); \ vmovdqu32 x12, (offset + 3 * (add))(dst); #define xor_src_dst(dst, src, offset, xreg) \ vpxord offset(src), xreg, xreg; \ vmovdqu32 xreg, offset(dst); #define clear_vec4(v0,v1,v2,v3) \ vpxord v0, v0, v0; \ vpxord v1, v1, v1; \ vpxord v2, v2, v2; \ vpxord v3, v3, v3; #define clear_zmm16_zmm31() \ - clear_vec4(%xmm16, %xmm20, %xmm24, %xmm28); \ - clear_vec4(%xmm17, %xmm21, %xmm25, %xmm29); \ - clear_vec4(%xmm18, %xmm22, %xmm26, %xmm30); \ - clear_vec4(%xmm19, %xmm23, %xmm27, %xmm31); + clear_vec4(%ymm16, %ymm20, %ymm24, %ymm28); \ + clear_vec4(%ymm17, %ymm21, %ymm25, %ymm29); \ + clear_vec4(%ymm18, %ymm22, %ymm26, %ymm30); \ + clear_vec4(%ymm19, %ymm23, %ymm27, %ymm31); /********************************************************************** 16-way (zmm), 8-way (ymm), 4-way (xmm) chacha20 **********************************************************************/ #define ROTATE2(v1,v2,c) \ vprold $(c), v1, v1; \ vprold $(c), v2, v2; #define XOR(ds,s) \ vpxord s, ds, ds; #define PLUS(ds,s) \ vpaddd s, ds, ds; #define QUARTERROUND2V(a1,b1,c1,d1,a2,b2,c2,d2) \ PLUS(a1,b1); PLUS(a2,b2); XOR(d1,a1); XOR(d2,a2); \ ROTATE2(d1, d2, 16); \ PLUS(c1,d1); PLUS(c2,d2); XOR(b1,c1); XOR(b2,c2); \ ROTATE2(b1, b2, 12); \ PLUS(a1,b1); PLUS(a2,b2); XOR(d1,a1); XOR(d2,a2); \ ROTATE2(d1, d2, 8); \ PLUS(c1,d1); PLUS(c2,d2); XOR(b1,c1); XOR(b2,c2); \ ROTATE2(b1, b2, 7); /********************************************************************** 1-way/2-way (xmm) chacha20 **********************************************************************/ #define ROTATE(v1,c) \ vprold $(c), v1, v1; \ #define WORD_SHUF(v1,shuf) \ vpshufd $shuf, v1, v1; #define QUARTERROUND1H(x0,x1,x2,x3,shuf_x1,shuf_x2,shuf_x3) \ PLUS(x0, x1); XOR(x3, x0); ROTATE(x3, 16); \ PLUS(x2, x3); XOR(x1, x2); ROTATE(x1, 12); \ PLUS(x0, x1); XOR(x3, x0); ROTATE(x3, 8); \ PLUS(x2, x3); \ WORD_SHUF(x3, shuf_x3); \ XOR(x1, x2); \ WORD_SHUF(x2, shuf_x2); \ ROTATE(x1, 7); \ WORD_SHUF(x1, shuf_x1); #define QUARTERROUND2H(x0,x1,x2,x3,y0,y1,y2,y3,shuf_x1,shuf_x2,shuf_x3) \ PLUS(x0, x1); PLUS(y0, y1); XOR(x3, x0); XOR(y3, y0); \ ROTATE(x3, 16); ROTATE(y3, 16); \ PLUS(x2, x3); PLUS(y2, y3); XOR(x1, x2); XOR(y1, y2); \ ROTATE(x1, 12); ROTATE(y1, 12); \ PLUS(x0, x1); PLUS(y0, y1); XOR(x3, x0); XOR(y3, y0); \ ROTATE(x3, 8); ROTATE(y3, 8); \ PLUS(x2, x3); PLUS(y2, y3); \ WORD_SHUF(x3, shuf_x3); WORD_SHUF(y3, shuf_x3); \ XOR(x1, x2); XOR(y1, y2); \ WORD_SHUF(x2, shuf_x2); WORD_SHUF(y2, shuf_x2); \ ROTATE(x1, 7); ROTATE(y1, 7); \ WORD_SHUF(x1, shuf_x1); WORD_SHUF(y1, shuf_x1); .align 64 ELF(.type _gcry_chacha20_amd64_avx512_data,@object;) _gcry_chacha20_amd64_avx512_data: .Lcounter_0_1_2_3: .Lcounter_0_1: .long 0,0,0,0 .Lone: .long 1,0,0,0 .Lcounter_2_3: .Ltwo: .long 2,0,0,0 .Lthree: .long 3,0,0,0 .Linc_counter: .byte 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 ELF(.size _gcry_chacha20_amd64_avx512_data,.-_gcry_chacha20_amd64_avx512_data) .align 16 .globl _gcry_chacha20_amd64_avx512_blocks ELF(.type _gcry_chacha20_amd64_avx512_blocks,@function;) _gcry_chacha20_amd64_avx512_blocks: /* input: * %rdi: input * %rsi: dst * %rdx: src * %rcx: nblks */ CFI_STARTPROC(); spec_stop_avx512; cmpq $4, NBLKS; jb .Lskip_vertical_handling; /* Load constants */ vpmovzxbd .Linc_counter rRIP, COUNTER_ADD; kxnorq %k1, %k1, %k1; cmpq $16, NBLKS; jae .Lprocess_16v; /* Preload state to YMM registers */ vpbroadcastd (0 * 4)(INPUT), S0y; vpbroadcastd (1 * 4)(INPUT), S1y; vpbroadcastd (2 * 4)(INPUT), S2y; vpbroadcastd (3 * 4)(INPUT), S3y; vpbroadcastd (4 * 4)(INPUT), S4y; vpbroadcastd (5 * 4)(INPUT), S5y; vpbroadcastd (6 * 4)(INPUT), S6y; vpbroadcastd (7 * 4)(INPUT), S7y; vpbroadcastd (8 * 4)(INPUT), S8y; vpbroadcastd (14 * 4)(INPUT), S14y; vpbroadcastd (15 * 4)(INPUT), S15y; jmp .Lskip16v; .align 16 .Lprocess_16v: /* Process 16 ChaCha20 blocks */ /* Preload state to ZMM registers */ vpbroadcastd (0 * 4)(INPUT), S0; vpbroadcastd (1 * 4)(INPUT), S1; vpbroadcastd (2 * 4)(INPUT), S2; vpbroadcastd (3 * 4)(INPUT), S3; vpbroadcastd (4 * 4)(INPUT), S4; vpbroadcastd (5 * 4)(INPUT), S5; vpbroadcastd (6 * 4)(INPUT), S6; vpbroadcastd (7 * 4)(INPUT), S7; vpbroadcastd (8 * 4)(INPUT), S8; vpbroadcastd (14 * 4)(INPUT), S14; vpbroadcastd (15 * 4)(INPUT), S15; movl $20, ROUND; subq $16, NBLKS; /* Construct counter vectors X12 and X13 */ vpmovm2d %k1, X9; vpaddd (12 * 4)(INPUT){1to16}, COUNTER_ADD, X12; vpbroadcastd (13 * 4)(INPUT), X13; vpcmpud $6, X12, COUNTER_ADD, %k2; vpsubd X9, X13, X13{%k2}; vmovdqa32 X12, X12_SAVE; vmovdqa32 X13, X13_SAVE; /* Load vectors */ vmovdqa32 S0, X0; vmovdqa32 S4, X4; vmovdqa32 S8, X8; vmovdqa32 S1, X1; vmovdqa32 S5, X5; vpbroadcastd (9 * 4)(INPUT), X9; QUARTERROUND2V(X0, X4, X8, X12, X1, X5, X9, X13) vmovdqa32 S2, X2; vmovdqa32 S6, X6; vpbroadcastd (10 * 4)(INPUT), X10; vmovdqa32 S14, X14; vmovdqa32 S3, X3; vmovdqa32 S7, X7; vpbroadcastd (11 * 4)(INPUT), X11; vmovdqa32 S15, X15; /* Update counter */ addq $16, (12 * 4)(INPUT); jmp .Lround2_entry_16v; .align 16 .Loop16v: movl $20, ROUND; subq $16, NBLKS; vmovdqa32 S0, X0; vmovdqa32 S4, X4; vmovdqa32 S8, X8; transpose_16byte_4x4(X1, X5, X9, X13, TMP0, TMP1); xor_src_dst_4x4(DST, SRC, (64 * 1), 256, X1, X5, X9, X13); vpmovm2d %k1, X9; vpaddd (12 * 4)(INPUT){1to16}, COUNTER_ADD, X12; vpbroadcastd (13 * 4)(INPUT), X13; vpcmpud $6, X12, COUNTER_ADD, %k2; vpsubd X9, X13, X13{%k2}; vmovdqa32 S1, X1; vmovdqa32 S5, X5; vpbroadcastd (9 * 4)(INPUT), X9; vmovdqa32 X12, X12_SAVE; vmovdqa32 X13, X13_SAVE; QUARTERROUND2V(X0, X4, X8, X12, X1, X5, X9, X13) transpose_16byte_4x4(X2, X6, X10, X14, TMP0, TMP1); xor_src_dst_4x4(DST, SRC, (64 * 2), 256, X2, X6, X10, X14); vmovdqa32 S2, X2; vmovdqa32 S6, X6; vpbroadcastd (10 * 4)(INPUT), X10; vmovdqa32 S14, X14; transpose_16byte_4x4(X3, X7, X11, X15, TMP0, TMP1); xor_src_dst_4x4(DST, SRC, (64 * 3), 256, X3, X7, X11, X15); leaq (16 * 64)(SRC), SRC; leaq (16 * 64)(DST), DST; vmovdqa32 S3, X3; vmovdqa32 S7, X7; vpbroadcastd (11 * 4)(INPUT), X11; vmovdqa32 S15, X15; /* Update counter */ addq $16, (12 * 4)(INPUT); jmp .Lround2_entry_16v; .align 16 .Lround2_16v: QUARTERROUND2V(X2, X7, X8, X13, X3, X4, X9, X14) QUARTERROUND2V(X0, X4, X8, X12, X1, X5, X9, X13) .align 16 .Lround2_entry_16v: QUARTERROUND2V(X2, X6, X10, X14, X3, X7, X11, X15) QUARTERROUND2V(X0, X5, X10, X15, X1, X6, X11, X12) subl $2, ROUND; jnz .Lround2_16v; PLUS(X0, S0); PLUS(X1, S1); QUARTERROUND2V(X2, X7, X8, X13, X3, X4, X9, X14) PLUS(X2, S2); PLUS(X3, S3); transpose_4x4(X0, X1, X2, X3, TMP0, TMP1); PLUS(X4, S4); PLUS(X5, S5); PLUS(X6, S6); PLUS(X7, S7); transpose_4x4(X4, X5, X6, X7, TMP0, TMP1); PLUS(X8, S8); PLUS(X9, (9 * 4)(INPUT){1to16}); PLUS(X10, (10 * 4)(INPUT){1to16}); PLUS(X11, (11 * 4)(INPUT){1to16}); transpose_4x4(X8, X9, X10, X11, TMP0, TMP1); PLUS(X12, X12_SAVE); PLUS(X13, X13_SAVE); PLUS(X14, S14); PLUS(X15, S15); transpose_4x4(X12, X13, X14, X15, TMP0, TMP1); transpose_16byte_4x4(X0, X4, X8, X12, TMP0, TMP1); xor_src_dst_4x4(DST, SRC, (64 * 0), 256, X0, X4, X8, X12); cmpq $16, NBLKS; jae .Loop16v; transpose_16byte_4x4(X1, X5, X9, X13, TMP0, TMP1); xor_src_dst_4x4(DST, SRC, (64 * 1), 256, X1, X5, X9, X13); transpose_16byte_4x4(X2, X6, X10, X14, TMP0, TMP1); xor_src_dst_4x4(DST, SRC, (64 * 2), 256, X2, X6, X10, X14); transpose_16byte_4x4(X3, X7, X11, X15, TMP0, TMP1); xor_src_dst_4x4(DST, SRC, (64 * 3), 256, X3, X7, X11, X15); leaq (16 * 64)(SRC), SRC; leaq (16 * 64)(DST), DST; .align 16 .Lskip16v: cmpq $8, NBLKS; jb .Lskip8v; /* Process 8 ChaCha20 blocks */ /* Construct counter vectors X12 and X13 */ vpmovm2d %k1, X9y; vpaddd (12 * 4)(INPUT){1to8}, COUNTER_ADDy, X12y; vpbroadcastd (13 * 4)(INPUT), X13y; vpcmpud $6, X12y, COUNTER_ADDy, %k2; vpsubd X9y, X13y, X13y{%k2}; vmovdqa32 X12y, X12_SAVEy; vmovdqa32 X13y, X13_SAVEy; /* Load vectors */ vmovdqa32 S0y, X0y; vmovdqa32 S4y, X4y; vmovdqa32 S8y, X8y; vmovdqa32 S1y, X1y; vmovdqa32 S5y, X5y; vpbroadcastd (9 * 4)(INPUT), X9y; vmovdqa32 S2y, X2y; vmovdqa32 S6y, X6y; vpbroadcastd (10 * 4)(INPUT), X10y; vmovdqa32 S14y, X14y; vmovdqa32 S3y, X3y; vmovdqa32 S7y, X7y; vpbroadcastd (11 * 4)(INPUT), X11y; vmovdqa32 S15y, X15y; /* Update counter */ addq $8, (12 * 4)(INPUT); movl $20, ROUND; subq $8, NBLKS; .align 16 .Lround2_8v: QUARTERROUND2V(X0y, X4y, X8y, X12y, X1y, X5y, X9y, X13y) QUARTERROUND2V(X2y, X6y, X10y, X14y, X3y, X7y, X11y, X15y) QUARTERROUND2V(X0y, X5y, X10y, X15y, X1y, X6y, X11y, X12y) QUARTERROUND2V(X2y, X7y, X8y, X13y, X3y, X4y, X9y, X14y) subl $2, ROUND; jnz .Lround2_8v; PLUS(X0y, S0y); PLUS(X1y, S1y); PLUS(X2y, S2y); PLUS(X3y, S3y); transpose_4x4(X0y, X1y, X2y, X3y, TMP0y, TMP1y); PLUS(X4y, S4y); PLUS(X5y, S5y); PLUS(X6y, S6y); PLUS(X7y, S7y); transpose_4x4(X4y, X5y, X6y, X7y, TMP0y, TMP1y); PLUS(X8y, S8y); transpose_16byte_2x2(X0y, X4y, TMP0y); PLUS(X9y, (9 * 4)(INPUT){1to8}); transpose_16byte_2x2(X1y, X5y, TMP0y); PLUS(X10y, (10 * 4)(INPUT){1to8}); transpose_16byte_2x2(X2y, X6y, TMP0y); PLUS(X11y, (11 * 4)(INPUT){1to8}); transpose_16byte_2x2(X3y, X7y, TMP0y); xor_src_dst_4x4(DST, SRC, (16 * 0), 64, X0y, X1y, X2y, X3y); transpose_4x4(X8y, X9y, X10y, X11y, TMP0y, TMP1y); PLUS(X12y, X12_SAVEy); PLUS(X13y, X13_SAVEy); PLUS(X14y, S14y); PLUS(X15y, S15y); xor_src_dst_4x4(DST, SRC, (16 * 16), 64, X4y, X5y, X6y, X7y); transpose_4x4(X12y, X13y, X14y, X15y, TMP0y, TMP1y); transpose_16byte_2x2(X8y, X12y, TMP0y); transpose_16byte_2x2(X9y, X13y, TMP0y); transpose_16byte_2x2(X10y, X14y, TMP0y); transpose_16byte_2x2(X11y, X15y, TMP0y); xor_src_dst_4x4(DST, SRC, (16 * 2), 64, X8y, X9y, X10y, X11y); xor_src_dst_4x4(DST, SRC, (16 * 18), 64, X12y, X13y, X14y, X15y); leaq (8 * 64)(SRC), SRC; leaq (8 * 64)(DST), DST; .align 16 .Lskip8v: cmpq $4, NBLKS; jb .Lskip4v; /* Process 4 ChaCha20 blocks */ /* Construct counter vectors X12 and X13 */ vpmovm2d %k1, X9x; vpaddd (12 * 4)(INPUT){1to4}, COUNTER_ADDx, X12x; vpbroadcastd (13 * 4)(INPUT), X13x; vpcmpud $6, X12x, COUNTER_ADDx, %k2; vpsubd X9x, X13x, X13x{%k2}; vmovdqa32 X12x, X12_SAVEx; vmovdqa32 X13x, X13_SAVEx; /* Load vectors */ vmovdqa32 S0x, X0x; vmovdqa32 S4x, X4x; vmovdqa32 S8x, X8x; vmovdqa32 S1x, X1x; vmovdqa32 S5x, X5x; vpbroadcastd (9 * 4)(INPUT), X9x; vmovdqa32 S2x, X2x; vmovdqa32 S6x, X6x; vpbroadcastd (10 * 4)(INPUT), X10x; vmovdqa32 S14x, X14x; vmovdqa32 S3x, X3x; vmovdqa32 S7x, X7x; vpbroadcastd (11 * 4)(INPUT), X11x; vmovdqa32 S15x, X15x; /* Update counter */ addq $4, (12 * 4)(INPUT); movl $20, ROUND; subq $4, NBLKS; .align 16 .Lround2_4v: QUARTERROUND2V(X0x, X4x, X8x, X12x, X1x, X5x, X9x, X13x) QUARTERROUND2V(X2x, X6x, X10x, X14x, X3x, X7x, X11x, X15x) QUARTERROUND2V(X0x, X5x, X10x, X15x, X1x, X6x, X11x, X12x) QUARTERROUND2V(X2x, X7x, X8x, X13x, X3x, X4x, X9x, X14x) subl $2, ROUND; jnz .Lround2_4v; PLUS(X0x, S0x); PLUS(X1x, S1x); PLUS(X2x, S2x); PLUS(X3x, S3x); transpose_4x4(X0x, X1x, X2x, X3x, TMP0x, TMP1x); PLUS(X4x, S4x); PLUS(X5x, S5x); PLUS(X6x, S6x); PLUS(X7x, S7x); xor_src_dst_4x4(DST, SRC, (16 * 0), 64, X0x, X1x, X2x, X3x); transpose_4x4(X4x, X5x, X6x, X7x, TMP0x, TMP1x); PLUS(X8x, S8x); PLUS(X9x, (9 * 4)(INPUT){1to4}); PLUS(X10x, (10 * 4)(INPUT){1to4}); PLUS(X11x, (11 * 4)(INPUT){1to4}); xor_src_dst_4x4(DST, SRC, (16 * 1), 64, X4x, X5x, X6x, X7x); transpose_4x4(X8x, X9x, X10x, X11x, TMP0x, TMP1x); PLUS(X12x, X12_SAVEx); PLUS(X13x, X13_SAVEx); PLUS(X14x, S14x); PLUS(X15x, S15x); xor_src_dst_4x4(DST, SRC, (16 * 2), 64, X8x, X9x, X10x, X11x); transpose_4x4(X12x, X13x, X14x, X15x, TMP0x, TMP1x); xor_src_dst_4x4(DST, SRC, (16 * 3), 64, X12x, X13x, X14x, X15x); leaq (4 * 64)(SRC), SRC; leaq (4 * 64)(DST), DST; .align 16 .Lskip4v: /* clear AVX512 registers */ kxorq %k2, %k2, %k2; vzeroupper; clear_zmm16_zmm31(); .align 16 .Lskip_vertical_handling: cmpq $0, NBLKS; je .Ldone; /* Load state */ vmovdqu (0 * 4)(INPUT), X10x; vmovdqu (4 * 4)(INPUT), X11x; vmovdqu (8 * 4)(INPUT), X12x; vmovdqu (12 * 4)(INPUT), X13x; /* Load constant */ vmovdqa .Lone rRIP, X4x; cmpq $1, NBLKS; je .Lhandle1; /* Process two ChaCha20 blocks (XMM) */ movl $20, ROUND; subq $2, NBLKS; vmovdqa X10x, X0x; vmovdqa X11x, X1x; vmovdqa X12x, X2x; vmovdqa X13x, X3x; vmovdqa X10x, X8x; vmovdqa X11x, X9x; vmovdqa X12x, X14x; vpaddq X4x, X13x, X15x; vmovdqa X15x, X7x; .align 16 .Lround2_2: QUARTERROUND2H(X0x, X1x, X2x, X3x, X8x, X9x, X14x, X15x, 0x39, 0x4e, 0x93); QUARTERROUND2H(X0x, X1x, X2x, X3x, X8x, X9x, X14x, X15x, 0x93, 0x4e, 0x39); subl $2, ROUND; jnz .Lround2_2; PLUS(X0x, X10x); PLUS(X1x, X11x); PLUS(X2x, X12x); PLUS(X3x, X13x); vpaddq .Ltwo rRIP, X13x, X13x; /* Update counter */ xor_src_dst_4x4(DST, SRC, 0 * 4, 4 * 4, X0x, X1x, X2x, X3x); PLUS(X8x, X10x); PLUS(X9x, X11x); PLUS(X14x, X12x); PLUS(X15x, X7x); xor_src_dst_4x4(DST, SRC, 16 * 4, 4 * 4, X8x, X9x, X14x, X15x); lea (2 * 64)(DST), DST; lea (2 * 64)(SRC), SRC; cmpq $0, NBLKS; je .Lskip1; .align 16 .Lhandle1: /* Process one ChaCha20 block (XMM) */ movl $20, ROUND; subq $1, NBLKS; vmovdqa X10x, X0x; vmovdqa X11x, X1x; vmovdqa X12x, X2x; vmovdqa X13x, X3x; .align 16 .Lround2_1: QUARTERROUND1H(X0x, X1x, X2x, X3x, 0x39, 0x4e, 0x93); QUARTERROUND1H(X0x, X1x, X2x, X3x, 0x93, 0x4e, 0x39); subl $2, ROUND; jnz .Lround2_1; PLUS(X0x, X10x); PLUS(X1x, X11x); PLUS(X2x, X12x); PLUS(X3x, X13x); vpaddq X4x, X13x, X13x; /* Update counter */ xor_src_dst_4x4(DST, SRC, 0 * 4, 4 * 4, X0x, X1x, X2x, X3x); .align 16 .Lskip1: /* Store counter */ vmovdqu X13x, (12 * 4)(INPUT); .align 16 .Ldone: vzeroall; /* clears ZMM0-ZMM15 */ xorl %eax, %eax; ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_chacha20_amd64_avx512_blocks, .-_gcry_chacha20_amd64_avx512_blocks;) #endif /*defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS)*/ #endif /*__x86_64*/ diff --git a/cipher/cipher-gcm-intel-pclmul.c b/cipher/cipher-gcm-intel-pclmul.c index ec00df09..391cbe6f 100644 --- a/cipher/cipher-gcm-intel-pclmul.c +++ b/cipher/cipher-gcm-intel-pclmul.c @@ -1,2025 +1,2025 @@ /* cipher-gcm-intel-pclmul.c - Intel PCLMUL accelerated Galois Counter Mode * implementation * Copyright (C) 2013-2014,2019,2022 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser general Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . */ #include #include #include #include #include #include "g10lib.h" #include "cipher.h" #include "bufhelp.h" #include "./cipher-internal.h" #ifdef GCM_USE_INTEL_PCLMUL #if _GCRY_GCC_VERSION >= 40400 /* 4.4 */ /* Prevent compiler from issuing SSE instructions between asm blocks. */ # pragma GCC target("no-sse") #endif #if __clang__ # pragma clang attribute push (__attribute__((target("no-sse"))), apply_to = function) #endif #define ALWAYS_INLINE inline __attribute__((always_inline)) #define NO_INSTRUMENT_FUNCTION __attribute__((no_instrument_function)) #define ASM_FUNC_ATTR NO_INSTRUMENT_FUNCTION #define ASM_FUNC_ATTR_INLINE ASM_FUNC_ATTR ALWAYS_INLINE #define GCM_INTEL_USE_VPCLMUL_AVX2 (1 << 0) #define GCM_INTEL_AGGR8_TABLE_INITIALIZED (1 << 1) #define GCM_INTEL_AGGR16_TABLE_INITIALIZED (1 << 2) #define GCM_INTEL_USE_VPCLMUL_AVX512 (1 << 3) #define GCM_INTEL_AGGR32_TABLE_INITIALIZED (1 << 4) /* Intel PCLMUL ghash based on white paper: "Intel® Carry-Less Multiplication Instruction and its Usage for Computing the GCM Mode - Rev 2.01"; Shay Gueron, Michael E. Kounavis. */ static ASM_FUNC_ATTR_INLINE void reduction(void) { /* input: */ asm volatile (/* first phase of the reduction */ "movdqa %%xmm3, %%xmm6\n\t" "movdqa %%xmm3, %%xmm5\n\t" "psllq $1, %%xmm6\n\t" /* packed right shifting << 63 */ "pxor %%xmm3, %%xmm6\n\t" "psllq $57, %%xmm5\n\t" /* packed right shifting << 57 */ "psllq $62, %%xmm6\n\t" /* packed right shifting << 62 */ "pxor %%xmm5, %%xmm6\n\t" /* xor the shifted versions */ "pshufd $0x6a, %%xmm6, %%xmm5\n\t" "pshufd $0xae, %%xmm6, %%xmm6\n\t" "pxor %%xmm5, %%xmm3\n\t" /* first phase of the reduction complete */ /* second phase of the reduction */ "pxor %%xmm3, %%xmm1\n\t" /* xor the shifted versions */ "psrlq $1, %%xmm3\n\t" /* packed left shifting >> 1 */ "pxor %%xmm3, %%xmm6\n\t" "psrlq $1, %%xmm3\n\t" /* packed left shifting >> 2 */ "pxor %%xmm3, %%xmm1\n\t" "psrlq $5, %%xmm3\n\t" /* packed left shifting >> 7 */ "pxor %%xmm3, %%xmm6\n\t" "pxor %%xmm6, %%xmm1\n\t" /* the result is in xmm1 */ ::: "memory" ); } static ASM_FUNC_ATTR_INLINE void gfmul_pclmul(void) { /* Input: XMM0 and XMM1, Output: XMM1. Input XMM0 stays unmodified. Input must be converted to little-endian. */ asm volatile (/* gfmul, xmm0 has operator a and xmm1 has operator b. */ "pshufd $78, %%xmm0, %%xmm2\n\t" "pshufd $78, %%xmm1, %%xmm4\n\t" "pxor %%xmm0, %%xmm2\n\t" /* xmm2 holds a0+a1 */ "pxor %%xmm1, %%xmm4\n\t" /* xmm4 holds b0+b1 */ "movdqa %%xmm0, %%xmm3\n\t" "pclmulqdq $0, %%xmm1, %%xmm3\n\t" /* xmm3 holds a0*b0 */ "pclmulqdq $17, %%xmm0, %%xmm1\n\t" /* xmm6 holds a1*b1 */ "movdqa %%xmm3, %%xmm5\n\t" "pclmulqdq $0, %%xmm2, %%xmm4\n\t" /* xmm4 holds (a0+a1)*(b0+b1) */ "pxor %%xmm1, %%xmm5\n\t" /* xmm5 holds a0*b0+a1*b1 */ "pxor %%xmm5, %%xmm4\n\t" /* xmm4 holds a0*b0+a1*b1+(a0+a1)*(b0+b1) */ "movdqa %%xmm4, %%xmm5\n\t" "psrldq $8, %%xmm4\n\t" "pslldq $8, %%xmm5\n\t" "pxor %%xmm5, %%xmm3\n\t" "pxor %%xmm4, %%xmm1\n\t" /* holds the result of the carry-less multiplication of xmm0 by xmm1 */ ::: "memory" ); reduction(); } #define GFMUL_AGGR4_ASM_1(be_to_le) \ /* perform clmul and merge results... */ \ "movdqu 2*16(%[h_table]), %%xmm2\n\t" /* Load H4 */ \ "movdqu 0*16(%[buf]), %%xmm5\n\t" \ be_to_le("pshufb %[be_mask], %%xmm5\n\t") /* be => le */ \ "pxor %%xmm5, %%xmm1\n\t" \ \ "pshufd $78, %%xmm2, %%xmm5\n\t" \ "pshufd $78, %%xmm1, %%xmm4\n\t" \ "pxor %%xmm2, %%xmm5\n\t" /* xmm5 holds 4:a0+a1 */ \ "pxor %%xmm1, %%xmm4\n\t" /* xmm4 holds 4:b0+b1 */ \ "movdqa %%xmm2, %%xmm3\n\t" \ "pclmulqdq $0, %%xmm1, %%xmm3\n\t" /* xmm3 holds 4:a0*b0 */ \ "pclmulqdq $17, %%xmm2, %%xmm1\n\t" /* xmm1 holds 4:a1*b1 */ \ "pclmulqdq $0, %%xmm5, %%xmm4\n\t" /* xmm4 holds 4:(a0+a1)*(b0+b1) */ \ \ "movdqu 1*16(%[h_table]), %%xmm5\n\t" /* Load H3 */ \ "movdqu 1*16(%[buf]), %%xmm2\n\t" \ be_to_le("pshufb %[be_mask], %%xmm2\n\t") /* be => le */ \ \ "pshufd $78, %%xmm5, %%xmm0\n\t" \ "pshufd $78, %%xmm2, %%xmm7\n\t" \ "pxor %%xmm5, %%xmm0\n\t" /* xmm0 holds 3:a0+a1 */ \ "pxor %%xmm2, %%xmm7\n\t" /* xmm7 holds 3:b0+b1 */ \ "movdqa %%xmm5, %%xmm6\n\t" \ "pclmulqdq $0, %%xmm2, %%xmm6\n\t" /* xmm6 holds 3:a0*b0 */ \ "pclmulqdq $17, %%xmm5, %%xmm2\n\t" /* xmm2 holds 3:a1*b1 */ \ "pclmulqdq $0, %%xmm0, %%xmm7\n\t" /* xmm7 holds 3:(a0+a1)*(b0+b1) */ \ \ "movdqu 2*16(%[buf]), %%xmm5\n\t" \ be_to_le("pshufb %[be_mask], %%xmm5\n\t") /* be => le */ \ \ "pxor %%xmm6, %%xmm3\n\t" /* xmm3 holds 3+4:a0*b0 */ \ "pxor %%xmm2, %%xmm1\n\t" /* xmm1 holds 3+4:a1*b1 */ \ "pxor %%xmm7, %%xmm4\n\t" /* xmm4 holds 3+4:(a0+a1)*(b0+b1) */ \ \ "movdqu 0*16(%[h_table]), %%xmm2\n\t" /* Load H2 */ \ \ "pshufd $78, %%xmm2, %%xmm0\n\t" \ "pshufd $78, %%xmm5, %%xmm7\n\t" \ "pxor %%xmm2, %%xmm0\n\t" /* xmm0 holds 2:a0+a1 */ \ "pxor %%xmm5, %%xmm7\n\t" /* xmm7 holds 2:b0+b1 */ \ "movdqa %%xmm2, %%xmm6\n\t" \ "pclmulqdq $0, %%xmm5, %%xmm6\n\t" /* xmm6 holds 2:a0*b0 */ \ "pclmulqdq $17, %%xmm2, %%xmm5\n\t" /* xmm5 holds 2:a1*b1 */ \ "pclmulqdq $0, %%xmm0, %%xmm7\n\t" /* xmm7 holds 2:(a0+a1)*(b0+b1) */ \ \ "movdqu 3*16(%[buf]), %%xmm2\n\t" \ be_to_le("pshufb %[be_mask], %%xmm2\n\t") /* be => le */ \ \ "pxor %%xmm6, %%xmm3\n\t" /* xmm3 holds 2+3+4:a0*b0 */ \ "pxor %%xmm5, %%xmm1\n\t" /* xmm1 holds 2+3+4:a1*b1 */ \ "pxor %%xmm7, %%xmm4\n\t" /* xmm4 holds 2+3+4:(a0+a1)*(b0+b1) */ #define GFMUL_AGGR4_ASM_2() \ "movdqu %[h_1], %%xmm5\n\t" /* Load H1 */ \ \ "pshufd $78, %%xmm5, %%xmm0\n\t" \ "pshufd $78, %%xmm2, %%xmm7\n\t" \ "pxor %%xmm5, %%xmm0\n\t" /* xmm0 holds 1:a0+a1 */ \ "pxor %%xmm2, %%xmm7\n\t" /* xmm7 holds 1:b0+b1 */ \ "movdqa %%xmm5, %%xmm6\n\t" \ "pclmulqdq $0, %%xmm2, %%xmm6\n\t" /* xmm6 holds 1:a0*b0 */ \ "pclmulqdq $17, %%xmm5, %%xmm2\n\t" /* xmm2 holds 1:a1*b1 */ \ "pclmulqdq $0, %%xmm0, %%xmm7\n\t" /* xmm7 holds 1:(a0+a1)*(b0+b1) */ \ \ "pxor %%xmm6, %%xmm3\n\t" /* xmm3 holds 1+2+3+4:a0*b0 */ \ "pxor %%xmm2, %%xmm1\n\t" /* xmm1 holds 1+2+3+4:a1*b1 */ \ "pxor %%xmm7, %%xmm4\n\t" /* xmm4 holds 1+2+3+4:(a0+a1)*(b0+b1) */ \ \ /* aggregated reduction... */ \ "movdqa %%xmm3, %%xmm5\n\t" \ "pxor %%xmm1, %%xmm5\n\t" /* xmm5 holds a0*b0+a1*b1 */ \ "pxor %%xmm5, %%xmm4\n\t" /* xmm4 holds a0*b0+a1*b1+(a0+a1)*(b0+b1) */ \ "movdqa %%xmm4, %%xmm5\n\t" \ "psrldq $8, %%xmm4\n\t" \ "pslldq $8, %%xmm5\n\t" \ "pxor %%xmm5, %%xmm3\n\t" \ "pxor %%xmm4, %%xmm1\n\t" /* holds the result of the \ carry-less multiplication of xmm0 \ by xmm1 */ #define be_to_le(...) __VA_ARGS__ #define le_to_le(...) /*_*/ static ASM_FUNC_ATTR_INLINE void gfmul_pclmul_aggr4(const void *buf, const void *h_1, const void *h_table, const unsigned char *be_mask) { /* Input: Hash: XMM1 Output: Hash: XMM1 */ asm volatile (GFMUL_AGGR4_ASM_1(be_to_le) : : [buf] "r" (buf), [h_table] "r" (h_table), [be_mask] "m" (*be_mask) : "memory" ); asm volatile (GFMUL_AGGR4_ASM_2() : : [h_1] "m" (*(const unsigned char *)h_1) : "memory" ); reduction(); } static ASM_FUNC_ATTR_INLINE void gfmul_pclmul_aggr4_le(const void *buf, const void *h_1, const void *h_table) { /* Input: Hash: XMM1 Output: Hash: XMM1 */ asm volatile (GFMUL_AGGR4_ASM_1(le_to_le) : : [buf] "r" (buf), [h_table] "r" (h_table) : "memory" ); asm volatile (GFMUL_AGGR4_ASM_2() : : [h_1] "m" (*(const unsigned char *)h_1) : "memory" ); reduction(); } #ifdef __x86_64__ #define GFMUL_AGGR8_ASM(be_to_le) \ /* Load H6, H7, H8. */ \ "movdqu 6*16(%[h_table]), %%xmm10\n\t" \ "movdqu 5*16(%[h_table]), %%xmm9\n\t" \ "movdqu 4*16(%[h_table]), %%xmm8\n\t" \ \ /* perform clmul and merge results... */ \ "movdqu 0*16(%[buf]), %%xmm5\n\t" \ "movdqu 1*16(%[buf]), %%xmm2\n\t" \ be_to_le("pshufb %%xmm15, %%xmm5\n\t") /* be => le */ \ be_to_le("pshufb %%xmm15, %%xmm2\n\t") /* be => le */ \ "pxor %%xmm5, %%xmm1\n\t" \ \ "pshufd $78, %%xmm10, %%xmm5\n\t" \ "pshufd $78, %%xmm1, %%xmm4\n\t" \ "pxor %%xmm10, %%xmm5\n\t" /* xmm5 holds 8:a0+a1 */ \ "pxor %%xmm1, %%xmm4\n\t" /* xmm4 holds 8:b0+b1 */ \ "movdqa %%xmm10, %%xmm3\n\t" \ "pclmulqdq $0, %%xmm1, %%xmm3\n\t" /* xmm3 holds 8:a0*b0 */ \ "pclmulqdq $17, %%xmm10, %%xmm1\n\t" /* xmm1 holds 8:a1*b1 */ \ "pclmulqdq $0, %%xmm5, %%xmm4\n\t" /* xmm4 holds 8:(a0+a1)*(b0+b1) */ \ \ "pshufd $78, %%xmm9, %%xmm11\n\t" \ "pshufd $78, %%xmm2, %%xmm7\n\t" \ "pxor %%xmm9, %%xmm11\n\t" /* xmm11 holds 7:a0+a1 */ \ "pxor %%xmm2, %%xmm7\n\t" /* xmm7 holds 7:b0+b1 */ \ "movdqa %%xmm9, %%xmm6\n\t" \ "pclmulqdq $0, %%xmm2, %%xmm6\n\t" /* xmm6 holds 7:a0*b0 */ \ "pclmulqdq $17, %%xmm9, %%xmm2\n\t" /* xmm2 holds 7:a1*b1 */ \ "pclmulqdq $0, %%xmm11, %%xmm7\n\t" /* xmm7 holds 7:(a0+a1)*(b0+b1) */ \ \ "pxor %%xmm6, %%xmm3\n\t" /* xmm3 holds 7+8:a0*b0 */ \ "pxor %%xmm2, %%xmm1\n\t" /* xmm1 holds 7+8:a1*b1 */ \ "pxor %%xmm7, %%xmm4\n\t" /* xmm4 holds 7+8:(a0+a1)*(b0+b1) */ \ \ "movdqu 2*16(%[buf]), %%xmm5\n\t" \ "movdqu 3*16(%[buf]), %%xmm2\n\t" \ be_to_le("pshufb %%xmm15, %%xmm5\n\t") /* be => le */ \ be_to_le("pshufb %%xmm15, %%xmm2\n\t") /* be => le */ \ \ "pshufd $78, %%xmm8, %%xmm11\n\t" \ "pshufd $78, %%xmm5, %%xmm7\n\t" \ "pxor %%xmm8, %%xmm11\n\t" /* xmm11 holds 6:a0+a1 */ \ "pxor %%xmm5, %%xmm7\n\t" /* xmm7 holds 6:b0+b1 */ \ "movdqa %%xmm8, %%xmm6\n\t" \ "pclmulqdq $0, %%xmm5, %%xmm6\n\t" /* xmm6 holds 6:a0*b0 */ \ "pclmulqdq $17, %%xmm8, %%xmm5\n\t" /* xmm5 holds 6:a1*b1 */ \ "pclmulqdq $0, %%xmm11, %%xmm7\n\t" /* xmm7 holds 6:(a0+a1)*(b0+b1) */ \ \ /* Load H3, H4, H5. */ \ "movdqu 3*16(%[h_table]), %%xmm10\n\t" \ "movdqu 2*16(%[h_table]), %%xmm9\n\t" \ "movdqu 1*16(%[h_table]), %%xmm8\n\t" \ \ "pxor %%xmm6, %%xmm3\n\t" /* xmm3 holds 6+7+8:a0*b0 */ \ "pxor %%xmm5, %%xmm1\n\t" /* xmm1 holds 6+7+8:a1*b1 */ \ "pxor %%xmm7, %%xmm4\n\t" /* xmm4 holds 6+7+8:(a0+a1)*(b0+b1) */ \ \ "pshufd $78, %%xmm10, %%xmm11\n\t" \ "pshufd $78, %%xmm2, %%xmm7\n\t" \ "pxor %%xmm10, %%xmm11\n\t" /* xmm11 holds 5:a0+a1 */ \ "pxor %%xmm2, %%xmm7\n\t" /* xmm7 holds 5:b0+b1 */ \ "movdqa %%xmm10, %%xmm6\n\t" \ "pclmulqdq $0, %%xmm2, %%xmm6\n\t" /* xmm6 holds 5:a0*b0 */ \ "pclmulqdq $17, %%xmm10, %%xmm2\n\t" /* xmm2 holds 5:a1*b1 */ \ "pclmulqdq $0, %%xmm11, %%xmm7\n\t" /* xmm7 holds 5:(a0+a1)*(b0+b1) */ \ \ "pxor %%xmm6, %%xmm3\n\t" /* xmm3 holds 5+6+7+8:a0*b0 */ \ "pxor %%xmm2, %%xmm1\n\t" /* xmm1 holds 5+6+7+8:a1*b1 */ \ "pxor %%xmm7, %%xmm4\n\t" /* xmm4 holds 5+6+7+8:(a0+a1)*(b0+b1) */ \ \ "movdqu 4*16(%[buf]), %%xmm5\n\t" \ "movdqu 5*16(%[buf]), %%xmm2\n\t" \ be_to_le("pshufb %%xmm15, %%xmm5\n\t") /* be => le */ \ be_to_le("pshufb %%xmm15, %%xmm2\n\t") /* be => le */ \ \ "pshufd $78, %%xmm9, %%xmm11\n\t" \ "pshufd $78, %%xmm5, %%xmm7\n\t" \ "pxor %%xmm9, %%xmm11\n\t" /* xmm11 holds 4:a0+a1 */ \ "pxor %%xmm5, %%xmm7\n\t" /* xmm7 holds 4:b0+b1 */ \ "movdqa %%xmm9, %%xmm6\n\t" \ "pclmulqdq $0, %%xmm5, %%xmm6\n\t" /* xmm6 holds 4:a0*b0 */ \ "pclmulqdq $17, %%xmm9, %%xmm5\n\t" /* xmm5 holds 4:a1*b1 */ \ "pclmulqdq $0, %%xmm11, %%xmm7\n\t" /* xmm7 holds 4:(a0+a1)*(b0+b1) */ \ \ "pxor %%xmm6, %%xmm3\n\t" /* xmm3 holds 4+5+6+7+8:a0*b0 */ \ "pxor %%xmm5, %%xmm1\n\t" /* xmm1 holds 4+5+6+7+8:a1*b1 */ \ "pxor %%xmm7, %%xmm4\n\t" /* xmm4 holds 4+5+6+7+8:(a0+a1)*(b0+b1) */ \ \ "pshufd $78, %%xmm8, %%xmm11\n\t" \ "pshufd $78, %%xmm2, %%xmm7\n\t" \ "pxor %%xmm8, %%xmm11\n\t" /* xmm11 holds 3:a0+a1 */ \ "pxor %%xmm2, %%xmm7\n\t" /* xmm7 holds 3:b0+b1 */ \ "movdqa %%xmm8, %%xmm6\n\t" \ "pclmulqdq $0, %%xmm2, %%xmm6\n\t" /* xmm6 holds 3:a0*b0 */ \ "pclmulqdq $17, %%xmm8, %%xmm2\n\t" /* xmm2 holds 3:a1*b1 */ \ "pclmulqdq $0, %%xmm11, %%xmm7\n\t" /* xmm7 holds 3:(a0+a1)*(b0+b1) */ \ \ "movdqu 0*16(%[h_table]), %%xmm8\n\t" /* Load H2 */ \ \ "pxor %%xmm6, %%xmm3\n\t" /* xmm3 holds 3+4+5+6+7+8:a0*b0 */ \ "pxor %%xmm2, %%xmm1\n\t" /* xmm1 holds 3+4+5+6+7+8:a1*b1 */ \ "pxor %%xmm7, %%xmm4\n\t" /* xmm4 holds 3+4+5+6+7+8:(a0+a1)*(b0+b1) */ \ \ "movdqu 6*16(%[buf]), %%xmm5\n\t" \ "movdqu 7*16(%[buf]), %%xmm2\n\t" \ be_to_le("pshufb %%xmm15, %%xmm5\n\t") /* be => le */ \ be_to_le("pshufb %%xmm15, %%xmm2\n\t") /* be => le */ \ \ "pshufd $78, %%xmm8, %%xmm11\n\t" \ "pshufd $78, %%xmm5, %%xmm7\n\t" \ "pxor %%xmm8, %%xmm11\n\t" /* xmm11 holds 2:a0+a1 */ \ "pxor %%xmm5, %%xmm7\n\t" /* xmm7 holds 2:b0+b1 */ \ "movdqa %%xmm8, %%xmm6\n\t" \ "pclmulqdq $0, %%xmm5, %%xmm6\n\t" /* xmm6 holds 2:a0*b0 */ \ "pclmulqdq $17, %%xmm8, %%xmm5\n\t" /* xmm5 holds 2:a1*b1 */ \ "pclmulqdq $0, %%xmm11, %%xmm7\n\t" /* xmm7 holds 2:(a0+a1)*(b0+b1) */ \ \ "pxor %%xmm6, %%xmm3\n\t" /* xmm3 holds 2+3+4+5+6+7+8:a0*b0 */ \ "pxor %%xmm5, %%xmm1\n\t" /* xmm1 holds 2+3+4+5+6+7+8:a1*b1 */ \ "pxor %%xmm7, %%xmm4\n\t" /* xmm4 holds 2+3+4+5+6+7+8:(a0+a1)*(b0+b1) */ \ \ "pshufd $78, %%xmm0, %%xmm11\n\t" \ "pshufd $78, %%xmm2, %%xmm7\n\t" \ "pxor %%xmm0, %%xmm11\n\t" /* xmm11 holds 1:a0+a1 */ \ "pxor %%xmm2, %%xmm7\n\t" /* xmm7 holds 1:b0+b1 */ \ "movdqa %%xmm0, %%xmm6\n\t" \ "pclmulqdq $0, %%xmm2, %%xmm6\n\t" /* xmm6 holds 1:a0*b0 */ \ "pclmulqdq $17, %%xmm0, %%xmm2\n\t" /* xmm2 holds 1:a1*b1 */ \ "pclmulqdq $0, %%xmm11, %%xmm7\n\t" /* xmm7 holds 1:(a0+a1)*(b0+b1) */ \ \ "pxor %%xmm6, %%xmm3\n\t" /* xmm3 holds 1+2+3+4+5+6+7+8:a0*b0 */ \ "pxor %%xmm2, %%xmm1\n\t" /* xmm1 holds 1+2+3+4+5+6+7+8:a1*b1 */ \ "pxor %%xmm7, %%xmm4\n\t"/* xmm4 holds 1+2+3+4+5+6+7+8:(a0+a1)*(b0+b1) */ \ \ /* aggregated reduction... */ \ "movdqa %%xmm3, %%xmm5\n\t" \ "pxor %%xmm1, %%xmm5\n\t" /* xmm5 holds a0*b0+a1*b1 */ \ "pxor %%xmm5, %%xmm4\n\t" /* xmm4 holds a0*b0+a1*b1+(a0+a1)*(b0+b1) */ \ "movdqa %%xmm4, %%xmm5\n\t" \ "psrldq $8, %%xmm4\n\t" \ "pslldq $8, %%xmm5\n\t" \ "pxor %%xmm5, %%xmm3\n\t" \ "pxor %%xmm4, %%xmm1\n\t" /* holds the result of the \ carry-less multiplication of xmm0 \ by xmm1 */ static ASM_FUNC_ATTR_INLINE void gfmul_pclmul_aggr8(const void *buf, const void *h_table) { /* Input: H¹: XMM0 bemask: XMM15 Hash: XMM1 Output: Hash: XMM1 Inputs XMM0 and XMM15 stays unmodified. */ asm volatile (GFMUL_AGGR8_ASM(be_to_le) : : [buf] "r" (buf), [h_table] "r" (h_table) : "memory" ); reduction(); } static ASM_FUNC_ATTR_INLINE void gfmul_pclmul_aggr8_le(const void *buf, const void *h_table) { /* Input: H¹: XMM0 Hash: XMM1 Output: Hash: XMM1 Inputs XMM0 and XMM15 stays unmodified. */ asm volatile (GFMUL_AGGR8_ASM(le_to_le) : : [buf] "r" (buf), [h_table] "r" (h_table) : "memory" ); reduction(); } #ifdef GCM_USE_INTEL_VPCLMUL_AVX2 #define GFMUL_AGGR16_ASM_VPCMUL_AVX2(be_to_le) \ /* perform clmul and merge results... */ \ "vmovdqu 0*16(%[buf]), %%ymm5\n\t" \ "vmovdqu 2*16(%[buf]), %%ymm2\n\t" \ be_to_le("vpshufb %%ymm15, %%ymm5, %%ymm5\n\t") /* be => le */ \ be_to_le("vpshufb %%ymm15, %%ymm2, %%ymm2\n\t") /* be => le */ \ "vpxor %%ymm5, %%ymm1, %%ymm1\n\t" \ \ "vpshufd $78, %%ymm0, %%ymm5\n\t" \ "vpshufd $78, %%ymm1, %%ymm4\n\t" \ "vpxor %%ymm0, %%ymm5, %%ymm5\n\t" /* ymm5 holds 15|16:a0+a1 */ \ "vpxor %%ymm1, %%ymm4, %%ymm4\n\t" /* ymm4 holds 15|16:b0+b1 */ \ "vpclmulqdq $0, %%ymm1, %%ymm0, %%ymm3\n\t" /* ymm3 holds 15|16:a0*b0 */ \ "vpclmulqdq $17, %%ymm0, %%ymm1, %%ymm1\n\t" /* ymm1 holds 15|16:a1*b1 */ \ "vpclmulqdq $0, %%ymm5, %%ymm4, %%ymm4\n\t" /* ymm4 holds 15|16:(a0+a1)*(b0+b1) */ \ \ "vmovdqu %[h1_h2], %%ymm0\n\t" \ \ "vpshufd $78, %%ymm13, %%ymm14\n\t" \ "vpshufd $78, %%ymm2, %%ymm7\n\t" \ "vpxor %%ymm13, %%ymm14, %%ymm14\n\t" /* ymm14 holds 13|14:a0+a1 */ \ "vpxor %%ymm2, %%ymm7, %%ymm7\n\t" /* ymm7 holds 13|14:b0+b1 */ \ "vpclmulqdq $0, %%ymm2, %%ymm13, %%ymm6\n\t" /* ymm6 holds 13|14:a0*b0 */ \ "vpclmulqdq $17, %%ymm13, %%ymm2, %%ymm2\n\t" /* ymm2 holds 13|14:a1*b1 */ \ "vpclmulqdq $0, %%ymm14, %%ymm7, %%ymm7\n\t" /* ymm7 holds 13|14:(a0+a1)*(b0+b1) */\ \ "vpxor %%ymm6, %%ymm3, %%ymm3\n\t" /* ymm3 holds 13+15|14+16:a0*b0 */ \ "vpxor %%ymm2, %%ymm1, %%ymm1\n\t" /* ymm1 holds 13+15|14+16:a1*b1 */ \ "vpxor %%ymm7, %%ymm4, %%ymm4\n\t" /* ymm4 holds 13+15|14+16:(a0+a1)*(b0+b1) */ \ \ "vmovdqu 4*16(%[buf]), %%ymm5\n\t" \ "vmovdqu 6*16(%[buf]), %%ymm2\n\t" \ be_to_le("vpshufb %%ymm15, %%ymm5, %%ymm5\n\t") /* be => le */ \ be_to_le("vpshufb %%ymm15, %%ymm2, %%ymm2\n\t") /* be => le */ \ \ "vpshufd $78, %%ymm12, %%ymm14\n\t" \ "vpshufd $78, %%ymm5, %%ymm7\n\t" \ "vpxor %%ymm12, %%ymm14, %%ymm14\n\t" /* ymm14 holds 11|12:a0+a1 */ \ "vpxor %%ymm5, %%ymm7, %%ymm7\n\t" /* ymm7 holds 11|12:b0+b1 */ \ "vpclmulqdq $0, %%ymm5, %%ymm12, %%ymm6\n\t" /* ymm6 holds 11|12:a0*b0 */ \ "vpclmulqdq $17, %%ymm12, %%ymm5, %%ymm5\n\t" /* ymm5 holds 11|12:a1*b1 */ \ "vpclmulqdq $0, %%ymm14, %%ymm7, %%ymm7\n\t" /* ymm7 holds 11|12:(a0+a1)*(b0+b1) */\ \ "vpxor %%ymm6, %%ymm3, %%ymm3\n\t" /* ymm3 holds 11+13+15|12+14+16:a0*b0 */ \ "vpxor %%ymm5, %%ymm1, %%ymm1\n\t" /* ymm1 holds 11+13+15|12+14+16:a1*b1 */ \ "vpxor %%ymm7, %%ymm4, %%ymm4\n\t" /* ymm4 holds 11+13+15|12+14+16:(a0+a1)*(b0+b1) */\ \ "vpshufd $78, %%ymm11, %%ymm14\n\t" \ "vpshufd $78, %%ymm2, %%ymm7\n\t" \ "vpxor %%ymm11, %%ymm14, %%ymm14\n\t" /* ymm14 holds 9|10:a0+a1 */ \ "vpxor %%ymm2, %%ymm7, %%ymm7\n\t" /* ymm7 holds 9|10:b0+b1 */ \ "vpclmulqdq $0, %%ymm2, %%ymm11, %%ymm6\n\t" /* ymm6 holds 9|10:a0*b0 */ \ "vpclmulqdq $17, %%ymm11, %%ymm2, %%ymm2\n\t" /* ymm2 holds 9|10:a1*b1 */ \ "vpclmulqdq $0, %%ymm14, %%ymm7, %%ymm7\n\t" /* ymm7 holds 9|10:(a0+a1)*(b0+b1) */ \ \ "vpxor %%ymm6, %%ymm3, %%ymm3\n\t" /* ymm3 holds 9+11+…+15|10+12+…+16:a0*b0 */ \ "vpxor %%ymm2, %%ymm1, %%ymm1\n\t" /* ymm1 holds 9+11+…+15|10+12+…+16:a1*b1 */ \ "vpxor %%ymm7, %%ymm4, %%ymm4\n\t" /* ymm4 holds 9+11+…+15|10+12+…+16:(a0+a1)*(b0+b1) */\ \ "vmovdqu 8*16(%[buf]), %%ymm5\n\t" \ "vmovdqu 10*16(%[buf]), %%ymm2\n\t" \ be_to_le("vpshufb %%ymm15, %%ymm5, %%ymm5\n\t") /* be => le */ \ be_to_le("vpshufb %%ymm15, %%ymm2, %%ymm2\n\t") /* be => le */ \ \ "vpshufd $78, %%ymm10, %%ymm14\n\t" \ "vpshufd $78, %%ymm5, %%ymm7\n\t" \ "vpxor %%ymm10, %%ymm14, %%ymm14\n\t" /* ymm14 holds 7|8:a0+a1 */ \ "vpxor %%ymm5, %%ymm7, %%ymm7\n\t" /* ymm7 holds 7|8:b0+b1 */ \ "vpclmulqdq $0, %%ymm5, %%ymm10, %%ymm6\n\t" /* ymm6 holds 7|8:a0*b0 */ \ "vpclmulqdq $17, %%ymm10, %%ymm5, %%ymm5\n\t" /* ymm5 holds 7|8:a1*b1 */ \ "vpclmulqdq $0, %%ymm14, %%ymm7, %%ymm7\n\t" /* ymm7 holds 7|8:(a0+a1)*(b0+b1) */ \ \ "vpxor %%ymm6, %%ymm3, %%ymm3\n\t" /* ymm3 holds 7+9+…+15|8+10+…+16:a0*b0 */ \ "vpxor %%ymm5, %%ymm1, %%ymm1\n\t" /* ymm1 holds 7+9+…+15|8+10+…+16:a1*b1 */ \ "vpxor %%ymm7, %%ymm4, %%ymm4\n\t" /* ymm4 holds 7+9+…+15|8+10+…+16:(a0+a1)*(b0+b1) */\ \ "vpshufd $78, %%ymm9, %%ymm14\n\t" \ "vpshufd $78, %%ymm2, %%ymm7\n\t" \ "vpxor %%ymm9, %%ymm14, %%ymm14\n\t" /* ymm14 holds 5|6:a0+a1 */ \ "vpxor %%ymm2, %%ymm7, %%ymm7\n\t" /* ymm7 holds 5|6:b0+b1 */ \ "vpclmulqdq $0, %%ymm2, %%ymm9, %%ymm6\n\t" /* ymm6 holds 5|6:a0*b0 */ \ "vpclmulqdq $17, %%ymm9, %%ymm2, %%ymm2\n\t" /* ymm2 holds 5|6:a1*b1 */ \ "vpclmulqdq $0, %%ymm14, %%ymm7, %%ymm7\n\t" /* ymm7 holds 5|6:(a0+a1)*(b0+b1) */ \ \ "vpxor %%ymm6, %%ymm3, %%ymm3\n\t" /* ymm3 holds 5+7+…+15|6+8+…+16:a0*b0 */ \ "vpxor %%ymm2, %%ymm1, %%ymm1\n\t" /* ymm1 holds 5+7+…+15|6+8+…+16:a1*b1 */ \ "vpxor %%ymm7, %%ymm4, %%ymm4\n\t" /* ymm4 holds 5+7+…+15|6+8+…+16:(a0+a1)*(b0+b1) */\ \ "vmovdqu 12*16(%[buf]), %%ymm5\n\t" \ "vmovdqu 14*16(%[buf]), %%ymm2\n\t" \ be_to_le("vpshufb %%ymm15, %%ymm5, %%ymm5\n\t") /* be => le */ \ be_to_le("vpshufb %%ymm15, %%ymm2, %%ymm2\n\t") /* be => le */ \ \ "vpshufd $78, %%ymm8, %%ymm14\n\t" \ "vpshufd $78, %%ymm5, %%ymm7\n\t" \ "vpxor %%ymm8, %%ymm14, %%ymm14\n\t" /* ymm14 holds 3|4:a0+a1 */ \ "vpxor %%ymm5, %%ymm7, %%ymm7\n\t" /* ymm7 holds 3|4:b0+b1 */ \ "vpclmulqdq $0, %%ymm5, %%ymm8, %%ymm6\n\t" /* ymm6 holds 3|4:a0*b0 */ \ "vpclmulqdq $17, %%ymm8, %%ymm5, %%ymm5\n\t" /* ymm5 holds 3|4:a1*b1 */ \ "vpclmulqdq $0, %%ymm14, %%ymm7, %%ymm7\n\t" /* ymm7 holds 3|4:(a0+a1)*(b0+b1) */ \ \ "vpxor %%ymm6, %%ymm3, %%ymm3\n\t" /* ymm3 holds 3+5+…+15|4+6+…+16:a0*b0 */ \ "vpxor %%ymm5, %%ymm1, %%ymm1\n\t" /* ymm1 holds 3+5+…+15|4+6+…+16:a1*b1 */ \ "vpxor %%ymm7, %%ymm4, %%ymm4\n\t" /* ymm4 holds 3+5+…+15|4+6+…+16:(a0+a1)*(b0+b1) */\ \ "vpshufd $78, %%ymm0, %%ymm14\n\t" \ "vpshufd $78, %%ymm2, %%ymm7\n\t" \ "vpxor %%ymm0, %%ymm14, %%ymm14\n\t" /* ymm14 holds 1|2:a0+a1 */ \ "vpxor %%ymm2, %%ymm7, %%ymm7\n\t" /* ymm7 holds 1|2:b0+b1 */ \ "vpclmulqdq $0, %%ymm2, %%ymm0, %%ymm6\n\t" /* ymm6 holds 1|2:a0*b0 */ \ "vpclmulqdq $17, %%ymm0, %%ymm2, %%ymm2\n\t" /* ymm2 holds 1|2:a1*b1 */ \ "vpclmulqdq $0, %%ymm14, %%ymm7, %%ymm7\n\t" /* ymm7 holds 1|2:(a0+a1)*(b0+b1) */ \ \ "vmovdqu %[h15_h16], %%ymm0\n\t" \ \ "vpxor %%ymm6, %%ymm3, %%ymm3\n\t" /* ymm3 holds 1+3+…+15|2+4+…+16:a0*b0 */ \ "vpxor %%ymm2, %%ymm1, %%ymm1\n\t" /* ymm1 holds 1+3+…+15|2+4+…+16:a1*b1 */ \ "vpxor %%ymm7, %%ymm4, %%ymm4\n\t" /* ymm4 holds 1+3+…+15|2+4+…+16:(a0+a1)*(b0+b1) */\ \ /* aggregated reduction... */ \ "vpxor %%ymm1, %%ymm3, %%ymm5\n\t" /* ymm5 holds a0*b0+a1*b1 */ \ "vpxor %%ymm5, %%ymm4, %%ymm4\n\t" /* ymm4 holds a0*b0+a1*b1+(a0+a1)*(b0+b1) */ \ "vpslldq $8, %%ymm4, %%ymm5\n\t" \ "vpsrldq $8, %%ymm4, %%ymm4\n\t" \ "vpxor %%ymm5, %%ymm3, %%ymm3\n\t" \ "vpxor %%ymm4, %%ymm1, %%ymm1\n\t" /* holds the result of the \ carry-less multiplication of ymm0 \ by ymm1 */ \ \ /* first phase of the reduction */ \ "vpsllq $1, %%ymm3, %%ymm6\n\t" /* packed right shifting << 63 */ \ "vpxor %%ymm3, %%ymm6, %%ymm6\n\t" \ "vpsllq $57, %%ymm3, %%ymm5\n\t" /* packed right shifting << 57 */ \ "vpsllq $62, %%ymm6, %%ymm6\n\t" /* packed right shifting << 62 */ \ "vpxor %%ymm5, %%ymm6, %%ymm6\n\t" /* xor the shifted versions */ \ "vpshufd $0x6a, %%ymm6, %%ymm5\n\t" \ "vpshufd $0xae, %%ymm6, %%ymm6\n\t" \ "vpxor %%ymm5, %%ymm3, %%ymm3\n\t" /* first phase of the reduction complete */ \ \ /* second phase of the reduction */ \ "vpxor %%ymm3, %%ymm1, %%ymm1\n\t" /* xor the shifted versions */ \ "vpsrlq $1, %%ymm3, %%ymm3\n\t" /* packed left shifting >> 1 */ \ "vpxor %%ymm3, %%ymm6, %%ymm6\n\t" \ "vpsrlq $1, %%ymm3, %%ymm3\n\t" /* packed left shifting >> 2 */ \ "vpxor %%ymm3, %%ymm1, %%ymm1\n\t" \ "vpsrlq $5, %%ymm3, %%ymm3\n\t" /* packed left shifting >> 7 */ \ "vpxor %%ymm3, %%ymm6, %%ymm6\n\t" \ "vpxor %%ymm6, %%ymm1, %%ymm1\n\t" /* the result is in ymm1 */ \ \ /* merge 128-bit halves */ \ "vextracti128 $1, %%ymm1, %%xmm2\n\t" \ "vpxor %%xmm2, %%xmm1, %%xmm1\n\t" static ASM_FUNC_ATTR_INLINE void gfmul_vpclmul_avx2_aggr16(const void *buf, const void *h_table, const u64 *h1_h2_h15_h16) { /* Input: Hx: YMM0, YMM8, YMM9, YMM10, YMM11, YMM12, YMM13 bemask: YMM15 Hash: XMM1 Output: Hash: XMM1 Inputs YMM0, YMM8, YMM9, YMM10, YMM11, YMM12, YMM13 and YMM15 stay unmodified. */ asm volatile (GFMUL_AGGR16_ASM_VPCMUL_AVX2(be_to_le) : : [buf] "r" (buf), [h_table] "r" (h_table), [h1_h2] "m" (h1_h2_h15_h16[0]), [h15_h16] "m" (h1_h2_h15_h16[4]) : "memory" ); } static ASM_FUNC_ATTR_INLINE void gfmul_vpclmul_avx2_aggr16_le(const void *buf, const void *h_table, const u64 *h1_h2_h15_h16) { /* Input: Hx: YMM0, YMM8, YMM9, YMM10, YMM11, YMM12, YMM13 bemask: YMM15 Hash: XMM1 Output: Hash: XMM1 Inputs YMM0, YMM8, YMM9, YMM10, YMM11, YMM12, YMM13 and YMM15 stay unmodified. */ asm volatile (GFMUL_AGGR16_ASM_VPCMUL_AVX2(le_to_le) : : [buf] "r" (buf), [h_table] "r" (h_table), [h1_h2] "m" (h1_h2_h15_h16[0]), [h15_h16] "m" (h1_h2_h15_h16[4]) : "memory" ); } static ASM_FUNC_ATTR_INLINE void gfmul_pclmul_avx2(void) { /* Input: YMM0 and YMM1, Output: YMM1. Input YMM0 stays unmodified. Input must be converted to little-endian. */ asm volatile (/* gfmul, ymm0 has operator a and ymm1 has operator b. */ "vpshufd $78, %%ymm0, %%ymm2\n\t" "vpshufd $78, %%ymm1, %%ymm4\n\t" "vpxor %%ymm0, %%ymm2, %%ymm2\n\t" /* ymm2 holds a0+a1 */ "vpxor %%ymm1, %%ymm4, %%ymm4\n\t" /* ymm4 holds b0+b1 */ "vpclmulqdq $0, %%ymm1, %%ymm0, %%ymm3\n\t" /* ymm3 holds a0*b0 */ "vpclmulqdq $17, %%ymm0, %%ymm1, %%ymm1\n\t" /* ymm6 holds a1*b1 */ "vpclmulqdq $0, %%ymm2, %%ymm4, %%ymm4\n\t" /* ymm4 holds (a0+a1)*(b0+b1) */ "vpxor %%ymm1, %%ymm3, %%ymm5\n\t" /* ymm5 holds a0*b0+a1*b1 */ "vpxor %%ymm5, %%ymm4, %%ymm4\n\t" /* ymm4 holds a0*b0+a1*b1+(a0+a1)*(b0+b1) */ "vpslldq $8, %%ymm4, %%ymm5\n\t" "vpsrldq $8, %%ymm4, %%ymm4\n\t" "vpxor %%ymm5, %%ymm3, %%ymm3\n\t" "vpxor %%ymm4, %%ymm1, %%ymm1\n\t" /* holds the result of the carry-less multiplication of ymm0 by ymm1 */ /* first phase of the reduction */ "vpsllq $1, %%ymm3, %%ymm6\n\t" /* packed right shifting << 63 */ "vpxor %%ymm3, %%ymm6, %%ymm6\n\t" "vpsllq $57, %%ymm3, %%ymm5\n\t" /* packed right shifting << 57 */ "vpsllq $62, %%ymm6, %%ymm6\n\t" /* packed right shifting << 62 */ "vpxor %%ymm5, %%ymm6, %%ymm6\n\t" /* xor the shifted versions */ "vpshufd $0x6a, %%ymm6, %%ymm5\n\t" "vpshufd $0xae, %%ymm6, %%ymm6\n\t" "vpxor %%ymm5, %%ymm3, %%ymm3\n\t" /* first phase of the reduction complete */ /* second phase of the reduction */ "vpxor %%ymm3, %%ymm1, %%ymm1\n\t" /* xor the shifted versions */ "vpsrlq $1, %%ymm3, %%ymm3\n\t" /* packed left shifting >> 1 */ "vpxor %%ymm3, %%ymm6, %%ymm6\n\t" "vpsrlq $1, %%ymm3, %%ymm3\n\t" /* packed left shifting >> 2 */ "vpxor %%ymm3, %%ymm1, %%ymm1\n\t" "vpsrlq $5, %%ymm3, %%ymm3\n\t" /* packed left shifting >> 7 */ "vpxor %%ymm3, %%ymm6, %%ymm6\n\t" "vpxor %%ymm6, %%ymm1, %%ymm1\n\t" /* the result is in ymm1 */ ::: "memory" ); } static ASM_FUNC_ATTR_INLINE void gcm_lsh_avx2(void *h, unsigned int hoffs) { static const u64 pconst[4] __attribute__ ((aligned (32))) = { U64_C(0x0000000000000001), U64_C(0xc200000000000000), U64_C(0x0000000000000001), U64_C(0xc200000000000000) }; asm volatile ("vmovdqu %[h], %%ymm2\n\t" "vpshufd $0xff, %%ymm2, %%ymm3\n\t" "vpsrad $31, %%ymm3, %%ymm3\n\t" "vpslldq $8, %%ymm2, %%ymm4\n\t" "vpand %[pconst], %%ymm3, %%ymm3\n\t" "vpaddq %%ymm2, %%ymm2, %%ymm2\n\t" "vpsrlq $63, %%ymm4, %%ymm4\n\t" "vpxor %%ymm3, %%ymm2, %%ymm2\n\t" "vpxor %%ymm4, %%ymm2, %%ymm2\n\t" "vmovdqu %%ymm2, %[h]\n\t" : [h] "+m" (*((byte *)h + hoffs)) : [pconst] "m" (*pconst) : "memory" ); } static ASM_FUNC_ATTR_INLINE void load_h1h2_to_ymm1(gcry_cipher_hd_t c) { unsigned int key_pos = offsetof(struct gcry_cipher_handle, u_mode.gcm.u_ghash_key.key); unsigned int table_pos = offsetof(struct gcry_cipher_handle, u_mode.gcm.gcm_table); if (key_pos + 16 == table_pos) { /* Optimization: Table follows immediately after key. */ asm volatile ("vmovdqu %[key], %%ymm1\n\t" : : [key] "m" (*c->u_mode.gcm.u_ghash_key.key) : "memory"); } else { asm volatile ("vmovdqa %[key], %%xmm1\n\t" "vinserti128 $1, 0*16(%[h_table]), %%ymm1, %%ymm1\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table), [key] "m" (*c->u_mode.gcm.u_ghash_key.key) : "memory"); } } static ASM_FUNC_ATTR void ghash_setup_aggr8_avx2(gcry_cipher_hd_t c) { c->u_mode.gcm.hw_impl_flags |= GCM_INTEL_AGGR8_TABLE_INITIALIZED; asm volatile (/* load H⁴ */ "vbroadcasti128 3*16(%[h_table]), %%ymm0\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); /* load H <<< 1, H² <<< 1 */ load_h1h2_to_ymm1 (c); gfmul_pclmul_avx2 (); /* H<<<1•H⁴ => H⁵, H²<<<1•H⁴ => H⁶ */ asm volatile ("vmovdqu %%ymm1, 3*16(%[h_table])\n\t" /* load H³ <<< 1, H⁴ <<< 1 */ "vmovdqu 1*16(%[h_table]), %%ymm1\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul_avx2 (); /* H³<<<1•H⁴ => H⁷, H⁴<<<1•H⁴ => H⁸ */ asm volatile ("vmovdqu %%ymm1, 6*16(%[h_table])\n\t" /* store H⁸ for aggr16 setup */ "vmovdqu %%ymm1, 5*16(%[h_table])\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gcm_lsh_avx2 (c->u_mode.gcm.gcm_table, 3 * 16); /* H⁵ <<< 1, H⁶ <<< 1 */ gcm_lsh_avx2 (c->u_mode.gcm.gcm_table, 5 * 16); /* H⁷ <<< 1, H⁸ <<< 1 */ } static ASM_FUNC_ATTR void ghash_setup_aggr16_avx2(gcry_cipher_hd_t c) { c->u_mode.gcm.hw_impl_flags |= GCM_INTEL_AGGR16_TABLE_INITIALIZED; asm volatile (/* load H⁸ */ "vbroadcasti128 7*16(%[h_table]), %%ymm0\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); /* load H <<< 1, H² <<< 1 */ load_h1h2_to_ymm1 (c); gfmul_pclmul_avx2 (); /* H<<<1•H⁸ => H⁹, H²<<<1•H⁸ => H¹⁰ */ asm volatile ("vmovdqu %%ymm1, 7*16(%[h_table])\n\t" /* load H³ <<< 1, H⁴ <<< 1 */ "vmovdqu 1*16(%[h_table]), %%ymm1\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul_avx2 (); /* H³<<<1•H⁸ => H¹¹, H⁴<<<1•H⁸ => H¹² */ asm volatile ("vmovdqu %%ymm1, 9*16(%[h_table])\n\t" /* load H⁵ <<< 1, H⁶ <<< 1 */ "vmovdqu 3*16(%[h_table]), %%ymm1\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul_avx2 (); /* H⁵<<<1•H⁸ => H¹³, H⁶<<<1•H⁸ => H¹⁴ */ asm volatile ("vmovdqu %%ymm1, 11*16(%[h_table])\n\t" /* load H⁷ <<< 1, H⁸ <<< 1 */ "vmovdqu 5*16(%[h_table]), %%ymm1\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul_avx2 (); /* H⁷<<<1•H⁸ => H¹⁵, H⁸<<<1•H⁸ => H¹⁶ */ asm volatile ("vmovdqu %%ymm1, 14*16(%[h_table])\n\t" /* store H¹⁶ for aggr32 setup */ "vmovdqu %%ymm1, 13*16(%[h_table])\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gcm_lsh_avx2 (c->u_mode.gcm.gcm_table, 7 * 16); /* H⁹ <<< 1, H¹⁰ <<< 1 */ gcm_lsh_avx2 (c->u_mode.gcm.gcm_table, 9 * 16); /* H¹¹ <<< 1, H¹² <<< 1 */ gcm_lsh_avx2 (c->u_mode.gcm.gcm_table, 11 * 16); /* H¹³ <<< 1, H¹⁴ <<< 1 */ gcm_lsh_avx2 (c->u_mode.gcm.gcm_table, 13 * 16); /* H¹⁵ <<< 1, H¹⁶ <<< 1 */ } #endif /* GCM_USE_INTEL_VPCLMUL_AVX2 */ #ifdef GCM_USE_INTEL_VPCLMUL_AVX512 #define GFMUL_AGGR32_ASM_VPCMUL_AVX512(be_to_le) \ /* perform clmul and merge results... */ \ "vmovdqu64 0*16(%[buf]), %%zmm5\n\t" \ "vmovdqu64 4*16(%[buf]), %%zmm2\n\t" \ be_to_le("vpshufb %%zmm15, %%zmm5, %%zmm5\n\t") /* be => le */ \ be_to_le("vpshufb %%zmm15, %%zmm2, %%zmm2\n\t") /* be => le */ \ "vpxorq %%zmm5, %%zmm1, %%zmm1\n\t" \ \ "vpshufd $78, %%zmm0, %%zmm5\n\t" \ "vpshufd $78, %%zmm1, %%zmm4\n\t" \ "vpxorq %%zmm0, %%zmm5, %%zmm5\n\t" /* zmm5 holds 29|…|32:a0+a1 */ \ "vpxorq %%zmm1, %%zmm4, %%zmm4\n\t" /* zmm4 holds 29|…|32:b0+b1 */ \ "vpclmulqdq $0, %%zmm1, %%zmm0, %%zmm3\n\t" /* zmm3 holds 29|…|32:a0*b0 */ \ "vpclmulqdq $17, %%zmm0, %%zmm1, %%zmm1\n\t" /* zmm1 holds 29|…|32:a1*b1 */ \ "vpclmulqdq $0, %%zmm5, %%zmm4, %%zmm4\n\t" /* zmm4 holds 29|…|32:(a0+a1)*(b0+b1) */ \ \ "vpshufd $78, %%zmm13, %%zmm14\n\t" \ "vpshufd $78, %%zmm2, %%zmm7\n\t" \ "vpxorq %%zmm13, %%zmm14, %%zmm14\n\t" /* zmm14 holds 25|…|28:a0+a1 */ \ "vpxorq %%zmm2, %%zmm7, %%zmm7\n\t" /* zmm7 holds 25|…|28:b0+b1 */ \ "vpclmulqdq $0, %%zmm2, %%zmm13, %%zmm17\n\t" /* zmm17 holds 25|…|28:a0*b0 */ \ "vpclmulqdq $17, %%zmm13, %%zmm2, %%zmm18\n\t" /* zmm18 holds 25|…|28:a1*b1 */ \ "vpclmulqdq $0, %%zmm14, %%zmm7, %%zmm19\n\t" /* zmm19 holds 25|…|28:(a0+a1)*(b0+b1) */\ \ "vmovdqu64 8*16(%[buf]), %%zmm5\n\t" \ "vmovdqu64 12*16(%[buf]), %%zmm2\n\t" \ be_to_le("vpshufb %%zmm15, %%zmm5, %%zmm5\n\t") /* be => le */ \ be_to_le("vpshufb %%zmm15, %%zmm2, %%zmm2\n\t") /* be => le */ \ \ "vpshufd $78, %%zmm12, %%zmm14\n\t" \ "vpshufd $78, %%zmm5, %%zmm7\n\t" \ "vpxorq %%zmm12, %%zmm14, %%zmm14\n\t" /* zmm14 holds 21|…|24:a0+a1 */ \ "vpxorq %%zmm5, %%zmm7, %%zmm7\n\t" /* zmm7 holds 21|…|24:b0+b1 */ \ "vpclmulqdq $0, %%zmm5, %%zmm12, %%zmm6\n\t" /* zmm6 holds 21|…|24:a0*b0 */ \ "vpclmulqdq $17, %%zmm12, %%zmm5, %%zmm5\n\t" /* zmm5 holds 21|…|24:a1*b1 */ \ "vpclmulqdq $0, %%zmm14, %%zmm7, %%zmm7\n\t" /* zmm7 holds 21|…|24:(a0+a1)*(b0+b1) */\ \ "vpternlogq $0x96, %%zmm6, %%zmm17, %%zmm3\n\t" /* zmm3 holds 21+…|…|…+32:a0*b0 */ \ "vpternlogq $0x96, %%zmm5, %%zmm18, %%zmm1\n\t" /* zmm1 holds 21+…|…|…+32:a1*b1 */ \ "vpternlogq $0x96, %%zmm7, %%zmm19, %%zmm4\n\t" /* zmm4 holds 21+…|…|…+32:(a0+a1)*(b0+b1) */\ \ "vpshufd $78, %%zmm11, %%zmm14\n\t" \ "vpshufd $78, %%zmm2, %%zmm7\n\t" \ "vpxorq %%zmm11, %%zmm14, %%zmm14\n\t" /* zmm14 holds 17|…|20:a0+a1 */ \ "vpxorq %%zmm2, %%zmm7, %%zmm7\n\t" /* zmm7 holds 17|…|20:b0+b1 */ \ "vpclmulqdq $0, %%zmm2, %%zmm11, %%zmm17\n\t" /* zmm17 holds 17|…|20:a0*b0 */ \ "vpclmulqdq $17, %%zmm11, %%zmm2, %%zmm18\n\t" /* zmm18 holds 17|…|20:a1*b1 */ \ "vpclmulqdq $0, %%zmm14, %%zmm7, %%zmm19\n\t" /* zmm19 holds 17|…|20:(a0+a1)*(b0+b1) */\ \ "vmovdqu64 16*16(%[buf]), %%zmm5\n\t" \ "vmovdqu64 20*16(%[buf]), %%zmm2\n\t" \ be_to_le("vpshufb %%zmm15, %%zmm5, %%zmm5\n\t") /* be => le */ \ be_to_le("vpshufb %%zmm15, %%zmm2, %%zmm2\n\t") /* be => le */ \ \ "vpshufd $78, %%zmm10, %%zmm14\n\t" \ "vpshufd $78, %%zmm5, %%zmm7\n\t" \ "vpxorq %%zmm10, %%zmm14, %%zmm14\n\t" /* zmm14 holds 13|…|16:a0+a1 */ \ "vpxorq %%zmm5, %%zmm7, %%zmm7\n\t" /* zmm7 holds 13|…|16:b0+b1 */ \ "vpclmulqdq $0, %%zmm5, %%zmm10, %%zmm6\n\t" /* zmm6 holds 13|…|16:a0*b0 */ \ "vpclmulqdq $17, %%zmm10, %%zmm5, %%zmm5\n\t" /* zmm5 holds 13|…|16:a1*b1 */ \ "vpclmulqdq $0, %%zmm14, %%zmm7, %%zmm7\n\t" /* zmm7 holds 13|…|16:(a0+a1)*(b0+b1) */ \ \ "vpternlogq $0x96, %%zmm6, %%zmm17, %%zmm3\n\t" /* zmm3 holds 13+…|…|…+32:a0*b0 */ \ "vpternlogq $0x96, %%zmm5, %%zmm18, %%zmm1\n\t" /* zmm1 holds 13+…|…|…+32:a1*b1 */ \ "vpternlogq $0x96, %%zmm7, %%zmm19, %%zmm4\n\t" /* zmm4 holds 13+…|…|…+32:(a0+a1)*(b0+b1) */\ \ "vpshufd $78, %%zmm9, %%zmm14\n\t" \ "vpshufd $78, %%zmm2, %%zmm7\n\t" \ "vpxorq %%zmm9, %%zmm14, %%zmm14\n\t" /* zmm14 holds 9|…|12:a0+a1 */ \ "vpxorq %%zmm2, %%zmm7, %%zmm7\n\t" /* zmm7 holds 9|…|12:b0+b1 */ \ "vpclmulqdq $0, %%zmm2, %%zmm9, %%zmm17\n\t" /* zmm17 holds 9|…|12:a0*b0 */ \ "vpclmulqdq $17, %%zmm9, %%zmm2, %%zmm18\n\t" /* zmm18 holds 9|…|12:a1*b1 */ \ "vpclmulqdq $0, %%zmm14, %%zmm7, %%zmm19\n\t" /* zmm19 holds 9|…|12:(a0+a1)*(b0+b1) */\ \ "vmovdqu64 24*16(%[buf]), %%zmm5\n\t" \ "vmovdqu64 28*16(%[buf]), %%zmm2\n\t" \ be_to_le("vpshufb %%zmm15, %%zmm5, %%zmm5\n\t") /* be => le */ \ be_to_le("vpshufb %%zmm15, %%zmm2, %%zmm2\n\t") /* be => le */ \ \ "vpshufd $78, %%zmm8, %%zmm14\n\t" \ "vpshufd $78, %%zmm5, %%zmm7\n\t" \ "vpxorq %%zmm8, %%zmm14, %%zmm14\n\t" /* zmm14 holds 5|…|8:a0+a1 */ \ "vpxorq %%zmm5, %%zmm7, %%zmm7\n\t" /* zmm7 holds 5|…|8:b0+b1 */ \ "vpclmulqdq $0, %%zmm5, %%zmm8, %%zmm6\n\t" /* zmm6 holds 5|…|8:a0*b0 */ \ "vpclmulqdq $17, %%zmm8, %%zmm5, %%zmm5\n\t" /* zmm5 holds 5|…|8:a1*b1 */ \ "vpclmulqdq $0, %%zmm14, %%zmm7, %%zmm7\n\t" /* zmm7 holds 5|…|8:(a0+a1)*(b0+b1) */ \ \ "vpternlogq $0x96, %%zmm6, %%zmm17, %%zmm3\n\t" /* zmm3 holds 5+…|…|…+32:a0*b0 */ \ "vpternlogq $0x96, %%zmm5, %%zmm18, %%zmm1\n\t" /* zmm1 holds 5+…|…|…+32:a1*b1 */ \ "vpternlogq $0x96, %%zmm7, %%zmm19, %%zmm4\n\t" /* zmm4 holds 5+…|…|…+32:(a0+a1)*(b0+b1) */\ \ "vpshufd $78, %%zmm16, %%zmm14\n\t" \ "vpshufd $78, %%zmm2, %%zmm7\n\t" \ "vpxorq %%zmm16, %%zmm14, %%zmm14\n\t" /* zmm14 holds 1|…|4:a0+a1 */ \ "vpxorq %%zmm2, %%zmm7, %%zmm7\n\t" /* zmm7 holds 1|2:b0+b1 */ \ "vpclmulqdq $0, %%zmm2, %%zmm16, %%zmm6\n\t" /* zmm6 holds 1|2:a0*b0 */ \ "vpclmulqdq $17, %%zmm16, %%zmm2, %%zmm2\n\t" /* zmm2 holds 1|2:a1*b1 */ \ "vpclmulqdq $0, %%zmm14, %%zmm7, %%zmm7\n\t" /* zmm7 holds 1|2:(a0+a1)*(b0+b1) */ \ \ "vpxorq %%zmm6, %%zmm3, %%zmm3\n\t" /* zmm3 holds 1+3+…+15|2+4+…+16:a0*b0 */ \ "vpxorq %%zmm2, %%zmm1, %%zmm1\n\t" /* zmm1 holds 1+3+…+15|2+4+…+16:a1*b1 */ \ "vpxorq %%zmm7, %%zmm4, %%zmm4\n\t" /* zmm4 holds 1+3+…+15|2+4+…+16:(a0+a1)*(b0+b1) */\ \ /* aggregated reduction... */ \ "vpternlogq $0x96, %%zmm1, %%zmm3, %%zmm4\n\t" /* zmm4 holds \ * a0*b0+a1*b1+(a0+a1)*(b0+b1) */ \ "vpslldq $8, %%zmm4, %%zmm5\n\t" \ "vpsrldq $8, %%zmm4, %%zmm4\n\t" \ "vpxorq %%zmm5, %%zmm3, %%zmm3\n\t" \ "vpxorq %%zmm4, %%zmm1, %%zmm1\n\t" /* holds the result of the \ carry-less multiplication of zmm0 \ by zmm1 */ \ \ /* first phase of the reduction */ \ "vpsllq $1, %%zmm3, %%zmm6\n\t" /* packed right shifting << 63 */ \ "vpxorq %%zmm3, %%zmm6, %%zmm6\n\t" \ "vpsllq $57, %%zmm3, %%zmm5\n\t" /* packed right shifting << 57 */ \ "vpsllq $62, %%zmm6, %%zmm6\n\t" /* packed right shifting << 62 */ \ "vpxorq %%zmm5, %%zmm6, %%zmm6\n\t" /* xor the shifted versions */ \ "vpshufd $0x6a, %%zmm6, %%zmm5\n\t" \ "vpshufd $0xae, %%zmm6, %%zmm6\n\t" \ "vpxorq %%zmm5, %%zmm3, %%zmm3\n\t" /* first phase of the reduction complete */ \ \ /* second phase of the reduction */ \ "vpsrlq $1, %%zmm3, %%zmm2\n\t" /* packed left shifting >> 1 */ \ "vpsrlq $2, %%zmm3, %%zmm4\n\t" /* packed left shifting >> 2 */ \ "vpsrlq $7, %%zmm3, %%zmm5\n\t" /* packed left shifting >> 7 */ \ "vpternlogq $0x96, %%zmm3, %%zmm2, %%zmm1\n\t" /* xor the shifted versions */ \ "vpternlogq $0x96, %%zmm4, %%zmm5, %%zmm6\n\t" \ "vpxorq %%zmm6, %%zmm1, %%zmm1\n\t" /* the result is in zmm1 */ \ \ /* merge 256-bit halves */ \ "vextracti64x4 $1, %%zmm1, %%ymm2\n\t" \ "vpxor %%ymm2, %%ymm1, %%ymm1\n\t" \ /* merge 128-bit halves */ \ "vextracti128 $1, %%ymm1, %%xmm2\n\t" \ "vpxor %%xmm2, %%xmm1, %%xmm1\n\t" static ASM_FUNC_ATTR_INLINE void gfmul_vpclmul_avx512_aggr32(const void *buf, const void *h_table) { /* Input: Hx: ZMM0, ZMM8, ZMM9, ZMM10, ZMM11, ZMM12, ZMM13, ZMM16 bemask: ZMM15 Hash: XMM1 Output: Hash: XMM1 Inputs ZMM0, ZMM8, ZMM9, ZMM10, ZMM11, ZMM12, ZMM13, ZMM16 and YMM15 stay unmodified. */ asm volatile (GFMUL_AGGR32_ASM_VPCMUL_AVX512(be_to_le) : : [buf] "r" (buf), [h_table] "r" (h_table) : "memory" ); } static ASM_FUNC_ATTR_INLINE void gfmul_vpclmul_avx512_aggr32_le(const void *buf, const void *h_table) { /* Input: Hx: ZMM0, ZMM8, ZMM9, ZMM10, ZMM11, ZMM12, ZMM13, ZMM16 bemask: ZMM15 Hash: XMM1 Output: Hash: XMM1 Inputs ZMM0, ZMM8, ZMM9, ZMM10, ZMM11, ZMM12, ZMM13, ZMM16 and YMM15 stay unmodified. */ asm volatile (GFMUL_AGGR32_ASM_VPCMUL_AVX512(le_to_le) : : [buf] "r" (buf), [h_table] "r" (h_table) : "memory" ); } static ASM_FUNC_ATTR_INLINE void gfmul_pclmul_avx512(void) { /* Input: ZMM0 and ZMM1, Output: ZMM1. Input ZMM0 stays unmodified. Input must be converted to little-endian. */ asm volatile (/* gfmul, zmm0 has operator a and zmm1 has operator b. */ "vpshufd $78, %%zmm0, %%zmm2\n\t" "vpshufd $78, %%zmm1, %%zmm4\n\t" "vpxorq %%zmm0, %%zmm2, %%zmm2\n\t" /* zmm2 holds a0+a1 */ "vpxorq %%zmm1, %%zmm4, %%zmm4\n\t" /* zmm4 holds b0+b1 */ "vpclmulqdq $0, %%zmm1, %%zmm0, %%zmm3\n\t" /* zmm3 holds a0*b0 */ "vpclmulqdq $17, %%zmm0, %%zmm1, %%zmm1\n\t" /* zmm6 holds a1*b1 */ "vpclmulqdq $0, %%zmm2, %%zmm4, %%zmm4\n\t" /* zmm4 holds (a0+a1)*(b0+b1) */ "vpternlogq $0x96, %%zmm1, %%zmm3, %%zmm4\n\t" /* zmm4 holds * a0*b0+a1*b1+(a0+a1)*(b0+b1) */ "vpslldq $8, %%zmm4, %%zmm5\n\t" "vpsrldq $8, %%zmm4, %%zmm4\n\t" "vpxorq %%zmm5, %%zmm3, %%zmm3\n\t" "vpxorq %%zmm4, %%zmm1, %%zmm1\n\t" /* holds the result of the carry-less multiplication of zmm0 by zmm1 */ /* first phase of the reduction */ "vpsllq $1, %%zmm3, %%zmm6\n\t" /* packed right shifting << 63 */ "vpxorq %%zmm3, %%zmm6, %%zmm6\n\t" "vpsllq $57, %%zmm3, %%zmm5\n\t" /* packed right shifting << 57 */ "vpsllq $62, %%zmm6, %%zmm6\n\t" /* packed right shifting << 62 */ "vpxorq %%zmm5, %%zmm6, %%zmm6\n\t" /* xor the shifted versions */ "vpshufd $0x6a, %%zmm6, %%zmm5\n\t" "vpshufd $0xae, %%zmm6, %%zmm6\n\t" "vpxorq %%zmm5, %%zmm3, %%zmm3\n\t" /* first phase of the reduction complete */ /* second phase of the reduction */ "vpsrlq $1, %%zmm3, %%zmm2\n\t" /* packed left shifting >> 1 */ "vpsrlq $2, %%zmm3, %%zmm4\n\t" /* packed left shifting >> 2 */ "vpsrlq $7, %%zmm3, %%zmm5\n\t" /* packed left shifting >> 7 */ "vpternlogq $0x96, %%zmm3, %%zmm2, %%zmm1\n\t" /* xor the shifted versions */ "vpternlogq $0x96, %%zmm4, %%zmm5, %%zmm6\n\t" "vpxorq %%zmm6, %%zmm1, %%zmm1\n\t" /* the result is in zmm1 */ ::: "memory" ); } static ASM_FUNC_ATTR_INLINE void gcm_lsh_avx512(void *h, unsigned int hoffs) { static const u64 pconst[8] __attribute__ ((aligned (64))) = { U64_C(0x0000000000000001), U64_C(0xc200000000000000), U64_C(0x0000000000000001), U64_C(0xc200000000000000), U64_C(0x0000000000000001), U64_C(0xc200000000000000), U64_C(0x0000000000000001), U64_C(0xc200000000000000) }; asm volatile ("vmovdqu64 %[h], %%zmm2\n\t" "vpshufd $0xff, %%zmm2, %%zmm3\n\t" "vpsrad $31, %%zmm3, %%zmm3\n\t" "vpslldq $8, %%zmm2, %%zmm4\n\t" "vpandq %[pconst], %%zmm3, %%zmm3\n\t" "vpaddq %%zmm2, %%zmm2, %%zmm2\n\t" "vpsrlq $63, %%zmm4, %%zmm4\n\t" "vpternlogq $0x96, %%zmm4, %%zmm3, %%zmm2\n\t" "vmovdqu64 %%zmm2, %[h]\n\t" : [h] "+m" (*((byte *)h + hoffs)) : [pconst] "m" (*pconst) : "memory" ); } static ASM_FUNC_ATTR_INLINE void load_h1h4_to_zmm1(gcry_cipher_hd_t c) { unsigned int key_pos = offsetof(struct gcry_cipher_handle, u_mode.gcm.u_ghash_key.key); unsigned int table_pos = offsetof(struct gcry_cipher_handle, u_mode.gcm.gcm_table); if (key_pos + 16 == table_pos) { /* Optimization: Table follows immediately after key. */ asm volatile ("vmovdqu64 %[key], %%zmm1\n\t" : : [key] "m" (*c->u_mode.gcm.u_ghash_key.key) : "memory"); } else { asm volatile ("vmovdqu64 -1*16(%[h_table]), %%zmm1\n\t" "vinserti64x2 $0, %[key], %%zmm1, %%zmm1\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table), [key] "m" (*c->u_mode.gcm.u_ghash_key.key) : "memory"); } } static ASM_FUNC_ATTR void ghash_setup_aggr8_avx512(gcry_cipher_hd_t c) { c->u_mode.gcm.hw_impl_flags |= GCM_INTEL_AGGR8_TABLE_INITIALIZED; asm volatile (/* load H⁴ */ "vbroadcasti64x2 3*16(%[h_table]), %%zmm0\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); /* load H <<< 1, H² <<< 1, H³ <<< 1, H⁴ <<< 1 */ load_h1h4_to_zmm1 (c); gfmul_pclmul_avx512 (); /* H<<<1•H⁴ => H⁵, …, H⁴<<<1•H⁴ => H⁸ */ asm volatile ("vmovdqu64 %%zmm1, 4*16(%[h_table])\n\t" /* store H⁸ for aggr16 setup */ "vmovdqu64 %%zmm1, 3*16(%[h_table])\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gcm_lsh_avx512 (c->u_mode.gcm.gcm_table, 3 * 16); /* H⁵ <<< 1, …, H⁸ <<< 1 */ } static ASM_FUNC_ATTR void ghash_setup_aggr16_avx512(gcry_cipher_hd_t c) { c->u_mode.gcm.hw_impl_flags |= GCM_INTEL_AGGR16_TABLE_INITIALIZED; asm volatile (/* load H⁸ */ "vbroadcasti64x2 7*16(%[h_table]), %%zmm0\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); /* load H <<< 1, H² <<< 1, H³ <<< 1, H⁴ <<< 1 */ load_h1h4_to_zmm1 (c); gfmul_pclmul_avx512 (); /* H<<<1•H⁸ => H⁹, … , H⁴<<<1•H⁸ => H¹² */ asm volatile ("vmovdqu64 %%zmm1, 7*16(%[h_table])\n\t" /* load H⁵ <<< 1, …, H⁸ <<< 1 */ "vmovdqu64 3*16(%[h_table]), %%zmm1\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul_avx512 (); /* H⁵<<<1•H⁸ => H¹¹, … , H⁸<<<1•H⁸ => H¹⁶ */ asm volatile ("vmovdqu64 %%zmm1, 12*16(%[h_table])\n\t" /* store H¹⁶ for aggr32 setup */ "vmovdqu64 %%zmm1, 11*16(%[h_table])\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gcm_lsh_avx512 (c->u_mode.gcm.gcm_table, 7 * 16); /* H⁹ <<< 1, …, H¹² <<< 1 */ gcm_lsh_avx512 (c->u_mode.gcm.gcm_table, 11 * 16); /* H¹³ <<< 1, …, H¹⁶ <<< 1 */ } static ASM_FUNC_ATTR void ghash_setup_aggr32_avx512(gcry_cipher_hd_t c) { c->u_mode.gcm.hw_impl_flags |= GCM_INTEL_AGGR32_TABLE_INITIALIZED; asm volatile (/* load H¹⁶ */ "vbroadcasti64x2 15*16(%[h_table]), %%zmm0\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); /* load H <<< 1, H² <<< 1, H³ <<< 1, H⁴ <<< 1 */ load_h1h4_to_zmm1 (c); gfmul_pclmul_avx512 (); /* H<<<1•H¹⁶ => H¹⁷, …, H⁴<<<1•H¹⁶ => H²⁰ */ asm volatile ("vmovdqu64 %%zmm1, 15*16(%[h_table])\n\t" /* load H⁵ <<< 1, …, H⁸ <<< 1 */ "vmovdqu64 3*16(%[h_table]), %%zmm1\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul_avx512 (); /* H⁵<<<1•H¹⁶ => H²¹, …, H⁹<<<1•H¹⁶ => H²⁴ */ asm volatile ("vmovdqu64 %%zmm1, 19*16(%[h_table])\n\t" /* load H⁹ <<< 1, …, H¹² <<< 1 */ "vmovdqu64 7*16(%[h_table]), %%zmm1\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul_avx512 (); /* H⁹<<<1•H¹⁶ => H²⁵, …, H¹²<<<1•H¹⁶ => H²⁸ */ asm volatile ("vmovdqu64 %%zmm1, 23*16(%[h_table])\n\t" /* load H¹³ <<< 1, …, H¹⁶ <<< 1 */ "vmovdqu64 11*16(%[h_table]), %%zmm1\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul_avx512 (); /* H¹³<<<1•H¹⁶ => H²⁹, …, H¹⁶<<<1•H¹⁶ => H³² */ asm volatile ("vmovdqu64 %%zmm1, 27*16(%[h_table])\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gcm_lsh_avx512 (c->u_mode.gcm.gcm_table, 15 * 16); gcm_lsh_avx512 (c->u_mode.gcm.gcm_table, 19 * 16); gcm_lsh_avx512 (c->u_mode.gcm.gcm_table, 23 * 16); gcm_lsh_avx512 (c->u_mode.gcm.gcm_table, 27 * 16); } static const u64 swap128b_perm[8] __attribute__ ((aligned (64))) = { /* For swapping order of 128bit lanes in 512bit register using vpermq. */ 6, 7, 4, 5, 2, 3, 0, 1 }; #endif /* GCM_USE_INTEL_VPCLMUL_AVX512 */ #endif /* __x86_64__ */ static unsigned int ASM_FUNC_ATTR _gcry_ghash_intel_pclmul (gcry_cipher_hd_t c, byte *result, const byte *buf, size_t nblocks); static unsigned int ASM_FUNC_ATTR _gcry_polyval_intel_pclmul (gcry_cipher_hd_t c, byte *result, const byte *buf, size_t nblocks); static ASM_FUNC_ATTR_INLINE void gcm_lsh(void *h, unsigned int hoffs) { static const u64 pconst[2] __attribute__ ((aligned (16))) = { U64_C(0x0000000000000001), U64_C(0xc200000000000000) }; asm volatile ("movdqu %[h], %%xmm2\n\t" "pshufd $0xff, %%xmm2, %%xmm3\n\t" "movdqa %%xmm2, %%xmm4\n\t" "psrad $31, %%xmm3\n\t" "pslldq $8, %%xmm4\n\t" "pand %[pconst], %%xmm3\n\t" "paddq %%xmm2, %%xmm2\n\t" "psrlq $63, %%xmm4\n\t" "pxor %%xmm3, %%xmm2\n\t" "pxor %%xmm4, %%xmm2\n\t" "movdqu %%xmm2, %[h]\n\t" : [h] "+m" (*((byte *)h + hoffs)) : [pconst] "m" (*pconst) : "memory" ); } void ASM_FUNC_ATTR _gcry_ghash_setup_intel_pclmul (gcry_cipher_hd_t c, unsigned int hw_features) { static const unsigned char be_mask[16] __attribute__ ((aligned (16))) = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }; #if defined(__x86_64__) && defined(__WIN64__) char win64tmp[10 * 16]; /* XMM6-XMM15 need to be restored after use. */ asm volatile ("movdqu %%xmm6, 0*16(%0)\n\t" "movdqu %%xmm7, 1*16(%0)\n\t" "movdqu %%xmm8, 2*16(%0)\n\t" "movdqu %%xmm9, 3*16(%0)\n\t" "movdqu %%xmm10, 4*16(%0)\n\t" "movdqu %%xmm11, 5*16(%0)\n\t" "movdqu %%xmm12, 6*16(%0)\n\t" "movdqu %%xmm13, 7*16(%0)\n\t" "movdqu %%xmm14, 8*16(%0)\n\t" "movdqu %%xmm15, 9*16(%0)\n\t" : : "r" (win64tmp) : "memory" ); #endif (void)hw_features; c->u_mode.gcm.hw_impl_flags = 0; c->u_mode.gcm.ghash_fn = _gcry_ghash_intel_pclmul; c->u_mode.gcm.polyval_fn = _gcry_polyval_intel_pclmul; /* Swap endianness of hsub. */ asm volatile ("movdqu (%[key]), %%xmm0\n\t" "pshufb %[be_mask], %%xmm0\n\t" "movdqu %%xmm0, (%[key])\n\t" : : [key] "r" (c->u_mode.gcm.u_ghash_key.key), [be_mask] "m" (*be_mask) : "memory"); gcm_lsh (c->u_mode.gcm.u_ghash_key.key, 0); /* H <<< 1 */ asm volatile ("movdqa %%xmm0, %%xmm1\n\t" "movdqu (%[key]), %%xmm0\n\t" /* load H <<< 1 */ : : [key] "r" (c->u_mode.gcm.u_ghash_key.key) : "memory"); gfmul_pclmul (); /* H<<<1•H => H² */ asm volatile ("movdqu %%xmm1, 0*16(%[h_table])\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gcm_lsh (c->u_mode.gcm.gcm_table, 0 * 16); /* H² <<< 1 */ if (0) { } #ifdef GCM_USE_INTEL_VPCLMUL_AVX2 else if ((hw_features & HWF_INTEL_VAES_VPCLMUL) && (hw_features & HWF_INTEL_AVX2)) { c->u_mode.gcm.hw_impl_flags |= GCM_INTEL_USE_VPCLMUL_AVX2; #ifdef GCM_USE_INTEL_VPCLMUL_AVX512 if (hw_features & HWF_INTEL_AVX512) c->u_mode.gcm.hw_impl_flags |= GCM_INTEL_USE_VPCLMUL_AVX512; #endif asm volatile (/* H² */ "vinserti128 $1, %%xmm1, %%ymm1, %%ymm1\n\t" /* load H <<< 1, H² <<< 1 */ "vinserti128 $1, 0*16(%[h_table]), %%ymm0, %%ymm0\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul_avx2 (); /* H<<<1•H² => H³, H²<<<1•H² => H⁴ */ asm volatile ("vmovdqu %%ymm1, 2*16(%[h_table])\n\t" /* store H⁴ for aggr8 setup */ "vmovdqu %%ymm1, 1*16(%[h_table])\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gcm_lsh_avx2 (c->u_mode.gcm.gcm_table, 1 * 16); /* H³ <<< 1, H⁴ <<< 1 */ asm volatile ("vzeroupper\n\t" ::: "memory" ); } #endif /* GCM_USE_INTEL_VPCLMUL_AVX2 */ else { asm volatile ("movdqa %%xmm1, %%xmm7\n\t" ::: "memory"); gfmul_pclmul (); /* H<<<1•H² => H³ */ asm volatile ("movdqa %%xmm7, %%xmm0\n\t" "movdqu %%xmm1, 1*16(%[h_table])\n\t" "movdqu 0*16(%[h_table]), %%xmm1\n\t" /* load H² <<< 1 */ : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul (); /* H²<<<1•H² => H⁴ */ asm volatile ("movdqu %%xmm1, 3*16(%[h_table])\n\t" /* store H⁴ for aggr8 setup */ "movdqu %%xmm1, 2*16(%[h_table])\n\t" : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gcm_lsh (c->u_mode.gcm.gcm_table, 1 * 16); /* H³ <<< 1 */ gcm_lsh (c->u_mode.gcm.gcm_table, 2 * 16); /* H⁴ <<< 1 */ } /* Clear/restore used registers. */ asm volatile ("pxor %%xmm0, %%xmm0\n\t" "pxor %%xmm1, %%xmm1\n\t" "pxor %%xmm2, %%xmm2\n\t" "pxor %%xmm3, %%xmm3\n\t" "pxor %%xmm4, %%xmm4\n\t" "pxor %%xmm5, %%xmm5\n\t" "pxor %%xmm6, %%xmm6\n\t" "pxor %%xmm7, %%xmm7\n\t" ::: "memory" ); #ifdef __x86_64__ #ifdef __WIN64__ asm volatile ("movdqu 0*16(%0), %%xmm6\n\t" "movdqu 1*16(%0), %%xmm7\n\t" "movdqu 2*16(%0), %%xmm8\n\t" "movdqu 3*16(%0), %%xmm9\n\t" "movdqu 4*16(%0), %%xmm10\n\t" "movdqu 5*16(%0), %%xmm11\n\t" "movdqu 6*16(%0), %%xmm12\n\t" "movdqu 7*16(%0), %%xmm13\n\t" "movdqu 8*16(%0), %%xmm14\n\t" "movdqu 9*16(%0), %%xmm15\n\t" : : "r" (win64tmp) : "memory" ); #else asm volatile ("pxor %%xmm8, %%xmm8\n\t" "pxor %%xmm9, %%xmm9\n\t" "pxor %%xmm10, %%xmm10\n\t" "pxor %%xmm11, %%xmm11\n\t" "pxor %%xmm12, %%xmm12\n\t" "pxor %%xmm13, %%xmm13\n\t" "pxor %%xmm14, %%xmm14\n\t" "pxor %%xmm15, %%xmm15\n\t" ::: "memory" ); #endif /* __WIN64__ */ #endif /* __x86_64__ */ } #ifdef __x86_64__ static ASM_FUNC_ATTR void ghash_setup_aggr8(gcry_cipher_hd_t c) { c->u_mode.gcm.hw_impl_flags |= GCM_INTEL_AGGR8_TABLE_INITIALIZED; asm volatile ("movdqa 3*16(%[h_table]), %%xmm0\n\t" /* load H⁴ */ "movdqu %[key], %%xmm1\n\t" /* load H <<< 1 */ : : [h_table] "r" (c->u_mode.gcm.gcm_table), [key] "m" (*c->u_mode.gcm.u_ghash_key.key) : "memory"); gfmul_pclmul (); /* H<<<1•H⁴ => H⁵ */ asm volatile ("movdqu %%xmm1, 3*16(%[h_table])\n\t" "movdqu 0*16(%[h_table]), %%xmm1\n\t" /* load H² <<< 1 */ : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul (); /* H²<<<1•H⁴ => H⁶ */ asm volatile ("movdqu %%xmm1, 4*16(%[h_table])\n\t" "movdqu 1*16(%[h_table]), %%xmm1\n\t" /* load H³ <<< 1 */ : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul (); /* H³<<<1•H⁴ => H⁷ */ asm volatile ("movdqu %%xmm1, 5*16(%[h_table])\n\t" "movdqu 2*16(%[h_table]), %%xmm1\n\t" /* load H⁴ <<< 1 */ : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gfmul_pclmul (); /* H⁴<<<1•H⁴ => H⁸ */ asm volatile ("movdqu %%xmm1, 6*16(%[h_table])\n\t" "movdqu %%xmm1, 7*16(%[h_table])\n\t" /* store H⁸ for aggr16 setup */ : : [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory"); gcm_lsh (c->u_mode.gcm.gcm_table, 3 * 16); /* H⁵ <<< 1 */ gcm_lsh (c->u_mode.gcm.gcm_table, 4 * 16); /* H⁶ <<< 1 */ gcm_lsh (c->u_mode.gcm.gcm_table, 5 * 16); /* H⁷ <<< 1 */ gcm_lsh (c->u_mode.gcm.gcm_table, 6 * 16); /* H⁸ <<< 1 */ } #endif /* __x86_64__ */ unsigned int ASM_FUNC_ATTR _gcry_ghash_intel_pclmul (gcry_cipher_hd_t c, byte *result, const byte *buf, size_t nblocks) { static const unsigned char be_mask[16] __attribute__ ((aligned (16))) = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }; const unsigned int blocksize = GCRY_GCM_BLOCK_LEN; #if defined(__x86_64__) && defined(__WIN64__) char win64tmp[10 * 16]; #endif if (nblocks == 0) return 0; #if defined(__x86_64__) && defined(__WIN64__) /* XMM6-XMM15 need to be restored after use. */ asm volatile ("movdqu %%xmm6, 0*16(%0)\n\t" "movdqu %%xmm7, 1*16(%0)\n\t" "movdqu %%xmm8, 2*16(%0)\n\t" "movdqu %%xmm9, 3*16(%0)\n\t" "movdqu %%xmm10, 4*16(%0)\n\t" "movdqu %%xmm11, 5*16(%0)\n\t" "movdqu %%xmm12, 6*16(%0)\n\t" "movdqu %%xmm13, 7*16(%0)\n\t" "movdqu %%xmm14, 8*16(%0)\n\t" "movdqu %%xmm15, 9*16(%0)\n\t" : : "r" (win64tmp) : "memory" ); #endif /* Preload hash. */ asm volatile ("movdqa %[be_mask], %%xmm7\n\t" "movdqu %[hash], %%xmm1\n\t" "pshufb %%xmm7, %%xmm1\n\t" /* be => le */ : : [hash] "m" (*result), [be_mask] "m" (*be_mask) : "memory" ); #if defined(GCM_USE_INTEL_VPCLMUL_AVX2) if (nblocks >= 16 && ((c->u_mode.gcm.hw_impl_flags & GCM_INTEL_USE_VPCLMUL_AVX2) || (c->u_mode.gcm.hw_impl_flags & GCM_INTEL_USE_VPCLMUL_AVX512))) { #if defined(GCM_USE_INTEL_VPCLMUL_AVX512) if (nblocks >= 32 && (c->u_mode.gcm.hw_impl_flags & GCM_INTEL_USE_VPCLMUL_AVX512)) { asm volatile ("vpopcntb %%xmm7, %%xmm16\n\t" /* spec stop for old AVX512 CPUs */ "vshufi64x2 $0, %%zmm7, %%zmm7, %%zmm15\n\t" "vmovdqa %%xmm1, %%xmm8\n\t" "vmovdqu64 %[swapperm], %%zmm14\n\t" : : [swapperm] "m" (swap128b_perm), [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory" ); if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR32_TABLE_INITIALIZED)) { if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR16_TABLE_INITIALIZED)) { if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR8_TABLE_INITIALIZED)) ghash_setup_aggr8_avx512 (c); /* Clobbers registers XMM0-XMM7. */ ghash_setup_aggr16_avx512 (c); /* Clobbers registers XMM0-XMM7. */ } ghash_setup_aggr32_avx512 (c); /* Clobbers registers XMM0-XMM7. */ } /* Preload H1-H32. */ load_h1h4_to_zmm1 (c); asm volatile ("vpermq %%zmm1, %%zmm14, %%zmm16\n\t" /* H1|H2|H3|H4 */ "vmovdqa %%xmm8, %%xmm1\n\t" "vpermq 27*16(%[h_table]), %%zmm14, %%zmm0\n\t" /* H28|H29|H31|H32 */ "vpermq 23*16(%[h_table]), %%zmm14, %%zmm13\n\t" /* H25|H26|H27|H28 */ "vpermq 19*16(%[h_table]), %%zmm14, %%zmm12\n\t" /* H21|H22|H23|H24 */ "vpermq 15*16(%[h_table]), %%zmm14, %%zmm11\n\t" /* H17|H18|H19|H20 */ "vpermq 11*16(%[h_table]), %%zmm14, %%zmm10\n\t" /* H13|H14|H15|H16 */ "vpermq 7*16(%[h_table]), %%zmm14, %%zmm9\n\t" /* H9|H10|H11|H12 */ "vpermq 3*16(%[h_table]), %%zmm14, %%zmm8\n\t" /* H4|H6|H7|H8 */ : : [h_1] "m" (*c->u_mode.gcm.u_ghash_key.key), [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory" ); while (nblocks >= 32) { gfmul_vpclmul_avx512_aggr32 (buf, c->u_mode.gcm.gcm_table); buf += 32 * blocksize; nblocks -= 32; } asm volatile ("vmovdqa %%xmm15, %%xmm7\n\t" - "vpxorq %%zmm16, %%zmm16, %%zmm16\n\t" - "vpxorq %%zmm17, %%zmm17, %%zmm17\n\t" - "vpxorq %%zmm18, %%zmm18, %%zmm18\n\t" - "vpxorq %%zmm19, %%zmm19, %%zmm19\n\t" + "vpxorq %%ymm16, %%ymm16, %%ymm16\n\t" + "vpxorq %%ymm17, %%ymm17, %%ymm17\n\t" + "vpxorq %%ymm18, %%ymm18, %%ymm18\n\t" + "vpxorq %%ymm19, %%ymm19, %%ymm19\n\t" : : : "memory" ); } #endif /* GCM_USE_INTEL_VPCLMUL_AVX512 */ if (nblocks >= 16) { u64 h1_h2_h15_h16[4*2]; asm volatile ("vinserti128 $1, %%xmm7, %%ymm7, %%ymm15\n\t" "vmovdqa %%xmm1, %%xmm8\n\t" ::: "memory" ); if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR16_TABLE_INITIALIZED)) { if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR8_TABLE_INITIALIZED)) ghash_setup_aggr8_avx2 (c); /* Clobbers registers XMM0-XMM7. */ ghash_setup_aggr16_avx2 (c); /* Clobbers registers XMM0-XMM7. */ } /* Preload H1-H16. */ load_h1h2_to_ymm1 (c); asm volatile ("vperm2i128 $0x23, %%ymm1, %%ymm1, %%ymm7\n\t" /* H1|H2 */ "vmovdqa %%xmm8, %%xmm1\n\t" "vpxor %%xmm8, %%xmm8, %%xmm8\n\t" "vperm2i128 $0x23, 13*16(%[h_table]), %%ymm8, %%ymm0\n\t" /* H15|H16 */ "vperm2i128 $0x23, 11*16(%[h_table]), %%ymm8, %%ymm13\n\t" /* H13|H14 */ "vperm2i128 $0x23, 9*16(%[h_table]), %%ymm8, %%ymm12\n\t" /* H11|H12 */ "vperm2i128 $0x23, 7*16(%[h_table]), %%ymm8, %%ymm11\n\t" /* H9|H10 */ "vperm2i128 $0x23, 5*16(%[h_table]), %%ymm8, %%ymm10\n\t" /* H7|H8 */ "vperm2i128 $0x23, 3*16(%[h_table]), %%ymm8, %%ymm9\n\t" /* H5|H6 */ "vperm2i128 $0x23, 1*16(%[h_table]), %%ymm8, %%ymm8\n\t" /* H3|H4 */ "vmovdqu %%ymm0, %[h15_h16]\n\t" "vmovdqu %%ymm7, %[h1_h2]\n\t" : [h1_h2] "=m" (h1_h2_h15_h16[0]), [h15_h16] "=m" (h1_h2_h15_h16[4]) : [h_1] "m" (*c->u_mode.gcm.u_ghash_key.key), [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory" ); while (nblocks >= 16) { gfmul_vpclmul_avx2_aggr16 (buf, c->u_mode.gcm.gcm_table, h1_h2_h15_h16); buf += 16 * blocksize; nblocks -= 16; } asm volatile ("vmovdqu %%ymm15, %[h15_h16]\n\t" "vmovdqu %%ymm15, %[h1_h2]\n\t" "vmovdqa %%xmm15, %%xmm7\n\t" : [h1_h2] "=m" (h1_h2_h15_h16[0]), [h15_h16] "=m" (h1_h2_h15_h16[4]) : : "memory" ); } asm volatile ("vzeroupper\n\t" ::: "memory" ); } #endif /* GCM_USE_INTEL_VPCLMUL_AVX2 */ #ifdef __x86_64__ if (nblocks >= 8) { asm volatile ("movdqa %%xmm7, %%xmm15\n\t" "movdqa %%xmm1, %%xmm8\n\t" ::: "memory" ); if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR8_TABLE_INITIALIZED)) ghash_setup_aggr8 (c); /* Clobbers registers XMM0-XMM7. */ /* Preload H1. */ asm volatile ("movdqa %%xmm8, %%xmm1\n\t" "movdqa %[h_1], %%xmm0\n\t" : : [h_1] "m" (*c->u_mode.gcm.u_ghash_key.key) : "memory" ); while (nblocks >= 8) { gfmul_pclmul_aggr8 (buf, c->u_mode.gcm.gcm_table); buf += 8 * blocksize; nblocks -= 8; } } #endif /* __x86_64__ */ while (nblocks >= 4) { gfmul_pclmul_aggr4 (buf, c->u_mode.gcm.u_ghash_key.key, c->u_mode.gcm.gcm_table, be_mask); buf += 4 * blocksize; nblocks -= 4; } if (nblocks) { /* Preload H1. */ asm volatile ("movdqa %[h_1], %%xmm0\n\t" : : [h_1] "m" (*c->u_mode.gcm.u_ghash_key.key) : "memory" ); while (nblocks) { asm volatile ("movdqu %[buf], %%xmm2\n\t" "pshufb %[be_mask], %%xmm2\n\t" /* be => le */ "pxor %%xmm2, %%xmm1\n\t" : : [buf] "m" (*buf), [be_mask] "m" (*be_mask) : "memory" ); gfmul_pclmul (); buf += blocksize; nblocks--; } } /* Store hash. */ asm volatile ("pshufb %[be_mask], %%xmm1\n\t" /* be => le */ "movdqu %%xmm1, %[hash]\n\t" : [hash] "=m" (*result) : [be_mask] "m" (*be_mask) : "memory" ); /* Clear/restore used registers. */ asm volatile ("pxor %%xmm0, %%xmm0\n\t" "pxor %%xmm1, %%xmm1\n\t" "pxor %%xmm2, %%xmm2\n\t" "pxor %%xmm3, %%xmm3\n\t" "pxor %%xmm4, %%xmm4\n\t" "pxor %%xmm5, %%xmm5\n\t" "pxor %%xmm6, %%xmm6\n\t" "pxor %%xmm7, %%xmm7\n\t" : : : "memory" ); #ifdef __x86_64__ #ifdef __WIN64__ asm volatile ("movdqu 0*16(%0), %%xmm6\n\t" "movdqu 1*16(%0), %%xmm7\n\t" "movdqu 2*16(%0), %%xmm8\n\t" "movdqu 3*16(%0), %%xmm9\n\t" "movdqu 4*16(%0), %%xmm10\n\t" "movdqu 5*16(%0), %%xmm11\n\t" "movdqu 6*16(%0), %%xmm12\n\t" "movdqu 7*16(%0), %%xmm13\n\t" "movdqu 8*16(%0), %%xmm14\n\t" "movdqu 9*16(%0), %%xmm15\n\t" : : "r" (win64tmp) : "memory" ); #else /* Clear used registers. */ asm volatile ( "pxor %%xmm8, %%xmm8\n\t" "pxor %%xmm9, %%xmm9\n\t" "pxor %%xmm10, %%xmm10\n\t" "pxor %%xmm11, %%xmm11\n\t" "pxor %%xmm12, %%xmm12\n\t" "pxor %%xmm13, %%xmm13\n\t" "pxor %%xmm14, %%xmm14\n\t" "pxor %%xmm15, %%xmm15\n\t" : : : "memory" ); #endif /* __WIN64__ */ #endif /* __x86_64__ */ return 0; } unsigned int ASM_FUNC_ATTR _gcry_polyval_intel_pclmul (gcry_cipher_hd_t c, byte *result, const byte *buf, size_t nblocks) { static const unsigned char be_mask[16] __attribute__ ((aligned (16))) = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }; const unsigned int blocksize = GCRY_GCM_BLOCK_LEN; #if defined(__x86_64__) && defined(__WIN64__) char win64tmp[10 * 16]; #endif if (nblocks == 0) return 0; #if defined(__x86_64__) && defined(__WIN64__) /* XMM6-XMM15 need to be restored after use. */ asm volatile ("movdqu %%xmm6, 0*16(%0)\n\t" "movdqu %%xmm7, 1*16(%0)\n\t" "movdqu %%xmm8, 2*16(%0)\n\t" "movdqu %%xmm9, 3*16(%0)\n\t" "movdqu %%xmm10, 4*16(%0)\n\t" "movdqu %%xmm11, 5*16(%0)\n\t" "movdqu %%xmm12, 6*16(%0)\n\t" "movdqu %%xmm13, 7*16(%0)\n\t" "movdqu %%xmm14, 8*16(%0)\n\t" "movdqu %%xmm15, 9*16(%0)\n\t" : : "r" (win64tmp) : "memory" ); #endif /* Preload hash. */ asm volatile ("pxor %%xmm7, %%xmm7\n\t" "movdqu %[hash], %%xmm1\n\t" "pshufb %[be_mask], %%xmm1\n\t" /* be => le */ : : [hash] "m" (*result), [be_mask] "m" (*be_mask) : "memory" ); #if defined(GCM_USE_INTEL_VPCLMUL_AVX2) if (nblocks >= 16 && ((c->u_mode.gcm.hw_impl_flags & GCM_INTEL_USE_VPCLMUL_AVX2) || (c->u_mode.gcm.hw_impl_flags & GCM_INTEL_USE_VPCLMUL_AVX512))) { #if defined(GCM_USE_INTEL_VPCLMUL_AVX512) if (nblocks >= 32 && (c->u_mode.gcm.hw_impl_flags & GCM_INTEL_USE_VPCLMUL_AVX512)) { asm volatile ("vpopcntb %%xmm1, %%xmm16\n\t" /* spec stop for old AVX512 CPUs */ "vmovdqa %%xmm1, %%xmm8\n\t" "vmovdqu64 %[swapperm], %%zmm14\n\t" : : [swapperm] "m" (swap128b_perm), [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory" ); if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR32_TABLE_INITIALIZED)) { if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR16_TABLE_INITIALIZED)) { if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR8_TABLE_INITIALIZED)) ghash_setup_aggr8_avx512 (c); /* Clobbers registers XMM0-XMM7. */ ghash_setup_aggr16_avx512 (c); /* Clobbers registers XMM0-XMM7. */ } ghash_setup_aggr32_avx512 (c); /* Clobbers registers XMM0-XMM7. */ } /* Preload H1-H32. */ load_h1h4_to_zmm1 (c); asm volatile ("vpermq %%zmm1, %%zmm14, %%zmm16\n\t" /* H1|H2|H3|H4 */ "vmovdqa %%xmm8, %%xmm1\n\t" "vpermq 27*16(%[h_table]), %%zmm14, %%zmm0\n\t" /* H28|H29|H31|H32 */ "vpermq 23*16(%[h_table]), %%zmm14, %%zmm13\n\t" /* H25|H26|H27|H28 */ "vpermq 19*16(%[h_table]), %%zmm14, %%zmm12\n\t" /* H21|H22|H23|H24 */ "vpermq 15*16(%[h_table]), %%zmm14, %%zmm11\n\t" /* H17|H18|H19|H20 */ "vpermq 11*16(%[h_table]), %%zmm14, %%zmm10\n\t" /* H13|H14|H15|H16 */ "vpermq 7*16(%[h_table]), %%zmm14, %%zmm9\n\t" /* H9|H10|H11|H12 */ "vpermq 3*16(%[h_table]), %%zmm14, %%zmm8\n\t" /* H4|H6|H7|H8 */ : : [h_1] "m" (*c->u_mode.gcm.u_ghash_key.key), [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory" ); while (nblocks >= 32) { gfmul_vpclmul_avx512_aggr32_le (buf, c->u_mode.gcm.gcm_table); buf += 32 * blocksize; nblocks -= 32; } asm volatile ("vpxor %%xmm7, %%xmm7, %%xmm7\n\t" - "vpxorq %%zmm16, %%zmm16, %%zmm16\n\t" - "vpxorq %%zmm17, %%zmm17, %%zmm17\n\t" - "vpxorq %%zmm18, %%zmm18, %%zmm18\n\t" - "vpxorq %%zmm19, %%zmm19, %%zmm19\n\t" + "vpxorq %%ymm16, %%ymm16, %%ymm16\n\t" + "vpxorq %%ymm17, %%ymm17, %%ymm17\n\t" + "vpxorq %%ymm18, %%ymm18, %%ymm18\n\t" + "vpxorq %%ymm19, %%ymm19, %%ymm19\n\t" : : : "memory" ); } -#endif +#endif /* GCM_USE_INTEL_VPCLMUL_AVX512 */ if (nblocks >= 16) { u64 h1_h2_h15_h16[4*2]; asm volatile ("vmovdqa %%xmm1, %%xmm8\n\t" ::: "memory" ); if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR16_TABLE_INITIALIZED)) { if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR8_TABLE_INITIALIZED)) ghash_setup_aggr8_avx2 (c); /* Clobbers registers XMM0-XMM7. */ ghash_setup_aggr16_avx2 (c); /* Clobbers registers XMM0-XMM7. */ } /* Preload H1-H16. */ load_h1h2_to_ymm1 (c); asm volatile ("vperm2i128 $0x23, %%ymm1, %%ymm1, %%ymm7\n\t" /* H1|H2 */ "vmovdqa %%xmm8, %%xmm1\n\t" "vpxor %%xmm8, %%xmm8, %%xmm8\n\t" "vperm2i128 $0x23, 13*16(%[h_table]), %%ymm8, %%ymm0\n\t" /* H15|H16 */ "vperm2i128 $0x23, 11*16(%[h_table]), %%ymm8, %%ymm13\n\t" /* H13|H14 */ "vperm2i128 $0x23, 9*16(%[h_table]), %%ymm8, %%ymm12\n\t" /* H11|H12 */ "vperm2i128 $0x23, 7*16(%[h_table]), %%ymm8, %%ymm11\n\t" /* H9|H10 */ "vperm2i128 $0x23, 5*16(%[h_table]), %%ymm8, %%ymm10\n\t" /* H7|H8 */ "vperm2i128 $0x23, 3*16(%[h_table]), %%ymm8, %%ymm9\n\t" /* H5|H6 */ "vperm2i128 $0x23, 1*16(%[h_table]), %%ymm8, %%ymm8\n\t" /* H3|H4 */ "vmovdqu %%ymm0, %[h15_h16]\n\t" "vmovdqu %%ymm7, %[h1_h2]\n\t" : [h1_h2] "=m" (h1_h2_h15_h16[0]), [h15_h16] "=m" (h1_h2_h15_h16[4]) : [h_1] "m" (*c->u_mode.gcm.u_ghash_key.key), [h_table] "r" (c->u_mode.gcm.gcm_table) : "memory" ); while (nblocks >= 16) { gfmul_vpclmul_avx2_aggr16_le (buf, c->u_mode.gcm.gcm_table, h1_h2_h15_h16); buf += 16 * blocksize; nblocks -= 16; } asm volatile ("vpxor %%xmm7, %%xmm7, %%xmm7\n\t" "vmovdqu %%ymm7, %[h15_h16]\n\t" "vmovdqu %%ymm7, %[h1_h2]\n\t" : [h1_h2] "=m" (h1_h2_h15_h16[0]), [h15_h16] "=m" (h1_h2_h15_h16[4]) : : "memory" ); } asm volatile ("vzeroupper\n\t" ::: "memory" ); } #endif /* GCM_USE_INTEL_VPCLMUL_AVX2 */ #ifdef __x86_64__ if (nblocks >= 8) { asm volatile ("movdqa %%xmm1, %%xmm8\n\t" ::: "memory" ); if (!(c->u_mode.gcm.hw_impl_flags & GCM_INTEL_AGGR8_TABLE_INITIALIZED)) ghash_setup_aggr8 (c); /* Clobbers registers XMM0-XMM7. */ /* Preload H1. */ asm volatile ("movdqa %%xmm8, %%xmm1\n\t" "pxor %%xmm15, %%xmm15\n\t" "movdqa %[h_1], %%xmm0\n\t" : : [h_1] "m" (*c->u_mode.gcm.u_ghash_key.key) : "memory" ); while (nblocks >= 8) { gfmul_pclmul_aggr8_le (buf, c->u_mode.gcm.gcm_table); buf += 8 * blocksize; nblocks -= 8; } } #endif while (nblocks >= 4) { gfmul_pclmul_aggr4_le (buf, c->u_mode.gcm.u_ghash_key.key, c->u_mode.gcm.gcm_table); buf += 4 * blocksize; nblocks -= 4; } if (nblocks) { /* Preload H1. */ asm volatile ("movdqa %[h_1], %%xmm0\n\t" : : [h_1] "m" (*c->u_mode.gcm.u_ghash_key.key) : "memory" ); while (nblocks) { asm volatile ("movdqu %[buf], %%xmm2\n\t" "pxor %%xmm2, %%xmm1\n\t" : : [buf] "m" (*buf) : "memory" ); gfmul_pclmul (); buf += blocksize; nblocks--; } } /* Store hash. */ asm volatile ("pshufb %[be_mask], %%xmm1\n\t" /* be => le */ "movdqu %%xmm1, %[hash]\n\t" : [hash] "=m" (*result) : [be_mask] "m" (*be_mask) : "memory" ); /* Clear/restore used registers. */ asm volatile ("pxor %%xmm0, %%xmm0\n\t" "pxor %%xmm1, %%xmm1\n\t" "pxor %%xmm2, %%xmm2\n\t" "pxor %%xmm3, %%xmm3\n\t" "pxor %%xmm4, %%xmm4\n\t" "pxor %%xmm5, %%xmm5\n\t" "pxor %%xmm6, %%xmm6\n\t" "pxor %%xmm7, %%xmm7\n\t" : : : "memory" ); #ifdef __x86_64__ #ifdef __WIN64__ asm volatile ("movdqu 0*16(%0), %%xmm6\n\t" "movdqu 1*16(%0), %%xmm7\n\t" "movdqu 2*16(%0), %%xmm8\n\t" "movdqu 3*16(%0), %%xmm9\n\t" "movdqu 4*16(%0), %%xmm10\n\t" "movdqu 5*16(%0), %%xmm11\n\t" "movdqu 6*16(%0), %%xmm12\n\t" "movdqu 7*16(%0), %%xmm13\n\t" "movdqu 8*16(%0), %%xmm14\n\t" "movdqu 9*16(%0), %%xmm15\n\t" : : "r" (win64tmp) : "memory" ); #else /* Clear used registers. */ asm volatile ( "pxor %%xmm8, %%xmm8\n\t" "pxor %%xmm9, %%xmm9\n\t" "pxor %%xmm10, %%xmm10\n\t" "pxor %%xmm11, %%xmm11\n\t" "pxor %%xmm12, %%xmm12\n\t" "pxor %%xmm13, %%xmm13\n\t" "pxor %%xmm14, %%xmm14\n\t" "pxor %%xmm15, %%xmm15\n\t" : : : "memory" ); #endif /* __WIN64__ */ #endif /* __x86_64__ */ return 0; } #if __clang__ # pragma clang attribute pop #endif #endif /* GCM_USE_INTEL_PCLMUL */ diff --git a/cipher/keccak-amd64-avx512.S b/cipher/keccak-amd64-avx512.S index 58b4150f..b1fc7b64 100644 --- a/cipher/keccak-amd64-avx512.S +++ b/cipher/keccak-amd64-avx512.S @@ -1,587 +1,587 @@ /* keccak-amd64-avx512.S - x86-64 AVX512 implementation of Keccak * * Copyright (C) 2022 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . * * --- * * Core function `KeccakF1600_ce` based on ARMv8-CE KeccakF1600 implementation * by Andy Polyakov from CRYPTOGAMS distribution `arm/keccak1600-armv8.pl`. * `KeccakF1600_ce` was ported to x86-64 AVX512 and converted to use GCC * preprocessed assembly and fitted with new absorb function optimized for * x86-64. SHA3-256 performance on Intel tigerlake, 5.72 cpB. * * Original copyright license follows: * * Copyright (c) 2006, CRYPTOGAMS by * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain copyright notices, * this list of conditions and the following disclaimer. * * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * * Neither the name of the CRYPTOGAMS nor the names of its * copyright holder and contributors may be used to endorse or * promote products derived from this software without specific * prior written permission. * * ALTERNATIVELY, provided that this notice is retained in full, this * product may be distributed under the terms of the GNU General Public * License (GPL), in which case the provisions of the GPL apply INSTEAD OF * those given above. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifdef __x86_64 #include #if defined(HAVE_GCC_INLINE_ASM_AVX512) && \ (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \ defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) #include "asm-common-amd64.h" .text /* Register macros. */ #define A_0_0 %xmm31 #define A_0_1 %xmm30 #define A_0_2 %xmm29 #define A_0_3 %xmm28 #define A_0_4 %xmm27 #define A_1_0 %xmm26 #define A_1_1 %xmm25 #define A_1_2 %xmm24 #define A_1_3 %xmm23 #define A_1_4 %xmm22 #define A_2_0 %xmm21 #define A_2_1 %xmm20 #define A_2_2 %xmm19 #define A_2_3 %xmm18 #define A_2_4 %xmm17 #define A_3_0 %xmm16 #define A_3_1 %xmm15 #define A_3_2 %xmm14 #define A_3_3 %xmm13 #define A_3_4 %xmm12 #define A_4_0 %xmm11 #define A_4_1 %xmm10 #define A_4_2 %xmm9 #define A_4_3 %xmm8 #define A_4_4 %xmm7 #define C_0 %xmm6 #define C_1 %xmm5 #define C_2 %xmm4 #define C_3 %xmm3 #define C_4 %xmm2 #define C_5 %xmm1 #define C_6 %xmm0 #define D_0 C_4 #define D_1 C_5 #define D_2 C_6 #define D_3 C_2 #define D_4 C_3 /* Helper macros for ARMv8-CE to x86-64/AVX512 conversion. */ #define eor3_d(dst_s1, s2, s3) \ vpternlogq $0x96, s3, s2, dst_s1; #define eor3(dst, s1, s2, s3) \ vmovdqa s1, dst; \ eor3_d(dst, s2, s3); #define rax1_c(dst, s1, s2_rol1) \ vprolq $1, s2_rol1, dst; \ vpxor s1, dst, dst; #define rax1_t(dst_s1, s2_rol1, tmp) \ vprolq $1, s2_rol1, tmp; \ vpxor tmp, dst_s1, dst_s1; #define rax1_s(dst_s1, s2_rol1) \ vprolq $1, s2_rol1, s2_rol1; \ vpxor s2_rol1, dst_s1, dst_s1; #define xar(dst, s1, s2, rol) \ vpxorq s2, s1, dst; \ vprolq $(rol), dst, dst; #define xar_x(dst, s1, s2, rol) \ vpxor s2, s1, dst; \ vprolq $(rol), dst, dst; #define bcax_d(dst_s1, s2, s3) \ vpternlogq $0xb4, s3, s2, dst_s1; #define bcax(dst, s1, s2, s3) \ vmovdqa64 s1, dst; \ bcax_d(dst, s2, s3); #define bcax_x(dst, s1, s2, s3) \ vmovdqa s1, dst; \ bcax_d(dst, s2, s3); #define eor(dst, s1, s2) \ vpxorq s2, s1, dst; /* Misc helper macros. */ #define clear_avx512_4regs(a, b, c, d) \ - eor(a, a, a); vmovdqa64 a, b; vmovdqa64 a, c; vmovdqa64 a, d; + eor(a, a, a); eor(b, b, b); eor(c, c, c); eor(d, d, d); #define clear_regs() \ vzeroall; /* xmm0-xmm15 */ \ - clear_avx512_4regs(%xmm16, %xmm17, %xmm18, %xmm19); \ - clear_avx512_4regs(%xmm20, %xmm21, %xmm22, %xmm23); \ - clear_avx512_4regs(%xmm24, %xmm25, %xmm26, %xmm27); \ - clear_avx512_4regs(%xmm28, %xmm29, %xmm30, %xmm31); + clear_avx512_4regs(%ymm16, %ymm17, %ymm18, %ymm19); \ + clear_avx512_4regs(%ymm20, %ymm21, %ymm22, %ymm23); \ + clear_avx512_4regs(%ymm24, %ymm25, %ymm26, %ymm27); \ + clear_avx512_4regs(%ymm28, %ymm29, %ymm30, %ymm31); ELF(.type KeccakF1600_ce,@function) .align 64, 0xcc KeccakF1600_ce: .Loop_ce: CFI_STARTPROC() ////////////////////////////////////////////////// Theta eor3( C_0, A_4_0, A_3_0, A_2_0) eor3( C_1, A_4_1, A_3_1, A_2_1) eor3( C_3, A_4_3, A_3_3, A_2_3) eor3( C_2, A_4_2, A_3_2, A_2_2) eor3( C_4, A_4_4, A_3_4, A_2_4) eor3_d( C_0, A_1_0, A_0_0) eor3_d( C_1, A_1_1, A_0_1) eor3_d( C_3, A_1_3, A_0_3) eor3_d( C_2, A_1_2, A_0_2) eor3_d( C_4, A_1_4, A_0_4) rax1_c( C_5, C_0, C_2) // D[1] rax1_t( C_2, C_4, C_6) // D[3] rax1_c( C_6, C_1, C_3) // D[2] rax1_s( C_3, C_0) // D[4] rax1_s( C_4, C_1) // D[0] ////////////////////////////////////////////////// Theta+Rho+Pi xar( C_0, A_0_1, D_1, 1) // C[0]=A[2][0] xar( A_0_1, A_1_1, D_1, 44) xar( A_1_1, A_1_4, D_4, 20) xar( A_1_4, A_4_2, D_2, 61) xar( A_4_2, A_2_4, D_4, 39) xar( A_2_4, A_4_0, D_0, 18) xar( C_1, A_0_2, D_2, 62) // C[1]=A[4][0] xar( A_0_2, A_2_2, D_2, 43) xar( A_2_2, A_2_3, D_3, 25) xar( A_2_3, A_3_4, D_4, 8) xar_x( A_3_4, A_4_3, D_3, 56) xar( A_4_3, A_3_0, D_0, 41) xar( A_3_0, A_0_4, D_4, 27) xar_x( D_4, A_4_4, D_4, 14) // D[4]=A[0][4] xar_x( A_4_4, A_4_1, D_1, 2) xar( A_1_3, A_1_3, D_3, 55) // A[1][3]=A[4][1] xar( A_0_4, A_3_1, D_1, 45) // A[0][4]=A[1][3] xar( A_3_1, A_1_0, D_0, 36) xar( A_1_0, A_0_3, D_3, 28) eor( A_0_0, A_0_0, D_0) xar_x( D_3, A_3_3, D_3, 21) // D[3]=A[0][3] xar( A_0_3, A_3_2, D_2, 15) // A[0][3]=A[3][3] xar( D_1, A_2_1, D_1, 10) // D[1]=A[3][2] xar( D_2, A_1_2, D_2, 6) // D[2]=A[2][1] xar( D_0, A_2_0, D_0, 3) // D[0]=A[1][2] ////////////////////////////////////////////////// Chi+Iota bcax_x( A_4_0, C_1, A_4_2, A_1_3) // A[1][3]=A[4][1] bcax( A_4_1, A_1_3, A_4_3, A_4_2) // A[1][3]=A[4][1] bcax_d( A_4_2, A_4_4, A_4_3) bcax_d( A_4_3, C_1, A_4_4) bcax_d( A_4_4, A_1_3, C_1) // A[1][3]=A[4][1] bcax_x( A_3_2, D_1, A_3_4, A_0_3) // A[0][3]=A[3][3] bcax( A_3_3, A_0_3, A_3_0, A_3_4) // A[0][3]=A[3][3] bcax_d( A_3_4, A_3_1, A_3_0) bcax_d( A_3_0, D_1, A_3_1) bcax_d( A_3_1, A_0_3, D_1) // A[0][3]=A[3][3] bcax( A_2_0, C_0, A_2_2, D_2) bcax( A_2_1, D_2, A_2_3, A_2_2) bcax_d( A_2_2, A_2_4, A_2_3) bcax_d( A_2_3, C_0, A_2_4) bcax_d( A_2_4, D_2, C_0) bcax( A_1_2, D_0, A_1_4, A_0_4) // A[0][4]=A[1][3] bcax( A_1_3, A_0_4, A_1_0, A_1_4) // A[0][4]=A[1][3] bcax_d( A_1_4, A_1_1, A_1_0) bcax_d( A_1_0, D_0, A_1_1) bcax_d( A_1_1, A_0_4, D_0) // A[0][4]=A[1][3] bcax( A_0_3, D_3, A_0_0, D_4) bcax( A_0_4, D_4, A_0_1, A_0_0) bcax_d( A_0_0, A_0_2, A_0_1) bcax_d( A_0_1, D_3, A_0_2) bcax_d( A_0_2, D_4, D_3) eor( A_0_0, A_0_0, (%r10)) cmpq %r10, %r11 je .Lend_ce addq $8, %r10 jmp .Loop_ce .align 64, 0xcc .Lend_ce: ret_spec_stop CFI_ENDPROC() ELF(.size KeccakF1600_ce,.-KeccakF1600_ce) .globl _gcry_keccak_f1600_state_permute64_avx512 ELF(.type _gcry_keccak_f1600_state_permute64_avx512,@function) .align 64, 0xcc _gcry_keccak_f1600_state_permute64_avx512: /* input: * %rdi: state * %rsi: round constants */ CFI_STARTPROC() spec_stop_avx512; leaq 12*8(%rdi), %rax leaq (24-1)*8(%rsi), %r11 vmovdqu64 0*8(%rdi), A_0_0 vmovdqu64 1*8(%rdi), A_0_1 vmovdqu64 2*8(%rdi), A_0_2 vmovdqu64 3*8(%rdi), A_0_3 vmovdqu64 4*8(%rdi), A_0_4 vmovdqu64 5*8(%rdi), A_1_0 vmovdqu64 6*8(%rdi), A_1_1 vmovdqu64 7*8(%rdi), A_1_2 vmovdqu64 8*8(%rdi), A_1_3 vmovdqu64 9*8(%rdi), A_1_4 vmovdqu64 10*8(%rdi), A_2_0 vmovdqu64 11*8(%rdi), A_2_1 vmovdqu64 0*8(%rax), A_2_2 vmovdqu64 1*8(%rax), A_2_3 vmovdqu64 2*8(%rax), A_2_4 vmovdqu64 3*8(%rax), A_3_0 vmovdqu 4*8(%rax), A_3_1 vmovdqu 5*8(%rax), A_3_2 vmovdqu 6*8(%rax), A_3_3 vmovdqu 7*8(%rax), A_3_4 vmovdqu 8*8(%rax), A_4_0 vmovdqu 9*8(%rax), A_4_1 vmovdqu 10*8(%rax), A_4_2 vmovdqu 11*8(%rax), A_4_3 vmovq 12*8(%rax), A_4_4 movq %rsi, %r10 call KeccakF1600_ce vpunpcklqdq A_0_1, A_0_0, A_0_0 vpunpcklqdq A_0_3, A_0_2, A_0_2 vpunpcklqdq A_1_0, A_0_4, A_0_4 vpunpcklqdq A_1_2, A_1_1, A_1_1 vpunpcklqdq A_1_4, A_1_3, A_1_3 vpunpcklqdq A_2_1, A_2_0, A_2_0 vpunpcklqdq A_2_3, A_2_2, A_2_2 vpunpcklqdq A_3_0, A_2_4, A_2_4 vpunpcklqdq A_3_2, A_3_1, A_3_1 vpunpcklqdq A_3_4, A_3_3, A_3_3 vpunpcklqdq A_4_1, A_4_0, A_4_0 vpunpcklqdq A_4_3, A_4_2, A_4_2 vmovdqu64 A_0_0, 0*8(%rdi) vmovdqu64 A_0_2, 2*8(%rdi) vmovdqu64 A_0_4, 4*8(%rdi) vmovdqu64 A_1_1, 6*8(%rdi) vmovdqu64 A_1_3, 8*8(%rdi) vmovdqu64 A_2_0, 10*8(%rdi) vmovdqu64 A_2_2, 0*8(%rax) vmovdqu64 A_2_4, 2*8(%rax) vmovdqu A_3_1, 4*8(%rax) vmovdqu A_3_3, 6*8(%rax) vmovdqu A_4_0, 8*8(%rax) vmovdqu A_4_2, 10*8(%rax) vmovq A_4_4, 12*8(%rax) xorl %eax, %eax clear_regs() ret_spec_stop CFI_ENDPROC() ELF(.size _gcry_keccak_f1600_state_permute64_avx512, .-_gcry_keccak_f1600_state_permute64_avx512) .globl _gcry_keccak_absorb_blocks_avx512 ELF(.type _gcry_keccak_absorb_blocks_avx512,@function) .align 64, 0xcc _gcry_keccak_absorb_blocks_avx512: /* input: * %rdi: state * %rsi: round constants * %rdx: lanes * %rcx: nlanes * %r8 : blocklanes * %r9 : lanes output pointer */ CFI_STARTPROC() spec_stop_avx512; leaq 12*8(%rdi), %rax leaq (24-1)*8(%rsi), %r11 vmovdqu64 0*8(%rdi), A_0_0 vmovdqu64 1*8(%rdi), A_0_1 vmovdqu64 2*8(%rdi), A_0_2 vmovdqu64 3*8(%rdi), A_0_3 vmovdqu64 4*8(%rdi), A_0_4 vmovdqu64 5*8(%rdi), A_1_0 vmovdqu64 6*8(%rdi), A_1_1 vmovdqu64 7*8(%rdi), A_1_2 vmovdqu64 8*8(%rdi), A_1_3 vmovdqu64 9*8(%rdi), A_1_4 vmovdqu64 10*8(%rdi), A_2_0 vmovdqu64 11*8(%rdi), A_2_1 vmovdqu64 0*8(%rax), A_2_2 vmovdqu64 1*8(%rax), A_2_3 vmovdqu64 2*8(%rax), A_2_4 vmovdqu64 3*8(%rax), A_3_0 vmovdqu 4*8(%rax), A_3_1 vmovdqu 5*8(%rax), A_3_2 vmovdqu 6*8(%rax), A_3_3 vmovdqu 7*8(%rax), A_3_4 vmovdqu 8*8(%rax), A_4_0 vmovdqu 9*8(%rax), A_4_1 vmovdqu 10*8(%rax), A_4_2 vmovdqu 11*8(%rax), A_4_3 vmovq 12*8(%rax), A_4_4 cmpq $(104 >> 3), %r8 jb .Loop_absorb_72_ce je .Loop_absorb_104_ce cmpq $(144 >> 3), %r8 jb .Loop_absorb_136_ce je .Loop_absorb_144_ce jmp .Loop_absorb_168_ce .align 64, 0xcc .Loop_absorb_168_ce: subq %r8, %rcx // len - bsz jb .Labsorbed_ce vpxorq 0*8(%rdx), A_0_0, A_0_0 vpxorq 1*8(%rdx), A_0_1, A_0_1 vpxorq 2*8(%rdx), A_0_2, A_0_2 vpxorq 3*8(%rdx), A_0_3, A_0_3 vpxorq 4*8(%rdx), A_0_4, A_0_4 vpxorq 5*8(%rdx), A_1_0, A_1_0 vpxorq 6*8(%rdx), A_1_1, A_1_1 vpxorq 7*8(%rdx), A_1_2, A_1_2 vpxorq 8*8(%rdx), A_1_3, A_1_3 vpxorq 9*8(%rdx), A_1_4, A_1_4 vpxorq 10*8(%rdx), A_2_0, A_2_0 vpxorq 11*8(%rdx), A_2_1, A_2_1 vpxorq 12*8(%rdx), A_2_2, A_2_2 vpxorq 13*8(%rdx), A_2_3, A_2_3 vpxorq 14*8(%rdx), A_2_4, A_2_4 vpxorq 15*8(%rdx), A_3_0, A_3_0 vpxor 16*8(%rdx), A_3_1, A_3_1 vpxor 17*8(%rdx), A_3_2, A_3_2 vpxor 18*8(%rdx), A_3_3, A_3_3 vpxor 19*8(%rdx), A_3_4, A_3_4 vmovq 20*8(%rdx), C_0 leaq 21*8(%rdx), %rdx vpxorq C_0, A_4_0, A_4_0 movq %rsi, %r10 call KeccakF1600_ce jmp .Loop_absorb_168_ce .align 64, 0xcc .Loop_absorb_144_ce: subq %r8, %rcx // len - bsz jb .Labsorbed_ce vpxorq 0*8(%rdx), A_0_0, A_0_0 vpxorq 1*8(%rdx), A_0_1, A_0_1 vpxorq 2*8(%rdx), A_0_2, A_0_2 vpxorq 3*8(%rdx), A_0_3, A_0_3 vpxorq 4*8(%rdx), A_0_4, A_0_4 vpxorq 5*8(%rdx), A_1_0, A_1_0 vpxorq 6*8(%rdx), A_1_1, A_1_1 vpxorq 7*8(%rdx), A_1_2, A_1_2 vpxorq 8*8(%rdx), A_1_3, A_1_3 vpxorq 9*8(%rdx), A_1_4, A_1_4 vpxorq 10*8(%rdx), A_2_0, A_2_0 vpxorq 11*8(%rdx), A_2_1, A_2_1 vpxorq 12*8(%rdx), A_2_2, A_2_2 vpxorq 13*8(%rdx), A_2_3, A_2_3 vpxorq 14*8(%rdx), A_2_4, A_2_4 vpxorq 15*8(%rdx), A_3_0, A_3_0 vpxor 16*8(%rdx), A_3_1, A_3_1 vmovq 17*8(%rdx), C_0 leaq 18*8(%rdx), %rdx vpxor C_0, A_3_2, A_3_2 movq %rsi, %r10 call KeccakF1600_ce jmp .Loop_absorb_144_ce .align 64, 0xcc .Loop_absorb_136_ce: subq %r8, %rcx // len - bsz jb .Labsorbed_ce vpxorq 0*8(%rdx), A_0_0, A_0_0 vpxorq 1*8(%rdx), A_0_1, A_0_1 vpxorq 2*8(%rdx), A_0_2, A_0_2 vpxorq 3*8(%rdx), A_0_3, A_0_3 vpxorq 4*8(%rdx), A_0_4, A_0_4 vpxorq 5*8(%rdx), A_1_0, A_1_0 vpxorq 6*8(%rdx), A_1_1, A_1_1 vpxorq 7*8(%rdx), A_1_2, A_1_2 vpxorq 8*8(%rdx), A_1_3, A_1_3 vpxorq 9*8(%rdx), A_1_4, A_1_4 vpxorq 10*8(%rdx), A_2_0, A_2_0 vpxorq 11*8(%rdx), A_2_1, A_2_1 vpxorq 12*8(%rdx), A_2_2, A_2_2 vpxorq 13*8(%rdx), A_2_3, A_2_3 vpxorq 14*8(%rdx), A_2_4, A_2_4 vpxorq 15*8(%rdx), A_3_0, A_3_0 vmovq 16*8(%rdx), C_0 leaq 17*8(%rdx), %rdx vpxor C_0, A_3_1, A_3_1 movq %rsi, %r10 call KeccakF1600_ce jmp .Loop_absorb_136_ce .align 64, 0xcc .Loop_absorb_104_ce: subq %r8, %rcx // len - bsz jb .Labsorbed_ce vpxorq 0*8(%rdx), A_0_0, A_0_0 vpxorq 1*8(%rdx), A_0_1, A_0_1 vpxorq 2*8(%rdx), A_0_2, A_0_2 vpxorq 3*8(%rdx), A_0_3, A_0_3 vpxorq 4*8(%rdx), A_0_4, A_0_4 vpxorq 5*8(%rdx), A_1_0, A_1_0 vpxorq 6*8(%rdx), A_1_1, A_1_1 vpxorq 7*8(%rdx), A_1_2, A_1_2 vpxorq 8*8(%rdx), A_1_3, A_1_3 vpxorq 9*8(%rdx), A_1_4, A_1_4 vpxorq 10*8(%rdx), A_2_0, A_2_0 vpxorq 11*8(%rdx), A_2_1, A_2_1 vmovq 12*8(%rdx), C_0 leaq 13*8(%rdx), %rdx vpxorq C_0, A_2_2, A_2_2 movq %rsi, %r10 call KeccakF1600_ce jmp .Loop_absorb_104_ce .align 64, 0xcc .Loop_absorb_72_ce: subq %r8, %rcx // len - bsz jb .Labsorbed_ce vpxorq 0*8(%rdx), A_0_0, A_0_0 vpxorq 1*8(%rdx), A_0_1, A_0_1 vpxorq 2*8(%rdx), A_0_2, A_0_2 vpxorq 3*8(%rdx), A_0_3, A_0_3 vpxorq 4*8(%rdx), A_0_4, A_0_4 vpxorq 5*8(%rdx), A_1_0, A_1_0 vpxorq 6*8(%rdx), A_1_1, A_1_1 vpxorq 7*8(%rdx), A_1_2, A_1_2 vmovq 8*8(%rdx), C_0 leaq 9*8(%rdx), %rdx vpxorq C_0, A_1_3, A_1_3 movq %rsi, %r10 call KeccakF1600_ce jmp .Loop_absorb_72_ce .align 64, 0xcc .Labsorbed_ce: vpunpcklqdq A_0_1, A_0_0, A_0_0 vpunpcklqdq A_0_3, A_0_2, A_0_2 vpunpcklqdq A_1_0, A_0_4, A_0_4 vpunpcklqdq A_1_2, A_1_1, A_1_1 vpunpcklqdq A_1_4, A_1_3, A_1_3 vpunpcklqdq A_2_1, A_2_0, A_2_0 vpunpcklqdq A_2_3, A_2_2, A_2_2 vpunpcklqdq A_3_0, A_2_4, A_2_4 vpunpcklqdq A_3_2, A_3_1, A_3_1 vpunpcklqdq A_3_4, A_3_3, A_3_3 vpunpcklqdq A_4_1, A_4_0, A_4_0 vpunpcklqdq A_4_3, A_4_2, A_4_2 vmovdqu64 A_0_0, 0*8(%rdi) vmovdqu64 A_0_2, 2*8(%rdi) vmovdqu64 A_0_4, 4*8(%rdi) vmovdqu64 A_1_1, 6*8(%rdi) vmovdqu64 A_1_3, 8*8(%rdi) vmovdqu64 A_2_0, 10*8(%rdi) vmovdqu64 A_2_2, 0*8(%rax) vmovdqu64 A_2_4, 2*8(%rax) vmovdqu A_3_1, 4*8(%rax) vmovdqu A_3_3, 6*8(%rax) vmovdqu A_4_0, 8*8(%rax) vmovdqu A_4_2, 10*8(%rax) vmovq A_4_4, 12*8(%rax) leaq (%r8, %rcx), %rax // return value movq %rdx, (%r9) // return buffer pointer clear_regs() ret_spec_stop CFI_ENDPROC() ELF(.size _gcry_keccak_absorb_blocks_avx512, .-_gcry_keccak_absorb_blocks_avx512) #endif /* HAVE_GCC_INLINE_ASM_AVX512 */ #endif /* __x86_64 */ diff --git a/cipher/poly1305-amd64-avx512.S b/cipher/poly1305-amd64-avx512.S index 6622861f..9beed8ad 100644 --- a/cipher/poly1305-amd64-avx512.S +++ b/cipher/poly1305-amd64-avx512.S @@ -1,1624 +1,1624 @@ /* ;; ;; Copyright (c) 2021-2022, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: ;; ;; * Redistributions of source code must retain the above copyright notice, ;; this list of conditions and the following disclaimer. ;; * Redistributions in binary form must reproduce the above copyright ;; notice, this list of conditions and the following disclaimer in the ;; documentation and/or other materials provided with the distribution. ;; * Neither the name of Intel Corporation nor the names of its contributors ;; may be used to endorse or promote products derived from this software ;; without specific prior written permission. ;; ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; */ /* * From: * https://github.com/intel/intel-ipsec-mb/blob/f0cad21a644231c0f5d4af51f56061a5796343fb/lib/avx512/poly_fma_avx512.asm * * Conversion to GAS assembly and integration to libgcrypt * by Jussi Kivilinna */ #ifdef __x86_64 #include #if (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \ defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) && \ defined(HAVE_INTEL_SYNTAX_PLATFORM_AS) && \ defined(HAVE_GCC_INLINE_ASM_AVX512) #include "asm-common-amd64.h" .intel_syntax noprefix .text ELF(.type _gcry_poly1305_avx512_consts,@object) _gcry_poly1305_avx512_consts: .align 64 .Lmask_44: .quad 0xfffffffffff, 0xfffffffffff, 0xfffffffffff, 0xfffffffffff .quad 0xfffffffffff, 0xfffffffffff, 0xfffffffffff, 0xfffffffffff .align 64 .Lmask_42: .quad 0x3ffffffffff, 0x3ffffffffff, 0x3ffffffffff, 0x3ffffffffff .quad 0x3ffffffffff, 0x3ffffffffff, 0x3ffffffffff, 0x3ffffffffff .align 64 .Lhigh_bit: .quad 0x10000000000, 0x10000000000, 0x10000000000, 0x10000000000 .quad 0x10000000000, 0x10000000000, 0x10000000000, 0x10000000000 .Lbyte_len_to_mask_table: .short 0x0000, 0x0001, 0x0003, 0x0007 .short 0x000f, 0x001f, 0x003f, 0x007f .short 0x00ff, 0x01ff, 0x03ff, 0x07ff .short 0x0fff, 0x1fff, 0x3fff, 0x7fff .short 0xffff .align 64 .Lbyte64_len_to_mask_table: .quad 0x0000000000000000, 0x0000000000000001 .quad 0x0000000000000003, 0x0000000000000007 .quad 0x000000000000000f, 0x000000000000001f .quad 0x000000000000003f, 0x000000000000007f .quad 0x00000000000000ff, 0x00000000000001ff .quad 0x00000000000003ff, 0x00000000000007ff .quad 0x0000000000000fff, 0x0000000000001fff .quad 0x0000000000003fff, 0x0000000000007fff .quad 0x000000000000ffff, 0x000000000001ffff .quad 0x000000000003ffff, 0x000000000007ffff .quad 0x00000000000fffff, 0x00000000001fffff .quad 0x00000000003fffff, 0x00000000007fffff .quad 0x0000000000ffffff, 0x0000000001ffffff .quad 0x0000000003ffffff, 0x0000000007ffffff .quad 0x000000000fffffff, 0x000000001fffffff .quad 0x000000003fffffff, 0x000000007fffffff .quad 0x00000000ffffffff, 0x00000001ffffffff .quad 0x00000003ffffffff, 0x00000007ffffffff .quad 0x0000000fffffffff, 0x0000001fffffffff .quad 0x0000003fffffffff, 0x0000007fffffffff .quad 0x000000ffffffffff, 0x000001ffffffffff .quad 0x000003ffffffffff, 0x000007ffffffffff .quad 0x00000fffffffffff, 0x00001fffffffffff .quad 0x00003fffffffffff, 0x00007fffffffffff .quad 0x0000ffffffffffff, 0x0001ffffffffffff .quad 0x0003ffffffffffff, 0x0007ffffffffffff .quad 0x000fffffffffffff, 0x001fffffffffffff .quad 0x003fffffffffffff, 0x007fffffffffffff .quad 0x00ffffffffffffff, 0x01ffffffffffffff .quad 0x03ffffffffffffff, 0x07ffffffffffffff .quad 0x0fffffffffffffff, 0x1fffffffffffffff .quad 0x3fffffffffffffff, 0x7fffffffffffffff .quad 0xffffffffffffffff .Lqword_high_bit_mask: .short 0, 0x1, 0x5, 0x15, 0x55, 0x57, 0x5f, 0x7f, 0xff ELF(.size _gcry_poly1305_avx512_consts,.-_gcry_poly1305_avx512_consts) #define raxd eax #define rbxd ebx #define rcxd ecx #define rdxd edx #define rsid esi #define rdid edi #define rbpd ebp #define rspd esp #define __DWORD(X) X##d #define DWORD(R) __DWORD(R) #define arg1 rdi #define arg2 rsi #define arg3 rdx #define arg4 rcx #define job arg1 #define gp1 rsi #define gp2 rcx /* ;; don't use rdx and rax - they are needed for multiply operation */ #define gp3 rbp #define gp4 r8 #define gp5 r9 #define gp6 r10 #define gp7 r11 #define gp8 r12 #define gp9 r13 #define gp10 r14 #define gp11 r15 #define len gp11 #define msg gp10 #define POLY1305_BLOCK_SIZE 16 #define STACK_r_save 0 #define STACK_r_save_size (6 * 64) #define STACK_gpr_save (STACK_r_save + STACK_r_save_size) #define STACK_gpr_save_size (8 * 8) #define STACK_rsp_save (STACK_gpr_save + STACK_gpr_save_size) #define STACK_rsp_save_size (1 * 8) #define STACK_SIZE (STACK_rsp_save + STACK_rsp_save_size) #define A2_ZERO(...) /**/ #define A2_ZERO_INVERT(...) __VA_ARGS__ #define A2_NOT_ZERO(...) __VA_ARGS__ #define A2_NOT_ZERO_INVERT(...) /**/ #define clear_zmm(vec) vpxord vec, vec, vec /* ;; ============================================================================= ;; ============================================================================= ;; Computes hash for message length being multiple of block size ;; ============================================================================= ;; Combining 64-bit x 64-bit multiplication with reduction steps ;; ;; NOTES: ;; 1) A2 here is only two bits so anything above is subject of reduction. ;; Constant C1 = R1 + (R1 >> 2) simplifies multiply with less operations ;; 2) Magic 5x comes from mod 2^130-5 property and incorporating ;; reduction into multiply phase. ;; See "Cheating at modular arithmetic" and "Poly1305's prime: 2^130 - 5" ;; paragraphs at https://loup-vaillant.fr/tutorials/poly1305-design for more details. ;; ;; Flow of the code below is as follows: ;; ;; A2 A1 A0 ;; x R1 R0 ;; ----------------------------- ;; A2×R0 A1×R0 A0×R0 ;; + A0×R1 ;; + 5xA2xR1 5xA1xR1 ;; ----------------------------- ;; [0|L2L] [L1H|L1L] [L0H|L0L] ;; ;; Registers: T3:T2 T1:A0 ;; ;; Completing the multiply and adding (with carry) 3x128-bit limbs into ;; 192-bits again (3x64-bits): ;; A0 = L0L ;; A1 = L0H + L1L ;; T3 = L1H + L2L ; A0 [in/out] GPR with accumulator bits 63:0 ; A1 [in/out] GPR with accumulator bits 127:64 ; A2 [in/out] GPR with accumulator bits 195:128 ; R0 [in] GPR with R constant bits 63:0 ; R1 [in] GPR with R constant bits 127:64 ; C1 [in] C1 = R1 + (R1 >> 2) ; T1 [clobbered] GPR register ; T2 [clobbered] GPR register ; T3 [clobbered] GPR register ; GP_RAX [clobbered] RAX register ; GP_RDX [clobbered] RDX register ; IF_A2 [in] Used if input A2 is not 0 */ #define POLY1305_MUL_REDUCE(A0, A1, A2, R0, R1, C1, T1, T2, T3, GP_RAX, GP_RDX, IF_A2) \ /* T3:T2 = (A0 * R1) */ \ mov GP_RAX, R1; \ mul A0; \ mov T2, GP_RAX; \ mov GP_RAX, R0; \ mov T3, GP_RDX; \ \ /* T1:A0 = (A0 * R0) */ \ mul A0; \ mov A0, GP_RAX; /* A0 not used in other operations */ \ mov GP_RAX, R0; \ mov T1, GP_RDX; \ \ /* T3:T2 += (A1 * R0) */ \ mul A1; \ add T2, GP_RAX; \ mov GP_RAX, C1; \ adc T3, GP_RDX; \ \ /* T1:A0 += (A1 * R1x5) */ \ mul A1; \ IF_A2(mov A1, A2); /* use A1 for A2 */ \ add A0, GP_RAX; \ adc T1, GP_RDX; \ \ /* NOTE: A2 is clamped to 2-bits, */ \ /* R1/R0 is clamped to 60-bits, */ \ /* their product is less than 2^64. */ \ \ IF_A2(/* T3:T2 += (A2 * R1x5) */); \ IF_A2(imul A1, C1); \ IF_A2(add T2, A1); \ IF_A2(mov A1, T1); /* T1:A0 => A1:A0 */ \ IF_A2(adc T3, 0); \ \ IF_A2(/* T3:A1 += (A2 * R0) */); \ IF_A2(imul A2, R0); \ IF_A2(add A1, T2); \ IF_A2(adc T3, A2); \ \ IF_A2##_INVERT(/* If A2 == 0, just move and add T1-T2 to A1 */); \ IF_A2##_INVERT(mov A1, T1); \ IF_A2##_INVERT(add A1, T2); \ IF_A2##_INVERT(adc T3, 0); \ \ /* At this point, 3 64-bit limbs are in T3:A1:A0 */ \ /* T3 can span over more than 2 bits so final partial reduction step is needed. */ \ \ /* Partial reduction (just to fit into 130 bits) */ \ /* A2 = T3 & 3 */ \ /* k = (T3 & ~3) + (T3 >> 2) */ \ /* Y x4 + Y x1 */ \ /* A2:A1:A0 += k */ \ \ /* Result will be in A2:A1:A0 */ \ mov T1, T3; \ mov DWORD(A2), DWORD(T3); \ and T1, ~3; \ shr T3, 2; \ and DWORD(A2), 3; \ add T1, T3; \ \ /* A2:A1:A0 += k (kept in T1) */ \ add A0, T1; \ adc A1, 0; \ adc DWORD(A2), 0 /* ;; ============================================================================= ;; ============================================================================= ;; Computes hash for 8 16-byte message blocks, ;; and adds new message blocks to accumulator. ;; ;; It first multiplies all 8 blocks with powers of R: ;; ;; a2 a1 a0 ;; × b2 b1 b0 ;; --------------------------------------- ;; a2×b0 a1×b0 a0×b0 ;; + a1×b1 a0×b1 5×a2×b1 ;; + a0×b2 5×a2×b2 5×a1×b2 ;; --------------------------------------- ;; p2 p1 p0 ;; ;; Then, it propagates the carry (higher bits after bit 43) from lower limbs into higher limbs, ;; multiplying by 5 in case of the carry of p2. ;; ;A0 [in/out] ZMM register containing 1st 44-bit limb of the 8 blocks ;A1 [in/out] ZMM register containing 2nd 44-bit limb of the 8 blocks ;A2 [in/out] ZMM register containing 3rd 44-bit limb of the 8 blocks ;R0 [in] ZMM register (R0) to include the 1st limb of R ;R1 [in] ZMM register (R1) to include the 2nd limb of R ;R2 [in] ZMM register (R2) to include the 3rd limb of R ;R1P [in] ZMM register (R1') to include the 2nd limb of R (multiplied by 5) ;R2P [in] ZMM register (R2') to include the 3rd limb of R (multiplied by 5) ;P0_L [clobbered] ZMM register to contain p[0] of the 8 blocks ;P0_H [clobbered] ZMM register to contain p[0] of the 8 blocks ;P1_L [clobbered] ZMM register to contain p[1] of the 8 blocks ;P1_H [clobbered] ZMM register to contain p[1] of the 8 blocks ;P2_L [clobbered] ZMM register to contain p[2] of the 8 blocks ;P2_H [clobbered] ZMM register to contain p[2] of the 8 blocks ;ZTMP1 [clobbered] Temporary ZMM register */ #define POLY1305_MUL_REDUCE_VEC(A0, A1, A2, R0, R1, R2, R1P, R2P, P0_L, P0_H, \ P1_L, P1_H, P2_L, P2_H, ZTMP1) \ /* ;; Reset accumulator */ \ vpxorq P0_L, P0_L, P0_L; \ vpxorq P0_H, P0_H, P0_H; \ vpxorq P1_L, P1_L, P1_L; \ vpxorq P1_H, P1_H, P1_H; \ vpxorq P2_L, P2_L, P2_L; \ vpxorq P2_H, P2_H, P2_H; \ \ /* ; Reset accumulator and calculate products */ \ vpmadd52luq P0_L, A2, R1P; \ vpmadd52huq P0_H, A2, R1P; \ vpmadd52luq P1_L, A2, R2P; \ vpmadd52huq P1_H, A2, R2P; \ vpmadd52luq P2_L, A2, R0; \ vpmadd52huq P2_H, A2, R0; \ \ vpmadd52luq P1_L, A0, R1; \ vpmadd52huq P1_H, A0, R1; \ vpmadd52luq P2_L, A0, R2; \ vpmadd52huq P2_H, A0, R2; \ vpmadd52luq P0_L, A0, R0; \ vpmadd52huq P0_H, A0, R0; \ \ vpmadd52luq P0_L, A1, R2P; \ vpmadd52huq P0_H, A1, R2P; \ vpmadd52luq P1_L, A1, R0; \ vpmadd52huq P1_H, A1, R0; \ vpmadd52luq P2_L, A1, R1; \ vpmadd52huq P2_H, A1, R1; \ \ /* ; Carry propagation (first pass) */ \ vpsrlq ZTMP1, P0_L, 44; \ vpandq A0, P0_L, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ vpsllq P0_H, P0_H, 8; \ vpaddq P0_H, P0_H, ZTMP1; \ vpaddq P1_L, P1_L, P0_H; \ vpandq A1, P1_L, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ vpsrlq ZTMP1, P1_L, 44; \ vpsllq P1_H, P1_H, 8; \ vpaddq P1_H, P1_H, ZTMP1; \ vpaddq P2_L, P2_L, P1_H; \ vpandq A2, P2_L, [.Lmask_42 ADD_RIP]; /* ; Clear top 22 bits */ \ vpsrlq ZTMP1, P2_L, 42; \ vpsllq P2_H, P2_H, 10; \ vpaddq P2_H, P2_H, ZTMP1; \ \ /* ; Carry propagation (second pass) */ \ \ /* ; Multiply by 5 the highest bits (above 130 bits) */ \ vpaddq A0, A0, P2_H; \ vpsllq P2_H, P2_H, 2; \ vpaddq A0, A0, P2_H; \ vpsrlq ZTMP1, A0, 44; \ vpandq A0, A0, [.Lmask_44 ADD_RIP]; \ vpaddq A1, A1, ZTMP1; /* ;; ============================================================================= ;; ============================================================================= ;; Computes hash for 16 16-byte message blocks, ;; and adds new message blocks to accumulator, ;; interleaving this computation with the loading and splatting ;; of new data. ;; ;; It first multiplies all 16 blocks with powers of R (8 blocks from A0-A2 ;; and 8 blocks from B0-B2, multiplied by R0-R2) ;; ;; a2 a1 a0 ;; × b2 b1 b0 ;; --------------------------------------- ;; a2×b0 a1×b0 a0×b0 ;; + a1×b1 a0×b1 5×a2×b1 ;; + a0×b2 5×a2×b2 5×a1×b2 ;; --------------------------------------- ;; p2 p1 p0 ;; ;; Then, it propagates the carry (higher bits after bit 43) ;; from lower limbs into higher limbs, ;; multiplying by 5 in case of the carry of p2, and adds ;; the results to A0-A2 and B0-B2. ;; ;; ============================================================================= ;A0 [in/out] ZMM register containing 1st 44-bit limb of blocks 1-8 ;A1 [in/out] ZMM register containing 2nd 44-bit limb of blocks 1-8 ;A2 [in/out] ZMM register containing 3rd 44-bit limb of blocks 1-8 ;B0 [in/out] ZMM register containing 1st 44-bit limb of blocks 9-16 ;B1 [in/out] ZMM register containing 2nd 44-bit limb of blocks 9-16 ;B2 [in/out] ZMM register containing 3rd 44-bit limb of blocks 9-16 ;R0 [in] ZMM register (R0) to include the 1st limb of R ;R1 [in] ZMM register (R1) to include the 2nd limb of R ;R2 [in] ZMM register (R2) to include the 3rd limb of R ;R1P [in] ZMM register (R1') to include the 2nd limb of R (multiplied by 5) ;R2P [in] ZMM register (R2') to include the 3rd limb of R (multiplied by 5) ;P0_L [clobbered] ZMM register to contain p[0] of the 8 blocks 1-8 ;P0_H [clobbered] ZMM register to contain p[0] of the 8 blocks 1-8 ;P1_L [clobbered] ZMM register to contain p[1] of the 8 blocks 1-8 ;P1_H [clobbered] ZMM register to contain p[1] of the 8 blocks 1-8 ;P2_L [clobbered] ZMM register to contain p[2] of the 8 blocks 1-8 ;P2_H [clobbered] ZMM register to contain p[2] of the 8 blocks 1-8 ;Q0_L [clobbered] ZMM register to contain p[0] of the 8 blocks 9-16 ;Q0_H [clobbered] ZMM register to contain p[0] of the 8 blocks 9-16 ;Q1_L [clobbered] ZMM register to contain p[1] of the 8 blocks 9-16 ;Q1_H [clobbered] ZMM register to contain p[1] of the 8 blocks 9-16 ;Q2_L [clobbered] ZMM register to contain p[2] of the 8 blocks 9-16 ;Q2_H [clobbered] ZMM register to contain p[2] of the 8 blocks 9-16 ;ZTMP1 [clobbered] Temporary ZMM register ;ZTMP2 [clobbered] Temporary ZMM register ;ZTMP3 [clobbered] Temporary ZMM register ;ZTMP4 [clobbered] Temporary ZMM register ;ZTMP5 [clobbered] Temporary ZMM register ;ZTMP6 [clobbered] Temporary ZMM register ;ZTMP7 [clobbered] Temporary ZMM register ;ZTMP8 [clobbered] Temporary ZMM register ;ZTMP9 [clobbered] Temporary ZMM register ;MSG [in/out] Pointer to message ;LEN [in/out] Length left of message */ #define POLY1305_MSG_MUL_REDUCE_VEC16(A0, A1, A2, B0, B1, B2, R0, R1, R2, R1P, \ R2P, P0_L, P0_H, P1_L, P1_H, P2_L, P2_H, \ Q0_L, Q0_H, Q1_L, Q1_H, Q2_L, Q2_H, \ ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, \ ZTMP6, ZTMP7, ZTMP8, ZTMP9, MSG, LEN) \ /* ;; Reset accumulator */ \ vpxorq P0_L, P0_L, P0_L; \ vpxorq P0_H, P0_H, P0_H; \ vpxorq P1_L, P1_L, P1_L; \ vpxorq P1_H, P1_H, P1_H; \ vpxorq P2_L, P2_L, P2_L; \ vpxorq P2_H, P2_H, P2_H; \ vpxorq Q0_L, Q0_L, Q0_L; \ vpxorq Q0_H, Q0_H, Q0_H; \ vpxorq Q1_L, Q1_L, Q1_L; \ vpxorq Q1_H, Q1_H, Q1_H; \ vpxorq Q2_L, Q2_L, Q2_L; \ vpxorq Q2_H, Q2_H, Q2_H; \ \ /* ;; This code interleaves hash computation with input loading/splatting */ \ \ /* ; Calculate products */ \ vpmadd52luq P0_L, A2, R1P; \ vpmadd52huq P0_H, A2, R1P; \ /* ;; input loading of new blocks */ \ add MSG, POLY1305_BLOCK_SIZE*16; \ sub LEN, POLY1305_BLOCK_SIZE*16; \ \ vpmadd52luq Q0_L, B2, R1P; \ vpmadd52huq Q0_H, B2, R1P; \ \ vpmadd52luq P1_L, A2, R2P; \ vpmadd52huq P1_H, A2, R2P; \ /* ; Load next block of data (128 bytes) */ \ vmovdqu64 ZTMP5, [MSG]; \ vmovdqu64 ZTMP2, [MSG + 64]; \ \ vpmadd52luq Q1_L, B2, R2P; \ vpmadd52huq Q1_H, B2, R2P; \ \ /* ; Interleave new blocks of data */ \ vpunpckhqdq ZTMP3, ZTMP5, ZTMP2; \ vpunpcklqdq ZTMP5, ZTMP5, ZTMP2; \ \ vpmadd52luq P0_L, A0, R0; \ vpmadd52huq P0_H, A0, R0; \ /* ; Highest 42-bit limbs of new blocks */ \ vpsrlq ZTMP6, ZTMP3, 24; \ vporq ZTMP6, ZTMP6, [.Lhigh_bit ADD_RIP]; /* ; Add 2^128 to all 8 final qwords of the message */ \ \ vpmadd52luq Q0_L, B0, R0; \ vpmadd52huq Q0_H, B0, R0; \ \ /* ; Middle 44-bit limbs of new blocks */ \ vpsrlq ZTMP2, ZTMP5, 44; \ vpsllq ZTMP4, ZTMP3, 20; \ \ vpmadd52luq P2_L, A2, R0; \ vpmadd52huq P2_H, A2, R0; \ vpternlogq ZTMP2, ZTMP4, [.Lmask_44 ADD_RIP], 0xA8; /* ; (A OR B AND C) */ \ \ /* ; Lowest 44-bit limbs of new blocks */ \ vpandq ZTMP5, ZTMP5, [.Lmask_44 ADD_RIP]; \ \ vpmadd52luq Q2_L, B2, R0; \ vpmadd52huq Q2_H, B2, R0; \ \ /* ; Load next block of data (128 bytes) */ \ vmovdqu64 ZTMP8, [MSG + 64*2]; \ vmovdqu64 ZTMP9, [MSG + 64*3]; \ \ vpmadd52luq P1_L, A0, R1; \ vpmadd52huq P1_H, A0, R1; \ /* ; Interleave new blocks of data */ \ vpunpckhqdq ZTMP3, ZTMP8, ZTMP9; \ vpunpcklqdq ZTMP8, ZTMP8, ZTMP9; \ \ vpmadd52luq Q1_L, B0, R1; \ vpmadd52huq Q1_H, B0, R1; \ \ /* ; Highest 42-bit limbs of new blocks */ \ vpsrlq ZTMP7, ZTMP3, 24; \ vporq ZTMP7, ZTMP7, [.Lhigh_bit ADD_RIP]; /* ; Add 2^128 to all 8 final qwords of the message */ \ \ vpmadd52luq P0_L, A1, R2P; \ vpmadd52huq P0_H, A1, R2P; \ \ /* ; Middle 44-bit limbs of new blocks */ \ vpsrlq ZTMP9, ZTMP8, 44; \ vpsllq ZTMP4, ZTMP3, 20; \ \ vpmadd52luq Q0_L, B1, R2P; \ vpmadd52huq Q0_H, B1, R2P; \ \ vpternlogq ZTMP9, ZTMP4, [.Lmask_44 ADD_RIP], 0xA8; /* ; (A OR B AND C) */ \ \ /* ; Lowest 44-bit limbs of new blocks */ \ vpandq ZTMP8, ZTMP8, [.Lmask_44 ADD_RIP]; \ \ vpmadd52luq P2_L, A0, R2; \ vpmadd52huq P2_H, A0, R2; \ /* ; Carry propagation (first pass) */ \ vpsrlq ZTMP1, P0_L, 44; \ vpsllq P0_H, P0_H, 8; \ vpmadd52luq Q2_L, B0, R2; \ vpmadd52huq Q2_H, B0, R2; \ \ vpsrlq ZTMP3, Q0_L, 44; \ vpsllq Q0_H, Q0_H, 8; \ \ vpmadd52luq P1_L, A1, R0; \ vpmadd52huq P1_H, A1, R0; \ /* ; Carry propagation (first pass) - continue */ \ vpandq A0, P0_L, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ vpaddq P0_H, P0_H, ZTMP1; \ vpmadd52luq Q1_L, B1, R0; \ vpmadd52huq Q1_H, B1, R0; \ \ vpandq B0, Q0_L, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ vpaddq Q0_H, Q0_H, ZTMP3; \ \ vpmadd52luq P2_L, A1, R1; \ vpmadd52huq P2_H, A1, R1; \ /* ; Carry propagation (first pass) - continue */ \ vpaddq P1_L, P1_L, P0_H; \ vpsllq P1_H, P1_H, 8; \ vpsrlq ZTMP1, P1_L, 44; \ vpmadd52luq Q2_L, B1, R1; \ vpmadd52huq Q2_H, B1, R1; \ \ vpandq A1, P1_L, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ vpaddq Q1_L, Q1_L, Q0_H; \ vpsllq Q1_H, Q1_H, 8; \ vpsrlq ZTMP3, Q1_L, 44; \ vpandq B1, Q1_L, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ \ vpaddq P2_L, P2_L, P1_H; /* ; P2_L += P1_H + P1_L[63:44] */ \ vpaddq P2_L, P2_L, ZTMP1; \ vpandq A2, P2_L, [.Lmask_42 ADD_RIP]; /* ; Clear top 22 bits */ \ vpaddq A2, A2, ZTMP6; /* ; Add highest bits from new blocks to accumulator */ \ vpsrlq ZTMP1, P2_L, 42; \ vpsllq P2_H, P2_H, 10; \ vpaddq P2_H, P2_H, ZTMP1; \ \ vpaddq Q2_L, Q2_L, Q1_H; /* ; Q2_L += P1_H + P1_L[63:44] */ \ vpaddq Q2_L, Q2_L, ZTMP3; \ vpandq B2, Q2_L, [.Lmask_42 ADD_RIP]; /* ; Clear top 22 bits */ \ vpaddq B2, B2, ZTMP7; /* ; Add highest bits from new blocks to accumulator */ \ vpsrlq ZTMP3, Q2_L, 42; \ vpsllq Q2_H, Q2_H, 10; \ vpaddq Q2_H, Q2_H, ZTMP3; \ \ /* ; Carry propagation (second pass) */ \ /* ; Multiply by 5 the highest bits (above 130 bits) */ \ vpaddq A0, A0, P2_H; \ vpsllq P2_H, P2_H, 2; \ vpaddq A0, A0, P2_H; \ vpaddq B0, B0, Q2_H; \ vpsllq Q2_H, Q2_H, 2; \ vpaddq B0, B0, Q2_H; \ \ vpsrlq ZTMP1, A0, 44; \ vpandq A0, A0, [.Lmask_44 ADD_RIP]; \ vpaddq A0, A0, ZTMP5; /* ; Add low 42-bit bits from new blocks to accumulator */ \ vpaddq A1, A1, ZTMP2; /* ; Add medium 42-bit bits from new blocks to accumulator */ \ vpaddq A1, A1, ZTMP1; \ vpsrlq ZTMP3, B0, 44; \ vpandq B0, B0, [.Lmask_44 ADD_RIP]; \ vpaddq B0, B0, ZTMP8; /* ; Add low 42-bit bits from new blocks to accumulator */ \ vpaddq B1, B1, ZTMP9; /* ; Add medium 42-bit bits from new blocks to accumulator */ \ vpaddq B1, B1, ZTMP3 /* ;; ============================================================================= ;; ============================================================================= ;; Computes hash for 16 16-byte message blocks. ;; ;; It first multiplies all 16 blocks with powers of R (8 blocks from A0-A2 ;; and 8 blocks from B0-B2, multiplied by R0-R2 and S0-S2) ;; ;; ;; a2 a1 a0 ;; × b2 b1 b0 ;; --------------------------------------- ;; a2×b0 a1×b0 a0×b0 ;; + a1×b1 a0×b1 5×a2×b1 ;; + a0×b2 5×a2×b2 5×a1×b2 ;; --------------------------------------- ;; p2 p1 p0 ;; ;; Then, it propagates the carry (higher bits after bit 43) from lower limbs into higher limbs, ;; multiplying by 5 in case of the carry of p2. ;; ;; ============================================================================= ;A0 [in/out] ZMM register containing 1st 44-bit limb of the 8 blocks ;A1 [in/out] ZMM register containing 2nd 44-bit limb of the 8 blocks ;A2 [in/out] ZMM register containing 3rd 44-bit limb of the 8 blocks ;B0 [in/out] ZMM register containing 1st 44-bit limb of the 8 blocks ;B1 [in/out] ZMM register containing 2nd 44-bit limb of the 8 blocks ;B2 [in/out] ZMM register containing 3rd 44-bit limb of the 8 blocks ;R0 [in] ZMM register (R0) to include the 1st limb in IDX ;R1 [in] ZMM register (R1) to include the 2nd limb in IDX ;R2 [in] ZMM register (R2) to include the 3rd limb in IDX ;R1P [in] ZMM register (R1') to include the 2nd limb (multiplied by 5) in IDX ;R2P [in] ZMM register (R2') to include the 3rd limb (multiplied by 5) in IDX ;S0 [in] ZMM register (R0) to include the 1st limb in IDX ;S1 [in] ZMM register (R1) to include the 2nd limb in IDX ;S2 [in] ZMM register (R2) to include the 3rd limb in IDX ;S1P [in] ZMM register (R1') to include the 2nd limb (multiplied by 5) in IDX ;S2P [in] ZMM register (R2') to include the 3rd limb (multiplied by 5) in IDX ;P0_L [clobbered] ZMM register to contain p[0] of the 8 blocks ;P0_H [clobbered] ZMM register to contain p[0] of the 8 blocks ;P1_L [clobbered] ZMM register to contain p[1] of the 8 blocks ;P1_H [clobbered] ZMM register to contain p[1] of the 8 blocks ;P2_L [clobbered] ZMM register to contain p[2] of the 8 blocks ;P2_H [clobbered] ZMM register to contain p[2] of the 8 blocks ;Q0_L [clobbered] ZMM register to contain p[0] of the 8 blocks ;Q0_H [clobbered] ZMM register to contain p[0] of the 8 blocks ;Q1_L [clobbered] ZMM register to contain p[1] of the 8 blocks ;Q1_H [clobbered] ZMM register to contain p[1] of the 8 blocks ;Q2_L [clobbered] ZMM register to contain p[2] of the 8 blocks ;Q2_H [clobbered] ZMM register to contain p[2] of the 8 blocks ;ZTMP1 [clobbered] Temporary ZMM register ;ZTMP2 [clobbered] Temporary ZMM register */ #define POLY1305_MUL_REDUCE_VEC16(A0, A1, A2, B0, B1, B2, R0, R1, R2, R1P, R2P,\ S0, S1, S2, S1P, S2P, P0_L, P0_H, P1_L, P1_H,\ P2_L, P2_H, Q0_L, Q0_H, Q1_L, Q1_H, Q2_L,\ Q2_H, ZTMP1, ZTMP2) \ /* ;; Reset accumulator */ \ vpxorq P0_L, P0_L, P0_L; \ vpxorq P0_H, P0_H, P0_H; \ vpxorq P1_L, P1_L, P1_L; \ vpxorq P1_H, P1_H, P1_H; \ vpxorq P2_L, P2_L, P2_L; \ vpxorq P2_H, P2_H, P2_H; \ vpxorq Q0_L, Q0_L, Q0_L; \ vpxorq Q0_H, Q0_H, Q0_H; \ vpxorq Q1_L, Q1_L, Q1_L; \ vpxorq Q1_H, Q1_H, Q1_H; \ vpxorq Q2_L, Q2_L, Q2_L; \ vpxorq Q2_H, Q2_H, Q2_H; \ \ /* ;; This code interleaves hash computation with input loading/splatting */ \ \ /* ; Calculate products */ \ vpmadd52luq P0_L, A2, R1P; \ vpmadd52huq P0_H, A2, R1P; \ \ vpmadd52luq Q0_L, B2, S1P; \ vpmadd52huq Q0_H, B2, S1P; \ \ vpmadd52luq P1_L, A2, R2P; \ vpmadd52huq P1_H, A2, R2P; \ \ vpmadd52luq Q1_L, B2, S2P; \ vpmadd52huq Q1_H, B2, S2P; \ \ vpmadd52luq P0_L, A0, R0; \ vpmadd52huq P0_H, A0, R0; \ \ vpmadd52luq Q0_L, B0, S0; \ vpmadd52huq Q0_H, B0, S0; \ \ vpmadd52luq P2_L, A2, R0; \ vpmadd52huq P2_H, A2, R0; \ vpmadd52luq Q2_L, B2, S0; \ vpmadd52huq Q2_H, B2, S0; \ \ vpmadd52luq P1_L, A0, R1; \ vpmadd52huq P1_H, A0, R1; \ vpmadd52luq Q1_L, B0, S1; \ vpmadd52huq Q1_H, B0, S1; \ \ vpmadd52luq P0_L, A1, R2P; \ vpmadd52huq P0_H, A1, R2P; \ \ vpmadd52luq Q0_L, B1, S2P; \ vpmadd52huq Q0_H, B1, S2P; \ \ vpmadd52luq P2_L, A0, R2; \ vpmadd52huq P2_H, A0, R2; \ \ vpmadd52luq Q2_L, B0, S2; \ vpmadd52huq Q2_H, B0, S2; \ \ /* ; Carry propagation (first pass) */ \ vpsrlq ZTMP1, P0_L, 44; \ vpsllq P0_H, P0_H, 8; \ vpsrlq ZTMP2, Q0_L, 44; \ vpsllq Q0_H, Q0_H, 8; \ \ vpmadd52luq P1_L, A1, R0; \ vpmadd52huq P1_H, A1, R0; \ vpmadd52luq Q1_L, B1, S0; \ vpmadd52huq Q1_H, B1, S0; \ \ /* ; Carry propagation (first pass) - continue */ \ vpandq A0, P0_L, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ vpaddq P0_H, P0_H, ZTMP1; \ vpandq B0, Q0_L, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ vpaddq Q0_H, Q0_H, ZTMP2; \ \ vpmadd52luq P2_L, A1, R1; \ vpmadd52huq P2_H, A1, R1; \ vpmadd52luq Q2_L, B1, S1; \ vpmadd52huq Q2_H, B1, S1; \ \ /* ; Carry propagation (first pass) - continue */ \ vpaddq P1_L, P1_L, P0_H; \ vpsllq P1_H, P1_H, 8; \ vpsrlq ZTMP1, P1_L, 44; \ vpandq A1, P1_L, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ vpaddq Q1_L, Q1_L, Q0_H; \ vpsllq Q1_H, Q1_H, 8; \ vpsrlq ZTMP2, Q1_L, 44; \ vpandq B1, Q1_L, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ \ vpaddq P2_L, P2_L, P1_H; /* ; P2_L += P1_H + P1_L[63:44] */ \ vpaddq P2_L, P2_L, ZTMP1; \ vpandq A2, P2_L, [.Lmask_42 ADD_RIP]; /* ; Clear top 22 bits */ \ vpsrlq ZTMP1, P2_L, 42; \ vpsllq P2_H, P2_H, 10; \ vpaddq P2_H, P2_H, ZTMP1; \ \ vpaddq Q2_L, Q2_L, Q1_H; /* ; Q2_L += P1_H + P1_L[63:44] */ \ vpaddq Q2_L, Q2_L, ZTMP2; \ vpandq B2, Q2_L, [.Lmask_42 ADD_RIP]; /* ; Clear top 22 bits */ \ vpsrlq ZTMP2, Q2_L, 42; \ vpsllq Q2_H, Q2_H, 10; \ vpaddq Q2_H, Q2_H, ZTMP2; \ \ /* ; Carry propagation (second pass) */ \ /* ; Multiply by 5 the highest bits (above 130 bits) */ \ vpaddq A0, A0, P2_H; \ vpsllq P2_H, P2_H, 2; \ vpaddq A0, A0, P2_H; \ vpaddq B0, B0, Q2_H; \ vpsllq Q2_H, Q2_H, 2; \ vpaddq B0, B0, Q2_H; \ \ vpsrlq ZTMP1, A0, 44; \ vpandq A0, A0, [.Lmask_44 ADD_RIP]; \ vpaddq A1, A1, ZTMP1; \ vpsrlq ZTMP2, B0, 44; \ vpandq B0, B0, [.Lmask_44 ADD_RIP]; \ vpaddq B1, B1, ZTMP2; /* ;; ============================================================================= ;; ============================================================================= ;; Shuffle data blocks, so they match the right power of R. ;; Powers of R are in this order: R^8 R^4 R^7 R^3 R^6 R^2 R^5 R ;; Data blocks are coming in this order: A0 A4 A1 A5 A2 A6 A3 A7 ;; Generally the computation is: A0*R^8 + A1*R^7 + A2*R^6 + A3*R^5 + ;; A4*R^4 + A5*R^3 + A6*R^2 + A7*R ;; When there are less data blocks, less powers of R are used, so data needs to ;; be shuffled. Example: if 4 blocks are left, only A0-A3 are available and only ;; R-R^4 are used (A0*R^4 + A1*R^3 + A2*R^2 + A3*R), so A0-A3 need to be shifted ;; ============================================================================= ;A_L [in/out] 0-43 bits of input data ;A_M [in/out] 44-87 bits of input data ;A_H [in/out] 88-129 bits of input data ;TMP [clobbered] Temporary GP register ;N_BLOCKS [in] Number of remaining input blocks */ #define SHUFFLE_DATA_SMASK_1 0x39 #define SHUFFLE_DATA_KMASK_1 0xffff #define SHUFFLE_DATA_SMASK_2 0x4E #define SHUFFLE_DATA_KMASK_2 0xffff #define SHUFFLE_DATA_SMASK_3 0x93 #define SHUFFLE_DATA_KMASK_3 0xffff #define SHUFFLE_DATA_KMASK_4 0xffff #define SHUFFLE_DATA_SMASK_5 0x39 #define SHUFFLE_DATA_KMASK_5 0xfff0 #define SHUFFLE_DATA_SMASK_6 0x4E #define SHUFFLE_DATA_KMASK_6 0xff00 #define SHUFFLE_DATA_SMASK_7 0x93 #define SHUFFLE_DATA_KMASK_7 0xf000 #define SHUFFLE_DATA_BLOCKS_GENERIC(A_L, A_M, A_H, TMP, N_BLOCKS) \ mov TMP, SHUFFLE_DATA_KMASK_##N_BLOCKS; \ kmovq k1, TMP; \ vpshufd A_L{k1}, A_L, 0x4E; \ vpshufd A_M{k1}, A_M, 0x4E; \ vpshufd A_H{k1}, A_H, 0x4E; \ vshufi64x2 A_L, A_L, A_L, SHUFFLE_DATA_SMASK_##N_BLOCKS; \ vshufi64x2 A_M, A_M, A_M, SHUFFLE_DATA_SMASK_##N_BLOCKS; \ vshufi64x2 A_H, A_H, A_H, SHUFFLE_DATA_SMASK_##N_BLOCKS #define SHUFFLE_DATA_BLOCKS_1(A_L, A_M, A_H, TMP) \ SHUFFLE_DATA_BLOCKS_GENERIC(A_L, A_M, A_H, TMP, 1) #define SHUFFLE_DATA_BLOCKS_2(A_L, A_M, A_H, TMP) \ SHUFFLE_DATA_BLOCKS_GENERIC(A_L, A_M, A_H, TMP, 2) #define SHUFFLE_DATA_BLOCKS_3(A_L, A_M, A_H, TMP) \ SHUFFLE_DATA_BLOCKS_GENERIC(A_L, A_M, A_H, TMP, 3) #define SHUFFLE_DATA_BLOCKS_4(A_L, A_M, A_H, TMP) \ mov TMP, SHUFFLE_DATA_KMASK_4; \ kmovq k1, TMP; \ vpshufd A_L{k1}, A_L, 0x4E; \ vpshufd A_M{k1}, A_M, 0x4E; \ vpshufd A_H{k1}, A_H, 0x4E; #define SHUFFLE_DATA_BLOCKS_5(A_L, A_M, A_H, TMP) \ SHUFFLE_DATA_BLOCKS_GENERIC(A_L, A_M, A_H, TMP, 5) #define SHUFFLE_DATA_BLOCKS_6(A_L, A_M, A_H, TMP) \ SHUFFLE_DATA_BLOCKS_GENERIC(A_L, A_M, A_H, TMP, 6) #define SHUFFLE_DATA_BLOCKS_7(A_L, A_M, A_H, TMP) \ SHUFFLE_DATA_BLOCKS_GENERIC(A_L, A_M, A_H, TMP, 7) /* ;; ============================================================================= ;; ============================================================================= ;; Computes hash for message length being multiple of block size ;; ============================================================================= ;MSG [in/out] GPR pointer to input message (updated) ;LEN [in/out] GPR in: length in bytes / out: length mod 16 ;A0 [in/out] accumulator bits 63..0 ;A1 [in/out] accumulator bits 127..64 ;A2 [in/out] accumulator bits 195..128 ;R0 [in] R constant bits 63..0 ;R1 [in] R constant bits 127..64 ;T0 [clobbered] GPR register ;T1 [clobbered] GPR register ;T2 [clobbered] GPR register ;T3 [clobbered] GPR register ;GP_RAX [clobbered] RAX register ;GP_RDX [clobbered] RDX register */ #define POLY1305_BLOCKS(MSG, LEN, A0, A1, A2, R0, R1, T0, T1, T2, T3, \ GP_RAX, GP_RDX) \ /* ; Minimum of 256 bytes to run vectorized code */ \ cmp LEN, POLY1305_BLOCK_SIZE*16; \ jb .L_final_loop; \ \ /* ; Spread accumulator into 44-bit limbs in quadwords */ \ mov T0, A0; \ and T0, [.Lmask_44 ADD_RIP]; /* ;; First limb (A[43:0]) */ \ vmovq xmm5, T0; \ \ mov T0, A1; \ shrd A0, T0, 44; \ and A0, [.Lmask_44 ADD_RIP]; /* ;; Second limb (A[77:52]) */ \ vmovq xmm6, A0; \ \ shrd A1, A2, 24; \ and A1, [.Lmask_42 ADD_RIP]; /* ;; Third limb (A[129:88]) */ \ vmovq xmm7, A1; \ \ /* ; Load first block of data (128 bytes) */ \ vmovdqu64 zmm0, [MSG]; \ vmovdqu64 zmm1, [MSG + 64]; \ \ /* ; Interleave the data to form 44-bit limbs */ \ /* ; */ \ /* ; zmm13 to have bits 0-43 of all 8 blocks in 8 qwords */ \ /* ; zmm14 to have bits 87-44 of all 8 blocks in 8 qwords */ \ /* ; zmm15 to have bits 127-88 of all 8 blocks in 8 qwords */ \ vpunpckhqdq zmm15, zmm0, zmm1; \ vpunpcklqdq zmm13, zmm0, zmm1; \ \ vpsrlq zmm14, zmm13, 44; \ vpsllq zmm18, zmm15, 20; \ vpternlogq zmm14, zmm18, [.Lmask_44 ADD_RIP], 0xA8; /* ; (A OR B AND C) */ \ \ vpandq zmm13, zmm13, [.Lmask_44 ADD_RIP]; \ vpsrlq zmm15, zmm15, 24; \ \ /* ; Add 2^128 to all 8 final qwords of the message */ \ vporq zmm15, zmm15, [.Lhigh_bit ADD_RIP]; \ \ vpaddq zmm13, zmm13, zmm5; \ vpaddq zmm14, zmm14, zmm6; \ vpaddq zmm15, zmm15, zmm7; \ \ /* ; Load next blocks of data (128 bytes) */ \ vmovdqu64 zmm0, [MSG + 64*2]; \ vmovdqu64 zmm1, [MSG + 64*3]; \ \ /* ; Interleave the data to form 44-bit limbs */ \ /* ; */ \ /* ; zmm13 to have bits 0-43 of all 8 blocks in 8 qwords */ \ /* ; zmm14 to have bits 87-44 of all 8 blocks in 8 qwords */ \ /* ; zmm15 to have bits 127-88 of all 8 blocks in 8 qwords */ \ vpunpckhqdq zmm18, zmm0, zmm1; \ vpunpcklqdq zmm16, zmm0, zmm1; \ \ vpsrlq zmm17, zmm16, 44; \ vpsllq zmm19, zmm18, 20; \ vpternlogq zmm17, zmm19, [.Lmask_44 ADD_RIP], 0xA8; /* ; (A OR B AND C) */ \ \ vpandq zmm16, zmm16, [.Lmask_44 ADD_RIP]; \ vpsrlq zmm18, zmm18, 24; \ \ /* ; Add 2^128 to all 8 final qwords of the message */ \ vporq zmm18, zmm18, [.Lhigh_bit ADD_RIP]; \ \ /* ; Use memory in stack to save powers of R, before loading them into ZMM registers */ \ /* ; The first 16*8 bytes will contain the 16 bytes of the 8 powers of R */ \ /* ; The last 64 bytes will contain the last 2 bits of powers of R, spread in 8 qwords, */ \ /* ; to be OR'd with the highest qwords (in zmm26) */ \ vmovq xmm3, R0; \ vpinsrq xmm3, xmm3, R1, 1; \ vinserti32x4 zmm1, zmm1, xmm3, 3; \ \ vpxorq zmm0, zmm0, zmm0; \ vpxorq zmm2, zmm2, zmm2; \ \ /* ; Calculate R^2 */ \ mov T0, R1; \ shr T0, 2; \ add T0, R1; /* ;; T0 = R1 + (R1 >> 2) */ \ \ mov A0, R0; \ mov A1, R1; \ \ POLY1305_MUL_REDUCE(A0, A1, A2, R0, R1, T0, T1, T2, T3, GP_RAX, GP_RDX, A2_ZERO); \ \ vmovq xmm3, A0; \ vpinsrq xmm3, xmm3, A1, 1; \ vinserti32x4 zmm1, zmm1, xmm3, 2; \ \ vmovq xmm4, A2; \ vinserti32x4 zmm2, zmm2, xmm4, 2; \ \ /* ; Calculate R^3 */ \ POLY1305_MUL_REDUCE(A0, A1, A2, R0, R1, T0, T1, T2, T3, GP_RAX, GP_RDX, A2_NOT_ZERO); \ \ vmovq xmm3, A0; \ vpinsrq xmm3, xmm3, A1, 1; \ vinserti32x4 zmm1, zmm1, xmm3, 1; \ \ vmovq xmm4, A2; \ vinserti32x4 zmm2, zmm2, xmm4, 1; \ \ /* ; Calculate R^4 */ \ POLY1305_MUL_REDUCE(A0, A1, A2, R0, R1, T0, T1, T2, T3, GP_RAX, GP_RDX, A2_NOT_ZERO); \ \ vmovq xmm3, A0; \ vpinsrq xmm3, xmm3, A1, 1; \ vinserti32x4 zmm1, zmm1, xmm3, 0; \ \ vmovq xmm4, A2; \ vinserti32x4 zmm2, zmm2, xmm4, 0; \ \ /* ; Move 2 MSbits to top 24 bits, to be OR'ed later */ \ vpsllq zmm2, zmm2, 40; \ \ vpunpckhqdq zmm21, zmm1, zmm0; \ vpunpcklqdq zmm19, zmm1, zmm0; \ \ vpsrlq zmm20, zmm19, 44; \ vpsllq zmm4, zmm21, 20; \ vpternlogq zmm20, zmm4, [.Lmask_44 ADD_RIP], 0xA8; /* ; (A OR B AND C) */ \ \ vpandq zmm19, zmm19, [.Lmask_44 ADD_RIP]; \ vpsrlq zmm21, zmm21, 24; \ \ /* ; zmm2 contains the 2 highest bits of the powers of R */ \ vporq zmm21, zmm21, zmm2; \ \ /* ; Broadcast 44-bit limbs of R^4 */ \ mov T0, A0; \ and T0, [.Lmask_44 ADD_RIP]; /* ;; First limb (R^4[43:0]) */ \ vpbroadcastq zmm22, T0; \ \ mov T0, A1; \ shrd A0, T0, 44; \ and A0, [.Lmask_44 ADD_RIP]; /* ;; Second limb (R^4[87:44]) */ \ vpbroadcastq zmm23, A0; \ \ shrd A1, A2, 24; \ and A1, [.Lmask_42 ADD_RIP]; /* ;; Third limb (R^4[129:88]) */ \ vpbroadcastq zmm24, A1; \ \ /* ; Generate 4*5*R^4 */ \ vpsllq zmm25, zmm23, 2; \ vpsllq zmm26, zmm24, 2; \ \ /* ; 5*R^4 */ \ vpaddq zmm25, zmm25, zmm23; \ vpaddq zmm26, zmm26, zmm24; \ \ /* ; 4*5*R^4 */ \ vpsllq zmm25, zmm25, 2; \ vpsllq zmm26, zmm26, 2; \ \ vpslldq zmm29, zmm19, 8; \ vpslldq zmm30, zmm20, 8; \ vpslldq zmm31, zmm21, 8; \ \ /* ; Calculate R^8-R^5 */ \ POLY1305_MUL_REDUCE_VEC(zmm19, zmm20, zmm21, \ zmm22, zmm23, zmm24, \ zmm25, zmm26, \ zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ zmm11); \ \ /* ; Interleave powers of R: R^8 R^4 R^7 R^3 R^6 R^2 R^5 R */ \ vporq zmm19, zmm19, zmm29; \ vporq zmm20, zmm20, zmm30; \ vporq zmm21, zmm21, zmm31; \ \ /* ; Broadcast R^8 */ \ vpbroadcastq zmm22, xmm19; \ vpbroadcastq zmm23, xmm20; \ vpbroadcastq zmm24, xmm21; \ \ /* ; Generate 4*5*R^8 */ \ vpsllq zmm25, zmm23, 2; \ vpsllq zmm26, zmm24, 2; \ \ /* ; 5*R^8 */ \ vpaddq zmm25, zmm25, zmm23; \ vpaddq zmm26, zmm26, zmm24; \ \ /* ; 4*5*R^8 */ \ vpsllq zmm25, zmm25, 2; \ vpsllq zmm26, zmm26, 2; \ \ cmp LEN, POLY1305_BLOCK_SIZE*32; \ jb .L_len_256_511; \ \ /* ; Store R^8-R for later use */ \ vmovdqa64 [rsp + STACK_r_save], zmm19; \ vmovdqa64 [rsp + STACK_r_save + 64], zmm20; \ vmovdqa64 [rsp + STACK_r_save + 64*2], zmm21; \ \ /* ; Calculate R^16-R^9 */ \ POLY1305_MUL_REDUCE_VEC(zmm19, zmm20, zmm21, \ zmm22, zmm23, zmm24, \ zmm25, zmm26, \ zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ zmm11); \ \ /* ; Store R^16-R^9 for later use */ \ vmovdqa64 [rsp + STACK_r_save + 64*3], zmm19; \ vmovdqa64 [rsp + STACK_r_save + 64*4], zmm20; \ vmovdqa64 [rsp + STACK_r_save + 64*5], zmm21; \ \ /* ; Broadcast R^16 */ \ vpbroadcastq zmm22, xmm19; \ vpbroadcastq zmm23, xmm20; \ vpbroadcastq zmm24, xmm21; \ \ /* ; Generate 4*5*R^16 */ \ vpsllq zmm25, zmm23, 2; \ vpsllq zmm26, zmm24, 2; \ \ /* ; 5*R^16 */ \ vpaddq zmm25, zmm25, zmm23; \ vpaddq zmm26, zmm26, zmm24; \ \ /* ; 4*5*R^16 */ \ vpsllq zmm25, zmm25, 2; \ vpsllq zmm26, zmm26, 2; \ \ mov T0, LEN; \ and T0, 0xffffffffffffff00; /* ; multiple of 256 bytes */ \ \ .L_poly1305_blocks_loop: \ cmp T0, POLY1305_BLOCK_SIZE*16; \ jbe .L_poly1305_blocks_loop_end; \ \ /* ; zmm13-zmm18 contain the 16 blocks of message plus the previous accumulator */ \ /* ; zmm22-24 contain the 5x44-bit limbs of the powers of R */ \ /* ; zmm25-26 contain the 5x44-bit limbs of the powers of R' (5*4*R) */ \ POLY1305_MSG_MUL_REDUCE_VEC16(zmm13, zmm14, zmm15, zmm16, zmm17, zmm18, \ zmm22, zmm23, zmm24, zmm25, zmm26, \ zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ zmm19, zmm20, zmm21, zmm27, zmm28, zmm29, \ zmm30, zmm31, zmm11, zmm0, zmm1, \ zmm2, zmm3, zmm4, zmm12, MSG, T0); \ \ jmp .L_poly1305_blocks_loop; \ \ .L_poly1305_blocks_loop_end: \ \ /* ;; Need to multiply by r^16, r^15, r^14... r */ \ \ /* ; First multiply by r^16-r^9 */ \ \ /* ; Read R^16-R^9 */ \ vmovdqa64 zmm19, [rsp + STACK_r_save + 64*3]; \ vmovdqa64 zmm20, [rsp + STACK_r_save + 64*4]; \ vmovdqa64 zmm21, [rsp + STACK_r_save + 64*5]; \ /* ; Read R^8-R */ \ vmovdqa64 zmm22, [rsp + STACK_r_save]; \ vmovdqa64 zmm23, [rsp + STACK_r_save + 64]; \ vmovdqa64 zmm24, [rsp + STACK_r_save + 64*2]; \ \ /* ; zmm27 to have bits 87-44 of all 9-16th powers of R' in 8 qwords */ \ /* ; zmm28 to have bits 129-88 of all 9-16th powers of R' in 8 qwords */ \ vpsllq zmm0, zmm20, 2; \ vpaddq zmm27, zmm20, zmm0; /* ; R1' (R1*5) */ \ vpsllq zmm1, zmm21, 2; \ vpaddq zmm28, zmm21, zmm1; /* ; R2' (R2*5) */ \ \ /* ; 4*5*R */ \ vpsllq zmm27, zmm27, 2; \ vpsllq zmm28, zmm28, 2; \ \ /* ; Then multiply by r^8-r */ \ \ /* ; zmm25 to have bits 87-44 of all 1-8th powers of R' in 8 qwords */ \ /* ; zmm26 to have bits 129-88 of all 1-8th powers of R' in 8 qwords */ \ vpsllq zmm2, zmm23, 2; \ vpaddq zmm25, zmm23, zmm2; /* ; R1' (R1*5) */ \ vpsllq zmm3, zmm24, 2; \ vpaddq zmm26, zmm24, zmm3; /* ; R2' (R2*5) */ \ \ /* ; 4*5*R */ \ vpsllq zmm25, zmm25, 2; \ vpsllq zmm26, zmm26, 2; \ \ POLY1305_MUL_REDUCE_VEC16(zmm13, zmm14, zmm15, zmm16, zmm17, zmm18, \ zmm19, zmm20, zmm21, zmm27, zmm28, \ zmm22, zmm23, zmm24, zmm25, zmm26, \ zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, \ zmm7, zmm8, zmm9, zmm10, zmm11, zmm12, zmm29); \ \ /* ;; Add all blocks (horizontally) */ \ vpaddq zmm13, zmm13, zmm16; \ vpaddq zmm14, zmm14, zmm17; \ vpaddq zmm15, zmm15, zmm18; \ \ vextracti64x4 ymm0, zmm13, 1; \ vextracti64x4 ymm1, zmm14, 1; \ vextracti64x4 ymm2, zmm15, 1; \ \ vpaddq ymm13, ymm13, ymm0; \ vpaddq ymm14, ymm14, ymm1; \ vpaddq ymm15, ymm15, ymm2; \ \ vextracti32x4 xmm10, ymm13, 1; \ vextracti32x4 xmm11, ymm14, 1; \ vextracti32x4 xmm12, ymm15, 1; \ \ vpaddq xmm13, xmm13, xmm10; \ vpaddq xmm14, xmm14, xmm11; \ vpaddq xmm15, xmm15, xmm12; \ \ vpsrldq xmm10, xmm13, 8; \ vpsrldq xmm11, xmm14, 8; \ vpsrldq xmm12, xmm15, 8; \ \ /* ; Finish folding and clear second qword */ \ mov T0, 0xfd; \ kmovq k1, T0; \ vpaddq xmm13{k1}{z}, xmm13, xmm10; \ vpaddq xmm14{k1}{z}, xmm14, xmm11; \ vpaddq xmm15{k1}{z}, xmm15, xmm12; \ \ add MSG, POLY1305_BLOCK_SIZE*16; \ \ and LEN, (POLY1305_BLOCK_SIZE*16 - 1); /* ; Get remaining lengths (LEN < 256 bytes) */ \ \ .L_less_than_256: \ \ cmp LEN, POLY1305_BLOCK_SIZE*8; \ jb .L_less_than_128; \ \ /* ; Read next 128 bytes */ \ /* ; Load first block of data (128 bytes) */ \ vmovdqu64 zmm0, [MSG]; \ vmovdqu64 zmm1, [MSG + 64]; \ \ /* ; Interleave the data to form 44-bit limbs */ \ /* ; */ \ /* ; zmm13 to have bits 0-43 of all 8 blocks in 8 qwords */ \ /* ; zmm14 to have bits 87-44 of all 8 blocks in 8 qwords */ \ /* ; zmm15 to have bits 127-88 of all 8 blocks in 8 qwords */ \ vpunpckhqdq zmm5, zmm0, zmm1; \ vpunpcklqdq zmm3, zmm0, zmm1; \ \ vpsrlq zmm4, zmm3, 44; \ vpsllq zmm8, zmm5, 20; \ vpternlogq zmm4, zmm8, [.Lmask_44 ADD_RIP], 0xA8; /* ; (A OR B AND C) */ \ \ vpandq zmm3, zmm3, [.Lmask_44 ADD_RIP]; \ vpsrlq zmm5, zmm5, 24; \ \ /* ; Add 2^128 to all 8 final qwords of the message */ \ vporq zmm5, zmm5, [.Lhigh_bit ADD_RIP]; \ \ vpaddq zmm13, zmm13, zmm3; \ vpaddq zmm14, zmm14, zmm4; \ vpaddq zmm15, zmm15, zmm5; \ \ add MSG, POLY1305_BLOCK_SIZE*8; \ sub LEN, POLY1305_BLOCK_SIZE*8; \ \ POLY1305_MUL_REDUCE_VEC(zmm13, zmm14, zmm15, \ zmm22, zmm23, zmm24, \ zmm25, zmm26, \ zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ zmm11); \ \ /* ;; Add all blocks (horizontally) */ \ vextracti64x4 ymm0, zmm13, 1; \ vextracti64x4 ymm1, zmm14, 1; \ vextracti64x4 ymm2, zmm15, 1; \ \ vpaddq ymm13, ymm13, ymm0; \ vpaddq ymm14, ymm14, ymm1; \ vpaddq ymm15, ymm15, ymm2; \ \ vextracti32x4 xmm10, ymm13, 1; \ vextracti32x4 xmm11, ymm14, 1; \ vextracti32x4 xmm12, ymm15, 1; \ \ vpaddq xmm13, xmm13, xmm10; \ vpaddq xmm14, xmm14, xmm11; \ vpaddq xmm15, xmm15, xmm12; \ \ vpsrldq xmm10, xmm13, 8; \ vpsrldq xmm11, xmm14, 8; \ vpsrldq xmm12, xmm15, 8; \ \ /* ; Finish folding and clear second qword */ \ mov T0, 0xfd; \ kmovq k1, T0; \ vpaddq xmm13{k1}{z}, xmm13, xmm10; \ vpaddq xmm14{k1}{z}, xmm14, xmm11; \ vpaddq xmm15{k1}{z}, xmm15, xmm12; \ \ .L_less_than_128: \ cmp LEN, 32; /* ; If remaining bytes is <= 32, perform last blocks in scalar */ \ jbe .L_simd_to_gp; \ \ mov T0, LEN; \ and T0, 0x3f; \ lea T1, [.Lbyte64_len_to_mask_table ADD_RIP]; \ mov T1, [T1 + 8*T0]; \ \ /* ; Load default byte masks */ \ mov T2, 0xffffffffffffffff; \ xor T3, T3; \ \ cmp LEN, 64; \ cmovb T2, T1; /* ; Load mask for first 64 bytes */ \ cmovg T3, T1; /* ; Load mask for second 64 bytes */ \ \ kmovq k1, T2; \ kmovq k2, T3; \ vmovdqu8 zmm0{k1}{z}, [MSG]; \ vmovdqu8 zmm1{k2}{z}, [MSG + 64]; \ \ /* ; Pad last block message, if partial */ \ mov T0, LEN; \ and T0, 0x70; /* ; Multiple of 16 bytes */ \ /* ; Load last block of data (up to 112 bytes) */ \ shr T0, 3; /* ; Get number of full qwords */ \ \ /* ; Interleave the data to form 44-bit limbs */ \ /* ; */ \ /* ; zmm13 to have bits 0-43 of all 8 blocks in 8 qwords */ \ /* ; zmm14 to have bits 87-44 of all 8 blocks in 8 qwords */ \ /* ; zmm15 to have bits 127-88 of all 8 blocks in 8 qwords */ \ vpunpckhqdq zmm4, zmm0, zmm1; \ vpunpcklqdq zmm2, zmm0, zmm1; \ \ vpsrlq zmm3, zmm2, 44; \ vpsllq zmm28, zmm4, 20; \ vpternlogq zmm3, zmm28, [.Lmask_44 ADD_RIP], 0xA8; /* ; (A OR B AND C) */ \ \ vpandq zmm2, zmm2, [.Lmask_44 ADD_RIP]; \ vpsrlq zmm4, zmm4, 24; \ \ lea T1, [.Lqword_high_bit_mask ADD_RIP]; \ kmovb k1, [T1 + T0]; \ /* ; Add 2^128 to final qwords of the message (all full blocks and partial block, */ \ /* ; if "pad_to_16" is selected) */ \ vporq zmm4{k1}, zmm4, [.Lhigh_bit ADD_RIP]; \ \ vpaddq zmm13, zmm13, zmm2; \ vpaddq zmm14, zmm14, zmm3; \ vpaddq zmm15, zmm15, zmm4; \ \ mov T0, LEN; \ add T0, 15; \ shr T0, 4; /* ; Get number of 16-byte blocks (including partial blocks) */ \ xor LEN, LEN; /* ; All length will be consumed */ \ \ /* ; No need to shuffle data blocks (data is in the right order) */ \ cmp T0, 8; \ je .L_end_shuffle; \ \ cmp T0, 4; \ je .L_shuffle_blocks_4; \ jb .L_shuffle_blocks_3; \ \ /* ; Number of 16-byte blocks > 4 */ \ cmp T0, 6; \ je .L_shuffle_blocks_6; \ ja .L_shuffle_blocks_7; \ jmp .L_shuffle_blocks_5; \ \ .L_shuffle_blocks_3: \ SHUFFLE_DATA_BLOCKS_3(zmm13, zmm14, zmm15, T1); \ jmp .L_end_shuffle; \ .L_shuffle_blocks_4: \ SHUFFLE_DATA_BLOCKS_4(zmm13, zmm14, zmm15, T1); \ jmp .L_end_shuffle; \ .L_shuffle_blocks_5: \ SHUFFLE_DATA_BLOCKS_5(zmm13, zmm14, zmm15, T1); \ jmp .L_end_shuffle; \ .L_shuffle_blocks_6: \ SHUFFLE_DATA_BLOCKS_6(zmm13, zmm14, zmm15, T1); \ jmp .L_end_shuffle; \ .L_shuffle_blocks_7: \ SHUFFLE_DATA_BLOCKS_7(zmm13, zmm14, zmm15, T1); \ \ .L_end_shuffle: \ \ /* ; zmm13-zmm15 contain the 8 blocks of message plus the previous accumulator */ \ /* ; zmm22-24 contain the 3x44-bit limbs of the powers of R */ \ /* ; zmm25-26 contain the 3x44-bit limbs of the powers of R' (5*4*R) */ \ POLY1305_MUL_REDUCE_VEC(zmm13, zmm14, zmm15, \ zmm22, zmm23, zmm24, \ zmm25, zmm26, \ zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ zmm11); \ \ /* ;; Add all blocks (horizontally) */ \ vextracti64x4 ymm0, zmm13, 1; \ vextracti64x4 ymm1, zmm14, 1; \ vextracti64x4 ymm2, zmm15, 1; \ \ vpaddq ymm13, ymm13, ymm0; \ vpaddq ymm14, ymm14, ymm1; \ vpaddq ymm15, ymm15, ymm2; \ \ vextracti32x4 xmm10, ymm13, 1; \ vextracti32x4 xmm11, ymm14, 1; \ vextracti32x4 xmm12, ymm15, 1; \ \ vpaddq xmm13, xmm13, xmm10; \ vpaddq xmm14, xmm14, xmm11; \ vpaddq xmm15, xmm15, xmm12; \ \ vpsrldq xmm10, xmm13, 8; \ vpsrldq xmm11, xmm14, 8; \ vpsrldq xmm12, xmm15, 8; \ \ vpaddq xmm13, xmm13, xmm10; \ vpaddq xmm14, xmm14, xmm11; \ vpaddq xmm15, xmm15, xmm12; \ \ .L_simd_to_gp: \ /* ; Carry propagation */ \ vpsrlq xmm0, xmm13, 44; \ vpandq xmm13, xmm13, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ vpaddq xmm14, xmm14, xmm0; \ vpsrlq xmm0, xmm14, 44; \ vpandq xmm14, xmm14, [.Lmask_44 ADD_RIP]; /* ; Clear top 20 bits */ \ vpaddq xmm15, xmm15, xmm0; \ vpsrlq xmm0, xmm15, 42; \ vpandq xmm15, xmm15, [.Lmask_42 ADD_RIP]; /* ; Clear top 22 bits */ \ vpsllq xmm1, xmm0, 2; \ vpaddq xmm0, xmm0, xmm1; \ vpaddq xmm13, xmm13, xmm0; \ \ /* ; Put together A */ \ vmovq A0, xmm13; \ \ vmovq T0, xmm14; \ mov T1, T0; \ shl T1, 44; \ or A0, T1; \ \ shr T0, 20; \ vmovq A2, xmm15; \ mov A1, A2; \ shl A1, 24; \ or A1, T0; \ shr A2, 40; \ \ /* ; Clear powers of R */ \ vpxorq zmm0, zmm0, zmm0; \ vmovdqa64 [rsp + STACK_r_save], zmm0; \ vmovdqa64 [rsp + STACK_r_save + 64], zmm0; \ vmovdqa64 [rsp + STACK_r_save + 64*2], zmm0; \ vmovdqa64 [rsp + STACK_r_save + 64*3], zmm0; \ vmovdqa64 [rsp + STACK_r_save + 64*4], zmm0; \ vmovdqa64 [rsp + STACK_r_save + 64*5], zmm0; \ \ vzeroall; \ - clear_zmm(xmm16); clear_zmm(xmm20); clear_zmm(xmm24); clear_zmm(xmm28); \ - clear_zmm(xmm17); clear_zmm(xmm21); clear_zmm(xmm25); clear_zmm(xmm29); \ - clear_zmm(xmm18); clear_zmm(xmm22); clear_zmm(xmm26); clear_zmm(xmm30); \ - clear_zmm(xmm19); clear_zmm(xmm23); clear_zmm(xmm27); clear_zmm(xmm31); \ + clear_zmm(ymm16); clear_zmm(ymm20); clear_zmm(ymm24); clear_zmm(ymm28); \ + clear_zmm(ymm17); clear_zmm(ymm21); clear_zmm(ymm25); clear_zmm(ymm29); \ + clear_zmm(ymm18); clear_zmm(ymm22); clear_zmm(ymm26); clear_zmm(ymm30); \ + clear_zmm(ymm19); clear_zmm(ymm23); clear_zmm(ymm27); clear_zmm(ymm31); \ \ .L_final_loop: \ cmp LEN, POLY1305_BLOCK_SIZE; \ jb .L_poly1305_blocks_exit; \ \ /* ;; A += MSG[i] */ \ add A0, [MSG + 0]; \ adc A1, [MSG + 8]; \ adc A2, 1; /* ;; no padding bit */ \ \ mov T0, R1; \ shr T0, 2; \ add T0, R1; /* ;; T0 = R1 + (R1 >> 2) */ \ \ POLY1305_MUL_REDUCE(A0, A1, A2, R0, R1, \ T0, T1, T2, T3, GP_RAX, GP_RDX, A2_NOT_ZERO); \ \ add MSG, POLY1305_BLOCK_SIZE; \ sub LEN, POLY1305_BLOCK_SIZE; \ \ jmp .L_final_loop; \ \ .L_len_256_511: \ \ /* ; zmm13-zmm15 contain the 8 blocks of message plus the previous accumulator */ \ /* ; zmm22-24 contain the 3x44-bit limbs of the powers of R */ \ /* ; zmm25-26 contain the 3x44-bit limbs of the powers of R' (5*4*R) */ \ POLY1305_MUL_REDUCE_VEC(zmm13, zmm14, zmm15, \ zmm22, zmm23, zmm24, \ zmm25, zmm26, \ zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ zmm11); \ \ /* ; Then multiply by r^8-r */ \ \ /* ; zmm19-zmm21 contains R^8-R, need to move it to zmm22-24, */ \ /* ; as it might be used in other part of the code */ \ vmovdqa64 zmm22, zmm19; \ vmovdqa64 zmm23, zmm20; \ vmovdqa64 zmm24, zmm21; \ \ /* ; zmm25 to have bits 87-44 of all 8 powers of R' in 8 qwords */ \ /* ; zmm26 to have bits 129-88 of all 8 powers of R' in 8 qwords */ \ vpsllq zmm0, zmm23, 2; \ vpaddq zmm25, zmm23, zmm0; /* ; R1' (R1*5) */ \ vpsllq zmm1, zmm24, 2; \ vpaddq zmm26, zmm24, zmm1; /* ; R2' (R2*5) */ \ \ /* ; 4*5*R^8 */ \ vpsllq zmm25, zmm25, 2; \ vpsllq zmm26, zmm26, 2; \ \ vpaddq zmm13, zmm13, zmm16; \ vpaddq zmm14, zmm14, zmm17; \ vpaddq zmm15, zmm15, zmm18; \ \ /* ; zmm13-zmm15 contain the 8 blocks of message plus the previous accumulator */ \ /* ; zmm22-24 contain the 3x44-bit limbs of the powers of R */ \ /* ; zmm25-26 contain the 3x44-bit limbs of the powers of R' (5*4*R) */ \ POLY1305_MUL_REDUCE_VEC(zmm13, zmm14, zmm15, \ zmm22, zmm23, zmm24, \ zmm25, zmm26, \ zmm5, zmm6, zmm7, zmm8, zmm9, zmm10, \ zmm11); \ \ /* ;; Add all blocks (horizontally) */ \ vextracti64x4 ymm0, zmm13, 1; \ vextracti64x4 ymm1, zmm14, 1; \ vextracti64x4 ymm2, zmm15, 1; \ \ vpaddq ymm13, ymm13, ymm0; \ vpaddq ymm14, ymm14, ymm1; \ vpaddq ymm15, ymm15, ymm2; \ \ vextracti32x4 xmm10, ymm13, 1; \ vextracti32x4 xmm11, ymm14, 1; \ vextracti32x4 xmm12, ymm15, 1; \ \ vpaddq xmm13, xmm13, xmm10; \ vpaddq xmm14, xmm14, xmm11; \ vpaddq xmm15, xmm15, xmm12; \ \ vpsrldq xmm10, xmm13, 8; \ vpsrldq xmm11, xmm14, 8; \ vpsrldq xmm12, xmm15, 8; \ \ /* ; Finish folding and clear second qword */ \ mov T0, 0xfd; \ kmovq k1, T0; \ vpaddq xmm13{k1}{z}, xmm13, xmm10; \ vpaddq xmm14{k1}{z}, xmm14, xmm11; \ vpaddq xmm15{k1}{z}, xmm15, xmm12; \ \ add MSG, POLY1305_BLOCK_SIZE*16; \ sub LEN, POLY1305_BLOCK_SIZE*16; \ \ jmp .L_less_than_256; \ .L_poly1305_blocks_exit: \ /* ;; ============================================================================= ;; ============================================================================= ;; Creates stack frame and saves registers ;; ============================================================================= */ #define FUNC_ENTRY() \ mov rax, rsp; \ CFI_DEF_CFA_REGISTER(rax); \ sub rsp, STACK_SIZE; \ and rsp, -64; \ \ mov [rsp + STACK_gpr_save + 8*0], rbx; \ mov [rsp + STACK_gpr_save + 8*1], rbp; \ mov [rsp + STACK_gpr_save + 8*2], r12; \ mov [rsp + STACK_gpr_save + 8*3], r13; \ mov [rsp + STACK_gpr_save + 8*4], r14; \ mov [rsp + STACK_gpr_save + 8*5], r15; \ mov [rsp + STACK_rsp_save], rax; \ CFI_CFA_ON_STACK(STACK_rsp_save, 0) /* ;; ============================================================================= ;; ============================================================================= ;; Restores registers and removes the stack frame ;; ============================================================================= */ #define FUNC_EXIT() \ mov rbx, [rsp + STACK_gpr_save + 8*0]; \ mov rbp, [rsp + STACK_gpr_save + 8*1]; \ mov r12, [rsp + STACK_gpr_save + 8*2]; \ mov r13, [rsp + STACK_gpr_save + 8*3]; \ mov r14, [rsp + STACK_gpr_save + 8*4]; \ mov r15, [rsp + STACK_gpr_save + 8*5]; \ mov rsp, [rsp + STACK_rsp_save]; \ CFI_DEF_CFA_REGISTER(rsp) /* ;; ============================================================================= ;; ============================================================================= ;; void poly1305_aead_update_fma_avx512(const void *msg, const uint64_t msg_len, ;; void *hash, const void *key) ;; arg1 - Input message ;; arg2 - Message length ;; arg3 - Input/output hash ;; arg4 - Poly1305 key */ .align 32 .globl _gcry_poly1305_amd64_avx512_blocks ELF(.type _gcry_poly1305_amd64_avx512_blocks,@function;) _gcry_poly1305_amd64_avx512_blocks: CFI_STARTPROC() spec_stop_avx512_intel_syntax; FUNC_ENTRY() #define _a0 gp3 #define _a0 gp3 #define _a1 gp4 #define _a2 gp5 #define _r0 gp6 #define _r1 gp7 #define _len arg2 #define _arg3 arg4 /* ; use rcx, arg3 = rdx */ /* ;; load R */ mov _r0, [arg4 + 0 * 8] mov _r1, [arg4 + 1 * 8] /* ;; load accumulator / current hash value */ /* ;; note: arg4 can't be used beyond this point */ mov _arg3, arg3 /* ; note: _arg3 = arg4 (linux) */ mov _a0, [_arg3 + 0 * 8] mov _a1, [_arg3 + 1 * 8] mov DWORD(_a2), [_arg3 + 2 * 8] /* ; note: _a2 = arg4 (win) */ POLY1305_BLOCKS(arg1, _len, _a0, _a1, _a2, _r0, _r1, gp10, gp11, gp8, gp9, rax, rdx) /* ;; save accumulator back */ mov [_arg3 + 0 * 8], _a0 mov [_arg3 + 1 * 8], _a1 mov [_arg3 + 2 * 8], DWORD(_a2) FUNC_EXIT() xor eax, eax kxorw k1, k1, k1 kxorw k2, k2, k2 ret_spec_stop CFI_ENDPROC() ELF(.size _gcry_poly1305_amd64_avx512_blocks, .-_gcry_poly1305_amd64_avx512_blocks;) #endif #endif diff --git a/cipher/sha512-avx512-amd64.S b/cipher/sha512-avx512-amd64.S index 65475422..431fb3e9 100644 --- a/cipher/sha512-avx512-amd64.S +++ b/cipher/sha512-avx512-amd64.S @@ -1,463 +1,463 @@ /* sha512-avx512-amd64.c - amd64/AVX512 implementation of SHA-512 transform * Copyright (C) 2022 Jussi Kivilinna * * This file is part of Libgcrypt. * * Libgcrypt is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * Libgcrypt is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program; if not, see . */ /* * Based on implementation from file "sha512-avx2-bmi2-amd64.S": ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; Copyright (c) 2012, Intel Corporation ; ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are ; met: ; ; * Redistributions of source code must retain the above copyright ; notice, this list of conditions and the following disclaimer. ; ; * Redistributions in binary form must reproduce the above copyright ; notice, this list of conditions and the following disclaimer in the ; documentation and/or other materials provided with the ; distribution. ; ; * Neither the name of the Intel Corporation nor the names of its ; contributors may be used to endorse or promote products derived from ; this software without specific prior written permission. ; ; ; THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS" AND ANY ; EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR ; PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR ; CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, ; EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, ; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR ; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF ; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING ; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; This code schedules 1 blocks at a time, with 4 lanes per block ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; */ #ifdef __x86_64 #include #if (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \ defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) && \ defined(HAVE_INTEL_SYNTAX_PLATFORM_AS) && \ defined(HAVE_GCC_INLINE_ASM_AVX512) && \ defined(USE_SHA512) #include "asm-common-amd64.h" .intel_syntax noprefix .text /* Virtual Registers */ #define Y_0 ymm0 #define Y_1 ymm1 #define Y_2 ymm2 #define Y_3 ymm3 #define YTMP0 ymm4 #define YTMP1 ymm5 #define YTMP2 ymm6 #define YTMP3 ymm7 #define YTMP4 ymm8 #define XFER YTMP0 #define BYTE_FLIP_MASK ymm9 #define PERM_VPALIGNR_8 ymm10 #define MASK_DC_00 k1 #define INP rdi /* 1st arg */ #define CTX rsi /* 2nd arg */ #define NUM_BLKS rdx /* 3rd arg */ #define SRND r8d #define RSP_SAVE r9 #define TBL rcx #define a xmm11 #define b xmm12 #define c xmm13 #define d xmm14 #define e xmm15 #define f xmm16 #define g xmm17 #define h xmm18 #define y0 xmm19 #define y1 xmm20 #define y2 xmm21 #define y3 xmm22 /* Local variables (stack frame) */ #define frame_XFER 0 #define frame_XFER_size (4*4*8) #define frame_size (frame_XFER + frame_XFER_size) #define clear_reg(x) vpxorq x,x,x /* addm [mem], reg */ /* Add reg to mem using reg-mem add and store */ #define addm(p1, p2) \ vmovq y0, p1; \ vpaddq p2, p2, y0; \ vmovq p1, p2; /* COPY_YMM_AND_BSWAP ymm, [mem], byte_flip_mask */ /* Load ymm with mem and byte swap each dword */ #define COPY_YMM_AND_BSWAP(p1, p2, p3) \ vmovdqu p1, p2; \ vpshufb p1, p1, p3 /* %macro MY_VPALIGNR YDST, YSRC1, YSRC2, RVAL */ /* YDST = {YSRC1, YSRC2} >> RVAL*8 */ #define MY_VPALIGNR(YDST_SRC1, YSRC2, RVAL) \ vpermt2q YDST_SRC1, PERM_VPALIGNR_##RVAL, YSRC2; #define ONE_ROUND_PART1(XFERIN, a, b, c, d, e, f, g, h) \ /* h += Sum1 (e) + Ch (e, f, g) + (k[t] + w[0]); \ * d += h; \ * h += Sum0 (a) + Maj (a, b, c); \ * \ * Ch(x, y, z) => ((x & y) + (~x & z)) \ * Maj(x, y, z) => ((x & y) + (z & (x ^ y))) \ */ \ \ vmovq y3, [XFERIN]; \ vmovdqa64 y2, e; \ vpaddq h, h, y3; \ vprorq y0, e, 41; \ vpternlogq y2, f, g, 0xca; /* Ch (e, f, g) */ \ vprorq y1, e, 18; \ vprorq y3, e, 14; \ vpaddq h, h, y2; \ vpternlogq y0, y1, y3, 0x96; /* Sum1 (e) */ \ vpaddq h, h, y0; /* h += Sum1 (e) + Ch (e, f, g) + (k[t] + w[0]) */ \ vpaddq d, d, h; /* d += h */ #define ONE_ROUND_PART2(a, b, c, d, e, f, g, h) \ vmovdqa64 y1, a; \ vprorq y0, a, 39; \ vpternlogq y1, b, c, 0xe8; /* Maj (a, b, c) */ \ vprorq y2, a, 34; \ vprorq y3, a, 28; \ vpternlogq y0, y2, y3, 0x96; /* Sum0 (a) */ \ vpaddq h, h, y1; \ vpaddq h, h, y0; /* h += Sum0 (a) + Maj (a, b, c) */ #define FOUR_ROUNDS_AND_SCHED(X, Y_0, Y_1, Y_2, Y_3, a, b, c, d, e, f, g, h) \ /*;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; RND N + 0 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; */; \ vmovdqa YTMP0, Y_3; \ vmovdqa YTMP1, Y_1; \ /* Extract w[t-7] */; \ vpermt2q YTMP0, PERM_VPALIGNR_8, Y_2 /* YTMP0 = W[-7] */; \ /* Calculate w[t-16] + w[t-7] */; \ vpaddq YTMP0, YTMP0, Y_0 /* YTMP0 = W[-7] + W[-16] */; \ /* Extract w[t-15] */; \ vpermt2q YTMP1, PERM_VPALIGNR_8, Y_0 /* YTMP1 = W[-15] */; \ ONE_ROUND_PART1(rsp+frame_XFER+0*8+X*32, a, b, c, d, e, f, g, h); \ \ /* Calculate sigma0 */; \ \ /* Calculate w[t-15] ror 1 */; \ vprorq YTMP3, YTMP1, 1; /* YTMP3 = W[-15] ror 1 */; \ /* Calculate w[t-15] shr 7 */; \ vpsrlq YTMP4, YTMP1, 7 /* YTMP4 = W[-15] >> 7 */; \ \ ONE_ROUND_PART2(a, b, c, d, e, f, g, h); \ \ /*;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; RND N + 1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; */; \ /* Calculate w[t-15] ror 8 */; \ vprorq YTMP1, YTMP1, 8 /* YTMP1 = W[-15] ror 8 */; \ /* XOR the three components */; \ vpternlogq YTMP1, YTMP3, YTMP4, 0x96 /* YTMP1 = s0 = W[-15] ror 1 ^ W[-15] >> 7 ^ W[-15] ror 8 */; \ \ /* Add three components, w[t-16], w[t-7] and sigma0 */; \ vpaddq YTMP0, YTMP0, YTMP1 /* YTMP0 = W[-16] + W[-7] + s0 */; \ ONE_ROUND_PART1(rsp+frame_XFER+1*8+X*32, h, a, b, c, d, e, f, g); \ /* Move to appropriate lanes for calculating w[16] and w[17] */; \ vshufi64x2 Y_0, YTMP0, YTMP0, 0x0 /* Y_0 = W[-16] + W[-7] + s0 {BABA} */; \ \ /* Calculate w[16] and w[17] in both 128 bit lanes */; \ \ /* Calculate sigma1 for w[16] and w[17] on both 128 bit lanes */; \ vshufi64x2 YTMP2, Y_3, Y_3, 0b11 /* YTMP2 = W[-2] {BABA} */; \ vpsrlq YTMP4, YTMP2, 6 /* YTMP4 = W[-2] >> 6 {BABA} */; \ \ ONE_ROUND_PART2(h, a, b, c, d, e, f, g); \ \ /*;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; RND N + 2 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; */; \ vprorq YTMP3, YTMP2, 19 /* YTMP3 = W[-2] ror 19 {BABA} */; \ vprorq YTMP1, YTMP2, 61 /* YTMP3 = W[-2] ror 61 {BABA} */; \ vpternlogq YTMP4, YTMP3, YTMP1, 0x96 /* YTMP4 = s1 = (W[-2] ror 19) ^ (W[-2] ror 61) ^ (W[-2] >> 6) {BABA} */; \ \ ONE_ROUND_PART1(rsp+frame_XFER+2*8+X*32, g, h, a, b, c, d, e, f); \ /* Add sigma1 to the other compunents to get w[16] and w[17] */; \ vpaddq Y_0, Y_0, YTMP4 /* Y_0 = {W[1], W[0], W[1], W[0]} */; \ \ /* Calculate sigma1 for w[18] and w[19] for upper 128 bit lane */; \ vpsrlq YTMP4, Y_0, 6 /* YTMP4 = W[-2] >> 6 {DC--} */; \ \ ONE_ROUND_PART2(g, h, a, b, c, d, e, f); \ \ /*;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; RND N + 3 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; */; \ vprorq YTMP3, Y_0, 19 /* YTMP3 = W[-2] ror 19 {DC--} */; \ vprorq YTMP1, Y_0, 61 /* YTMP1 = W[-2] ror 61 {DC--} */; \ vpternlogq YTMP4, YTMP3, YTMP1, 0x96 /* YTMP4 = s1 = (W[-2] ror 19) ^ (W[-2] ror 61) ^ (W[-2] >> 6) {DC--} */; \ \ ONE_ROUND_PART1(rsp+frame_XFER+3*8+X*32, f, g, h, a, b, c, d, e); \ /* Add the sigma0 + w[t-7] + w[t-16] for w[18] and w[19] to newly calculated sigma1 to get w[18] and w[19] */; \ /* Form w[19, w[18], w17], w[16] */; \ vpaddq Y_0{MASK_DC_00}, YTMP0, YTMP4 /* YTMP2 = {W[3], W[2], W[1], W[0]} */; \ \ vpaddq XFER, Y_0, [TBL + (4+X)*32]; \ vmovdqa [rsp + frame_XFER + X*32], XFER; \ ONE_ROUND_PART2(f, g, h, a, b, c, d, e) #define ONE_ROUND(XFERIN, a, b, c, d, e, f, g, h) \ ONE_ROUND_PART1(XFERIN, a, b, c, d, e, f, g, h); \ ONE_ROUND_PART2(a, b, c, d, e, f, g, h) #define DO_4ROUNDS(X, a, b, c, d, e, f, g, h) \ ONE_ROUND(rsp+frame_XFER+0*8+X*32, a, b, c, d, e, f, g, h); \ ONE_ROUND(rsp+frame_XFER+1*8+X*32, h, a, b, c, d, e, f, g); \ ONE_ROUND(rsp+frame_XFER+2*8+X*32, g, h, a, b, c, d, e, f); \ ONE_ROUND(rsp+frame_XFER+3*8+X*32, f, g, h, a, b, c, d, e) /* ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; void sha512_avx512(const void* M, void* D, uint64_t L); ; Purpose: Updates the SHA512 digest stored at D with the message stored in M. ; The size of the message pointed to by M must be an integer multiple of SHA512 ; message blocks. ; L is the message length in SHA512 blocks */ .globl _gcry_sha512_transform_amd64_avx512 ELF(.type _gcry_sha512_transform_amd64_avx512,@function;) .align 16 _gcry_sha512_transform_amd64_avx512: CFI_STARTPROC() xor eax, eax cmp rdx, 0 je .Lnowork spec_stop_avx512_intel_syntax; /* Setup mask register for DC:BA merging. */ mov eax, 0b1100 kmovd MASK_DC_00, eax /* Allocate Stack Space */ mov RSP_SAVE, rsp CFI_DEF_CFA_REGISTER(RSP_SAVE); sub rsp, frame_size and rsp, ~(0x40 - 1) /*; load initial digest */ vmovq a,[8*0 + CTX] vmovq b,[8*1 + CTX] vmovq c,[8*2 + CTX] vmovq d,[8*3 + CTX] vmovq e,[8*4 + CTX] vmovq f,[8*5 + CTX] vmovq g,[8*6 + CTX] vmovq h,[8*7 + CTX] vmovdqa BYTE_FLIP_MASK, [.LPSHUFFLE_BYTE_FLIP_MASK ADD_RIP] vpmovzxbq PERM_VPALIGNR_8, [.LPERM_VPALIGNR_8 ADD_RIP] lea TBL,[.LK512 ADD_RIP] /*; byte swap first 16 dwords */ COPY_YMM_AND_BSWAP(Y_0, [INP + 0*32], BYTE_FLIP_MASK) COPY_YMM_AND_BSWAP(Y_1, [INP + 1*32], BYTE_FLIP_MASK) COPY_YMM_AND_BSWAP(Y_2, [INP + 2*32], BYTE_FLIP_MASK) COPY_YMM_AND_BSWAP(Y_3, [INP + 3*32], BYTE_FLIP_MASK) lea INP, [INP + 128] vpaddq XFER, Y_0, [TBL + 0*32] vmovdqa [rsp + frame_XFER + 0*32], XFER vpaddq XFER, Y_1, [TBL + 1*32] vmovdqa [rsp + frame_XFER + 1*32], XFER vpaddq XFER, Y_2, [TBL + 2*32] vmovdqa [rsp + frame_XFER + 2*32], XFER vpaddq XFER, Y_3, [TBL + 3*32] vmovdqa [rsp + frame_XFER + 3*32], XFER /*; schedule 64 input dwords, by doing 12 rounds of 4 each */ mov SRND, 4 .align 16 .Loop0: FOUR_ROUNDS_AND_SCHED(0, Y_0, Y_1, Y_2, Y_3, a, b, c, d, e, f, g, h) FOUR_ROUNDS_AND_SCHED(1, Y_1, Y_2, Y_3, Y_0, e, f, g, h, a, b, c, d) FOUR_ROUNDS_AND_SCHED(2, Y_2, Y_3, Y_0, Y_1, a, b, c, d, e, f, g, h) FOUR_ROUNDS_AND_SCHED(3, Y_3, Y_0, Y_1, Y_2, e, f, g, h, a, b, c, d) lea TBL, [TBL + 4*32] sub SRND, 1 jne .Loop0 sub NUM_BLKS, 1 je .Ldone_hash lea TBL, [.LK512 ADD_RIP] /* load next block and byte swap */ COPY_YMM_AND_BSWAP(Y_0, [INP + 0*32], BYTE_FLIP_MASK) COPY_YMM_AND_BSWAP(Y_1, [INP + 1*32], BYTE_FLIP_MASK) COPY_YMM_AND_BSWAP(Y_2, [INP + 2*32], BYTE_FLIP_MASK) COPY_YMM_AND_BSWAP(Y_3, [INP + 3*32], BYTE_FLIP_MASK) lea INP, [INP + 128] DO_4ROUNDS(0, a, b, c, d, e, f, g, h) vpaddq XFER, Y_0, [TBL + 0*32] vmovdqa [rsp + frame_XFER + 0*32], XFER DO_4ROUNDS(1, e, f, g, h, a, b, c, d) vpaddq XFER, Y_1, [TBL + 1*32] vmovdqa [rsp + frame_XFER + 1*32], XFER DO_4ROUNDS(2, a, b, c, d, e, f, g, h) vpaddq XFER, Y_2, [TBL + 2*32] vmovdqa [rsp + frame_XFER + 2*32], XFER DO_4ROUNDS(3, e, f, g, h, a, b, c, d) vpaddq XFER, Y_3, [TBL + 3*32] vmovdqa [rsp + frame_XFER + 3*32], XFER addm([8*0 + CTX],a) addm([8*1 + CTX],b) addm([8*2 + CTX],c) addm([8*3 + CTX],d) addm([8*4 + CTX],e) addm([8*5 + CTX],f) addm([8*6 + CTX],g) addm([8*7 + CTX],h) /*; schedule 64 input dwords, by doing 12 rounds of 4 each */ mov SRND, 4 jmp .Loop0 .Ldone_hash: DO_4ROUNDS(0, a, b, c, d, e, f, g, h) DO_4ROUNDS(1, e, f, g, h, a, b, c, d) DO_4ROUNDS(2, a, b, c, d, e, f, g, h) DO_4ROUNDS(3, e, f, g, h, a, b, c, d) addm([8*0 + CTX],a) xor eax, eax /* burn stack */ addm([8*1 + CTX],b) addm([8*2 + CTX],c) addm([8*3 + CTX],d) addm([8*4 + CTX],e) addm([8*5 + CTX],f) addm([8*6 + CTX],g) addm([8*7 + CTX],h) kxord MASK_DC_00, MASK_DC_00, MASK_DC_00 vzeroall vmovdqa [rsp + frame_XFER + 0*32], ymm0 /* burn stack */ vmovdqa [rsp + frame_XFER + 1*32], ymm0 /* burn stack */ vmovdqa [rsp + frame_XFER + 2*32], ymm0 /* burn stack */ vmovdqa [rsp + frame_XFER + 3*32], ymm0 /* burn stack */ - clear_reg(xmm16); - clear_reg(xmm17); - clear_reg(xmm18); - clear_reg(xmm19); - clear_reg(xmm20); - clear_reg(xmm21); - clear_reg(xmm22); + clear_reg(ymm16); + clear_reg(ymm17); + clear_reg(ymm18); + clear_reg(ymm19); + clear_reg(ymm20); + clear_reg(ymm21); + clear_reg(ymm22); /* Restore Stack Pointer */ mov rsp, RSP_SAVE CFI_DEF_CFA_REGISTER(rsp) .Lnowork: ret_spec_stop CFI_ENDPROC() ELF(.size _gcry_sha512_transform_amd64_avx512,.-_gcry_sha512_transform_amd64_avx512) /*;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; */ /*;; Binary Data */ ELF(.type _gcry_sha512_avx512_consts,@object) _gcry_sha512_avx512_consts: .align 64 /* K[t] used in SHA512 hashing */ .LK512: .quad 0x428a2f98d728ae22,0x7137449123ef65cd .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc .quad 0x3956c25bf348b538,0x59f111f1b605d019 .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 .quad 0xd807aa98a3030242,0x12835b0145706fbe .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 .quad 0x9bdc06a725c71235,0xc19bf174cf692694 .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65 .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 .quad 0x983e5152ee66dfab,0xa831c66d2db43210 .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 .quad 0x06ca6351e003826f,0x142929670a0e6e70 .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df .quad 0x650a73548baf63de,0x766a0abb3c77b2a8 .quad 0x81c2c92e47edaee6,0x92722c851482353b .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 .quad 0xc24b8b70d0f89791,0xc76c51a30654be30 .quad 0xd192e819d6ef5218,0xd69906245565a910 .quad 0xf40e35855771202a,0x106aa07032bbd1b8 .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 .quad 0x748f82ee5defb2fc,0x78a5636f43172f60 .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec .quad 0x90befffa23631e28,0xa4506cebde82bde9 .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b .quad 0xca273eceea26619c,0xd186b8c721c0c207 .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6 .quad 0x113f9804bef90dae,0x1b710b35131c471b .quad 0x28db77f523047d84,0x32caab7b40c72493 .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 /* Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb. */ .align 32 .LPSHUFFLE_BYTE_FLIP_MASK: .octa 0x08090a0b0c0d0e0f0001020304050607 .octa 0x18191a1b1c1d1e1f1011121314151617 .align 4 .LPERM_VPALIGNR_8: .byte 5, 6, 7, 0 ELF(.size _gcry_sha512_avx512_consts,.-_gcry_sha512_avx512_consts) #endif #endif