From 5e9e228727ebe0acabce5df791f4bc1d69200595 Mon Sep 17 00:00:00 2001 From: Jeffrey Walton Date: Sun, 30 Jul 2017 22:55:50 -0400 Subject: [PATCH] Fix ARIA under SSSE3 --- GNUmakefile | 2 + aria-simd.cpp | 66 ++++++++- aria.cpp | 355 +++++++++--------------------------------------- bench1.cpp | 8 +- blake2.cpp | 12 +- config.h | 71 +++++----- cpu.cpp | 6 +- cpu.h | 6 +- crc-simd.cpp | 8 +- crc.cpp | 6 +- gcm-simd.cpp | 14 +- gcm.cpp | 30 ++-- rdrand-masm.cmd | 0 rijndael.cpp | 14 +- 14 files changed, 221 insertions(+), 377 deletions(-) mode change 100644 => 100755 GNUmakefile mode change 100644 => 100755 rdrand-masm.cmd diff --git a/GNUmakefile b/GNUmakefile old mode 100644 new mode 100755 index 8bb5526f..9e32ce53 --- a/GNUmakefile +++ b/GNUmakefile @@ -199,6 +199,8 @@ endif # -DCRYPTOPP_DISABLE_SSSE3 endif # -DCRYPTOPP_DISABLE_ASM endif # CXXFLAGS +SSSE3_FLAG = $(shell echo | $(CXX) $(CXXFLAGS) -mssse3 -dM -E - | grep -i -c -q __SSSE3__ && echo "-mssse3") +ARIA_FLAG = $(SSSE3_FLAG) ifeq ($(findstring -DCRYPTOPP_DISABLE_SSE4,$(CXXFLAGS)),) SSE42_FLAG = $(shell echo | $(CXX) $(CXXFLAGS) -msse4.2 -dM -E - | grep -i -c -q __SSE4_2__ && echo "-msse4.2") ifeq ($(findstring -DCRYPTOPP_DISABLE_AESNI,$(CXXFLAGS)),) diff --git a/aria-simd.cpp b/aria-simd.cpp index fe50f650..86a62914 100644 --- a/aria-simd.cpp +++ b/aria-simd.cpp @@ -14,9 +14,28 @@ # include "arm_neon.h" #endif +#if (CRYPTOPP_SSSE3_AVAILABLE) +# include "tmmintrin.h" +#endif + +NAMESPACE_BEGIN(CryptoPP) +NAMESPACE_BEGIN(ARIATab) + +extern const word32 S1[256]; +extern const word32 S2[256]; +extern const word32 X1[256]; +extern const word32 X2[256]; +extern const word32 KRK[3][4]; + +NAMESPACE_END +NAMESPACE_END + NAMESPACE_BEGIN(CryptoPP) +using namespace ARIATab; + #if (CRYPTOPP_ARM_NEON_AVAILABLE) + template inline void ARIA_GSRK_NEON(const uint32x4_t X, const uint32x4_t Y, byte RK[16]) { @@ -70,6 +89,51 @@ void ARIA_ProcessAndXorBlock_Xor_NEON(const byte* xorBlock, byte* outBlock) vld1q_u32(reinterpret_cast(outBlock)), vld1q_u32(reinterpret_cast(xorBlock)))); } -#endif + +#endif // CRYPTOPP_ARM_NEON_AVAILABLE + +#if (CRYPTOPP_SSSE3_AVAILABLE) + +inline byte ARIA_BRF(const word32 x, const int y) { + return GETBYTE(x, y); +} + +void ARIA_ProcessAndXorBlock_Xor_SSSE3(const byte* xorBlock, byte* outBlock, const byte *rk, word32 *t) +{ + const __m128i MASK = _mm_set_epi8(12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3); + + outBlock[ 0] = (byte)(X1[ARIA_BRF(t[0],3)] ); + outBlock[ 1] = (byte)(X2[ARIA_BRF(t[0],2)]>>8); + outBlock[ 2] = (byte)(S1[ARIA_BRF(t[0],1)] ); + outBlock[ 3] = (byte)(S2[ARIA_BRF(t[0],0)] ); + outBlock[ 4] = (byte)(X1[ARIA_BRF(t[1],3)] ); + outBlock[ 5] = (byte)(X2[ARIA_BRF(t[1],2)]>>8); + outBlock[ 6] = (byte)(S1[ARIA_BRF(t[1],1)] ); + outBlock[ 7] = (byte)(S2[ARIA_BRF(t[1],0)] ); + outBlock[ 8] = (byte)(X1[ARIA_BRF(t[2],3)] ); + outBlock[ 9] = (byte)(X2[ARIA_BRF(t[2],2)]>>8); + outBlock[10] = (byte)(S1[ARIA_BRF(t[2],1)] ); + outBlock[11] = (byte)(S2[ARIA_BRF(t[2],0)] ); + outBlock[12] = (byte)(X1[ARIA_BRF(t[3],3)] ); + outBlock[13] = (byte)(X2[ARIA_BRF(t[3],2)]>>8); + outBlock[14] = (byte)(S1[ARIA_BRF(t[3],1)] ); + outBlock[15] = (byte)(S2[ARIA_BRF(t[3],0)] ); + + // 'outBlock' may be unaligned. + _mm_storeu_si128(reinterpret_cast<__m128i*>(outBlock), + _mm_xor_si128(_mm_loadu_si128((const __m128i*)(outBlock)), + _mm_shuffle_epi8(_mm_load_si128((const __m128i*)(rk)), MASK))); + + // 'outBlock' and 'xorBlock' may be unaligned. + if (xorBlock != NULLPTR) + { + _mm_storeu_si128((__m128i*)(outBlock), + _mm_xor_si128( + _mm_loadu_si128((const __m128i*)(outBlock)), + _mm_loadu_si128((const __m128i*)(xorBlock)))); + } +} + +#endif // CRYPTOPP_SSSE3_AVAILABLE NAMESPACE_END diff --git a/aria.cpp b/aria.cpp index 6f9e2a15..80ee7dbd 100644 --- a/aria.cpp +++ b/aria.cpp @@ -7,175 +7,30 @@ #include "misc.h" #include "cpu.h" -#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +#if CRYPTOPP_SSE2_AVAILABLE # define CRYPTOPP_ENABLE_ARIA_SSE2_INTRINSICS 1 #endif -#if CRYPTOPP_BOOL_SSSE3_INTRINSICS_AVAILABLE +#if CRYPTOPP_SSSE3_AVAILABLE # define CRYPTOPP_ENABLE_ARIA_SSSE3_INTRINSICS 1 #endif -#if CRYPTOPP_ARM_NEON_AVAILABLE -# define CRYPTOPP_ENABLE_ARIA_NEON_INTRINSICS 1 -#endif +NAMESPACE_BEGIN(CryptoPP) +NAMESPACE_BEGIN(ARIATab) -ANONYMOUS_NAMESPACE_BEGIN +extern const word32 S1[256]; +extern const word32 S2[256]; +extern const word32 X1[256]; +extern const word32 X2[256]; +extern const word32 KRK[3][4]; -CRYPTOPP_ALIGN_DATA(16) -const CryptoPP::word32 S1[256]={ - 0x00636363,0x007c7c7c,0x00777777,0x007b7b7b,0x00f2f2f2,0x006b6b6b,0x006f6f6f,0x00c5c5c5, - 0x00303030,0x00010101,0x00676767,0x002b2b2b,0x00fefefe,0x00d7d7d7,0x00ababab,0x00767676, - 0x00cacaca,0x00828282,0x00c9c9c9,0x007d7d7d,0x00fafafa,0x00595959,0x00474747,0x00f0f0f0, - 0x00adadad,0x00d4d4d4,0x00a2a2a2,0x00afafaf,0x009c9c9c,0x00a4a4a4,0x00727272,0x00c0c0c0, - 0x00b7b7b7,0x00fdfdfd,0x00939393,0x00262626,0x00363636,0x003f3f3f,0x00f7f7f7,0x00cccccc, - 0x00343434,0x00a5a5a5,0x00e5e5e5,0x00f1f1f1,0x00717171,0x00d8d8d8,0x00313131,0x00151515, - 0x00040404,0x00c7c7c7,0x00232323,0x00c3c3c3,0x00181818,0x00969696,0x00050505,0x009a9a9a, - 0x00070707,0x00121212,0x00808080,0x00e2e2e2,0x00ebebeb,0x00272727,0x00b2b2b2,0x00757575, - 0x00090909,0x00838383,0x002c2c2c,0x001a1a1a,0x001b1b1b,0x006e6e6e,0x005a5a5a,0x00a0a0a0, - 0x00525252,0x003b3b3b,0x00d6d6d6,0x00b3b3b3,0x00292929,0x00e3e3e3,0x002f2f2f,0x00848484, - 0x00535353,0x00d1d1d1,0x00000000,0x00ededed,0x00202020,0x00fcfcfc,0x00b1b1b1,0x005b5b5b, - 0x006a6a6a,0x00cbcbcb,0x00bebebe,0x00393939,0x004a4a4a,0x004c4c4c,0x00585858,0x00cfcfcf, - 0x00d0d0d0,0x00efefef,0x00aaaaaa,0x00fbfbfb,0x00434343,0x004d4d4d,0x00333333,0x00858585, - 0x00454545,0x00f9f9f9,0x00020202,0x007f7f7f,0x00505050,0x003c3c3c,0x009f9f9f,0x00a8a8a8, - 0x00515151,0x00a3a3a3,0x00404040,0x008f8f8f,0x00929292,0x009d9d9d,0x00383838,0x00f5f5f5, - 0x00bcbcbc,0x00b6b6b6,0x00dadada,0x00212121,0x00101010,0x00ffffff,0x00f3f3f3,0x00d2d2d2, - 0x00cdcdcd,0x000c0c0c,0x00131313,0x00ececec,0x005f5f5f,0x00979797,0x00444444,0x00171717, - 0x00c4c4c4,0x00a7a7a7,0x007e7e7e,0x003d3d3d,0x00646464,0x005d5d5d,0x00191919,0x00737373, - 0x00606060,0x00818181,0x004f4f4f,0x00dcdcdc,0x00222222,0x002a2a2a,0x00909090,0x00888888, - 0x00464646,0x00eeeeee,0x00b8b8b8,0x00141414,0x00dedede,0x005e5e5e,0x000b0b0b,0x00dbdbdb, - 0x00e0e0e0,0x00323232,0x003a3a3a,0x000a0a0a,0x00494949,0x00060606,0x00242424,0x005c5c5c, - 0x00c2c2c2,0x00d3d3d3,0x00acacac,0x00626262,0x00919191,0x00959595,0x00e4e4e4,0x00797979, - 0x00e7e7e7,0x00c8c8c8,0x00373737,0x006d6d6d,0x008d8d8d,0x00d5d5d5,0x004e4e4e,0x00a9a9a9, - 0x006c6c6c,0x00565656,0x00f4f4f4,0x00eaeaea,0x00656565,0x007a7a7a,0x00aeaeae,0x00080808, - 0x00bababa,0x00787878,0x00252525,0x002e2e2e,0x001c1c1c,0x00a6a6a6,0x00b4b4b4,0x00c6c6c6, - 0x00e8e8e8,0x00dddddd,0x00747474,0x001f1f1f,0x004b4b4b,0x00bdbdbd,0x008b8b8b,0x008a8a8a, - 0x00707070,0x003e3e3e,0x00b5b5b5,0x00666666,0x00484848,0x00030303,0x00f6f6f6,0x000e0e0e, - 0x00616161,0x00353535,0x00575757,0x00b9b9b9,0x00868686,0x00c1c1c1,0x001d1d1d,0x009e9e9e, - 0x00e1e1e1,0x00f8f8f8,0x00989898,0x00111111,0x00696969,0x00d9d9d9,0x008e8e8e,0x00949494, - 0x009b9b9b,0x001e1e1e,0x00878787,0x00e9e9e9,0x00cecece,0x00555555,0x00282828,0x00dfdfdf, - 0x008c8c8c,0x00a1a1a1,0x00898989,0x000d0d0d,0x00bfbfbf,0x00e6e6e6,0x00424242,0x00686868, - 0x00414141,0x00999999,0x002d2d2d,0x000f0f0f,0x00b0b0b0,0x00545454,0x00bbbbbb,0x00161616 -}; - -CRYPTOPP_ALIGN_DATA(16) -const CryptoPP::word32 S2[256]={ - 0xe200e2e2,0x4e004e4e,0x54005454,0xfc00fcfc,0x94009494,0xc200c2c2,0x4a004a4a,0xcc00cccc, - 0x62006262,0x0d000d0d,0x6a006a6a,0x46004646,0x3c003c3c,0x4d004d4d,0x8b008b8b,0xd100d1d1, - 0x5e005e5e,0xfa00fafa,0x64006464,0xcb00cbcb,0xb400b4b4,0x97009797,0xbe00bebe,0x2b002b2b, - 0xbc00bcbc,0x77007777,0x2e002e2e,0x03000303,0xd300d3d3,0x19001919,0x59005959,0xc100c1c1, - 0x1d001d1d,0x06000606,0x41004141,0x6b006b6b,0x55005555,0xf000f0f0,0x99009999,0x69006969, - 0xea00eaea,0x9c009c9c,0x18001818,0xae00aeae,0x63006363,0xdf00dfdf,0xe700e7e7,0xbb00bbbb, - 0x00000000,0x73007373,0x66006666,0xfb00fbfb,0x96009696,0x4c004c4c,0x85008585,0xe400e4e4, - 0x3a003a3a,0x09000909,0x45004545,0xaa00aaaa,0x0f000f0f,0xee00eeee,0x10001010,0xeb00ebeb, - 0x2d002d2d,0x7f007f7f,0xf400f4f4,0x29002929,0xac00acac,0xcf00cfcf,0xad00adad,0x91009191, - 0x8d008d8d,0x78007878,0xc800c8c8,0x95009595,0xf900f9f9,0x2f002f2f,0xce00cece,0xcd00cdcd, - 0x08000808,0x7a007a7a,0x88008888,0x38003838,0x5c005c5c,0x83008383,0x2a002a2a,0x28002828, - 0x47004747,0xdb00dbdb,0xb800b8b8,0xc700c7c7,0x93009393,0xa400a4a4,0x12001212,0x53005353, - 0xff00ffff,0x87008787,0x0e000e0e,0x31003131,0x36003636,0x21002121,0x58005858,0x48004848, - 0x01000101,0x8e008e8e,0x37003737,0x74007474,0x32003232,0xca00caca,0xe900e9e9,0xb100b1b1, - 0xb700b7b7,0xab00abab,0x0c000c0c,0xd700d7d7,0xc400c4c4,0x56005656,0x42004242,0x26002626, - 0x07000707,0x98009898,0x60006060,0xd900d9d9,0xb600b6b6,0xb900b9b9,0x11001111,0x40004040, - 0xec00ecec,0x20002020,0x8c008c8c,0xbd00bdbd,0xa000a0a0,0xc900c9c9,0x84008484,0x04000404, - 0x49004949,0x23002323,0xf100f1f1,0x4f004f4f,0x50005050,0x1f001f1f,0x13001313,0xdc00dcdc, - 0xd800d8d8,0xc000c0c0,0x9e009e9e,0x57005757,0xe300e3e3,0xc300c3c3,0x7b007b7b,0x65006565, - 0x3b003b3b,0x02000202,0x8f008f8f,0x3e003e3e,0xe800e8e8,0x25002525,0x92009292,0xe500e5e5, - 0x15001515,0xdd00dddd,0xfd00fdfd,0x17001717,0xa900a9a9,0xbf00bfbf,0xd400d4d4,0x9a009a9a, - 0x7e007e7e,0xc500c5c5,0x39003939,0x67006767,0xfe00fefe,0x76007676,0x9d009d9d,0x43004343, - 0xa700a7a7,0xe100e1e1,0xd000d0d0,0xf500f5f5,0x68006868,0xf200f2f2,0x1b001b1b,0x34003434, - 0x70007070,0x05000505,0xa300a3a3,0x8a008a8a,0xd500d5d5,0x79007979,0x86008686,0xa800a8a8, - 0x30003030,0xc600c6c6,0x51005151,0x4b004b4b,0x1e001e1e,0xa600a6a6,0x27002727,0xf600f6f6, - 0x35003535,0xd200d2d2,0x6e006e6e,0x24002424,0x16001616,0x82008282,0x5f005f5f,0xda00dada, - 0xe600e6e6,0x75007575,0xa200a2a2,0xef00efef,0x2c002c2c,0xb200b2b2,0x1c001c1c,0x9f009f9f, - 0x5d005d5d,0x6f006f6f,0x80008080,0x0a000a0a,0x72007272,0x44004444,0x9b009b9b,0x6c006c6c, - 0x90009090,0x0b000b0b,0x5b005b5b,0x33003333,0x7d007d7d,0x5a005a5a,0x52005252,0xf300f3f3, - 0x61006161,0xa100a1a1,0xf700f7f7,0xb000b0b0,0xd600d6d6,0x3f003f3f,0x7c007c7c,0x6d006d6d, - 0xed00eded,0x14001414,0xe000e0e0,0xa500a5a5,0x3d003d3d,0x22002222,0xb300b3b3,0xf800f8f8, - 0x89008989,0xde00dede,0x71007171,0x1a001a1a,0xaf00afaf,0xba00baba,0xb500b5b5,0x81008181 -}; - -CRYPTOPP_ALIGN_DATA(16) -const CryptoPP::word32 X1[256]={ - 0x52520052,0x09090009,0x6a6a006a,0xd5d500d5,0x30300030,0x36360036,0xa5a500a5,0x38380038, - 0xbfbf00bf,0x40400040,0xa3a300a3,0x9e9e009e,0x81810081,0xf3f300f3,0xd7d700d7,0xfbfb00fb, - 0x7c7c007c,0xe3e300e3,0x39390039,0x82820082,0x9b9b009b,0x2f2f002f,0xffff00ff,0x87870087, - 0x34340034,0x8e8e008e,0x43430043,0x44440044,0xc4c400c4,0xdede00de,0xe9e900e9,0xcbcb00cb, - 0x54540054,0x7b7b007b,0x94940094,0x32320032,0xa6a600a6,0xc2c200c2,0x23230023,0x3d3d003d, - 0xeeee00ee,0x4c4c004c,0x95950095,0x0b0b000b,0x42420042,0xfafa00fa,0xc3c300c3,0x4e4e004e, - 0x08080008,0x2e2e002e,0xa1a100a1,0x66660066,0x28280028,0xd9d900d9,0x24240024,0xb2b200b2, - 0x76760076,0x5b5b005b,0xa2a200a2,0x49490049,0x6d6d006d,0x8b8b008b,0xd1d100d1,0x25250025, - 0x72720072,0xf8f800f8,0xf6f600f6,0x64640064,0x86860086,0x68680068,0x98980098,0x16160016, - 0xd4d400d4,0xa4a400a4,0x5c5c005c,0xcccc00cc,0x5d5d005d,0x65650065,0xb6b600b6,0x92920092, - 0x6c6c006c,0x70700070,0x48480048,0x50500050,0xfdfd00fd,0xeded00ed,0xb9b900b9,0xdada00da, - 0x5e5e005e,0x15150015,0x46460046,0x57570057,0xa7a700a7,0x8d8d008d,0x9d9d009d,0x84840084, - 0x90900090,0xd8d800d8,0xabab00ab,0x00000000,0x8c8c008c,0xbcbc00bc,0xd3d300d3,0x0a0a000a, - 0xf7f700f7,0xe4e400e4,0x58580058,0x05050005,0xb8b800b8,0xb3b300b3,0x45450045,0x06060006, - 0xd0d000d0,0x2c2c002c,0x1e1e001e,0x8f8f008f,0xcaca00ca,0x3f3f003f,0x0f0f000f,0x02020002, - 0xc1c100c1,0xafaf00af,0xbdbd00bd,0x03030003,0x01010001,0x13130013,0x8a8a008a,0x6b6b006b, - 0x3a3a003a,0x91910091,0x11110011,0x41410041,0x4f4f004f,0x67670067,0xdcdc00dc,0xeaea00ea, - 0x97970097,0xf2f200f2,0xcfcf00cf,0xcece00ce,0xf0f000f0,0xb4b400b4,0xe6e600e6,0x73730073, - 0x96960096,0xacac00ac,0x74740074,0x22220022,0xe7e700e7,0xadad00ad,0x35350035,0x85850085, - 0xe2e200e2,0xf9f900f9,0x37370037,0xe8e800e8,0x1c1c001c,0x75750075,0xdfdf00df,0x6e6e006e, - 0x47470047,0xf1f100f1,0x1a1a001a,0x71710071,0x1d1d001d,0x29290029,0xc5c500c5,0x89890089, - 0x6f6f006f,0xb7b700b7,0x62620062,0x0e0e000e,0xaaaa00aa,0x18180018,0xbebe00be,0x1b1b001b, - 0xfcfc00fc,0x56560056,0x3e3e003e,0x4b4b004b,0xc6c600c6,0xd2d200d2,0x79790079,0x20200020, - 0x9a9a009a,0xdbdb00db,0xc0c000c0,0xfefe00fe,0x78780078,0xcdcd00cd,0x5a5a005a,0xf4f400f4, - 0x1f1f001f,0xdddd00dd,0xa8a800a8,0x33330033,0x88880088,0x07070007,0xc7c700c7,0x31310031, - 0xb1b100b1,0x12120012,0x10100010,0x59590059,0x27270027,0x80800080,0xecec00ec,0x5f5f005f, - 0x60600060,0x51510051,0x7f7f007f,0xa9a900a9,0x19190019,0xb5b500b5,0x4a4a004a,0x0d0d000d, - 0x2d2d002d,0xe5e500e5,0x7a7a007a,0x9f9f009f,0x93930093,0xc9c900c9,0x9c9c009c,0xefef00ef, - 0xa0a000a0,0xe0e000e0,0x3b3b003b,0x4d4d004d,0xaeae00ae,0x2a2a002a,0xf5f500f5,0xb0b000b0, - 0xc8c800c8,0xebeb00eb,0xbbbb00bb,0x3c3c003c,0x83830083,0x53530053,0x99990099,0x61610061, - 0x17170017,0x2b2b002b,0x04040004,0x7e7e007e,0xbaba00ba,0x77770077,0xd6d600d6,0x26260026, - 0xe1e100e1,0x69690069,0x14140014,0x63630063,0x55550055,0x21210021,0x0c0c000c,0x7d7d007d -}; - -CRYPTOPP_ALIGN_DATA(16) -const CryptoPP::word32 X2[256]={ - 0x30303000,0x68686800,0x99999900,0x1b1b1b00,0x87878700,0xb9b9b900,0x21212100,0x78787800, - 0x50505000,0x39393900,0xdbdbdb00,0xe1e1e100,0x72727200,0x09090900,0x62626200,0x3c3c3c00, - 0x3e3e3e00,0x7e7e7e00,0x5e5e5e00,0x8e8e8e00,0xf1f1f100,0xa0a0a000,0xcccccc00,0xa3a3a300, - 0x2a2a2a00,0x1d1d1d00,0xfbfbfb00,0xb6b6b600,0xd6d6d600,0x20202000,0xc4c4c400,0x8d8d8d00, - 0x81818100,0x65656500,0xf5f5f500,0x89898900,0xcbcbcb00,0x9d9d9d00,0x77777700,0xc6c6c600, - 0x57575700,0x43434300,0x56565600,0x17171700,0xd4d4d400,0x40404000,0x1a1a1a00,0x4d4d4d00, - 0xc0c0c000,0x63636300,0x6c6c6c00,0xe3e3e300,0xb7b7b700,0xc8c8c800,0x64646400,0x6a6a6a00, - 0x53535300,0xaaaaaa00,0x38383800,0x98989800,0x0c0c0c00,0xf4f4f400,0x9b9b9b00,0xededed00, - 0x7f7f7f00,0x22222200,0x76767600,0xafafaf00,0xdddddd00,0x3a3a3a00,0x0b0b0b00,0x58585800, - 0x67676700,0x88888800,0x06060600,0xc3c3c300,0x35353500,0x0d0d0d00,0x01010100,0x8b8b8b00, - 0x8c8c8c00,0xc2c2c200,0xe6e6e600,0x5f5f5f00,0x02020200,0x24242400,0x75757500,0x93939300, - 0x66666600,0x1e1e1e00,0xe5e5e500,0xe2e2e200,0x54545400,0xd8d8d800,0x10101000,0xcecece00, - 0x7a7a7a00,0xe8e8e800,0x08080800,0x2c2c2c00,0x12121200,0x97979700,0x32323200,0xababab00, - 0xb4b4b400,0x27272700,0x0a0a0a00,0x23232300,0xdfdfdf00,0xefefef00,0xcacaca00,0xd9d9d900, - 0xb8b8b800,0xfafafa00,0xdcdcdc00,0x31313100,0x6b6b6b00,0xd1d1d100,0xadadad00,0x19191900, - 0x49494900,0xbdbdbd00,0x51515100,0x96969600,0xeeeeee00,0xe4e4e400,0xa8a8a800,0x41414100, - 0xdadada00,0xffffff00,0xcdcdcd00,0x55555500,0x86868600,0x36363600,0xbebebe00,0x61616100, - 0x52525200,0xf8f8f800,0xbbbbbb00,0x0e0e0e00,0x82828200,0x48484800,0x69696900,0x9a9a9a00, - 0xe0e0e000,0x47474700,0x9e9e9e00,0x5c5c5c00,0x04040400,0x4b4b4b00,0x34343400,0x15151500, - 0x79797900,0x26262600,0xa7a7a700,0xdedede00,0x29292900,0xaeaeae00,0x92929200,0xd7d7d700, - 0x84848400,0xe9e9e900,0xd2d2d200,0xbababa00,0x5d5d5d00,0xf3f3f300,0xc5c5c500,0xb0b0b000, - 0xbfbfbf00,0xa4a4a400,0x3b3b3b00,0x71717100,0x44444400,0x46464600,0x2b2b2b00,0xfcfcfc00, - 0xebebeb00,0x6f6f6f00,0xd5d5d500,0xf6f6f600,0x14141400,0xfefefe00,0x7c7c7c00,0x70707000, - 0x5a5a5a00,0x7d7d7d00,0xfdfdfd00,0x2f2f2f00,0x18181800,0x83838300,0x16161600,0xa5a5a500, - 0x91919100,0x1f1f1f00,0x05050500,0x95959500,0x74747400,0xa9a9a900,0xc1c1c100,0x5b5b5b00, - 0x4a4a4a00,0x85858500,0x6d6d6d00,0x13131300,0x07070700,0x4f4f4f00,0x4e4e4e00,0x45454500, - 0xb2b2b200,0x0f0f0f00,0xc9c9c900,0x1c1c1c00,0xa6a6a600,0xbcbcbc00,0xececec00,0x73737300, - 0x90909000,0x7b7b7b00,0xcfcfcf00,0x59595900,0x8f8f8f00,0xa1a1a100,0xf9f9f900,0x2d2d2d00, - 0xf2f2f200,0xb1b1b100,0x00000000,0x94949400,0x37373700,0x9f9f9f00,0xd0d0d000,0x2e2e2e00, - 0x9c9c9c00,0x6e6e6e00,0x28282800,0x3f3f3f00,0x80808000,0xf0f0f000,0x3d3d3d00,0xd3d3d300, - 0x25252500,0x8a8a8a00,0xb5b5b500,0xe7e7e700,0x42424200,0xb3b3b300,0xc7c7c700,0xeaeaea00, - 0xf7f7f700,0x4c4c4c00,0x11111100,0x33333300,0x03030300,0xa2a2a200,0xacacac00,0x60606000 -}; - -CRYPTOPP_ALIGN_DATA(16) -const CryptoPP::word32 KRK[3][4] = { - {0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0}, - {0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0}, - {0xdb92371d, 0x2126e970, 0x03249775, 0x04e8c90e} -}; - -ANONYMOUS_NAMESPACE_END +NAMESPACE_END +NAMESPACE_END NAMESPACE_BEGIN(CryptoPP) +using namespace ARIATab; + typedef BlockGetAndPut BigEndianBlock; typedef BlockGetAndPut NativeEndianBlock; @@ -227,6 +82,10 @@ extern void ARIA_UncheckedSetKey_Schedule_NEON(byte* rk, word32* ws, unsigned in extern void ARIA_ProcessAndXorBlock_Xor_NEON(const byte* xorBlock, byte* outblock); #endif +#if (CRYPTOPP_SSSE3_AVAILABLE) +extern void ARIA_ProcessAndXorBlock_Xor_SSSE3(const byte* xorBlock, byte* outBlock, const byte *rk, word32 *t); +#endif + // n-bit right shift of Y XORed to X template inline void ARIA_GSRK(const word32 X[4], const word32 Y[4], byte RK[16]) @@ -270,115 +129,51 @@ void ARIA::Base::UncheckedSetKey(const byte *key, unsigned int keylen, const Nam // w0 has room for 32 bytes. w1-w3 each has room for 16 bytes. t and u are 16 byte temp areas. word32 *w0 = m_w.data(), *w1 = m_w.data()+8, *w2 = m_w.data()+12, *w3 = m_w.data()+16, *t = m_w.data()+20; -#if CRYPTOPP_ENABLE_ARIA_SSSE3_INTRINSICS - const __m128i MASK = _mm_set_epi8(12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3); - if (HasSSSE3()) + BigEndianBlock::Get(mk)(w0[0])(w0[1])(w0[2])(w0[3]); + t[0]=w0[0]^KRK[q][0]; t[1]=w0[1]^KRK[q][1]; + t[2]=w0[2]^KRK[q][2]; t[3]=w0[3]^KRK[q][3]; + + ARIA_FO; + + if (keylen == 32) { - // 'mk' may be unaligned. - const __m128i w = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)(mk)), MASK); - _mm_store_si128((__m128i*)w0, w); - _mm_store_si128((__m128i*)t, _mm_xor_si128(w, _mm_load_si128((const __m128i*)(KRK[q])))); - - ARIA_FO; - - if (keylen == 32) - { - // 'mk' may be unaligned. - _mm_store_si128(reinterpret_cast<__m128i*>(w1), - _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)(mk+16)), MASK)); - } - else if (keylen == 24) - { - BigEndianBlock::Get(mk+16)(w1[0])(w1[1]); - w1[2] = w1[3] = 0; - } - else - { - w1[0]=w1[1]=w1[2]=w1[3]=0; - } + BigEndianBlock::Get(mk+16)(w1[0])(w1[1])(w1[2])(w1[3]); + } + else if (keylen == 24) + { + BigEndianBlock::Get(mk+16)(w1[0])(w1[1]); + w1[2] = w1[3] = 0; } else -#endif // CRYPTOPP_ENABLE_ARIA_SSSE3_INTRINSICS { - BigEndianBlock::Get(mk)(w0[0])(w0[1])(w0[2])(w0[3]); - t[0]=w0[0]^KRK[q][0]; t[1]=w0[1]^KRK[q][1]; - t[2]=w0[2]^KRK[q][2]; t[3]=w0[3]^KRK[q][3]; - - ARIA_FO; - - if (keylen == 32) - { - BigEndianBlock::Get(mk+16)(w1[0])(w1[1])(w1[2])(w1[3]); - } - else if (keylen == 24) - { - BigEndianBlock::Get(mk+16)(w1[0])(w1[1]); - w1[2] = w1[3] = 0; - } - else - { - w1[0]=w1[1]=w1[2]=w1[3]=0; - } + w1[0]=w1[1]=w1[2]=w1[3]=0; } -#if CRYPTOPP_ENABLE_ARIA_SSE2_INTRINSICS - if (HasSSE2()) - { - const __m128i x = _mm_xor_si128( - _mm_load_si128((const __m128i*)(w1)), - _mm_load_si128((const __m128i*)(t))); - _mm_store_si128((__m128i*)(w1), x); + w1[0]^=t[0]; w1[1]^=t[1]; w1[2]^=t[2]; w1[3]^=t[3]; + ::memcpy(t, w1, 16); - q = (q==2) ? 0 : (q+1); - _mm_store_si128((__m128i*)(t), _mm_xor_si128(x, - _mm_load_si128((const __m128i*)(KRK[q])))); + q = (q==2) ? 0 : (q+1); + t[0]^=KRK[q][0]; t[1]^=KRK[q][1]; t[2]^=KRK[q][2]; t[3]^=KRK[q][3]; - ARIA_FE; + ARIA_FE; - const __m128i y = _mm_xor_si128( - _mm_load_si128((const __m128i*)(w0)), - _mm_load_si128((const __m128i*)(t))); - _mm_store_si128((__m128i*)(w2), y); + t[0]^=w0[0]; t[1]^=w0[1]; t[2]^=w0[2]; t[3]^=w0[3]; + ::memcpy(w2, t, 16); - q = (q==2) ? 0 : (q+1); - _mm_store_si128((__m128i*)(t), _mm_xor_si128(y, - _mm_load_si128((const __m128i*)(KRK[q])))); + q = (q==2) ? 0 : (q+1); + t[0]^=KRK[q][0]; t[1]^=KRK[q][1]; t[2]^=KRK[q][2]; t[3]^=KRK[q][3]; - ARIA_FO; + ARIA_FO; - _mm_store_si128((__m128i*)(w3), _mm_xor_si128( - _mm_load_si128((const __m128i*)(w1)), - _mm_load_si128((const __m128i*)(t)))); - } - else -#endif // CRYPTOPP_ENABLE_ARIA_SSE2_INTRINSICS - { - w1[0]^=t[0]; w1[1]^=t[1]; w1[2]^=t[2]; w1[3]^=t[3]; - ::memcpy(t, w1, 16); + w3[0]=t[0]^w1[0]; w3[1]=t[1]^w1[1]; w3[2]=t[2]^w1[2]; w3[3]=t[3]^w1[3]; - q = (q==2) ? 0 : (q+1); - t[0]^=KRK[q][0]; t[1]^=KRK[q][1]; t[2]^=KRK[q][2]; t[3]^=KRK[q][3]; - - ARIA_FE; - - t[0]^=w0[0]; t[1]^=w0[1]; t[2]^=w0[2]; t[3]^=w0[3]; - ::memcpy(w2, t, 16); - - q = (q==2) ? 0 : (q+1); - t[0]^=KRK[q][0]; t[1]^=KRK[q][1]; t[2]^=KRK[q][2]; t[3]^=KRK[q][3]; - - ARIA_FO; - - w3[0]=t[0]^w1[0]; w3[1]=t[1]^w1[1]; w3[2]=t[2]^w1[2]; w3[3]=t[3]^w1[3]; - } - -#if CRYPTOPP_ENABLE_ARIA_NEON_INTRINSICS +#if CRYPTOPP_ARM_NEON_AVAILABLE if (HasNEON()) { ARIA_UncheckedSetKey_Schedule_NEON(rk, m_w, keylen); } else -#endif // CRYPTOPP_ENABLE_ARIA_NEON_INTRINSICS +#endif // CRYPTOPP_ARM_NEON_AVAILABLE { ARIA_GSRK<19>(w0, w1, rk + 0); ARIA_GSRK<19>(w1, w2, rk + 16); @@ -414,53 +209,24 @@ void ARIA::Base::UncheckedSetKey(const byte *key, unsigned int keylen, const Nam rk = m_rk.data(); r = R; q = Q; -#if CRYPTOPP_ENABLE_ARIA_SSE2_INTRINSICS && !defined(__SUNPRO_CC) - if (HasSSE2()) + a=reinterpret_cast(rk); s=m_w.data()+24; z=a+r*4; + ::memcpy(t, a, 16); ::memcpy(a, z, 16); ::memcpy(z, t, 16); + + a+=4; z-=4; + for (; a(rk); s=m_w.data()+24; z=a+r*4; - _mm_store_si128((__m128i*)t, _mm_load_si128((const __m128i*)a)); - _mm_store_si128((__m128i*)a, _mm_load_si128((const __m128i*)z)); - _mm_store_si128((__m128i*)z, _mm_load_si128((const __m128i*)t)); - - a+=4; z-=4; - for (; a(rk); s=m_w.data()+24; z=a+r*4; - ::memcpy(t, a, 16); ::memcpy(a, z, 16); ::memcpy(z, t, 16); + ::memcpy(s, t, 16); - a+=4; z-=4; - for (; a>8); outBlock[ 2] = (byte)(S1[ARIA_BRF(t[0],1)] ); @@ -535,6 +302,10 @@ void ARIA::Base::ProcessAndXorBlock(const byte *inBlock, const byte *xorBlock, b _mm_loadu_si128((const __m128i*)(xorBlock)))); } return; + #endif + + ARIA_ProcessAndXorBlock_Xor_SSSE3(xorBlock, outBlock, rk, t); + return; } else # endif // CRYPTOPP_ENABLE_ARIA_SSSE3_INTRINSICS @@ -578,14 +349,14 @@ void ARIA::Base::ProcessAndXorBlock(const byte *inBlock, const byte *xorBlock, b BigEndianBlock::Put(rk, t)(t[0])(t[1])(t[2])(t[3]); #endif -#if CRYPTOPP_ENABLE_ARIA_NEON_INTRINSICS +#if CRYPTOPP_ARM_NEON_AVAILABLE if (HasNEON()) { if (xorBlock != NULLPTR) ARIA_ProcessAndXorBlock_Xor_NEON(xorBlock, outBlock); } else -#endif // CRYPTOPP_ENABLE_ARIA_NEON_INTRINSICS +#endif // CRYPTOPP_ARM_NEON_AVAILABLE { if (xorBlock != NULLPTR) for (unsigned int n=0; n"; { -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE if (HasCLMUL()) BenchMarkByName2("AES/GCM", 0, "GMAC(AES)"); else -#elif CRYPTOPP_ARMV8A_PMULL_AVAILABLE +#elif CRYPTOPP_ARMV_PMULL_AVAILABLE if (HasPMULL()) BenchMarkByName2("AES/GCM", 0, "GMAC(AES)"); else @@ -594,11 +594,11 @@ void Benchmark2(double t, double hertz) std::cout << "\n"; { -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE if (HasCLMUL()) BenchMarkByName2("AES/GCM", 0, "AES/GCM"); else -#elif CRYPTOPP_ARMV8A_PMULL_AVAILABLE +#elif CRYPTOPP_ARMV_PMULL_AVAILABLE if (HasPMULL()) BenchMarkByName2("AES/GCM", 0, "AES/GCM"); else diff --git a/blake2.cpp b/blake2.cpp index 10c74d3c..4a7f280c 100644 --- a/blake2.cpp +++ b/blake2.cpp @@ -24,7 +24,7 @@ NAMESPACE_BEGIN(CryptoPP) // Sun Studio 12.3 and earlier lack SSE2's _mm_set_epi64x. Win32 lacks _mm_set_epi64x, Win64 supplies it except for VS2008. // Also see http://stackoverflow.com/a/38547909/608639 -#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE && ((__SUNPRO_CC >= 0x5100 && __SUNPRO_CC < 0x5130) || (defined(_MSC_VER) && _MSC_VER < 1600) || (defined(_M_IX86) && _MSC_VER >= 1600)) +#if CRYPTOPP_SSE2_AVAILABLE && ((__SUNPRO_CC >= 0x5100 && __SUNPRO_CC < 0x5130) || (defined(_MSC_VER) && _MSC_VER < 1600) || (defined(_M_IX86) && _MSC_VER >= 1600)) inline __m128i MM_SET_EPI64X(const word64 a, const word64 b) { const word64 t[2] = {b,a}; __m128i r; @@ -40,7 +40,7 @@ static void BLAKE2_Compress32_CXX(const byte* input, BLAKE2_State static void BLAKE2_Compress64_CXX(const byte* input, BLAKE2_State& state); // Also see http://github.com/weidai11/cryptopp/issues/247 for SunCC 5.12 -#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +#if CRYPTOPP_SSE2_AVAILABLE static void BLAKE2_Compress32_SSE2(const byte* input, BLAKE2_State& state); # if (__SUNPRO_CC != 0x5120) static void BLAKE2_Compress64_SSE2(const byte* input, BLAKE2_State& state); @@ -114,7 +114,7 @@ pfnCompress64 InitializeCompress64Fn() return &BLAKE2_Compress64_SSE4; else #endif -#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +#if CRYPTOPP_SSE2_AVAILABLE # if (__SUNPRO_CC != 0x5120) if (HasSSE2()) return &BLAKE2_Compress64_SSE2; @@ -136,7 +136,7 @@ pfnCompress32 InitializeCompress32Fn() return &BLAKE2_Compress32_SSE4; else #endif -#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +#if CRYPTOPP_SSE2_AVAILABLE if (HasSSE2()) return &BLAKE2_Compress32_SSE2; else @@ -569,7 +569,7 @@ void BLAKE2_Compress32_CXX(const byte* input, BLAKE2_State& state state.h[i] = state.h[i] ^ ConditionalByteReverse(LittleEndian::ToEnum(), v[i] ^ v[i + 8]); } -#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +#if CRYPTOPP_SSE2_AVAILABLE static void BLAKE2_Compress32_SSE2(const byte* input, BLAKE2_State& state) { word32 m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14, m15; @@ -1881,7 +1881,7 @@ static void BLAKE2_Compress64_SSE2(const byte* input, BLAKE2_State _mm_storeu_si128((__m128i *)(void*)(&state.h[6]), _mm_xor_si128(_mm_loadu_si128((const __m128i*)(const void*)(&state.h[6])), row2h)); } # endif // (__SUNPRO_CC != 0x5120) -#endif // CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +#endif // CRYPTOPP_SSE2_AVAILABLE template class BLAKE2_Base; template class BLAKE2_Base; diff --git a/config.h b/config.h index d6af3c9d..862c7589 100644 --- a/config.h +++ b/config.h @@ -80,8 +80,15 @@ // #endif // File system code to write to GZIP archive. +// http://www.gzip.org/format.txt #if !defined(GZIP_OS_CODE) -# define GZIP_OS_CODE 0 +# if defined(__macintosh__) +# define GZIP_OS_CODE 7 +# elif defined(__unix__) || defined(__linux__) +# define GZIP_OS_CODE 3 +# else +# define GZIP_OS_CODE 0 +# endif #endif // Try this if your CPU has 256K internal cache or a slow multiply instruction @@ -458,55 +465,55 @@ NAMESPACE_END #if !defined(CRYPTOPP_DISABLE_SSE2) && (defined(_MSC_VER) || CRYPTOPP_GCC_VERSION >= 30300 || defined(__SSE2__)) #define CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 1 - #else - #define CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 0 #endif - #if !defined(CRYPTOPP_DISABLE_SSSE3) && (_MSC_VER >= 1500 || (defined(__SSE3__) && defined(__SSSE3__))) + #if !defined(CRYPTOPP_DISABLE_SSSE3) && (_MSC_VER >= 1500 || defined(__SSSE3__)) #define CRYPTOPP_BOOL_SSSE3_ASM_AVAILABLE 1 - #else - #define CRYPTOPP_BOOL_SSSE3_ASM_AVAILABLE 0 #endif #endif #if !defined(CRYPTOPP_DISABLE_ASM) && defined(_MSC_VER) && defined(_M_X64) - #define CRYPTOPP_X64_MASM_AVAILABLE + #define CRYPTOPP_X64_MASM_AVAILABLE 1 #endif #if !defined(CRYPTOPP_DISABLE_ASM) && defined(__GNUC__) && defined(__x86_64__) - #define CRYPTOPP_X64_ASM_AVAILABLE + #define CRYPTOPP_X64_ASM_AVAILABLE 1 #endif #if !defined(CRYPTOPP_DISABLE_ASM) && (defined(_MSC_VER) || defined(__SSE2__)) - #define CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE 1 -#else - #define CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE 0 + #define CRYPTOPP_SSE2_AVAILABLE 1 #endif -#if !defined(CRYPTOPP_DISABLE_ASM) && !defined(CRYPTOPP_DISABLE_SSSE3) && (_MSC_VER >= 1500 || (defined(__SSSE3__) && defined(__SSSE3__))) - #define CRYPTOPP_BOOL_SSSE3_INTRINSICS_AVAILABLE 1 -#else - #define CRYPTOPP_BOOL_SSSE3_INTRINSICS_AVAILABLE 0 +#if !defined(CRYPTOPP_DISABLE_ASM) && !defined(CRYPTOPP_DISABLE_SSSE3) +# if defined(__SSSE3__) || (_MSC_VER >= 1500) || (CRYPTOPP_GCC_VERSION >= 40300) + #define CRYPTOPP_SSSE3_AVAILABLE 1 +# endif #endif // Intrinsics availible in GCC 4.3 (http://gcc.gnu.org/gcc-4.3/changes.html) and // MSVC 2008 (http://msdn.microsoft.com/en-us/library/bb892950%28v=vs.90%29.aspx) // SunCC could generate SSE4 at 12.1, but the intrinsics are missing until 12.4. -#if !defined(CRYPTOPP_DISABLE_ASM) && !defined(CRYPTOPP_DISABLE_SSE4) && ((_MSC_VER >= 1500) || (defined(__SSE4_1__) && defined(__SSE4_2__))) +#if !defined(CRYPTOPP_DISABLE_SSE4) && defined(CRYPTOPP_SSSE3_AVAILABLE) && \ + (defined(__SSE4_1__) || (CRYPTOPP_MSC_VERSION >= 1500) || \ + (CRYPTOPP_GCC_VERSION >= 40300) || (__INTEL_COMPILER >= 1000) || \ + (CRYPTOPP_LLVM_CLANG_VERSION >= 20300) || (CRYPTOPP_APPLE_CLANG_VERSION >= 40000)) + #define CRYPTOPP_SSE41_AVAILABLE 1 +#endif + +#if !defined(CRYPTOPP_DISABLE_SSE4) && defined(CRYPTOPP_SSSE3_AVAILABLE) && \ + (defined(__SSE4_2__) || (CRYPTOPP_MSC_VERSION >= 1500) || \ + (CRYPTOPP_GCC_VERSION >= 40300) || (__INTEL_COMPILER >= 1000) || \ + (CRYPTOPP_LLVM_CLANG_VERSION >= 20300) || (CRYPTOPP_APPLE_CLANG_VERSION >= 40000)) #define CRYPTOPP_SSE42_AVAILABLE 1 #endif // Don't disgorge AES-NI from CLMUL. There will be two to four subtle breaks #if !defined(CRYPTOPP_DISABLE_ASM) && !defined(CRYPTOPP_DISABLE_AESNI) && (_MSC_FULL_VER >= 150030729 || __INTEL_COMPILER >= 1110 || (defined(__AES__) && defined(__PCLMUL__))) - #define CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE 1 -#else - #define CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE 0 + #define CRYPTOPPL_AESNI_AES_AVAILABLE 1 #endif #if !defined(CRYPTOPP_DISABLE_ASM) && !defined(CRYPTOPP_DISABLE_SHA) && ((_MSC_VER >= 1900) || defined(__SHA__)) #define CRYPTOPP_SHANI_SHA_AVAILABLE 1 -#else - #define CRYPTOPP_SHANI_SHA_AVAILABLE 0 #endif #endif // X86, X32, X64 @@ -527,21 +534,21 @@ NAMESPACE_END // LLVM Clang requires 3.5. Apple Clang is unknown at the moment. // Microsoft plans to support ARM-64, but its not clear how to detect it. // TODO: Add MSC_VER and ARM-64 platform define when available -#if !defined(CRYPTOPP_ARMV8A_CRC32_AVAILABLE) && !defined(CRYPTOPP_DISABLE_ASM) +#if !defined(CRYPTOPP_ARMV_CRC32_AVAILABLE) && !defined(CRYPTOPP_DISABLE_ASM) # if defined(__ARM_FEATURE_CRYPTO) || (CRYPTOPP_MSC_VER >= 2000) || \ (CRYPTOPP_GCC_VERSION >= 40800) || (CRYPTOPP_LLVM_CLANG_VERSION >= 30500) -# define CRYPTOPP_ARMV8A_CRC32_AVAILABLE 1 +# define CRYPTOPP_ARMV_CRC32_AVAILABLE 1 # endif #endif -// Requires ARMv8, but we are not sure of the define because the ACLE does not discuss it. -// GCC seems to requires 4.8 and above. LLVM Clang requires 3.5. Apple Clang does not support -// it at the moment. Microsoft plans to support ARM-64, but its not clear how to detect it. +// Requires ARMv8 and ACLE 2.0. GCC requires 4.8 and above. +// LLVM Clang requires 3.5. Apple Clang is unknown at the moment. +// Microsoft plans to support ARM-64, but its not clear how to detect it. // TODO: Add MSC_VER and ARM-64 platform define when available -#if !defined(CRYPTOPP_ARMV8A_PMULL_AVAILABLE) && !defined(CRYPTOPP_DISABLE_ASM) && !defined(__apple_build_version__) +#if !defined(CRYPTOPP_ARMV_PMULL_AVAILABLE) && !defined(CRYPTOPP_DISABLE_ASM) && !defined(__apple_build_version__) # if defined(__ARM_FEATURE_CRYPTO) || (CRYPTOPP_MSC_VER >= 2000) || \ (CRYPTOPP_GCC_VERSION >= 40800) || (CRYPTOPP_LLVM_CLANG_VERSION >= 30500) -# define CRYPTOPP_ARMV8A_PMULL_AVAILABLE 1 +# define CRYPTOPP_ARMV_PMULL_AVAILABLE 1 # endif #endif @@ -552,21 +559,21 @@ NAMESPACE_END #if !defined(CRYPTOPP_ARMV8A_CRYPTO_AVAILABLE) && !defined(CRYPTOPP_DISABLE_ASM) # if defined(__ARM_FEATURE_CRYPTO) || (CRYPTOPP_MSC_VER >= 2000) || \ (CRYPTOPP_GCC_VERSION >= 40800) || (CRYPTOPP_LLVM_CLANG_VERSION >= 30500) -# define CRYPTOPP_ARMV8A_AES_AVAILABLE 1 -# define CRYPTOPP_ARMV8A_PMULL_AVAILABLE 1 +# define CRYPTOPP_ARMV_AES_AVAILABLE 1 +# define CRYPTOPP_ARMV_PMULL_AVAILABLE 1 # define CRYPTOPP_ARMV8A_SHA_AVAILABLE 1 # define CRYPTOPP_ARMV8A_CRYPTO_AVAILABLE 1 # endif #endif // TODO... -#undef CRYPTOPP_ARMV8A_AES_AVAILABLE +#undef CRYPTOPP_ARMV_AES_AVAILABLE #endif // ARM32, ARM64 // ***************** Miscellaneous ******************** -#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || CRYPTOPP_ARM_NEON_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE) +#if CRYPTOPP_SSE2_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || CRYPTOPP_ARM_NEON_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE) #define CRYPTOPP_BOOL_ALIGN16 1 #else #define CRYPTOPP_BOOL_ALIGN16 0 diff --git a/cpu.cpp b/cpu.cpp index b9e2ea9b..570c4917 100644 --- a/cpu.cpp +++ b/cpu.cpp @@ -137,7 +137,7 @@ static bool TrySSE2() { #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE AS2(por xmm0, xmm0) // executing SSE2 instruction -#elif CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +#elif CRYPTOPP_SSE2_AVAILABLE __m128i x = _mm_setzero_si128(); return _mm_cvtsi128_si32(x) == 0; #endif @@ -169,7 +169,7 @@ static bool TrySSE2() { #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE __asm __volatile ("por %xmm0, %xmm0"); -#elif CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +#elif CRYPTOPP_SSE2_AVAILABLE __m128i x = _mm_setzero_si128(); result = _mm_cvtsi128_si32(x) == 0; #endif @@ -354,7 +354,7 @@ extern "C" static bool TryAES() { -#if (CRYPTOPP_ARMV8A_AES_AVAILABLE) +#if (CRYPTOPP_ARMV_AES_AVAILABLE) # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY) volatile bool result = true; __try diff --git a/cpu.h b/cpu.h index 8bc19e9d..3c9d942f 100644 --- a/cpu.h +++ b/cpu.h @@ -56,17 +56,17 @@ #endif // Baseline include -#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +#if CRYPTOPP_SSE2_AVAILABLE # include // __m64, __m128i, _mm_set_epi64x #endif -#if CRYPTOPP_BOOL_SSSE3_INTRINSICS_AVAILABLE +#if CRYPTOPP_SSSE3_AVAILABLE # include // _mm_shuffle_pi8, _mm_shuffle_epi8 #endif // tmmintrin.h #if CRYPTOPP_SSE42_AVAILABLE # include // _mm_blend_epi16 # include // _mm_crc32_u{8|16|32} #endif // smmintrin.h -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE # include // aesenc, aesdec, etc #endif // wmmintrin.h #if CRYPTOPP_SHANI_SHA_AVAILABLE diff --git a/crc-simd.cpp b/crc-simd.cpp index a8428b48..3c41c94f 100644 --- a/crc-simd.cpp +++ b/crc-simd.cpp @@ -14,7 +14,7 @@ # include "nmmintrin.h" #endif -#if (CRYPTOPP_ARMV8A_CRC32_AVAILABLE) +#if (CRYPTOPP_ARMV_CRC32_AVAILABLE) # include "arm_neon.h" #if defined(__GNUC__) # include "arm_acle.h" @@ -40,7 +40,7 @@ extern "C" { }; #endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY -#if (CRYPTOPP_ARMV8A_CRC32_AVAILABLE) +#if (CRYPTOPP_ARMV_CRC32_AVAILABLE) bool CPU_TryCRC32_ARMV8() { # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY) @@ -96,9 +96,9 @@ bool CPU_TryCRC32_ARMV8() return result; # endif } -#endif // CRYPTOPP_ARMV8A_CRC32_AVAILABLE +#endif // CRYPTOPP_ARMV_CRC32_AVAILABLE -#if (CRYPTOPP_ARMV8A_CRC32_AVAILABLE) +#if (CRYPTOPP_ARMV_CRC32_AVAILABLE) void CRC32_Update_ARMV8(const byte *s, size_t n, word32& c) { for(; !IsAligned(s) && n > 0; s++, n--) diff --git a/crc.cpp b/crc.cpp index 7dee01c3..0859f279 100644 --- a/crc.cpp +++ b/crc.cpp @@ -8,7 +8,7 @@ NAMESPACE_BEGIN(CryptoPP) // crc-simd.cpp -#if (CRYPTOPP_ARMV8A_CRC32_AVAILABLE) +#if (CRYPTOPP_ARMV_CRC32_AVAILABLE) extern void CRC32_Update_ARMV8(const byte *s, size_t n, word32& c); extern void CRC32C_Update_ARMV8(const byte *s, size_t n, word32& c); #endif @@ -136,7 +136,7 @@ CRC32::CRC32() void CRC32::Update(const byte *s, size_t n) { -#if (CRYPTOPP_ARMV8A_CRC32_AVAILABLE) +#if (CRYPTOPP_ARMV_CRC32_AVAILABLE) if (HasCRC32()) { CRC32_Update_ARMV8(s, n, m_crc); @@ -302,7 +302,7 @@ void CRC32C::Update(const byte *s, size_t n) CRC32C_Update_SSE42(s, n, m_crc); return; } -#elif (CRYPTOPP_ARMV8A_CRC32_AVAILABLE) +#elif (CRYPTOPP_ARMV_CRC32_AVAILABLE) if (HasCRC32()) { CRC32C_Update_ARMV8(s, n, m_crc); diff --git a/gcm-simd.cpp b/gcm-simd.cpp index 3e57fc23..38de5e27 100644 --- a/gcm-simd.cpp +++ b/gcm-simd.cpp @@ -16,7 +16,7 @@ #if (CRYPTOPP_ARM_NEON_AVAILABLE) # include "arm_neon.h" -#if (CRYPTOPP_ARMV8A_PMULL_AVAILABLE) +#if (CRYPTOPP_ARMV_PMULL_AVAILABLE) # include "arm_acle.h" #endif #endif @@ -29,7 +29,7 @@ ANONYMOUS_NAMESPACE_BEGIN // GCC 4.8 and 4.9 are missing PMULL gear -#if (CRYPTOPP_ARMV8A_PMULL_AVAILABLE) +#if (CRYPTOPP_ARMV_PMULL_AVAILABLE) # if (CRYPTOPP_GCC_VERSION >= 40800) && (CRYPTOPP_GCC_VERSION < 50000) inline poly128_t VMULL_P64(poly64_t a, poly64_t b) { @@ -43,7 +43,7 @@ inline poly128_t VMULL_HIGH_P64(poly64x2_t a, poly64x2_t b) # endif #endif -#if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARM64) && CRYPTOPP_ARMV8A_PMULL_AVAILABLE +#if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARM64) && CRYPTOPP_ARMV_PMULL_AVAILABLE #if defined(__GNUC__) // Schneiders, Hovsmith and O'Rourke used this trick. // It results in much better code generation in production code @@ -137,7 +137,7 @@ inline uint64x2_t VEXT_U8(uint64x2_t a, uint64x2_t b) return (uint64x2_t)vextq_u8(vreinterpretq_u8_u64(a), vreinterpretq_u8_u64(b), C); } #endif // Microsoft and compatibles -#endif // CRYPTOPP_ARMV8A_PMULL_AVAILABLE +#endif // CRYPTOPP_ARMV_PMULL_AVAILABLE ANONYMOUS_NAMESPACE_END @@ -155,7 +155,7 @@ extern "C" { }; #endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY -#if (CRYPTOPP_ARMV8A_PMULL_AVAILABLE) +#if (CRYPTOPP_ARMV_PMULL_AVAILABLE) bool CPU_TryPMULL_ARMV8() { # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY) @@ -218,7 +218,7 @@ bool CPU_TryPMULL_ARMV8() return result; # endif } -#endif // CRYPTOPP_ARMV8A_PMULL_AVAILABLE +#endif // CRYPTOPP_ARMV_PMULL_AVAILABLE #if CRYPTOPP_ARM_NEON_AVAILABLE void GCM_Xor16_NEON(byte *a, const byte *b, const byte *c) @@ -230,7 +230,7 @@ void GCM_Xor16_NEON(byte *a, const byte *b, const byte *c) } #endif -#if CRYPTOPP_ARMV8A_PMULL_AVAILABLE +#if CRYPTOPP_ARMV_PMULL_AVAILABLE ANONYMOUS_NAMESPACE_BEGIN diff --git a/gcm.cpp b/gcm.cpp index da1ea1f4..323f5dd7 100644 --- a/gcm.cpp +++ b/gcm.cpp @@ -24,7 +24,7 @@ // SunCC 5.13 and below crash with AES-NI/CLMUL and C++{03|11}. Disable one or the other. // Also see http://github.com/weidai11/cryptopp/issues/226 #if defined(__SUNPRO_CC) && (__SUNPRO_CC <= 0x513) -# undef CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +# undef CRYPTOPPL_AESNI_AES_AVAILABLE #endif #include "gcm.h" @@ -110,14 +110,14 @@ inline static void Xor16(byte *a, const byte *b, const byte *c) ((word64 *)(void *)a)[1] = ((word64 *)(void *)b)[1] ^ ((word64 *)(void *)c)[1]; } -#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE +#if CRYPTOPP_SSE2_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE inline static void SSE2_Xor16(byte *a, const byte *b, const byte *c) { // SunCC 5.14 crash (bewildering since asserts are not in effect in release builds) // Also see http://github.com/weidai11/cryptopp/issues/226 and http://github.com/weidai11/cryptopp/issues/284 # if __SUNPRO_CC *(__m128i *)(void *)a = _mm_xor_si128(*(__m128i *)(void *)b, *(__m128i *)(void *)c); -# elif CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE +# elif CRYPTOPP_SSE2_AVAILABLE CRYPTOPP_ASSERT(IsAlignedOn(a,GetAlignmentOf<__m128i>())); CRYPTOPP_ASSERT(IsAlignedOn(b,GetAlignmentOf<__m128i>())); CRYPTOPP_ASSERT(IsAlignedOn(c,GetAlignmentOf<__m128i>())); @@ -128,7 +128,7 @@ inline static void SSE2_Xor16(byte *a, const byte *b, const byte *c) } #endif -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE CRYPTOPP_ALIGN_DATA(16) static const word64 s_clmulConstants64[] = { W64LIT(0xe100000000000000), W64LIT(0xc200000000000000), @@ -180,7 +180,7 @@ inline __m128i CLMUL_GF_Mul(const __m128i &x, const __m128i &h, const __m128i &r } #endif -#if CRYPTOPP_ARMV8A_PMULL_AVAILABLE +#if CRYPTOPP_ARMV_PMULL_AVAILABLE extern size_t GCM_AuthenticateBlocks_ARMV8(const byte *data, size_t len, const byte *mtable, byte *hbuffer); extern uint64x2_t GCM_Multiply_ARMV8A(const uint64x2_t &x, const uint64x2_t &h, const uint64x2_t &r); @@ -206,7 +206,7 @@ void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const int tableSize, i, j, k; -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE if (HasCLMUL()) { // Avoid "parameter not used" error and suppress Coverity finding @@ -214,7 +214,7 @@ void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const tableSize = s_clmulTableSizeInBlocks * REQUIRED_BLOCKSIZE; } else -#elif CRYPTOPP_ARMV8A_PMULL_AVAILABLE +#elif CRYPTOPP_ARMV_PMULL_AVAILABLE if (HasPMULL()) { // Avoid "parameter not used" error and suppress Coverity finding @@ -241,7 +241,7 @@ void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const memset(hashKey, 0, REQUIRED_BLOCKSIZE); blockCipher.ProcessBlock(hashKey); -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE if (HasCLMUL()) { const __m128i r = s_clmulConstants[0]; @@ -260,7 +260,7 @@ void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const return; } -#elif CRYPTOPP_ARMV8A_PMULL_AVAILABLE +#elif CRYPTOPP_ARMV_PMULL_AVAILABLE if (HasPMULL()) { const uint64x2_t r = s_clmulConstants[0]; @@ -307,7 +307,7 @@ void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const for (i=0; i<16; i++) { memset(table+i*256*16, 0, 16); -#if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE +#if CRYPTOPP_SSE2_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE if (HasSSE2()) for (j=2; j<=0x80; j*=2) for (k=1; k= 1600 || CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32)) +#if (CRYPTOPPL_AESNI_AES_AVAILABLE && CRYPTOPP_SSE42_AVAILABLE && (!defined(_MSC_VER) || _MSC_VER >= 1600 || CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32)) // MSVC 2008 SP1 generates bad code for _mm_extract_epi32() when compiling for X64 if (HasAESNI() && HasSSE4()) { @@ -379,7 +379,7 @@ void Rijndael::Base::UncheckedSetKey(const byte *userKey, unsigned int keylen, c temp = ConditionalByteReverse(BIG_ENDIAN_ORDER, rk[3]); rk[3] = ConditionalByteReverse(BIG_ENDIAN_ORDER, rk[4*m_rounds+3]); rk[4*m_rounds+3] = temp; } -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE if (HasAESNI()) ConditionalByteReverse(BIG_ENDIAN_ORDER, rk+4, rk+4, (m_rounds-1)*16); #endif @@ -387,7 +387,7 @@ void Rijndael::Base::UncheckedSetKey(const byte *userKey, unsigned int keylen, c void Rijndael::Enc::ProcessAndXorBlock(const byte *inBlock, const byte *xorBlock, byte *outBlock) const { -#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE) || CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE) || CRYPTOPPL_AESNI_AES_AVAILABLE #if (CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)) && !defined(CRYPTOPP_DISABLE_RIJNDAEL_ASM) if (HasSSE2()) #else @@ -468,7 +468,7 @@ void Rijndael::Enc::ProcessAndXorBlock(const byte *inBlock, const byte *xorBlock void Rijndael::Dec::ProcessAndXorBlock(const byte *inBlock, const byte *xorBlock, byte *outBlock) const { -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE if (HasAESNI()) { Rijndael::Dec::AdvancedProcessBlocks(inBlock, xorBlock, outBlock, 16, 0); @@ -1082,7 +1082,7 @@ static inline bool AliasedWithTable(const byte *begin, const byte *end) return (s0 < t1 || s1 <= t1) || (s0 >= t0 || s1 > t0); } -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE inline void AESNI_Enc_Block(__m128i &block, MAYBE_CONST __m128i *subkeys, unsigned int rounds) { @@ -1285,7 +1285,7 @@ Rijndael::Enc::Enc() : m_aliasBlock(s_sizeToAllocate) { } size_t Rijndael::Enc::AdvancedProcessBlocks(const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags) const { -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE if (HasAESNI()) return AESNI_AdvancedProcessBlocks(AESNI_Enc_Block, AESNI_Enc_4_Blocks, (MAYBE_CONST __m128i *)(const void *)m_key.begin(), m_rounds, inBlocks, xorBlocks, outBlocks, length, flags); #endif @@ -1347,7 +1347,7 @@ size_t Rijndael::Enc::AdvancedProcessBlocks(const byte *inBlocks, const byte *xo #if CRYPTOPP_BOOL_X64 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X86 size_t Rijndael::Dec::AdvancedProcessBlocks(const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags) const { -#if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE +#if CRYPTOPPL_AESNI_AES_AVAILABLE if (HasAESNI()) return AESNI_AdvancedProcessBlocks(AESNI_Dec_Block, AESNI_Dec_4_Blocks, (MAYBE_CONST __m128i *)(const void *)m_key.begin(), m_rounds, inBlocks, xorBlocks, outBlocks, length, flags); #endif