13 #if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARM64)
14 # if defined(__GNUC__)
17 # if CRYPTOPP_BOOL_NEON_INTRINSICS_AVAILABLE || defined(__ARM_NEON)
18 # include <arm_neon.h>
20 # if (CRYPTOPP_BOOL_ARM_CRYPTO_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_ARM_CRC32_INTRINSICS_AVAILABLE) || defined(__ARM_ACLE)
21 # include <arm_acle.h>
23 #endif // ARM32 and ARM64
26 #if defined(_MSC_VER) || defined(__BORLANDC__)
27 # define CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
29 # define CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
33 #if defined(CRYPTOPP_LLVM_CLANG_VERSION) || defined(CRYPTOPP_APPLE_CLANG_VERSION) || defined(CRYPTOPP_CLANG_INTEGRATED_ASSEMBLER)
35 #define INTEL_PREFIX ".intel_syntax;"
36 #define INTEL_NOPREFIX ".intel_syntax;"
37 #define ATT_PREFIX ".att_syntax;"
38 #define ATT_NOPREFIX ".att_syntax;"
39 #elif defined(__GNUC__)
41 #define INTEL_PREFIX ".intel_syntax prefix;"
42 #define INTEL_NOPREFIX ".intel_syntax noprefix;"
43 #define ATT_PREFIX ".att_syntax prefix;"
44 #define ATT_NOPREFIX ".att_syntax noprefix;"
48 #define INTEL_NOPREFIX
53 #ifdef CRYPTOPP_GENERATE_X64_MASM
55 #define CRYPTOPP_X86_ASM_AVAILABLE
56 #define CRYPTOPP_BOOL_X64 1
57 #define CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 1
62 # if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
63 # include <emmintrin.h>
66 #if CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
70 #if defined(__has_include)
71 # if __has_include(<xmmintrin.h>)
72 # include <xmmintrin.h>
77 #if !defined(__GNUC__) || defined(__SSSE3__)|| defined(__INTEL_COMPILER) || (CRYPTOPP_LLVM_CLANG_VERSION >= 30300) || (CRYPTOPP_APPLE_CLANG_VERSION >= 50000)
78 #include <tmmintrin.h>
81 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
82 _mm_shuffle_epi8 (__m128i a, __m128i b)
84 asm (
"pshufb %1, %0" :
"+x"(a) :
"xm"(b));
91 #if !defined(__GNUC__) || defined(__SSE4_1__)|| defined(__INTEL_COMPILER) || (CRYPTOPP_LLVM_CLANG_VERSION >= 30300) || (CRYPTOPP_APPLE_CLANG_VERSION >= 50000)
92 #include <smmintrin.h>
95 __inline
int __attribute__((__gnu_inline__, __always_inline__, __artificial__))
96 _mm_extract_epi32 (__m128i a,
const int i)
99 asm (
"pextrd %2, %1, %0" :
"=rm"(r) :
"x"(a),
"i"(i));
102 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
103 _mm_insert_epi32 (__m128i a,
int b,
const int i)
105 asm (
"pinsrd %2, %1, %0" :
"+x"(a) :
"rm"(b),
"i"(i));
109 #endif // smmintrin.h
112 #if !defined(__GNUC__) || (defined(__AES__) && defined(__PCLMUL__)) || defined(__INTEL_COMPILER) || (CRYPTOPP_LLVM_CLANG_VERSION >= 30400) || (CRYPTOPP_APPLE_CLANG_VERSION >= 60000)
113 #include <wmmintrin.h>
116 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
117 _mm_clmulepi64_si128 (__m128i a, __m128i b,
const int i)
119 asm (
"pclmulqdq %2, %1, %0" :
"+x"(a) :
"xm"(b),
"i"(i));
122 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
123 _mm_aeskeygenassist_si128 (__m128i a,
const int i)
126 asm (
"aeskeygenassist %2, %1, %0" :
"=x"(r) :
"xm"(a),
"i"(i));
129 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
130 _mm_aesimc_si128 (__m128i a)
133 asm (
"aesimc %1, %0" :
"=x"(r) :
"xm"(a));
136 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
137 _mm_aesenc_si128 (__m128i a, __m128i b)
139 asm (
"aesenc %1, %0" :
"+x"(a) :
"xm"(b));
142 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
143 _mm_aesenclast_si128 (__m128i a, __m128i b)
145 asm (
"aesenclast %1, %0" :
"+x"(a) :
"xm"(b));
148 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
149 _mm_aesdec_si128 (__m128i a, __m128i b)
151 asm (
"aesdec %1, %0" :
"+x"(a) :
"xm"(b));
154 __inline __m128i __attribute__((__gnu_inline__, __always_inline__, __artificial__))
155 _mm_aesdeclast_si128 (__m128i a, __m128i b)
157 asm (
"aesdeclast %1, %0" :
"+x"(a) :
"xm"(b));
161 #endif // wmmintrin.h
162 #endif // CRYPTOPP_BOOL_AESNI_INTRINSICS_AVAILABLE
164 #if (CRYPTOPP_BOOL_SSE4_INTRINSICS_AVAILABLE) && ((__SUNPRO_CC >= 0x5110) || defined(__clang__) || defined(__INTEL_COMPILER))
165 # include <emmintrin.h>
166 # include <smmintrin.h>
167 # include <tmmintrin.h>
168 # include <nmmintrin.h>
173 #if CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64 || CRYPTOPP_DOXYGEN_PROCESSING
175 #define CRYPTOPP_CPUID_AVAILABLE
178 #ifndef CRYPTOPP_DOXYGEN_PROCESSING
180 extern CRYPTOPP_DLL
bool g_x86DetectionDone;
181 extern CRYPTOPP_DLL
bool g_hasMMX;
182 extern CRYPTOPP_DLL
bool g_hasISSE;
183 extern CRYPTOPP_DLL
bool g_hasSSE2;
184 extern CRYPTOPP_DLL
bool g_hasSSSE3;
185 extern CRYPTOPP_DLL
bool g_hasSSE4;
186 extern CRYPTOPP_DLL
bool g_hasAESNI;
187 extern CRYPTOPP_DLL
bool g_hasCLMUL;
188 extern CRYPTOPP_DLL
bool g_isP4;
189 extern CRYPTOPP_DLL
bool g_hasRDRAND;
190 extern CRYPTOPP_DLL
bool g_hasRDSEED;
191 extern CRYPTOPP_DLL
bool g_hasPadlockRNG;
192 extern CRYPTOPP_DLL
bool g_hasPadlockACE;
193 extern CRYPTOPP_DLL
bool g_hasPadlockACE2;
194 extern CRYPTOPP_DLL
bool g_hasPadlockPHE;
195 extern CRYPTOPP_DLL
bool g_hasPadlockPMM;
196 extern CRYPTOPP_DLL word32 g_cacheLineSize;
198 CRYPTOPP_DLL
void CRYPTOPP_API DetectX86Features();
199 CRYPTOPP_DLL
bool CRYPTOPP_API CpuId(word32 input, word32 output[4]);
200 #endif // CRYPTOPP_DOXYGEN_PROCESSING
208 #if CRYPTOPP_BOOL_X64
211 if (!g_x86DetectionDone)
223 #if CRYPTOPP_BOOL_X64
226 if (!g_x86DetectionDone)
238 #if CRYPTOPP_BOOL_X64
241 if (!g_x86DetectionDone)
253 if (!g_x86DetectionDone)
263 if (!g_x86DetectionDone)
273 if (!g_x86DetectionDone)
283 if (!g_x86DetectionDone)
293 if (!g_x86DetectionDone)
303 if (!g_x86DetectionDone)
313 if (!g_x86DetectionDone)
323 if (!g_x86DetectionDone)
325 return g_hasPadlockRNG;
333 if (!g_x86DetectionDone)
335 return g_hasPadlockACE;
343 if (!g_x86DetectionDone)
345 return g_hasPadlockACE2;
353 if (!g_x86DetectionDone)
355 return g_hasPadlockPHE;
363 if (!g_x86DetectionDone)
365 return g_hasPadlockPMM;
377 if (!g_x86DetectionDone)
379 return g_cacheLineSize;
382 #elif (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARM64)
384 extern bool g_ArmDetectionDone;
385 extern bool g_hasNEON, g_hasPMULL, g_hasCRC32, g_hasAES, g_hasSHA1, g_hasSHA2;
386 void CRYPTOPP_API DetectArmFeatures();
394 inline bool HasNEON()
396 if (!g_ArmDetectionDone)
407 inline bool HasPMULL()
409 if (!g_ArmDetectionDone)
422 inline bool HasCRC32()
424 if (!g_ArmDetectionDone)
439 if (!g_ArmDetectionDone)
452 inline bool HasSHA1()
454 if (!g_ArmDetectionDone)
467 inline bool HasSHA2()
469 if (!g_ArmDetectionDone)
480 return CRYPTOPP_L1_CACHE_LINE_SIZE;
487 return CRYPTOPP_L1_CACHE_LINE_SIZE;
490 #endif // X86/X32/X64 and ARM
494 #if CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64
496 #ifdef CRYPTOPP_GENERATE_X64_MASM
497 #define AS1(x) x*newline*
498 #define AS2(x, y) x, y*newline*
499 #define AS3(x, y, z) x, y, z*newline*
500 #define ASS(x, y, a, b, c, d) x, y, a*64+b*16+c*4+d*newline*
501 #define ASL(x) label##x:*newline*
502 #define ASJ(x, y, z) x label##y*newline*
503 #define ASC(x, y) x label##y*newline*
504 #define AS_HEX(y) 0##y##h
505 #elif defined(_MSC_VER) || defined(__BORLANDC__)
506 #define CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
507 #define AS1(x) __asm {x}
508 #define AS2(x, y) __asm {x, y}
509 #define AS3(x, y, z) __asm {x, y, z}
510 #define ASS(x, y, a, b, c, d) __asm {x, y, (a)*64+(b)*16+(c)*4+(d)}
511 #define ASL(x) __asm {label##x:}
512 #define ASJ(x, y, z) __asm {x label##y}
513 #define ASC(x, y) __asm {x label##y}
514 #define CRYPTOPP_NAKED __declspec(naked)
515 #define AS_HEX(y) 0x##y
517 #define CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
520 #define GNU_AS1(x) #x ";" NEW_LINE
521 #define GNU_AS2(x, y) #x ", " #y ";" NEW_LINE
522 #define GNU_AS3(x, y, z) #x ", " #y ", " #z ";" NEW_LINE
523 #define GNU_ASL(x) "\n" #x ":" NEW_LINE
524 #define GNU_ASJ(x, y, z) #x " " #y #z ";" NEW_LINE
525 #define AS1(x) GNU_AS1(x)
526 #define AS2(x, y) GNU_AS2(x, y)
527 #define AS3(x, y, z) GNU_AS3(x, y, z)
528 #define ASS(x, y, a, b, c, d) #x ", " #y ", " #a "*64+" #b "*16+" #c "*4+" #d ";"
529 #define ASL(x) GNU_ASL(x)
530 #define ASJ(x, y, z) GNU_ASJ(x, y, z)
531 #define ASC(x, y) #x " " #y ";"
532 #define CRYPTOPP_NAKED
533 #define AS_HEX(y) 0x##y
539 #ifdef CRYPTOPP_GENERATE_X64_MASM
540 #define ASM_MOD(x, y) ((x) MOD (y))
541 #define XMMWORD_PTR XMMWORD PTR
544 #define ASM_MOD(x, y) ((x)-((x)/(y))*(y))
549 #if CRYPTOPP_BOOL_X86
557 #define AS_REG_1d ecx
558 #define AS_REG_2d edx
559 #define AS_REG_3d esi
560 #define AS_REG_4d edi
561 #define AS_REG_5d eax
562 #define AS_REG_6d ebx
563 #define AS_REG_7d ebp
565 #define WORD_REG(x) e##x
566 #define WORD_PTR DWORD PTR
567 #define AS_PUSH_IF86(x) AS1(push e##x)
568 #define AS_POP_IF86(x) AS1(pop e##x)
569 #define AS_JCXZ jecxz
570 #elif CRYPTOPP_BOOL_X32
576 #define AS_REG_6 r10d
577 #define AS_REG_7 r11d
578 #define AS_REG_1d ecx
579 #define AS_REG_2d edx
580 #define AS_REG_3d r8d
581 #define AS_REG_4d r9d
582 #define AS_REG_5d eax
583 #define AS_REG_6d r10d
584 #define AS_REG_7d r11d
586 #define WORD_REG(x) e##x
587 #define WORD_PTR DWORD PTR
588 #define AS_PUSH_IF86(x) AS1(push r##x)
589 #define AS_POP_IF86(x) AS1(pop r##x)
590 #define AS_JCXZ jecxz
591 #elif CRYPTOPP_BOOL_X64
592 #ifdef CRYPTOPP_GENERATE_X64_MASM
600 #define AS_REG_1d ecx
601 #define AS_REG_2d edx
602 #define AS_REG_3d r8d
603 #define AS_REG_4d r9d
604 #define AS_REG_5d eax
605 #define AS_REG_6d r10d
606 #define AS_REG_7d r11d
615 #define AS_REG_1d edi
616 #define AS_REG_2d esi
617 #define AS_REG_3d edx
618 #define AS_REG_4d ecx
619 #define AS_REG_5d r8d
620 #define AS_REG_6d r9d
621 #define AS_REG_7d r10d
624 #define WORD_REG(x) r##x
625 #define WORD_PTR QWORD PTR
626 #define AS_PUSH_IF86(x)
627 #define AS_POP_IF86(x)
628 #define AS_JCXZ jrcxz
632 #define AS_XMM_OUTPUT4(labelPrefix, inputPtr, outputPtr, x0, x1, x2, x3, t, p0, p1, p2, p3, increment)\
633 AS2( test inputPtr, inputPtr)\
634 ASC( jz, labelPrefix##3)\
635 AS2( test inputPtr, 15)\
636 ASC( jnz, labelPrefix##7)\
637 AS2( pxor xmm##x0, [inputPtr+p0*16])\
638 AS2( pxor xmm##x1, [inputPtr+p1*16])\
639 AS2( pxor xmm##x2, [inputPtr+p2*16])\
640 AS2( pxor xmm##x3, [inputPtr+p3*16])\
641 AS2( add inputPtr, increment*16)\
642 ASC( jmp, labelPrefix##3)\
644 AS2( movdqu xmm##t, [inputPtr+p0*16])\
645 AS2( pxor xmm##x0, xmm##t)\
646 AS2( movdqu xmm##t, [inputPtr+p1*16])\
647 AS2( pxor xmm##x1, xmm##t)\
648 AS2( movdqu xmm##t, [inputPtr+p2*16])\
649 AS2( pxor xmm##x2, xmm##t)\
650 AS2( movdqu xmm##t, [inputPtr+p3*16])\
651 AS2( pxor xmm##x3, xmm##t)\
652 AS2( add inputPtr, increment*16)\
654 AS2( test outputPtr, 15)\
655 ASC( jnz, labelPrefix##8)\
656 AS2( movdqa [outputPtr+p0*16], xmm##x0)\
657 AS2( movdqa [outputPtr+p1*16], xmm##x1)\
658 AS2( movdqa [outputPtr+p2*16], xmm##x2)\
659 AS2( movdqa [outputPtr+p3*16], xmm##x3)\
660 ASC( jmp, labelPrefix##9)\
662 AS2( movdqu [outputPtr+p0*16], xmm##x0)\
663 AS2( movdqu [outputPtr+p1*16], xmm##x1)\
664 AS2( movdqu [outputPtr+p2*16], xmm##x2)\
665 AS2( movdqu [outputPtr+p3*16], xmm##x3)\
667 AS2( add outputPtr, increment*16)
669 #endif // X86/X32/X64
673 #endif // CRYPTOPP_CPU_H