/src/libsodium/src/libsodium/sodium/runtime.c
Line | Count | Source |
1 | | #include <stddef.h> |
2 | | #include <stdint.h> |
3 | | #ifdef HAVE_ANDROID_GETCPUFEATURES |
4 | | # include <cpu-features.h> |
5 | | #endif |
6 | | #ifdef __APPLE__ |
7 | | # include <sys/types.h> |
8 | | # include <sys/sysctl.h> |
9 | | # include <mach/machine.h> |
10 | | #endif |
11 | | #ifdef HAVE_SYS_AUXV_H |
12 | | # include <sys/auxv.h> |
13 | | #endif |
14 | | |
15 | | #include "private/common.h" |
16 | | #include "runtime.h" |
17 | | |
18 | | typedef struct CPUFeatures_ { |
19 | | int initialized; |
20 | | int has_neon; |
21 | | int has_armcrypto; |
22 | | int has_sse2; |
23 | | int has_sse3; |
24 | | int has_ssse3; |
25 | | int has_sse41; |
26 | | int has_avx; |
27 | | int has_avx2; |
28 | | int has_avx512f; |
29 | | int has_pclmul; |
30 | | int has_aesni; |
31 | | int has_rdrand; |
32 | | } CPUFeatures; |
33 | | |
34 | | static CPUFeatures _cpu_features; |
35 | | |
36 | 0 | #define CPUID_EBX_AVX2 0x00000020 |
37 | 0 | #define CPUID_EBX_AVX512F 0x00010000 |
38 | | |
39 | 0 | #define CPUID_ECX_SSE3 0x00000001 |
40 | 0 | #define CPUID_ECX_PCLMUL 0x00000002 |
41 | 0 | #define CPUID_ECX_SSSE3 0x00000200 |
42 | 0 | #define CPUID_ECX_SSE41 0x00080000 |
43 | 0 | #define CPUID_ECX_AESNI 0x02000000 |
44 | 0 | #define CPUID_ECX_XSAVE 0x04000000 |
45 | 0 | #define CPUID_ECX_OSXSAVE 0x08000000 |
46 | 0 | #define CPUID_ECX_AVX 0x10000000 |
47 | 0 | #define CPUID_ECX_RDRAND 0x40000000 |
48 | | |
49 | 0 | #define CPUID_EDX_SSE2 0x04000000 |
50 | | |
51 | 0 | #define XCR0_SSE 0x00000002 |
52 | 0 | #define XCR0_AVX 0x00000004 |
53 | 0 | #define XCR0_OPMASK 0x00000020 |
54 | 0 | #define XCR0_ZMM_HI256 0x00000040 |
55 | 0 | #define XCR0_HI16_ZMM 0x00000080 |
56 | | |
57 | | static int |
58 | | _sodium_runtime_arm_cpu_features(CPUFeatures * const cpu_features) |
59 | 3 | { |
60 | 3 | cpu_features->has_neon = 0; |
61 | 3 | cpu_features->has_armcrypto = 0; |
62 | | |
63 | 3 | #ifndef __ARM_ARCH |
64 | 3 | return -1; /* LCOV_EXCL_LINE */ |
65 | 0 | #endif |
66 | | |
67 | | #if defined(__ARM_NEON) || defined(__aarch64__) || defined(_M_ARM64) |
68 | | cpu_features->has_neon = 1; |
69 | | #elif defined(HAVE_ANDROID_GETCPUFEATURES) |
70 | | cpu_features->has_neon = |
71 | | (android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_ASIMD) != 0x0; |
72 | | #elif (defined(__aarch64__) || defined(_M_ARM64)) && defined(AT_HWCAP) |
73 | | # ifdef HAVE_GETAUXVAL |
74 | | cpu_features->has_neon = (getauxval(AT_HWCAP) & (1L << 1)) != 0; |
75 | | # elif defined(HAVE_ELF_AUX_INFO) |
76 | | { |
77 | | unsigned long buf; |
78 | | if (elf_aux_info(AT_HWCAP, (void *) &buf, (int) sizeof buf) == 0) { |
79 | | cpu_features->has_neon = (buf & (1L << 1)) != 0; |
80 | | } |
81 | | } |
82 | | # endif |
83 | | #elif defined(__arm__) && defined(AT_HWCAP) |
84 | | # ifdef HAVE_GETAUXVAL |
85 | | cpu_features->has_neon = (getauxval(AT_HWCAP) & (1L << 12)) != 0; |
86 | | # elif defined(HAVE_ELF_AUX_INFO) |
87 | | { |
88 | | unsigned long buf; |
89 | | if (elf_aux_info(AT_HWCAP, (void *) &buf, (int) sizeof buf) == 0) { |
90 | | cpu_features->has_neon = (buf & (1L << 12)) != 0; |
91 | | } |
92 | | } |
93 | | # endif |
94 | | #endif |
95 | | |
96 | 0 | if (cpu_features->has_neon == 0) { |
97 | 0 | return 0; |
98 | 0 | } |
99 | | |
100 | | #if defined(__ARM_FEATURE_CRYPTO) && defined(__ARM_FEATURE_AES) |
101 | | cpu_features->has_armcrypto = 1; |
102 | | #elif defined(_M_ARM64) |
103 | | cpu_features->has_armcrypto = 1; /* assuming all CPUs supported by ARM Windows have the crypto extensions */ |
104 | | #elif defined(__APPLE__) && defined(CPU_TYPE_ARM64) && defined(CPU_SUBTYPE_ARM64E) |
105 | | { |
106 | | cpu_type_t cpu_type; |
107 | | cpu_subtype_t cpu_subtype; |
108 | | size_t cpu_type_len = sizeof cpu_type; |
109 | | size_t cpu_subtype_len = sizeof cpu_subtype; |
110 | | |
111 | | if (sysctlbyname("hw.cputype", &cpu_type, &cpu_type_len, |
112 | | NULL, 0) == 0 && cpu_type == CPU_TYPE_ARM64 && |
113 | | sysctlbyname("hw.cpusubtype", &cpu_subtype, &cpu_subtype_len, |
114 | | NULL, 0) == 0 && |
115 | | (cpu_subtype == CPU_SUBTYPE_ARM64E || |
116 | | cpu_subtype == CPU_SUBTYPE_ARM64_V8)) { |
117 | | cpu_features->has_armcrypto = 1; |
118 | | } |
119 | | } |
120 | | #elif defined(HAVE_ANDROID_GETCPUFEATURES) |
121 | | cpu_features->has_armcrypto = |
122 | | (android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_AES) != 0x0; |
123 | | #elif (defined(__aarch64__) || defined(_M_ARM64)) && defined(AT_HWCAP) |
124 | | # ifdef HAVE_GETAUXVAL |
125 | | cpu_features->has_armcrypto = (getauxval(AT_HWCAP) & (1L << 3)) != 0; |
126 | | # elif defined(HAVE_ELF_AUX_INFO) |
127 | | { |
128 | | unsigned long buf; |
129 | | if (elf_aux_info(AT_HWCAP, (void *) &buf, (int) sizeof buf) == 0) { |
130 | | cpu_features->has_armcrypto = (buf & (1L << 3)) != 0; |
131 | | } |
132 | | } |
133 | | # endif |
134 | | #elif defined(__arm__) && defined(AT_HWCAP2) |
135 | | # ifdef HAVE_GETAUXVAL |
136 | | cpu_features->has_armcrypto = (getauxval(AT_HWCAP2) & (1L << 0)) != 0; |
137 | | # elif defined(HAVE_ELF_AUX_INFO) |
138 | | { |
139 | | unsigned long buf; |
140 | | if (elf_aux_info(AT_HWCAP2, (void *) &buf, (int) sizeof buf) == 0) { |
141 | | cpu_features->has_armcrypto = (buf & (1L << 0)) != 0; |
142 | | } |
143 | | } |
144 | | # endif |
145 | | #endif |
146 | | |
147 | 0 | return 0; |
148 | 0 | } |
149 | | |
150 | | static void |
151 | | _cpuid(unsigned int cpu_info[4U], const unsigned int cpu_info_type) |
152 | 3 | { |
153 | | #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) |
154 | | __cpuid((int *) cpu_info, cpu_info_type); |
155 | | #elif defined(HAVE_CPUID) |
156 | | cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0; |
157 | | # ifdef __i386__ |
158 | | __asm__ __volatile__( |
159 | | "pushfl; pushfl; " |
160 | | "popl %0; " |
161 | | "movl %0, %1; xorl %2, %0; " |
162 | | "pushl %0; " |
163 | | "popfl; pushfl; popl %0; popfl" |
164 | | : "=&r"(cpu_info[0]), "=&r"(cpu_info[1]) |
165 | | : "i"(0x200000)); |
166 | | if (((cpu_info[0] ^ cpu_info[1]) & 0x200000) == 0x0) { |
167 | | return; /* LCOV_EXCL_LINE */ |
168 | | } |
169 | | # endif |
170 | | # ifdef __i386__ |
171 | | __asm__ __volatile__("xchgl %%ebx, %k1; cpuid; xchgl %%ebx, %k1" |
172 | | : "=a"(cpu_info[0]), "=&r"(cpu_info[1]), |
173 | | "=c"(cpu_info[2]), "=d"(cpu_info[3]) |
174 | | : "0"(cpu_info_type), "2"(0U)); |
175 | | # elif defined(__x86_64__) |
176 | | __asm__ __volatile__("xchgq %%rbx, %q1; cpuid; xchgq %%rbx, %q1" |
177 | | : "=a"(cpu_info[0]), "=&r"(cpu_info[1]), |
178 | | "=c"(cpu_info[2]), "=d"(cpu_info[3]) |
179 | | : "0"(cpu_info_type), "2"(0U)); |
180 | | # else |
181 | | __asm__ __volatile__("cpuid" |
182 | | : "=a"(cpu_info[0]), "=b"(cpu_info[1]), |
183 | | "=c"(cpu_info[2]), "=d"(cpu_info[3]) |
184 | | : "0"(cpu_info_type), "2"(0U)); |
185 | | # endif |
186 | | #else |
187 | 3 | (void) cpu_info_type; |
188 | 3 | cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0; |
189 | 3 | #endif |
190 | 3 | } |
191 | | |
192 | | static int |
193 | | _sodium_runtime_intel_cpu_features(CPUFeatures * const cpu_features) |
194 | 3 | { |
195 | 3 | unsigned int cpu_info[4]; |
196 | 3 | uint32_t xcr0 = 0U; |
197 | | |
198 | 3 | _cpuid(cpu_info, 0x0); |
199 | 3 | if (cpu_info[0] == 0U) { |
200 | 3 | return -1; /* LCOV_EXCL_LINE */ |
201 | 3 | } |
202 | 0 | _cpuid(cpu_info, 0x00000001); |
203 | 0 | #ifdef HAVE_EMMINTRIN_H |
204 | 0 | cpu_features->has_sse2 = ((cpu_info[3] & CPUID_EDX_SSE2) != 0x0); |
205 | | #else |
206 | | cpu_features->has_sse2 = 0; |
207 | | #endif |
208 | |
|
209 | 0 | #ifdef HAVE_PMMINTRIN_H |
210 | 0 | cpu_features->has_sse3 = ((cpu_info[2] & CPUID_ECX_SSE3) != 0x0); |
211 | | #else |
212 | | cpu_features->has_sse3 = 0; |
213 | | #endif |
214 | |
|
215 | 0 | #ifdef HAVE_TMMINTRIN_H |
216 | 0 | cpu_features->has_ssse3 = ((cpu_info[2] & CPUID_ECX_SSSE3) != 0x0); |
217 | | #else |
218 | | cpu_features->has_ssse3 = 0; |
219 | | #endif |
220 | |
|
221 | 0 | #ifdef HAVE_SMMINTRIN_H |
222 | 0 | cpu_features->has_sse41 = ((cpu_info[2] & CPUID_ECX_SSE41) != 0x0); |
223 | | #else |
224 | | cpu_features->has_sse41 = 0; |
225 | | #endif |
226 | |
|
227 | 0 | cpu_features->has_avx = 0; |
228 | |
|
229 | 0 | (void) xcr0; |
230 | 0 | #ifdef HAVE_AVXINTRIN_H |
231 | 0 | if ((cpu_info[2] & (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) == |
232 | 0 | (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) { |
233 | 0 | xcr0 = 0U; |
234 | | # if defined(HAVE__XGETBV) || \ |
235 | | (defined(_MSC_VER) && defined(_XCR_XFEATURE_ENABLED_MASK) && _MSC_FULL_VER >= 160040219) |
236 | | xcr0 = (uint32_t) _xgetbv(0); |
237 | | # elif defined(_MSC_VER) && defined(_M_IX86) |
238 | | /* |
239 | | * Visual Studio documentation states that eax/ecx/edx don't need to |
240 | | * be preserved in inline assembly code. But that doesn't seem to |
241 | | * always hold true on Visual Studio 2010. |
242 | | */ |
243 | | __asm { |
244 | | push eax |
245 | | push ecx |
246 | | push edx |
247 | | xor ecx, ecx |
248 | | _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0 |
249 | | mov xcr0, eax |
250 | | pop edx |
251 | | pop ecx |
252 | | pop eax |
253 | | } |
254 | | # elif defined(HAVE_AVX_ASM) |
255 | | __asm__ __volatile__(".byte 0x0f, 0x01, 0xd0" /* XGETBV */ |
256 | | : "=a"(xcr0) |
257 | | : "c"((uint32_t) 0U) |
258 | | : "%edx"); |
259 | | # endif |
260 | 0 | if ((xcr0 & (XCR0_SSE | XCR0_AVX)) == (XCR0_SSE | XCR0_AVX)) { |
261 | 0 | cpu_features->has_avx = 1; |
262 | 0 | } |
263 | 0 | } |
264 | 0 | #endif |
265 | |
|
266 | 0 | cpu_features->has_avx2 = 0; |
267 | 0 | #ifdef HAVE_AVX2INTRIN_H |
268 | 0 | if (cpu_features->has_avx) { |
269 | 0 | unsigned int cpu_info7[4]; |
270 | |
|
271 | 0 | _cpuid(cpu_info7, 0x00000007); |
272 | 0 | cpu_features->has_avx2 = ((cpu_info7[1] & CPUID_EBX_AVX2) != 0x0); |
273 | 0 | } |
274 | 0 | #endif |
275 | |
|
276 | 0 | cpu_features->has_avx512f = 0; |
277 | 0 | #ifdef HAVE_AVX512FINTRIN_H |
278 | 0 | if (cpu_features->has_avx2) { |
279 | 0 | unsigned int cpu_info7[4]; |
280 | |
|
281 | 0 | _cpuid(cpu_info7, 0x00000007); |
282 | | /* LCOV_EXCL_START */ |
283 | 0 | if ((cpu_info7[1] & CPUID_EBX_AVX512F) == CPUID_EBX_AVX512F && |
284 | 0 | (xcr0 & (XCR0_OPMASK | XCR0_ZMM_HI256 | XCR0_HI16_ZMM)) |
285 | 0 | == (XCR0_OPMASK | XCR0_ZMM_HI256 | XCR0_HI16_ZMM)) { |
286 | 0 | cpu_features->has_avx512f = 1; |
287 | 0 | } |
288 | | /* LCOV_EXCL_STOP */ |
289 | 0 | } |
290 | 0 | #endif |
291 | |
|
292 | 0 | #ifdef HAVE_WMMINTRIN_H |
293 | 0 | cpu_features->has_pclmul = ((cpu_info[2] & CPUID_ECX_PCLMUL) != 0x0); |
294 | 0 | cpu_features->has_aesni = ((cpu_info[2] & CPUID_ECX_AESNI) != 0x0); |
295 | | #else |
296 | | cpu_features->has_pclmul = 0; |
297 | | cpu_features->has_aesni = 0; |
298 | | #endif |
299 | |
|
300 | 0 | #ifdef HAVE_RDRAND |
301 | 0 | cpu_features->has_rdrand = ((cpu_info[2] & CPUID_ECX_RDRAND) != 0x0); |
302 | | #else |
303 | | cpu_features->has_rdrand = 0; |
304 | | #endif |
305 | |
|
306 | 0 | return 0; |
307 | 3 | } |
308 | | |
309 | | int |
310 | | _sodium_runtime_get_cpu_features(void) |
311 | 3 | { |
312 | 3 | int ret = -1; |
313 | | |
314 | 3 | ret &= _sodium_runtime_arm_cpu_features(&_cpu_features); |
315 | 3 | ret &= _sodium_runtime_intel_cpu_features(&_cpu_features); |
316 | 3 | _cpu_features.initialized = 1; |
317 | | |
318 | 3 | return ret; |
319 | 3 | } |
320 | | |
321 | | int |
322 | | sodium_runtime_has_neon(void) |
323 | 0 | { |
324 | 0 | return _cpu_features.has_neon; |
325 | 0 | } |
326 | | |
327 | | int |
328 | | sodium_runtime_has_armcrypto(void) |
329 | 0 | { |
330 | 0 | return _cpu_features.has_armcrypto; |
331 | 0 | } |
332 | | |
333 | | int |
334 | | sodium_runtime_has_sse2(void) |
335 | 6 | { |
336 | 6 | return _cpu_features.has_sse2; |
337 | 6 | } |
338 | | |
339 | | int |
340 | | sodium_runtime_has_sse3(void) |
341 | 0 | { |
342 | 0 | return _cpu_features.has_sse3; |
343 | 0 | } |
344 | | |
345 | | int |
346 | | sodium_runtime_has_ssse3(void) |
347 | 9 | { |
348 | 9 | return _cpu_features.has_ssse3; |
349 | 9 | } |
350 | | |
351 | | int |
352 | | sodium_runtime_has_sse41(void) |
353 | 3 | { |
354 | 3 | return _cpu_features.has_sse41; |
355 | 3 | } |
356 | | |
357 | | int |
358 | | sodium_runtime_has_avx(void) |
359 | 6 | { |
360 | 6 | return _cpu_features.has_avx; |
361 | 6 | } |
362 | | |
363 | | int |
364 | | sodium_runtime_has_avx2(void) |
365 | 12 | { |
366 | 12 | return _cpu_features.has_avx2; |
367 | 12 | } |
368 | | |
369 | | int |
370 | | sodium_runtime_has_avx512f(void) |
371 | 3 | { |
372 | 3 | return _cpu_features.has_avx512f; |
373 | 3 | } |
374 | | |
375 | | int |
376 | | sodium_runtime_has_pclmul(void) |
377 | 0 | { |
378 | 0 | return _cpu_features.has_pclmul; |
379 | 0 | } |
380 | | |
381 | | int |
382 | | sodium_runtime_has_aesni(void) |
383 | 6 | { |
384 | 6 | return _cpu_features.has_aesni; |
385 | 6 | } |
386 | | |
387 | | int |
388 | | sodium_runtime_has_rdrand(void) |
389 | 0 | { |
390 | 0 | return _cpu_features.has_rdrand; |
391 | 0 | } |