1 /* cpu_features.c -- Processor features detection.
2 *
3 * Copyright 2018 The Chromium Authors
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the Chromium source repository LICENSE file.
6 */
7
8 #include "cpu_features.h"
9 #include "zutil.h"
10
11 #include <stdint.h>
12 #if defined(_MSC_VER)
13 #include <intrin.h>
14 #elif defined(ADLER32_SIMD_SSSE3)
15 #include <cpuid.h>
16 #endif
17
18 /* TODO(cavalcantii): remove checks for x86_flags on deflate.
19 */
20 #if defined(ARMV8_OS_MACOS)
21 /* Crypto extensions (crc32/pmull) are a baseline feature in ARMv8.1-A, and
22 * OSX running on arm64 is new enough that these can be assumed without
23 * runtime detection.
24 */
25 int ZLIB_INTERNAL arm_cpu_enable_crc32 = 1;
26 int ZLIB_INTERNAL arm_cpu_enable_pmull = 1;
27 #else
28 int ZLIB_INTERNAL arm_cpu_enable_crc32 = 0;
29 int ZLIB_INTERNAL arm_cpu_enable_pmull = 0;
30 #endif
31 int ZLIB_INTERNAL x86_cpu_enable_sse2 = 0;
32 int ZLIB_INTERNAL x86_cpu_enable_ssse3 = 0;
33 int ZLIB_INTERNAL x86_cpu_enable_simd = 0;
34 int ZLIB_INTERNAL x86_cpu_enable_avx512 = 0;
35
36 int ZLIB_INTERNAL riscv_cpu_enable_rvv = 0;
37 int ZLIB_INTERNAL riscv_cpu_enable_vclmul = 0;
38
39 #ifndef CPU_NO_SIMD
40
41 #if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || \
42 defined(ARMV8_OS_FUCHSIA) || defined(ARMV8_OS_IOS)
43 #include <pthread.h>
44 #endif
45
46 #if defined(ARMV8_OS_ANDROID)
47 #include <cpu-features.h>
48 #elif defined(ARMV8_OS_LINUX)
49 #include <asm/hwcap.h>
50 #include <sys/auxv.h>
51 #elif defined(ARMV8_OS_FUCHSIA)
52 #include <zircon/features.h>
53 #include <zircon/syscalls.h>
54 #include <zircon/types.h>
55 #elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS)
56 #include <windows.h>
57 #elif defined(ARMV8_OS_IOS)
58 #include <sys/sysctl.h>
59 #elif !defined(_MSC_VER)
60 #include <pthread.h>
61 #else
62 #error cpu_features.c CPU feature detection in not defined for your platform
63 #endif
64
65 #if !defined(CPU_NO_SIMD) && !defined(ARMV8_OS_MACOS)
66 static void _cpu_check_features(void);
67 #endif
68
69 #if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || \
70 defined(ARMV8_OS_MACOS) || defined(ARMV8_OS_FUCHSIA) || \
71 defined(X86_NOT_WINDOWS) || defined(ARMV8_OS_IOS) || \
72 defined(RISCV_RVV)
73 #if !defined(ARMV8_OS_MACOS)
74 // _cpu_check_features() doesn't need to do anything on mac/arm since all
75 // features are known at build time, so don't call it.
76 // Do provide cpu_check_features() (with a no-op implementation) so that we
77 // don't have to make all callers of it check for mac/arm.
78 static pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT;
79 #endif
cpu_check_features(void)80 void ZLIB_INTERNAL cpu_check_features(void)
81 {
82 #if !defined(ARMV8_OS_MACOS)
83 pthread_once(&cpu_check_inited_once, _cpu_check_features);
84 #endif
85 }
86 #elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS)
87 static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT;
_cpu_check_features_forwarder(PINIT_ONCE once,PVOID param,PVOID * context)88 static BOOL CALLBACK _cpu_check_features_forwarder(PINIT_ONCE once, PVOID param, PVOID* context)
89 {
90 _cpu_check_features();
91 return TRUE;
92 }
cpu_check_features(void)93 void ZLIB_INTERNAL cpu_check_features(void)
94 {
95 InitOnceExecuteOnce(&cpu_check_inited_once, _cpu_check_features_forwarder,
96 NULL, NULL);
97 }
98 #endif
99
100 #if (defined(__ARM_NEON__) || defined(__ARM_NEON))
101 #if !defined(ARMV8_OS_MACOS)
102 /*
103 * See http://bit.ly/2CcoEsr for run-time detection of ARM features and also
104 * crbug.com/931275 for android_getCpuFeatures() use in the Android sandbox.
105 */
_cpu_check_features(void)106 static void _cpu_check_features(void)
107 {
108 #if defined(ARMV8_OS_ANDROID) && defined(__aarch64__)
109 uint64_t features = android_getCpuFeatures();
110 arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM64_FEATURE_CRC32);
111 arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM64_FEATURE_PMULL);
112 #elif defined(ARMV8_OS_ANDROID) /* aarch32 */
113 uint64_t features = android_getCpuFeatures();
114 arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM_FEATURE_CRC32);
115 arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM_FEATURE_PMULL);
116 #elif defined(ARMV8_OS_LINUX) && defined(__aarch64__)
117 unsigned long features = getauxval(AT_HWCAP);
118 arm_cpu_enable_crc32 = !!(features & HWCAP_CRC32);
119 arm_cpu_enable_pmull = !!(features & HWCAP_PMULL);
120 #elif defined(ARMV8_OS_LINUX) && (defined(__ARM_NEON) || defined(__ARM_NEON__))
121 /* Query HWCAP2 for ARMV8-A SoCs running in aarch32 mode */
122 unsigned long features = getauxval(AT_HWCAP2);
123 arm_cpu_enable_crc32 = !!(features & HWCAP2_CRC32);
124 arm_cpu_enable_pmull = !!(features & HWCAP2_PMULL);
125 #elif defined(ARMV8_OS_FUCHSIA)
126 uint32_t features;
127 zx_status_t rc = zx_system_get_features(ZX_FEATURE_KIND_CPU, &features);
128 if (rc != ZX_OK || (features & ZX_ARM64_FEATURE_ISA_ASIMD) == 0)
129 return; /* Report nothing if ASIMD(NEON) is missing */
130 arm_cpu_enable_crc32 = !!(features & ZX_ARM64_FEATURE_ISA_CRC32);
131 arm_cpu_enable_pmull = !!(features & ZX_ARM64_FEATURE_ISA_PMULL);
132 #elif defined(ARMV8_OS_WINDOWS)
133 arm_cpu_enable_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE);
134 arm_cpu_enable_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE);
135 #elif defined(ARMV8_OS_IOS)
136 // Determine what features are supported dynamically. This code is applicable to macOS
137 // as well if we wish to do that dynamically on that platform in the future.
138 // See https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics
139 int val = 0;
140 size_t len = sizeof(val);
141 arm_cpu_enable_crc32 = sysctlbyname("hw.optional.armv8_crc32", &val, &len, 0, 0) == 0
142 && val != 0;
143 val = 0;
144 len = sizeof(val);
145 arm_cpu_enable_pmull = sysctlbyname("hw.optional.arm.FEAT_PMULL", &val, &len, 0, 0) == 0
146 && val != 0;
147 #endif
148 }
149 #endif
150 #elif defined(X86_NOT_WINDOWS) || defined(X86_WINDOWS)
151 /*
152 * iOS@x86 (i.e. emulator) is another special case where we disable
153 * SIMD optimizations.
154 */
155 #ifndef CPU_NO_SIMD
156 /* On x86 we simply use a instruction to check the CPU features.
157 * (i.e. CPUID).
158 */
159 #ifdef CRC32_SIMD_AVX512_PCLMUL
160 #include <immintrin.h>
161 #include <xsaveintrin.h>
162 #endif
_cpu_check_features(void)163 static void _cpu_check_features(void)
164 {
165 int x86_cpu_has_sse2;
166 int x86_cpu_has_ssse3;
167 int x86_cpu_has_sse42;
168 int x86_cpu_has_pclmulqdq;
169 int abcd[4];
170
171 #ifdef _MSC_VER
172 __cpuid(abcd, 1);
173 #else
174 __cpuid(1, abcd[0], abcd[1], abcd[2], abcd[3]);
175 #endif
176
177 x86_cpu_has_sse2 = abcd[3] & 0x4000000;
178 x86_cpu_has_ssse3 = abcd[2] & 0x000200;
179 x86_cpu_has_sse42 = abcd[2] & 0x100000;
180 x86_cpu_has_pclmulqdq = abcd[2] & 0x2;
181
182 x86_cpu_enable_sse2 = x86_cpu_has_sse2;
183
184 x86_cpu_enable_ssse3 = x86_cpu_has_ssse3;
185
186 x86_cpu_enable_simd = x86_cpu_has_sse2 &&
187 x86_cpu_has_sse42 &&
188 x86_cpu_has_pclmulqdq;
189
190 #ifdef CRC32_SIMD_AVX512_PCLMUL
191 x86_cpu_enable_avx512 = _xgetbv(0) & 0x00000040;
192 #endif
193 }
194 #endif // x86 & NO_SIMD
195
196 #elif defined(RISCV_RVV)
197 #include <sys/auxv.h>
198
199 #ifndef ZLIB_HWCAP_RVV
200 #define ZLIB_HWCAP_RVV (1 << ('v' - 'a'))
201 #endif
202
203 /* TODO(cavalcantii)
204 * - add support for Android@RISCV i.e. __riscv_hwprobe().
205 * - detect vclmul (crypto extensions).
206 */
_cpu_check_features(void)207 static void _cpu_check_features(void)
208 {
209 unsigned long features = getauxval(AT_HWCAP);
210 riscv_cpu_enable_rvv = !!(features & ZLIB_HWCAP_RVV);
211 }
212 #endif // ARM | x86 | RISCV
213 #endif // NO SIMD CPU
214