1// Copyright 2017 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5//go:build 386 || amd64
6
7package cpu
8
9const CacheLinePadSize = 64
10
11// cpuid is implemented in cpu_x86.s.
12func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32)
13
14// xgetbv with ecx = 0 is implemented in cpu_x86.s.
15func xgetbv() (eax, edx uint32)
16
17// getGOAMD64level is implemented in cpu_x86.s. Returns number in [1,4].
18func getGOAMD64level() int32
19
20const (
21	// ecx bits
22	cpuid_SSE3      = 1 << 0
23	cpuid_PCLMULQDQ = 1 << 1
24	cpuid_SSSE3     = 1 << 9
25	cpuid_FMA       = 1 << 12
26	cpuid_SSE41     = 1 << 19
27	cpuid_SSE42     = 1 << 20
28	cpuid_POPCNT    = 1 << 23
29	cpuid_AES       = 1 << 25
30	cpuid_OSXSAVE   = 1 << 27
31	cpuid_AVX       = 1 << 28
32
33	// ebx bits
34	cpuid_BMI1     = 1 << 3
35	cpuid_AVX2     = 1 << 5
36	cpuid_BMI2     = 1 << 8
37	cpuid_ERMS     = 1 << 9
38	cpuid_AVX512F  = 1 << 16
39	cpuid_ADX      = 1 << 19
40	cpuid_SHA      = 1 << 29
41	cpuid_AVX512BW = 1 << 30
42	cpuid_AVX512VL = 1 << 31
43
44	// edx bits for CPUID 0x80000001
45	cpuid_RDTSCP = 1 << 27
46)
47
48var maxExtendedFunctionInformation uint32
49
50func doinit() {
51	options = []option{
52		{Name: "adx", Feature: &X86.HasADX},
53		{Name: "aes", Feature: &X86.HasAES},
54		{Name: "erms", Feature: &X86.HasERMS},
55		{Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ},
56		{Name: "rdtscp", Feature: &X86.HasRDTSCP},
57		{Name: "sha", Feature: &X86.HasSHA},
58	}
59	level := getGOAMD64level()
60	if level < 2 {
61		// These options are required at level 2. At lower levels
62		// they can be turned off.
63		options = append(options,
64			option{Name: "popcnt", Feature: &X86.HasPOPCNT},
65			option{Name: "sse3", Feature: &X86.HasSSE3},
66			option{Name: "sse41", Feature: &X86.HasSSE41},
67			option{Name: "sse42", Feature: &X86.HasSSE42},
68			option{Name: "ssse3", Feature: &X86.HasSSSE3})
69	}
70	if level < 3 {
71		// These options are required at level 3. At lower levels
72		// they can be turned off.
73		options = append(options,
74			option{Name: "avx", Feature: &X86.HasAVX},
75			option{Name: "avx2", Feature: &X86.HasAVX2},
76			option{Name: "bmi1", Feature: &X86.HasBMI1},
77			option{Name: "bmi2", Feature: &X86.HasBMI2},
78			option{Name: "fma", Feature: &X86.HasFMA})
79	}
80	if level < 4 {
81		// These options are required at level 4. At lower levels
82		// they can be turned off.
83		options = append(options,
84			option{Name: "avx512f", Feature: &X86.HasAVX512F},
85			option{Name: "avx512bw", Feature: &X86.HasAVX512BW},
86			option{Name: "avx512vl", Feature: &X86.HasAVX512VL},
87		)
88	}
89
90	maxID, _, _, _ := cpuid(0, 0)
91
92	if maxID < 1 {
93		return
94	}
95
96	maxExtendedFunctionInformation, _, _, _ = cpuid(0x80000000, 0)
97
98	_, _, ecx1, _ := cpuid(1, 0)
99
100	X86.HasSSE3 = isSet(ecx1, cpuid_SSE3)
101	X86.HasPCLMULQDQ = isSet(ecx1, cpuid_PCLMULQDQ)
102	X86.HasSSSE3 = isSet(ecx1, cpuid_SSSE3)
103	X86.HasSSE41 = isSet(ecx1, cpuid_SSE41)
104	X86.HasSSE42 = isSet(ecx1, cpuid_SSE42)
105	X86.HasPOPCNT = isSet(ecx1, cpuid_POPCNT)
106	X86.HasAES = isSet(ecx1, cpuid_AES)
107
108	// OSXSAVE can be false when using older Operating Systems
109	// or when explicitly disabled on newer Operating Systems by
110	// e.g. setting the xsavedisable boot option on Windows 10.
111	X86.HasOSXSAVE = isSet(ecx1, cpuid_OSXSAVE)
112
113	// The FMA instruction set extension only has VEX prefixed instructions.
114	// VEX prefixed instructions require OSXSAVE to be enabled.
115	// See Intel 64 and IA-32 Architecture Software Developer’s Manual Volume 2
116	// Section 2.4 "AVX and SSE Instruction Exception Specification"
117	X86.HasFMA = isSet(ecx1, cpuid_FMA) && X86.HasOSXSAVE
118
119	osSupportsAVX := false
120	osSupportsAVX512 := false
121	// For XGETBV, OSXSAVE bit is required and sufficient.
122	if X86.HasOSXSAVE {
123		eax, _ := xgetbv()
124		// Check if XMM and YMM registers have OS support.
125		osSupportsAVX = isSet(eax, 1<<1) && isSet(eax, 1<<2)
126
127		// AVX512 detection does not work on Darwin,
128		// see https://github.com/golang/go/issues/49233
129		//
130		// Check if opmask, ZMMhi256 and Hi16_ZMM have OS support.
131		osSupportsAVX512 = osSupportsAVX && isSet(eax, 1<<5) && isSet(eax, 1<<6) && isSet(eax, 1<<7)
132	}
133
134	X86.HasAVX = isSet(ecx1, cpuid_AVX) && osSupportsAVX
135
136	if maxID < 7 {
137		return
138	}
139
140	_, ebx7, _, _ := cpuid(7, 0)
141	X86.HasBMI1 = isSet(ebx7, cpuid_BMI1)
142	X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX
143	X86.HasBMI2 = isSet(ebx7, cpuid_BMI2)
144	X86.HasERMS = isSet(ebx7, cpuid_ERMS)
145	X86.HasADX = isSet(ebx7, cpuid_ADX)
146	X86.HasSHA = isSet(ebx7, cpuid_SHA)
147
148	X86.HasAVX512F = isSet(ebx7, cpuid_AVX512F) && osSupportsAVX512
149	if X86.HasAVX512F {
150		X86.HasAVX512BW = isSet(ebx7, cpuid_AVX512BW)
151		X86.HasAVX512VL = isSet(ebx7, cpuid_AVX512VL)
152	}
153
154	var maxExtendedInformation uint32
155	maxExtendedInformation, _, _, _ = cpuid(0x80000000, 0)
156
157	if maxExtendedInformation < 0x80000001 {
158		return
159	}
160
161	_, _, _, edxExt1 := cpuid(0x80000001, 0)
162	X86.HasRDTSCP = isSet(edxExt1, cpuid_RDTSCP)
163}
164
165func isSet(hwc uint32, value uint32) bool {
166	return hwc&value != 0
167}
168
169// Name returns the CPU name given by the vendor.
170// If the CPU name can not be determined an
171// empty string is returned.
172func Name() string {
173	if maxExtendedFunctionInformation < 0x80000004 {
174		return ""
175	}
176
177	data := make([]byte, 0, 3*4*4)
178
179	var eax, ebx, ecx, edx uint32
180	eax, ebx, ecx, edx = cpuid(0x80000002, 0)
181	data = appendBytes(data, eax, ebx, ecx, edx)
182	eax, ebx, ecx, edx = cpuid(0x80000003, 0)
183	data = appendBytes(data, eax, ebx, ecx, edx)
184	eax, ebx, ecx, edx = cpuid(0x80000004, 0)
185	data = appendBytes(data, eax, ebx, ecx, edx)
186
187	// Trim leading spaces.
188	for len(data) > 0 && data[0] == ' ' {
189		data = data[1:]
190	}
191
192	// Trim tail after and including the first null byte.
193	for i, c := range data {
194		if c == '\x00' {
195			data = data[:i]
196			break
197		}
198	}
199
200	return string(data)
201}
202
203func appendBytes(b []byte, args ...uint32) []byte {
204	for _, arg := range args {
205		b = append(b,
206			byte((arg >> 0)),
207			byte((arg >> 8)),
208			byte((arg >> 16)),
209			byte((arg >> 24)))
210	}
211	return b
212}
213