1// Copyright 2018 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5//go:build 386 || amd64 || amd64p32
6// +build 386 amd64 amd64p32
7
8package cpu
9
10import "runtime"
11
12const cacheLineSize = 64
13
14func initOptions() {
15	options = []option{
16		{Name: "adx", Feature: &X86.HasADX},
17		{Name: "aes", Feature: &X86.HasAES},
18		{Name: "avx", Feature: &X86.HasAVX},
19		{Name: "avx2", Feature: &X86.HasAVX2},
20		{Name: "avx512", Feature: &X86.HasAVX512},
21		{Name: "avx512f", Feature: &X86.HasAVX512F},
22		{Name: "avx512cd", Feature: &X86.HasAVX512CD},
23		{Name: "avx512er", Feature: &X86.HasAVX512ER},
24		{Name: "avx512pf", Feature: &X86.HasAVX512PF},
25		{Name: "avx512vl", Feature: &X86.HasAVX512VL},
26		{Name: "avx512bw", Feature: &X86.HasAVX512BW},
27		{Name: "avx512dq", Feature: &X86.HasAVX512DQ},
28		{Name: "avx512ifma", Feature: &X86.HasAVX512IFMA},
29		{Name: "avx512vbmi", Feature: &X86.HasAVX512VBMI},
30		{Name: "avx512vnniw", Feature: &X86.HasAVX5124VNNIW},
31		{Name: "avx5124fmaps", Feature: &X86.HasAVX5124FMAPS},
32		{Name: "avx512vpopcntdq", Feature: &X86.HasAVX512VPOPCNTDQ},
33		{Name: "avx512vpclmulqdq", Feature: &X86.HasAVX512VPCLMULQDQ},
34		{Name: "avx512vnni", Feature: &X86.HasAVX512VNNI},
35		{Name: "avx512gfni", Feature: &X86.HasAVX512GFNI},
36		{Name: "avx512vaes", Feature: &X86.HasAVX512VAES},
37		{Name: "avx512vbmi2", Feature: &X86.HasAVX512VBMI2},
38		{Name: "avx512bitalg", Feature: &X86.HasAVX512BITALG},
39		{Name: "avx512bf16", Feature: &X86.HasAVX512BF16},
40		{Name: "bmi1", Feature: &X86.HasBMI1},
41		{Name: "bmi2", Feature: &X86.HasBMI2},
42		{Name: "cx16", Feature: &X86.HasCX16},
43		{Name: "erms", Feature: &X86.HasERMS},
44		{Name: "fma", Feature: &X86.HasFMA},
45		{Name: "osxsave", Feature: &X86.HasOSXSAVE},
46		{Name: "pclmulqdq", Feature: &X86.HasPCLMULQDQ},
47		{Name: "popcnt", Feature: &X86.HasPOPCNT},
48		{Name: "rdrand", Feature: &X86.HasRDRAND},
49		{Name: "rdseed", Feature: &X86.HasRDSEED},
50		{Name: "sse3", Feature: &X86.HasSSE3},
51		{Name: "sse41", Feature: &X86.HasSSE41},
52		{Name: "sse42", Feature: &X86.HasSSE42},
53		{Name: "ssse3", Feature: &X86.HasSSSE3},
54
55		// These capabilities should always be enabled on amd64:
56		{Name: "sse2", Feature: &X86.HasSSE2, Required: runtime.GOARCH == "amd64"},
57	}
58}
59
60func archInit() {
61
62	Initialized = true
63
64	maxID, _, _, _ := cpuid(0, 0)
65
66	if maxID < 1 {
67		return
68	}
69
70	_, _, ecx1, edx1 := cpuid(1, 0)
71	X86.HasSSE2 = isSet(26, edx1)
72
73	X86.HasSSE3 = isSet(0, ecx1)
74	X86.HasPCLMULQDQ = isSet(1, ecx1)
75	X86.HasSSSE3 = isSet(9, ecx1)
76	X86.HasFMA = isSet(12, ecx1)
77	X86.HasCX16 = isSet(13, ecx1)
78	X86.HasSSE41 = isSet(19, ecx1)
79	X86.HasSSE42 = isSet(20, ecx1)
80	X86.HasPOPCNT = isSet(23, ecx1)
81	X86.HasAES = isSet(25, ecx1)
82	X86.HasOSXSAVE = isSet(27, ecx1)
83	X86.HasRDRAND = isSet(30, ecx1)
84
85	var osSupportsAVX, osSupportsAVX512 bool
86	// For XGETBV, OSXSAVE bit is required and sufficient.
87	if X86.HasOSXSAVE {
88		eax, _ := xgetbv()
89		// Check if XMM and YMM registers have OS support.
90		osSupportsAVX = isSet(1, eax) && isSet(2, eax)
91
92		if runtime.GOOS == "darwin" {
93			// Check darwin commpage for AVX512 support. Necessary because:
94			// https://github.com/apple/darwin-xnu/blob/0a798f6738bc1db01281fc08ae024145e84df927/osfmk/i386/fpu.c#L175-L201
95			osSupportsAVX512 = osSupportsAVX && darwinSupportsAVX512()
96		} else {
97			// Check if OPMASK and ZMM registers have OS support.
98			osSupportsAVX512 = osSupportsAVX && isSet(5, eax) && isSet(6, eax) && isSet(7, eax)
99		}
100	}
101
102	X86.HasAVX = isSet(28, ecx1) && osSupportsAVX
103
104	if maxID < 7 {
105		return
106	}
107
108	_, ebx7, ecx7, edx7 := cpuid(7, 0)
109	X86.HasBMI1 = isSet(3, ebx7)
110	X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX
111	X86.HasBMI2 = isSet(8, ebx7)
112	X86.HasERMS = isSet(9, ebx7)
113	X86.HasRDSEED = isSet(18, ebx7)
114	X86.HasADX = isSet(19, ebx7)
115
116	X86.HasAVX512 = isSet(16, ebx7) && osSupportsAVX512 // Because avx-512 foundation is the core required extension
117	if X86.HasAVX512 {
118		X86.HasAVX512F = true
119		X86.HasAVX512CD = isSet(28, ebx7)
120		X86.HasAVX512ER = isSet(27, ebx7)
121		X86.HasAVX512PF = isSet(26, ebx7)
122		X86.HasAVX512VL = isSet(31, ebx7)
123		X86.HasAVX512BW = isSet(30, ebx7)
124		X86.HasAVX512DQ = isSet(17, ebx7)
125		X86.HasAVX512IFMA = isSet(21, ebx7)
126		X86.HasAVX512VBMI = isSet(1, ecx7)
127		X86.HasAVX5124VNNIW = isSet(2, edx7)
128		X86.HasAVX5124FMAPS = isSet(3, edx7)
129		X86.HasAVX512VPOPCNTDQ = isSet(14, ecx7)
130		X86.HasAVX512VPCLMULQDQ = isSet(10, ecx7)
131		X86.HasAVX512VNNI = isSet(11, ecx7)
132		X86.HasAVX512GFNI = isSet(8, ecx7)
133		X86.HasAVX512VAES = isSet(9, ecx7)
134		X86.HasAVX512VBMI2 = isSet(6, ecx7)
135		X86.HasAVX512BITALG = isSet(12, ecx7)
136
137		eax71, _, _, _ := cpuid(7, 1)
138		X86.HasAVX512BF16 = isSet(5, eax71)
139	}
140}
141
142func isSet(bitpos uint, value uint32) bool {
143	return value&(1<<bitpos) != 0
144}
145