1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "mozilla/Maybe.h"
8
9 #include <algorithm>
10
11 #include "jit/AutoWritableJitCode.h"
12 #if defined(JS_CODEGEN_X86)
13 # include "jit/x86/MacroAssembler-x86.h"
14 #elif defined(JS_CODEGEN_X64)
15 # include "jit/x64/MacroAssembler-x64.h"
16 #else
17 # error "Wrong architecture. Only x86 and x64 should build this file!"
18 #endif
19
20 #ifdef _MSC_VER
21 # include <intrin.h> // for __cpuid
22 # if defined(_M_X64) && (_MSC_FULL_VER >= 160040219)
23 # include <immintrin.h> // for _xgetbv
24 # endif
25 #endif
26
27 using namespace js;
28 using namespace js::jit;
29
copyJumpRelocationTable(uint8_t * dest)30 void AssemblerX86Shared::copyJumpRelocationTable(uint8_t* dest) {
31 if (jumpRelocations_.length()) {
32 memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
33 }
34 }
35
copyDataRelocationTable(uint8_t * dest)36 void AssemblerX86Shared::copyDataRelocationTable(uint8_t* dest) {
37 if (dataRelocations_.length()) {
38 memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
39 }
40 }
41
42 /* static */
TraceDataRelocations(JSTracer * trc,JitCode * code,CompactBufferReader & reader)43 void AssemblerX86Shared::TraceDataRelocations(JSTracer* trc, JitCode* code,
44 CompactBufferReader& reader) {
45 mozilla::Maybe<AutoWritableJitCode> awjc;
46
47 while (reader.more()) {
48 size_t offset = reader.readUnsigned();
49 MOZ_ASSERT(offset >= sizeof(void*) && offset <= code->instructionsSize());
50
51 uint8_t* src = code->raw() + offset;
52 void* data = X86Encoding::GetPointer(src);
53
54 #ifdef JS_PUNBOX64
55 // Data relocations can be for Values or for raw pointers. If a Value is
56 // zero-tagged, we can trace it as if it were a raw pointer. If a Value
57 // is not zero-tagged, we have to interpret it as a Value to ensure that the
58 // tag bits are masked off to recover the actual pointer.
59
60 uintptr_t word = reinterpret_cast<uintptr_t>(data);
61 if (word >> JSVAL_TAG_SHIFT) {
62 // This relocation is a Value with a non-zero tag.
63 Value value = Value::fromRawBits(word);
64 MOZ_ASSERT_IF(value.isGCThing(),
65 gc::IsCellPointerValid(value.toGCThing()));
66 TraceManuallyBarrieredEdge(trc, &value, "jit-masm-value");
67 if (word != value.asRawBits()) {
68 if (awjc.isNothing()) {
69 awjc.emplace(code);
70 }
71 X86Encoding::SetPointer(src, value.bitsAsPunboxPointer());
72 }
73 continue;
74 }
75 #endif
76
77 // This relocation is a raw pointer or a Value with a zero tag.
78 gc::Cell* cell = static_cast<gc::Cell*>(data);
79 MOZ_ASSERT(gc::IsCellPointerValid(cell));
80 TraceManuallyBarrieredGenericPointerEdge(trc, &cell, "jit-masm-ptr");
81 if (cell != data) {
82 if (awjc.isNothing()) {
83 awjc.emplace(code);
84 }
85 X86Encoding::SetPointer(src, cell);
86 }
87 }
88 }
89
executableCopy(void * buffer)90 void AssemblerX86Shared::executableCopy(void* buffer) {
91 masm.executableCopy(buffer);
92 }
93
processCodeLabels(uint8_t * rawCode)94 void AssemblerX86Shared::processCodeLabels(uint8_t* rawCode) {
95 for (const CodeLabel& label : codeLabels_) {
96 Bind(rawCode, label);
97 }
98 }
99
InvertCondition(Condition cond)100 AssemblerX86Shared::Condition AssemblerX86Shared::InvertCondition(
101 Condition cond) {
102 switch (cond) {
103 case Zero:
104 return NonZero;
105 case NonZero:
106 return Zero;
107 case LessThan:
108 return GreaterThanOrEqual;
109 case LessThanOrEqual:
110 return GreaterThan;
111 case GreaterThan:
112 return LessThanOrEqual;
113 case GreaterThanOrEqual:
114 return LessThan;
115 case Above:
116 return BelowOrEqual;
117 case AboveOrEqual:
118 return Below;
119 case Below:
120 return AboveOrEqual;
121 case BelowOrEqual:
122 return Above;
123 default:
124 MOZ_CRASH("unexpected condition");
125 }
126 }
127
UnsignedCondition(Condition cond)128 AssemblerX86Shared::Condition AssemblerX86Shared::UnsignedCondition(
129 Condition cond) {
130 switch (cond) {
131 case Zero:
132 case NonZero:
133 return cond;
134 case LessThan:
135 case Below:
136 return Below;
137 case LessThanOrEqual:
138 case BelowOrEqual:
139 return BelowOrEqual;
140 case GreaterThan:
141 case Above:
142 return Above;
143 case AboveOrEqual:
144 case GreaterThanOrEqual:
145 return AboveOrEqual;
146 default:
147 MOZ_CRASH("unexpected condition");
148 }
149 }
150
ConditionWithoutEqual(Condition cond)151 AssemblerX86Shared::Condition AssemblerX86Shared::ConditionWithoutEqual(
152 Condition cond) {
153 switch (cond) {
154 case LessThan:
155 case LessThanOrEqual:
156 return LessThan;
157 case Below:
158 case BelowOrEqual:
159 return Below;
160 case GreaterThan:
161 case GreaterThanOrEqual:
162 return GreaterThan;
163 case Above:
164 case AboveOrEqual:
165 return Above;
166 default:
167 MOZ_CRASH("unexpected condition");
168 }
169 }
170
InvertCondition(DoubleCondition cond)171 AssemblerX86Shared::DoubleCondition AssemblerX86Shared::InvertCondition(
172 DoubleCondition cond) {
173 switch (cond) {
174 case DoubleEqual:
175 return DoubleNotEqualOrUnordered;
176 case DoubleEqualOrUnordered:
177 return DoubleNotEqual;
178 case DoubleNotEqualOrUnordered:
179 return DoubleEqual;
180 case DoubleNotEqual:
181 return DoubleEqualOrUnordered;
182 case DoubleLessThan:
183 return DoubleGreaterThanOrEqualOrUnordered;
184 case DoubleLessThanOrUnordered:
185 return DoubleGreaterThanOrEqual;
186 case DoubleLessThanOrEqual:
187 return DoubleGreaterThanOrUnordered;
188 case DoubleLessThanOrEqualOrUnordered:
189 return DoubleGreaterThan;
190 case DoubleGreaterThan:
191 return DoubleLessThanOrEqualOrUnordered;
192 case DoubleGreaterThanOrUnordered:
193 return DoubleLessThanOrEqual;
194 case DoubleGreaterThanOrEqual:
195 return DoubleLessThanOrUnordered;
196 case DoubleGreaterThanOrEqualOrUnordered:
197 return DoubleLessThan;
198 default:
199 MOZ_CRASH("unexpected condition");
200 }
201 }
202
203 CPUInfo::SSEVersion CPUInfo::maxSSEVersion = UnknownSSE;
204 CPUInfo::SSEVersion CPUInfo::maxEnabledSSEVersion = UnknownSSE;
205 bool CPUInfo::avxPresent = false;
206 bool CPUInfo::avxEnabled = false;
207 bool CPUInfo::popcntPresent = false;
208 bool CPUInfo::bmi1Present = false;
209 bool CPUInfo::bmi2Present = false;
210 bool CPUInfo::lzcntPresent = false;
211 bool CPUInfo::avx2Present = false;
212
213 namespace js {
214 namespace jit {
CPUFlagsHaveBeenComputed()215 bool CPUFlagsHaveBeenComputed() { return CPUInfo::FlagsHaveBeenComputed(); }
216 } // namespace jit
217 } // namespace js
218
ReadXGETBV()219 static uintptr_t ReadXGETBV() {
220 // We use a variety of low-level mechanisms to get at the xgetbv
221 // instruction, including spelling out the xgetbv instruction as bytes,
222 // because older compilers and assemblers may not recognize the instruction
223 // by name.
224 size_t xcr0EAX = 0;
225 #if defined(_XCR_XFEATURE_ENABLED_MASK)
226 xcr0EAX = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
227 #elif defined(__GNUC__)
228 // xgetbv returns its results in %eax and %edx, and for our purposes here,
229 // we're only interested in the %eax value.
230 asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0EAX) : "c"(0) : "%edx");
231 #elif defined(_MSC_VER) && defined(_M_IX86)
232 __asm {
233 xor ecx, ecx
234 _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0
235 mov xcr0EAX, eax
236 }
237 #endif
238 return xcr0EAX;
239 }
240
ReadCPUInfo(int * flagsEax,int * flagsEbx,int * flagsEcx,int * flagsEdx)241 static void ReadCPUInfo(int* flagsEax, int* flagsEbx, int* flagsEcx,
242 int* flagsEdx) {
243 #ifdef _MSC_VER
244 int cpuinfo[4];
245 __cpuid(cpuinfo, *flagsEax);
246 *flagsEax = cpuinfo[0];
247 *flagsEbx = cpuinfo[1];
248 *flagsEcx = cpuinfo[2];
249 *flagsEdx = cpuinfo[3];
250 #elif defined(__GNUC__)
251 // Some older 32-bits processors don't fill the ecx register with cpuid, so
252 // clobber it before calling cpuid, so that there's no risk of picking
253 // random bits indicating SSE3/SSE4 are present. Also make sure that it's
254 // set to 0 as an input for BMI detection on all platforms.
255 *flagsEcx = 0;
256 # ifdef JS_CODEGEN_X64
257 asm("cpuid;"
258 : "+a"(*flagsEax), "=b"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx));
259 # else
260 // On x86, preserve ebx. The compiler needs it for PIC mode.
261 asm("mov %%ebx, %%edi;"
262 "cpuid;"
263 "xchg %%edi, %%ebx;"
264 : "+a"(*flagsEax), "=D"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx));
265 # endif
266 #else
267 # error "Unsupported compiler"
268 #endif
269 }
270
ComputeFlags()271 void CPUInfo::ComputeFlags() {
272 MOZ_ASSERT(!FlagsHaveBeenComputed());
273
274 int flagsEax = 1;
275 int flagsEbx = 0;
276 int flagsEcx = 0;
277 int flagsEdx = 0;
278 ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
279
280 static constexpr int SSEBit = 1 << 25;
281 static constexpr int SSE2Bit = 1 << 26;
282 static constexpr int SSE3Bit = 1 << 0;
283 static constexpr int SSSE3Bit = 1 << 9;
284 static constexpr int SSE41Bit = 1 << 19;
285 static constexpr int SSE42Bit = 1 << 20;
286
287 if (flagsEcx & SSE42Bit) {
288 maxSSEVersion = SSE4_2;
289 } else if (flagsEcx & SSE41Bit) {
290 maxSSEVersion = SSE4_1;
291 } else if (flagsEcx & SSSE3Bit) {
292 maxSSEVersion = SSSE3;
293 } else if (flagsEcx & SSE3Bit) {
294 maxSSEVersion = SSE3;
295 } else if (flagsEdx & SSE2Bit) {
296 maxSSEVersion = SSE2;
297 } else if (flagsEdx & SSEBit) {
298 maxSSEVersion = SSE;
299 } else {
300 maxSSEVersion = NoSSE;
301 }
302
303 if (maxEnabledSSEVersion != UnknownSSE) {
304 maxSSEVersion = std::min(maxSSEVersion, maxEnabledSSEVersion);
305 }
306
307 static constexpr int AVXBit = 1 << 28;
308 static constexpr int XSAVEBit = 1 << 27;
309 avxPresent = (flagsEcx & AVXBit) && (flagsEcx & XSAVEBit) && avxEnabled;
310
311 // If the hardware supports AVX, check whether the OS supports it too.
312 if (avxPresent) {
313 size_t xcr0EAX = ReadXGETBV();
314 static constexpr int xcr0SSEBit = 1 << 1;
315 static constexpr int xcr0AVXBit = 1 << 2;
316 avxPresent = (xcr0EAX & xcr0SSEBit) && (xcr0EAX & xcr0AVXBit);
317 }
318
319 // CMOV instruction are supposed to be supported by all CPU which have SSE2
320 // enabled. While this might be true, this is not guaranteed by any
321 // documentation, nor AMD, nor Intel.
322 static constexpr int CMOVBit = 1 << 15;
323 MOZ_RELEASE_ASSERT(flagsEdx & CMOVBit,
324 "CMOVcc instruction is not recognized by this CPU.");
325
326 static constexpr int POPCNTBit = 1 << 23;
327 popcntPresent = (flagsEcx & POPCNTBit);
328
329 flagsEax = 0x80000001;
330 ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
331
332 static constexpr int LZCNTBit = 1 << 5;
333 lzcntPresent = (flagsEcx & LZCNTBit);
334
335 flagsEax = 0x7;
336 ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
337
338 static constexpr int BMI1Bit = 1 << 3;
339 static constexpr int BMI2Bit = 1 << 8;
340 static constexpr int AVX2Bit = 1 << 5;
341 bmi1Present = (flagsEbx & BMI1Bit);
342 bmi2Present = bmi1Present && (flagsEbx & BMI2Bit);
343 avx2Present = avxPresent && (flagsEbx & AVX2Bit);
344
345 MOZ_ASSERT(FlagsHaveBeenComputed());
346 }
347