1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "mozilla/Maybe.h"
8
9 #include <algorithm>
10
11 #include "gc/Marking.h"
12 #include "jit/AutoWritableJitCode.h"
13 #if defined(JS_CODEGEN_X86)
14 # include "jit/x86/MacroAssembler-x86.h"
15 #elif defined(JS_CODEGEN_X64)
16 # include "jit/x64/MacroAssembler-x64.h"
17 #else
18 # error "Wrong architecture. Only x86 and x64 should build this file!"
19 #endif
20
21 #ifdef _MSC_VER
22 # include <intrin.h> // for __cpuid
23 # if defined(_M_X64) && (_MSC_FULL_VER >= 160040219)
24 # include <immintrin.h> // for _xgetbv
25 # endif
26 #endif
27
28 using namespace js;
29 using namespace js::jit;
30
copyJumpRelocationTable(uint8_t * dest)31 void AssemblerX86Shared::copyJumpRelocationTable(uint8_t* dest) {
32 if (jumpRelocations_.length()) {
33 memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
34 }
35 }
36
copyDataRelocationTable(uint8_t * dest)37 void AssemblerX86Shared::copyDataRelocationTable(uint8_t* dest) {
38 if (dataRelocations_.length()) {
39 memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
40 }
41 }
42
43 /* static */
TraceDataRelocations(JSTracer * trc,JitCode * code,CompactBufferReader & reader)44 void AssemblerX86Shared::TraceDataRelocations(JSTracer* trc, JitCode* code,
45 CompactBufferReader& reader) {
46 mozilla::Maybe<AutoWritableJitCode> awjc;
47
48 while (reader.more()) {
49 size_t offset = reader.readUnsigned();
50 MOZ_ASSERT(offset >= sizeof(void*) && offset <= code->instructionsSize());
51
52 uint8_t* src = code->raw() + offset;
53 void* data = X86Encoding::GetPointer(src);
54
55 #ifdef JS_PUNBOX64
56 // Data relocations can be for Values or for raw pointers. If a Value is
57 // zero-tagged, we can trace it as if it were a raw pointer. If a Value
58 // is not zero-tagged, we have to interpret it as a Value to ensure that the
59 // tag bits are masked off to recover the actual pointer.
60
61 uintptr_t word = reinterpret_cast<uintptr_t>(data);
62 if (word >> JSVAL_TAG_SHIFT) {
63 // This relocation is a Value with a non-zero tag.
64 Value value = Value::fromRawBits(word);
65 MOZ_ASSERT_IF(value.isGCThing(),
66 gc::IsCellPointerValid(value.toGCThing()));
67 TraceManuallyBarrieredEdge(trc, &value, "jit-masm-value");
68 if (word != value.asRawBits()) {
69 if (awjc.isNothing()) {
70 awjc.emplace(code);
71 }
72 X86Encoding::SetPointer(src, value.bitsAsPunboxPointer());
73 }
74 continue;
75 }
76 #endif
77
78 // This relocation is a raw pointer or a Value with a zero tag.
79 gc::Cell* cell = static_cast<gc::Cell*>(data);
80 MOZ_ASSERT(gc::IsCellPointerValid(cell));
81 TraceManuallyBarrieredGenericPointerEdge(trc, &cell, "jit-masm-ptr");
82 if (cell != data) {
83 if (awjc.isNothing()) {
84 awjc.emplace(code);
85 }
86 X86Encoding::SetPointer(src, cell);
87 }
88 }
89 }
90
executableCopy(void * buffer)91 void AssemblerX86Shared::executableCopy(void* buffer) {
92 masm.executableCopy(buffer);
93
94 // Crash diagnostics for bug 1124397. Check the code buffer has not been
95 // poisoned with 0xE5 bytes.
96 static const size_t MinPoisoned = 16;
97 const uint8_t* bytes = (const uint8_t*)buffer;
98 size_t len = size();
99
100 for (size_t i = 0; i < len; i += MinPoisoned) {
101 if (bytes[i] != 0xE5) {
102 continue;
103 }
104
105 size_t startOffset = i;
106 while (startOffset > 0 && bytes[startOffset - 1] == 0xE5) {
107 startOffset--;
108 }
109
110 size_t endOffset = i;
111 while (endOffset + 1 < len && bytes[endOffset + 1] == 0xE5) {
112 endOffset++;
113 }
114
115 if (endOffset - startOffset < MinPoisoned) {
116 continue;
117 }
118
119 volatile uintptr_t dump[5];
120 blackbox = dump;
121 blackbox[0] = uintptr_t(0xABCD4321);
122 blackbox[1] = uintptr_t(len);
123 blackbox[2] = uintptr_t(startOffset);
124 blackbox[3] = uintptr_t(endOffset);
125 blackbox[4] = uintptr_t(0xFFFF8888);
126 MOZ_CRASH("Corrupt code buffer");
127 }
128 }
129
processCodeLabels(uint8_t * rawCode)130 void AssemblerX86Shared::processCodeLabels(uint8_t* rawCode) {
131 for (const CodeLabel& label : codeLabels_) {
132 Bind(rawCode, label);
133 }
134 }
135
InvertCondition(Condition cond)136 AssemblerX86Shared::Condition AssemblerX86Shared::InvertCondition(
137 Condition cond) {
138 switch (cond) {
139 case Zero:
140 return NonZero;
141 case NonZero:
142 return Zero;
143 case LessThan:
144 return GreaterThanOrEqual;
145 case LessThanOrEqual:
146 return GreaterThan;
147 case GreaterThan:
148 return LessThanOrEqual;
149 case GreaterThanOrEqual:
150 return LessThan;
151 case Above:
152 return BelowOrEqual;
153 case AboveOrEqual:
154 return Below;
155 case Below:
156 return AboveOrEqual;
157 case BelowOrEqual:
158 return Above;
159 default:
160 MOZ_CRASH("unexpected condition");
161 }
162 }
163
UnsignedCondition(Condition cond)164 AssemblerX86Shared::Condition AssemblerX86Shared::UnsignedCondition(
165 Condition cond) {
166 switch (cond) {
167 case Zero:
168 case NonZero:
169 return cond;
170 case LessThan:
171 case Below:
172 return Below;
173 case LessThanOrEqual:
174 case BelowOrEqual:
175 return BelowOrEqual;
176 case GreaterThan:
177 case Above:
178 return Above;
179 case AboveOrEqual:
180 case GreaterThanOrEqual:
181 return AboveOrEqual;
182 default:
183 MOZ_CRASH("unexpected condition");
184 }
185 }
186
ConditionWithoutEqual(Condition cond)187 AssemblerX86Shared::Condition AssemblerX86Shared::ConditionWithoutEqual(
188 Condition cond) {
189 switch (cond) {
190 case LessThan:
191 case LessThanOrEqual:
192 return LessThan;
193 case Below:
194 case BelowOrEqual:
195 return Below;
196 case GreaterThan:
197 case GreaterThanOrEqual:
198 return GreaterThan;
199 case Above:
200 case AboveOrEqual:
201 return Above;
202 default:
203 MOZ_CRASH("unexpected condition");
204 }
205 }
206
InvertCondition(DoubleCondition cond)207 AssemblerX86Shared::DoubleCondition AssemblerX86Shared::InvertCondition(
208 DoubleCondition cond) {
209 switch (cond) {
210 case DoubleEqual:
211 return DoubleNotEqualOrUnordered;
212 case DoubleEqualOrUnordered:
213 return DoubleNotEqual;
214 case DoubleNotEqualOrUnordered:
215 return DoubleEqual;
216 case DoubleNotEqual:
217 return DoubleEqualOrUnordered;
218 case DoubleLessThan:
219 return DoubleGreaterThanOrEqualOrUnordered;
220 case DoubleLessThanOrUnordered:
221 return DoubleGreaterThanOrEqual;
222 case DoubleLessThanOrEqual:
223 return DoubleGreaterThanOrUnordered;
224 case DoubleLessThanOrEqualOrUnordered:
225 return DoubleGreaterThan;
226 case DoubleGreaterThan:
227 return DoubleLessThanOrEqualOrUnordered;
228 case DoubleGreaterThanOrUnordered:
229 return DoubleLessThanOrEqual;
230 case DoubleGreaterThanOrEqual:
231 return DoubleLessThanOrUnordered;
232 case DoubleGreaterThanOrEqualOrUnordered:
233 return DoubleLessThan;
234 default:
235 MOZ_CRASH("unexpected condition");
236 }
237 }
238
239 CPUInfo::SSEVersion CPUInfo::maxSSEVersion = UnknownSSE;
240 CPUInfo::SSEVersion CPUInfo::maxEnabledSSEVersion = UnknownSSE;
241 bool CPUInfo::avxPresent = false;
242 bool CPUInfo::avxEnabled = false;
243 bool CPUInfo::popcntPresent = false;
244 bool CPUInfo::bmi1Present = false;
245 bool CPUInfo::bmi2Present = false;
246 bool CPUInfo::lzcntPresent = false;
247
ReadXGETBV()248 static uintptr_t ReadXGETBV() {
249 // We use a variety of low-level mechanisms to get at the xgetbv
250 // instruction, including spelling out the xgetbv instruction as bytes,
251 // because older compilers and assemblers may not recognize the instruction
252 // by name.
253 size_t xcr0EAX = 0;
254 #if defined(_XCR_XFEATURE_ENABLED_MASK)
255 xcr0EAX = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
256 #elif defined(__GNUC__)
257 // xgetbv returns its results in %eax and %edx, and for our purposes here,
258 // we're only interested in the %eax value.
259 asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0EAX) : "c"(0) : "%edx");
260 #elif defined(_MSC_VER) && defined(_M_IX86)
261 __asm {
262 xor ecx, ecx
263 _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0
264 mov xcr0EAX, eax
265 }
266 #endif
267 return xcr0EAX;
268 }
269
ReadCPUInfo(int * flagsEax,int * flagsEbx,int * flagsEcx,int * flagsEdx)270 static void ReadCPUInfo(int* flagsEax, int* flagsEbx, int* flagsEcx,
271 int* flagsEdx) {
272 #ifdef _MSC_VER
273 int cpuinfo[4];
274 __cpuid(cpuinfo, *flagsEax);
275 *flagsEax = cpuinfo[0];
276 *flagsEbx = cpuinfo[1];
277 *flagsEcx = cpuinfo[2];
278 *flagsEdx = cpuinfo[3];
279 #elif defined(__GNUC__)
280 // Some older 32-bits processors don't fill the ecx register with cpuid, so
281 // clobber it before calling cpuid, so that there's no risk of picking
282 // random bits indicating SSE3/SSE4 are present. Also make sure that it's
283 // set to 0 as an input for BMI detection on all platforms.
284 *flagsEcx = 0;
285 # ifdef JS_CODEGEN_X64
286 asm("cpuid;"
287 : "+a"(*flagsEax), "=b"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx));
288 # else
289 // On x86, preserve ebx. The compiler needs it for PIC mode.
290 asm("mov %%ebx, %%edi;"
291 "cpuid;"
292 "xchg %%edi, %%ebx;"
293 : "+a"(*flagsEax), "=D"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx));
294 # endif
295 #else
296 # error "Unsupported compiler"
297 #endif
298 }
299
SetSSEVersion()300 void CPUInfo::SetSSEVersion() {
301 int flagsEax = 1;
302 int flagsEbx = 0;
303 int flagsEcx = 0;
304 int flagsEdx = 0;
305 ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
306
307 static constexpr int SSEBit = 1 << 25;
308 static constexpr int SSE2Bit = 1 << 26;
309 static constexpr int SSE3Bit = 1 << 0;
310 static constexpr int SSSE3Bit = 1 << 9;
311 static constexpr int SSE41Bit = 1 << 19;
312 static constexpr int SSE42Bit = 1 << 20;
313
314 if (flagsEcx & SSE42Bit) {
315 maxSSEVersion = SSE4_2;
316 } else if (flagsEcx & SSE41Bit) {
317 maxSSEVersion = SSE4_1;
318 } else if (flagsEcx & SSSE3Bit) {
319 maxSSEVersion = SSSE3;
320 } else if (flagsEcx & SSE3Bit) {
321 maxSSEVersion = SSE3;
322 } else if (flagsEdx & SSE2Bit) {
323 maxSSEVersion = SSE2;
324 } else if (flagsEdx & SSEBit) {
325 maxSSEVersion = SSE;
326 } else {
327 maxSSEVersion = NoSSE;
328 }
329
330 if (maxEnabledSSEVersion != UnknownSSE) {
331 maxSSEVersion = std::min(maxSSEVersion, maxEnabledSSEVersion);
332 }
333
334 static constexpr int AVXBit = 1 << 28;
335 static constexpr int XSAVEBit = 1 << 27;
336 avxPresent = (flagsEcx & AVXBit) && (flagsEcx & XSAVEBit) && avxEnabled;
337
338 // If the hardware supports AVX, check whether the OS supports it too.
339 if (avxPresent) {
340 size_t xcr0EAX = ReadXGETBV();
341 static constexpr int xcr0SSEBit = 1 << 1;
342 static constexpr int xcr0AVXBit = 1 << 2;
343 avxPresent = (xcr0EAX & xcr0SSEBit) && (xcr0EAX & xcr0AVXBit);
344 }
345
346 // CMOV instruction are supposed to be supported by all CPU which have SSE2
347 // enabled. While this might be true, this is not guaranteed by any
348 // documentation, nor AMD, nor Intel.
349 static constexpr int CMOVBit = 1 << 15;
350 MOZ_RELEASE_ASSERT(flagsEdx & CMOVBit,
351 "CMOVcc instruction is not recognized by this CPU.");
352
353 static constexpr int POPCNTBit = 1 << 23;
354 popcntPresent = (flagsEcx & POPCNTBit);
355
356 flagsEax = 0x80000001;
357 ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
358
359 static constexpr int LZCNTBit = 1 << 5;
360 lzcntPresent = (flagsEcx & LZCNTBit);
361
362 flagsEax = 0x7;
363 ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
364
365 static constexpr int BMI1Bit = 1 << 3;
366 static constexpr int BMI2Bit = 1 << 8;
367 bmi1Present = (flagsEbx & BMI1Bit);
368 bmi2Present = bmi1Present && (flagsEbx & BMI2Bit);
369 }
370
371 volatile uintptr_t* blackbox = nullptr;
372