1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 * vim: set ts=8 sts=4 et sw=4 tw=99:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "jit/shared/CodeGenerator-shared-inl.h"
8
9 #include "mozilla/DebugOnly.h"
10 #include "mozilla/SizePrintfMacros.h"
11
12 #include "jit/CompactBuffer.h"
13 #include "jit/IonCaches.h"
14 #include "jit/JitcodeMap.h"
15 #include "jit/JitSpewer.h"
16 #include "jit/MacroAssembler.h"
17 #include "jit/MIR.h"
18 #include "jit/MIRGenerator.h"
19 #include "jit/OptimizationTracking.h"
20 #include "js/Conversions.h"
21 #include "vm/TraceLogging.h"
22
23 #include "jit/JitFrames-inl.h"
24 #include "jit/MacroAssembler-inl.h"
25
26 using namespace js;
27 using namespace js::jit;
28
29 using mozilla::BitwiseCast;
30 using mozilla::DebugOnly;
31
32 namespace js {
33 namespace jit {
34
35 MacroAssembler&
ensureMasm(MacroAssembler * masmArg)36 CodeGeneratorShared::ensureMasm(MacroAssembler* masmArg)
37 {
38 if (masmArg)
39 return *masmArg;
40 maybeMasm_.emplace();
41 return *maybeMasm_;
42 }
43
CodeGeneratorShared(MIRGenerator * gen,LIRGraph * graph,MacroAssembler * masmArg)44 CodeGeneratorShared::CodeGeneratorShared(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masmArg)
45 : maybeMasm_(),
46 masm(ensureMasm(masmArg)),
47 gen(gen),
48 graph(*graph),
49 current(nullptr),
50 snapshots_(),
51 recovers_(),
52 deoptTable_(nullptr),
53 #ifdef DEBUG
54 pushedArgs_(0),
55 #endif
56 lastOsiPointOffset_(0),
57 safepoints_(graph->totalSlotCount(), (gen->info().nargs() + 1) * sizeof(Value)),
58 returnLabel_(),
59 stubSpace_(),
60 nativeToBytecodeMap_(nullptr),
61 nativeToBytecodeMapSize_(0),
62 nativeToBytecodeTableOffset_(0),
63 nativeToBytecodeNumRegions_(0),
64 nativeToBytecodeScriptList_(nullptr),
65 nativeToBytecodeScriptListLength_(0),
66 trackedOptimizationsMap_(nullptr),
67 trackedOptimizationsMapSize_(0),
68 trackedOptimizationsRegionTableOffset_(0),
69 trackedOptimizationsTypesTableOffset_(0),
70 trackedOptimizationsAttemptsTableOffset_(0),
71 osrEntryOffset_(0),
72 skipArgCheckEntryOffset_(0),
73 #ifdef CHECK_OSIPOINT_REGISTERS
74 checkOsiPointRegisters(JitOptions.checkOsiPointRegisters),
75 #endif
76 frameDepth_(graph->paddedLocalSlotsSize() + graph->argumentsSize()),
77 frameInitialAdjustment_(0)
78 {
79 if (gen->isProfilerInstrumentationEnabled())
80 masm.enableProfilingInstrumentation();
81
82 if (gen->compilingWasm()) {
83 // Since wasm uses the system ABI which does not necessarily use a
84 // regular array where all slots are sizeof(Value), it maintains the max
85 // argument stack depth separately.
86 MOZ_ASSERT(graph->argumentSlotCount() == 0);
87 frameDepth_ += gen->wasmMaxStackArgBytes();
88
89 if (gen->usesSimd()) {
90 // If the function uses any SIMD then we may need to insert padding
91 // so that local slots are aligned for SIMD.
92 frameInitialAdjustment_ = ComputeByteAlignment(sizeof(wasm::Frame),
93 WasmStackAlignment);
94 frameDepth_ += frameInitialAdjustment_;
95 // Keep the stack aligned. Some SIMD sequences build values on the
96 // stack and need the stack aligned.
97 frameDepth_ += ComputeByteAlignment(sizeof(wasm::Frame) + frameDepth_,
98 WasmStackAlignment);
99 } else if (gen->performsCall()) {
100 // An MWasmCall does not align the stack pointer at calls sites but
101 // instead relies on the a priori stack adjustment. This must be the
102 // last adjustment of frameDepth_.
103 frameDepth_ += ComputeByteAlignment(sizeof(wasm::Frame) + frameDepth_,
104 WasmStackAlignment);
105 }
106
107 // FrameSizeClass is only used for bailing, which cannot happen in
108 // wasm code.
109 frameClass_ = FrameSizeClass::None();
110 } else {
111 frameClass_ = FrameSizeClass::FromDepth(frameDepth_);
112 }
113 }
114
115 bool
generatePrologue()116 CodeGeneratorShared::generatePrologue()
117 {
118 MOZ_ASSERT(masm.framePushed() == 0);
119 MOZ_ASSERT(!gen->compilingWasm());
120
121 #ifdef JS_USE_LINK_REGISTER
122 masm.pushReturnAddress();
123 #endif
124
125 // If profiling, save the current frame pointer to a per-thread global field.
126 if (isProfilerInstrumentationEnabled())
127 masm.profilerEnterFrame(masm.getStackPointer(), CallTempReg0);
128
129 // Ensure that the Ion frame is properly aligned.
130 masm.assertStackAlignment(JitStackAlignment, 0);
131
132 // Note that this automatically sets MacroAssembler::framePushed().
133 masm.reserveStack(frameSize());
134 masm.checkStackAlignment();
135
136 emitTracelogIonStart();
137 return true;
138 }
139
140 bool
generateEpilogue()141 CodeGeneratorShared::generateEpilogue()
142 {
143 MOZ_ASSERT(!gen->compilingWasm());
144 masm.bind(&returnLabel_);
145
146 emitTracelogIonStop();
147
148 masm.freeStack(frameSize());
149 MOZ_ASSERT(masm.framePushed() == 0);
150
151 // If profiling, reset the per-thread global lastJitFrame to point to
152 // the previous frame.
153 if (isProfilerInstrumentationEnabled())
154 masm.profilerExitFrame();
155
156 masm.ret();
157
158 // On systems that use a constant pool, this is a good time to emit.
159 masm.flushBuffer();
160 return true;
161 }
162
163 bool
generateOutOfLineCode()164 CodeGeneratorShared::generateOutOfLineCode()
165 {
166 for (size_t i = 0; i < outOfLineCode_.length(); i++) {
167 // Add native => bytecode mapping entries for OOL sites.
168 // Not enabled on wasm yet since it doesn't contain bytecode mappings.
169 if (!gen->compilingWasm()) {
170 if (!addNativeToBytecodeEntry(outOfLineCode_[i]->bytecodeSite()))
171 return false;
172 }
173
174 if (!gen->alloc().ensureBallast())
175 return false;
176
177 JitSpew(JitSpew_Codegen, "# Emitting out of line code");
178
179 masm.setFramePushed(outOfLineCode_[i]->framePushed());
180 lastPC_ = outOfLineCode_[i]->pc();
181 outOfLineCode_[i]->bind(&masm);
182
183 outOfLineCode_[i]->generate(this);
184 }
185
186 return !masm.oom();
187 }
188
189 void
addOutOfLineCode(OutOfLineCode * code,const MInstruction * mir)190 CodeGeneratorShared::addOutOfLineCode(OutOfLineCode* code, const MInstruction* mir)
191 {
192 MOZ_ASSERT(mir);
193 addOutOfLineCode(code, mir->trackedSite());
194 }
195
196 void
addOutOfLineCode(OutOfLineCode * code,const BytecodeSite * site)197 CodeGeneratorShared::addOutOfLineCode(OutOfLineCode* code, const BytecodeSite* site)
198 {
199 code->setFramePushed(masm.framePushed());
200 code->setBytecodeSite(site);
201 MOZ_ASSERT_IF(!gen->compilingWasm(), code->script()->containsPC(code->pc()));
202 masm.propagateOOM(outOfLineCode_.append(code));
203 }
204
205 bool
addNativeToBytecodeEntry(const BytecodeSite * site)206 CodeGeneratorShared::addNativeToBytecodeEntry(const BytecodeSite* site)
207 {
208 // Skip the table entirely if profiling is not enabled.
209 if (!isProfilerInstrumentationEnabled())
210 return true;
211
212 // Fails early if the last added instruction caused the macro assembler to
213 // run out of memory as continuity assumption below do not hold.
214 if (masm.oom())
215 return false;
216
217 MOZ_ASSERT(site);
218 MOZ_ASSERT(site->tree());
219 MOZ_ASSERT(site->pc());
220
221 InlineScriptTree* tree = site->tree();
222 jsbytecode* pc = site->pc();
223 uint32_t nativeOffset = masm.currentOffset();
224
225 MOZ_ASSERT_IF(nativeToBytecodeList_.empty(), nativeOffset == 0);
226
227 if (!nativeToBytecodeList_.empty()) {
228 size_t lastIdx = nativeToBytecodeList_.length() - 1;
229 NativeToBytecode& lastEntry = nativeToBytecodeList_[lastIdx];
230
231 MOZ_ASSERT(nativeOffset >= lastEntry.nativeOffset.offset());
232
233 // If the new entry is for the same inlineScriptTree and same
234 // bytecodeOffset, but the nativeOffset has changed, do nothing.
235 // The same site just generated some more code.
236 if (lastEntry.tree == tree && lastEntry.pc == pc) {
237 JitSpew(JitSpew_Profiling, " => In-place update [%" PRIuSIZE "-%" PRIu32 "]",
238 lastEntry.nativeOffset.offset(), nativeOffset);
239 return true;
240 }
241
242 // If the new entry is for the same native offset, then update the
243 // previous entry with the new bytecode site, since the previous
244 // bytecode site did not generate any native code.
245 if (lastEntry.nativeOffset.offset() == nativeOffset) {
246 lastEntry.tree = tree;
247 lastEntry.pc = pc;
248 JitSpew(JitSpew_Profiling, " => Overwriting zero-length native region.");
249
250 // This overwrite might have made the entry merge-able with a
251 // previous one. If so, merge it.
252 if (lastIdx > 0) {
253 NativeToBytecode& nextToLastEntry = nativeToBytecodeList_[lastIdx - 1];
254 if (nextToLastEntry.tree == lastEntry.tree && nextToLastEntry.pc == lastEntry.pc) {
255 JitSpew(JitSpew_Profiling, " => Merging with previous region");
256 nativeToBytecodeList_.erase(&lastEntry);
257 }
258 }
259
260 dumpNativeToBytecodeEntry(nativeToBytecodeList_.length() - 1);
261 return true;
262 }
263 }
264
265 // Otherwise, some native code was generated for the previous bytecode site.
266 // Add a new entry for code that is about to be generated.
267 NativeToBytecode entry;
268 entry.nativeOffset = CodeOffset(nativeOffset);
269 entry.tree = tree;
270 entry.pc = pc;
271 if (!nativeToBytecodeList_.append(entry))
272 return false;
273
274 JitSpew(JitSpew_Profiling, " => Push new entry.");
275 dumpNativeToBytecodeEntry(nativeToBytecodeList_.length() - 1);
276 return true;
277 }
278
279 void
dumpNativeToBytecodeEntries()280 CodeGeneratorShared::dumpNativeToBytecodeEntries()
281 {
282 #ifdef JS_JITSPEW
283 InlineScriptTree* topTree = gen->info().inlineScriptTree();
284 JitSpewStart(JitSpew_Profiling, "Native To Bytecode Entries for %s:%" PRIuSIZE "\n",
285 topTree->script()->filename(), topTree->script()->lineno());
286 for (unsigned i = 0; i < nativeToBytecodeList_.length(); i++)
287 dumpNativeToBytecodeEntry(i);
288 #endif
289 }
290
291 void
dumpNativeToBytecodeEntry(uint32_t idx)292 CodeGeneratorShared::dumpNativeToBytecodeEntry(uint32_t idx)
293 {
294 #ifdef JS_JITSPEW
295 NativeToBytecode& ref = nativeToBytecodeList_[idx];
296 InlineScriptTree* tree = ref.tree;
297 JSScript* script = tree->script();
298 uint32_t nativeOffset = ref.nativeOffset.offset();
299 unsigned nativeDelta = 0;
300 unsigned pcDelta = 0;
301 if (idx + 1 < nativeToBytecodeList_.length()) {
302 NativeToBytecode* nextRef = &ref + 1;
303 nativeDelta = nextRef->nativeOffset.offset() - nativeOffset;
304 if (nextRef->tree == ref.tree)
305 pcDelta = nextRef->pc - ref.pc;
306 }
307 JitSpewStart(JitSpew_Profiling, " %08" PRIxSIZE " [+%-6d] => %-6ld [%-4d] {%-10s} (%s:%" PRIuSIZE,
308 ref.nativeOffset.offset(),
309 nativeDelta,
310 (long) (ref.pc - script->code()),
311 pcDelta,
312 CodeName[JSOp(*ref.pc)],
313 script->filename(), script->lineno());
314
315 for (tree = tree->caller(); tree; tree = tree->caller()) {
316 JitSpewCont(JitSpew_Profiling, " <= %s:%" PRIuSIZE, tree->script()->filename(),
317 tree->script()->lineno());
318 }
319 JitSpewCont(JitSpew_Profiling, ")");
320 JitSpewFin(JitSpew_Profiling);
321 #endif
322 }
323
324 bool
addTrackedOptimizationsEntry(const TrackedOptimizations * optimizations)325 CodeGeneratorShared::addTrackedOptimizationsEntry(const TrackedOptimizations* optimizations)
326 {
327 if (!isOptimizationTrackingEnabled())
328 return true;
329
330 MOZ_ASSERT(optimizations);
331
332 uint32_t nativeOffset = masm.currentOffset();
333
334 if (!trackedOptimizations_.empty()) {
335 NativeToTrackedOptimizations& lastEntry = trackedOptimizations_.back();
336 MOZ_ASSERT_IF(!masm.oom(), nativeOffset >= lastEntry.endOffset.offset());
337
338 // If we're still generating code for the same set of optimizations,
339 // we are done.
340 if (lastEntry.optimizations == optimizations)
341 return true;
342 }
343
344 // If we're generating code for a new set of optimizations, add a new
345 // entry.
346 NativeToTrackedOptimizations entry;
347 entry.startOffset = CodeOffset(nativeOffset);
348 entry.endOffset = CodeOffset(nativeOffset);
349 entry.optimizations = optimizations;
350 return trackedOptimizations_.append(entry);
351 }
352
353 void
extendTrackedOptimizationsEntry(const TrackedOptimizations * optimizations)354 CodeGeneratorShared::extendTrackedOptimizationsEntry(const TrackedOptimizations* optimizations)
355 {
356 if (!isOptimizationTrackingEnabled())
357 return;
358
359 uint32_t nativeOffset = masm.currentOffset();
360 NativeToTrackedOptimizations& entry = trackedOptimizations_.back();
361 MOZ_ASSERT(entry.optimizations == optimizations);
362 MOZ_ASSERT_IF(!masm.oom(), nativeOffset >= entry.endOffset.offset());
363
364 entry.endOffset = CodeOffset(nativeOffset);
365
366 // If we generated no code, remove the last entry.
367 if (nativeOffset == entry.startOffset.offset())
368 trackedOptimizations_.popBack();
369 }
370
371 // see OffsetOfFrameSlot
372 static inline int32_t
ToStackIndex(LAllocation * a)373 ToStackIndex(LAllocation* a)
374 {
375 if (a->isStackSlot()) {
376 MOZ_ASSERT(a->toStackSlot()->slot() >= 1);
377 return a->toStackSlot()->slot();
378 }
379 return -int32_t(sizeof(JitFrameLayout) + a->toArgument()->index());
380 }
381
382 void
encodeAllocation(LSnapshot * snapshot,MDefinition * mir,uint32_t * allocIndex)383 CodeGeneratorShared::encodeAllocation(LSnapshot* snapshot, MDefinition* mir,
384 uint32_t* allocIndex)
385 {
386 if (mir->isBox())
387 mir = mir->toBox()->getOperand(0);
388
389 MIRType type =
390 mir->isRecoveredOnBailout() ? MIRType::None :
391 mir->isUnused() ? MIRType::MagicOptimizedOut :
392 mir->type();
393
394 RValueAllocation alloc;
395
396 switch (type) {
397 case MIRType::None:
398 {
399 MOZ_ASSERT(mir->isRecoveredOnBailout());
400 uint32_t index = 0;
401 LRecoverInfo* recoverInfo = snapshot->recoverInfo();
402 MNode** it = recoverInfo->begin();
403 MNode** end = recoverInfo->end();
404 while (it != end && mir != *it) {
405 ++it;
406 ++index;
407 }
408
409 // This MDefinition is recovered, thus it should be listed in the
410 // LRecoverInfo.
411 MOZ_ASSERT(it != end && mir == *it);
412
413 // Lambda should have a default value readable for iterating over the
414 // inner frames.
415 if (mir->isLambda()) {
416 MConstant* constant = mir->toLambda()->functionOperand();
417 uint32_t cstIndex;
418 masm.propagateOOM(graph.addConstantToPool(constant->toJSValue(), &cstIndex));
419 alloc = RValueAllocation::RecoverInstruction(index, cstIndex);
420 break;
421 }
422
423 alloc = RValueAllocation::RecoverInstruction(index);
424 break;
425 }
426 case MIRType::Undefined:
427 alloc = RValueAllocation::Undefined();
428 break;
429 case MIRType::Null:
430 alloc = RValueAllocation::Null();
431 break;
432 case MIRType::Int32:
433 case MIRType::String:
434 case MIRType::Symbol:
435 case MIRType::Object:
436 case MIRType::ObjectOrNull:
437 case MIRType::Boolean:
438 case MIRType::Double:
439 {
440 LAllocation* payload = snapshot->payloadOfSlot(*allocIndex);
441 if (payload->isConstant()) {
442 MConstant* constant = mir->toConstant();
443 uint32_t index;
444 masm.propagateOOM(graph.addConstantToPool(constant->toJSValue(), &index));
445 alloc = RValueAllocation::ConstantPool(index);
446 break;
447 }
448
449 JSValueType valueType =
450 (type == MIRType::ObjectOrNull) ? JSVAL_TYPE_OBJECT : ValueTypeFromMIRType(type);
451
452 MOZ_ASSERT(payload->isMemory() || payload->isRegister());
453 if (payload->isMemory())
454 alloc = RValueAllocation::Typed(valueType, ToStackIndex(payload));
455 else if (payload->isGeneralReg())
456 alloc = RValueAllocation::Typed(valueType, ToRegister(payload));
457 else if (payload->isFloatReg())
458 alloc = RValueAllocation::Double(ToFloatRegister(payload));
459 break;
460 }
461 case MIRType::Float32:
462 case MIRType::Int8x16:
463 case MIRType::Int16x8:
464 case MIRType::Int32x4:
465 case MIRType::Float32x4:
466 case MIRType::Bool8x16:
467 case MIRType::Bool16x8:
468 case MIRType::Bool32x4:
469 {
470 LAllocation* payload = snapshot->payloadOfSlot(*allocIndex);
471 if (payload->isConstant()) {
472 MConstant* constant = mir->toConstant();
473 uint32_t index;
474 masm.propagateOOM(graph.addConstantToPool(constant->toJSValue(), &index));
475 alloc = RValueAllocation::ConstantPool(index);
476 break;
477 }
478
479 MOZ_ASSERT(payload->isMemory() || payload->isFloatReg());
480 if (payload->isFloatReg())
481 alloc = RValueAllocation::AnyFloat(ToFloatRegister(payload));
482 else
483 alloc = RValueAllocation::AnyFloat(ToStackIndex(payload));
484 break;
485 }
486 case MIRType::MagicOptimizedArguments:
487 case MIRType::MagicOptimizedOut:
488 case MIRType::MagicUninitializedLexical:
489 case MIRType::MagicIsConstructing:
490 {
491 uint32_t index;
492 JSWhyMagic why = JS_GENERIC_MAGIC;
493 switch (type) {
494 case MIRType::MagicOptimizedArguments:
495 why = JS_OPTIMIZED_ARGUMENTS;
496 break;
497 case MIRType::MagicOptimizedOut:
498 why = JS_OPTIMIZED_OUT;
499 break;
500 case MIRType::MagicUninitializedLexical:
501 why = JS_UNINITIALIZED_LEXICAL;
502 break;
503 case MIRType::MagicIsConstructing:
504 why = JS_IS_CONSTRUCTING;
505 break;
506 default:
507 MOZ_CRASH("Invalid Magic MIRType");
508 }
509
510 Value v = MagicValue(why);
511 masm.propagateOOM(graph.addConstantToPool(v, &index));
512 alloc = RValueAllocation::ConstantPool(index);
513 break;
514 }
515 default:
516 {
517 MOZ_ASSERT(mir->type() == MIRType::Value);
518 LAllocation* payload = snapshot->payloadOfSlot(*allocIndex);
519 #ifdef JS_NUNBOX32
520 LAllocation* type = snapshot->typeOfSlot(*allocIndex);
521 if (type->isRegister()) {
522 if (payload->isRegister())
523 alloc = RValueAllocation::Untyped(ToRegister(type), ToRegister(payload));
524 else
525 alloc = RValueAllocation::Untyped(ToRegister(type), ToStackIndex(payload));
526 } else {
527 if (payload->isRegister())
528 alloc = RValueAllocation::Untyped(ToStackIndex(type), ToRegister(payload));
529 else
530 alloc = RValueAllocation::Untyped(ToStackIndex(type), ToStackIndex(payload));
531 }
532 #elif JS_PUNBOX64
533 if (payload->isRegister())
534 alloc = RValueAllocation::Untyped(ToRegister(payload));
535 else
536 alloc = RValueAllocation::Untyped(ToStackIndex(payload));
537 #endif
538 break;
539 }
540 }
541
542 // This set an extra bit as part of the RValueAllocation, such that we know
543 // that recover instruction have to be executed without wrapping the
544 // instruction in a no-op recover instruction.
545 if (mir->isIncompleteObject())
546 alloc.setNeedSideEffect();
547
548 masm.propagateOOM(snapshots_.add(alloc));
549
550 *allocIndex += mir->isRecoveredOnBailout() ? 0 : 1;
551 }
552
553 void
encode(LRecoverInfo * recover)554 CodeGeneratorShared::encode(LRecoverInfo* recover)
555 {
556 if (recover->recoverOffset() != INVALID_RECOVER_OFFSET)
557 return;
558
559 uint32_t numInstructions = recover->numInstructions();
560 JitSpew(JitSpew_IonSnapshots, "Encoding LRecoverInfo %p (frameCount %u, instructions %u)",
561 (void*)recover, recover->mir()->frameCount(), numInstructions);
562
563 MResumePoint::Mode mode = recover->mir()->mode();
564 MOZ_ASSERT(mode != MResumePoint::Outer);
565 bool resumeAfter = (mode == MResumePoint::ResumeAfter);
566
567 RecoverOffset offset = recovers_.startRecover(numInstructions, resumeAfter);
568
569 for (MNode* insn : *recover)
570 recovers_.writeInstruction(insn);
571
572 recovers_.endRecover();
573 recover->setRecoverOffset(offset);
574 masm.propagateOOM(!recovers_.oom());
575 }
576
577 void
encode(LSnapshot * snapshot)578 CodeGeneratorShared::encode(LSnapshot* snapshot)
579 {
580 if (snapshot->snapshotOffset() != INVALID_SNAPSHOT_OFFSET)
581 return;
582
583 LRecoverInfo* recoverInfo = snapshot->recoverInfo();
584 encode(recoverInfo);
585
586 RecoverOffset recoverOffset = recoverInfo->recoverOffset();
587 MOZ_ASSERT(recoverOffset != INVALID_RECOVER_OFFSET);
588
589 JitSpew(JitSpew_IonSnapshots, "Encoding LSnapshot %p (LRecover %p)",
590 (void*)snapshot, (void*) recoverInfo);
591
592 SnapshotOffset offset = snapshots_.startSnapshot(recoverOffset, snapshot->bailoutKind());
593
594 #ifdef TRACK_SNAPSHOTS
595 uint32_t pcOpcode = 0;
596 uint32_t lirOpcode = 0;
597 uint32_t lirId = 0;
598 uint32_t mirOpcode = 0;
599 uint32_t mirId = 0;
600
601 if (LNode* ins = instruction()) {
602 lirOpcode = ins->op();
603 lirId = ins->id();
604 if (ins->mirRaw()) {
605 mirOpcode = ins->mirRaw()->op();
606 mirId = ins->mirRaw()->id();
607 if (ins->mirRaw()->trackedPc())
608 pcOpcode = *ins->mirRaw()->trackedPc();
609 }
610 }
611 snapshots_.trackSnapshot(pcOpcode, mirOpcode, mirId, lirOpcode, lirId);
612 #endif
613
614 uint32_t allocIndex = 0;
615 for (LRecoverInfo::OperandIter it(recoverInfo); !it; ++it) {
616 DebugOnly<uint32_t> allocWritten = snapshots_.allocWritten();
617 encodeAllocation(snapshot, *it, &allocIndex);
618 MOZ_ASSERT_IF(!snapshots_.oom(), allocWritten + 1 == snapshots_.allocWritten());
619 }
620
621 MOZ_ASSERT(allocIndex == snapshot->numSlots());
622 snapshots_.endSnapshot();
623 snapshot->setSnapshotOffset(offset);
624 masm.propagateOOM(!snapshots_.oom());
625 }
626
627 bool
assignBailoutId(LSnapshot * snapshot)628 CodeGeneratorShared::assignBailoutId(LSnapshot* snapshot)
629 {
630 MOZ_ASSERT(snapshot->snapshotOffset() != INVALID_SNAPSHOT_OFFSET);
631
632 // Can we not use bailout tables at all?
633 if (!deoptTable_)
634 return false;
635
636 MOZ_ASSERT(frameClass_ != FrameSizeClass::None());
637
638 if (snapshot->bailoutId() != INVALID_BAILOUT_ID)
639 return true;
640
641 // Is the bailout table full?
642 if (bailouts_.length() >= BAILOUT_TABLE_SIZE)
643 return false;
644
645 unsigned bailoutId = bailouts_.length();
646 snapshot->setBailoutId(bailoutId);
647 JitSpew(JitSpew_IonSnapshots, "Assigned snapshot bailout id %u", bailoutId);
648 masm.propagateOOM(bailouts_.append(snapshot->snapshotOffset()));
649 return true;
650 }
651
652 bool
encodeSafepoints()653 CodeGeneratorShared::encodeSafepoints()
654 {
655 for (SafepointIndex& index : safepointIndices_) {
656 LSafepoint* safepoint = index.safepoint();
657
658 if (!safepoint->encoded())
659 safepoints_.encode(safepoint);
660
661 index.resolve();
662 }
663
664 return !safepoints_.oom();
665 }
666
667 bool
createNativeToBytecodeScriptList(JSContext * cx)668 CodeGeneratorShared::createNativeToBytecodeScriptList(JSContext* cx)
669 {
670 js::Vector<JSScript*, 0, SystemAllocPolicy> scriptList;
671 InlineScriptTree* tree = gen->info().inlineScriptTree();
672 for (;;) {
673 // Add script from current tree.
674 bool found = false;
675 for (uint32_t i = 0; i < scriptList.length(); i++) {
676 if (scriptList[i] == tree->script()) {
677 found = true;
678 break;
679 }
680 }
681 if (!found) {
682 if (!scriptList.append(tree->script()))
683 return false;
684 }
685
686 // Process rest of tree
687
688 // If children exist, emit children.
689 if (tree->hasChildren()) {
690 tree = tree->firstChild();
691 continue;
692 }
693
694 // Otherwise, find the first tree up the chain (including this one)
695 // that contains a next sibling.
696 while (!tree->hasNextCallee() && tree->hasCaller())
697 tree = tree->caller();
698
699 // If we found a sibling, use it.
700 if (tree->hasNextCallee()) {
701 tree = tree->nextCallee();
702 continue;
703 }
704
705 // Otherwise, we must have reached the top without finding any siblings.
706 MOZ_ASSERT(tree->isOutermostCaller());
707 break;
708 }
709
710 // Allocate array for list.
711 JSScript** data = cx->runtime()->pod_malloc<JSScript*>(scriptList.length());
712 if (!data)
713 return false;
714
715 for (uint32_t i = 0; i < scriptList.length(); i++)
716 data[i] = scriptList[i];
717
718 // Success.
719 nativeToBytecodeScriptListLength_ = scriptList.length();
720 nativeToBytecodeScriptList_ = data;
721 return true;
722 }
723
724 bool
generateCompactNativeToBytecodeMap(JSContext * cx,JitCode * code)725 CodeGeneratorShared::generateCompactNativeToBytecodeMap(JSContext* cx, JitCode* code)
726 {
727 MOZ_ASSERT(nativeToBytecodeScriptListLength_ == 0);
728 MOZ_ASSERT(nativeToBytecodeScriptList_ == nullptr);
729 MOZ_ASSERT(nativeToBytecodeMap_ == nullptr);
730 MOZ_ASSERT(nativeToBytecodeMapSize_ == 0);
731 MOZ_ASSERT(nativeToBytecodeTableOffset_ == 0);
732 MOZ_ASSERT(nativeToBytecodeNumRegions_ == 0);
733
734 if (!createNativeToBytecodeScriptList(cx))
735 return false;
736
737 MOZ_ASSERT(nativeToBytecodeScriptListLength_ > 0);
738 MOZ_ASSERT(nativeToBytecodeScriptList_ != nullptr);
739
740 CompactBufferWriter writer;
741 uint32_t tableOffset = 0;
742 uint32_t numRegions = 0;
743
744 if (!JitcodeIonTable::WriteIonTable(
745 writer, nativeToBytecodeScriptList_, nativeToBytecodeScriptListLength_,
746 &nativeToBytecodeList_[0],
747 &nativeToBytecodeList_[0] + nativeToBytecodeList_.length(),
748 &tableOffset, &numRegions))
749 {
750 js_free(nativeToBytecodeScriptList_);
751 return false;
752 }
753
754 MOZ_ASSERT(tableOffset > 0);
755 MOZ_ASSERT(numRegions > 0);
756
757 // Writer is done, copy it to sized buffer.
758 uint8_t* data = cx->runtime()->pod_malloc<uint8_t>(writer.length());
759 if (!data) {
760 js_free(nativeToBytecodeScriptList_);
761 return false;
762 }
763
764 memcpy(data, writer.buffer(), writer.length());
765 nativeToBytecodeMap_ = data;
766 nativeToBytecodeMapSize_ = writer.length();
767 nativeToBytecodeTableOffset_ = tableOffset;
768 nativeToBytecodeNumRegions_ = numRegions;
769
770 verifyCompactNativeToBytecodeMap(code);
771
772 JitSpew(JitSpew_Profiling, "Compact Native To Bytecode Map [%p-%p]",
773 data, data + nativeToBytecodeMapSize_);
774
775 return true;
776 }
777
778 void
verifyCompactNativeToBytecodeMap(JitCode * code)779 CodeGeneratorShared::verifyCompactNativeToBytecodeMap(JitCode* code)
780 {
781 #ifdef DEBUG
782 MOZ_ASSERT(nativeToBytecodeScriptListLength_ > 0);
783 MOZ_ASSERT(nativeToBytecodeScriptList_ != nullptr);
784 MOZ_ASSERT(nativeToBytecodeMap_ != nullptr);
785 MOZ_ASSERT(nativeToBytecodeMapSize_ > 0);
786 MOZ_ASSERT(nativeToBytecodeTableOffset_ > 0);
787 MOZ_ASSERT(nativeToBytecodeNumRegions_ > 0);
788
789 // The pointer to the table must be 4-byte aligned
790 const uint8_t* tablePtr = nativeToBytecodeMap_ + nativeToBytecodeTableOffset_;
791 MOZ_ASSERT(uintptr_t(tablePtr) % sizeof(uint32_t) == 0);
792
793 // Verify that numRegions was encoded correctly.
794 const JitcodeIonTable* ionTable = reinterpret_cast<const JitcodeIonTable*>(tablePtr);
795 MOZ_ASSERT(ionTable->numRegions() == nativeToBytecodeNumRegions_);
796
797 // Region offset for first region should be at the start of the payload region.
798 // Since the offsets are backward from the start of the table, the first entry
799 // backoffset should be equal to the forward table offset from the start of the
800 // allocated data.
801 MOZ_ASSERT(ionTable->regionOffset(0) == nativeToBytecodeTableOffset_);
802
803 // Verify each region.
804 for (uint32_t i = 0; i < ionTable->numRegions(); i++) {
805 // Back-offset must point into the payload region preceding the table, not before it.
806 MOZ_ASSERT(ionTable->regionOffset(i) <= nativeToBytecodeTableOffset_);
807
808 // Back-offset must point to a later area in the payload region than previous
809 // back-offset. This means that back-offsets decrease monotonically.
810 MOZ_ASSERT_IF(i > 0, ionTable->regionOffset(i) < ionTable->regionOffset(i - 1));
811
812 JitcodeRegionEntry entry = ionTable->regionEntry(i);
813
814 // Ensure native code offset for region falls within jitcode.
815 MOZ_ASSERT(entry.nativeOffset() <= code->instructionsSize());
816
817 // Read out script/pc stack and verify.
818 JitcodeRegionEntry::ScriptPcIterator scriptPcIter = entry.scriptPcIterator();
819 while (scriptPcIter.hasMore()) {
820 uint32_t scriptIdx = 0, pcOffset = 0;
821 scriptPcIter.readNext(&scriptIdx, &pcOffset);
822
823 // Ensure scriptIdx refers to a valid script in the list.
824 MOZ_ASSERT(scriptIdx < nativeToBytecodeScriptListLength_);
825 JSScript* script = nativeToBytecodeScriptList_[scriptIdx];
826
827 // Ensure pcOffset falls within the script.
828 MOZ_ASSERT(pcOffset < script->length());
829 }
830
831 // Obtain the original nativeOffset and pcOffset and script.
832 uint32_t curNativeOffset = entry.nativeOffset();
833 JSScript* script = nullptr;
834 uint32_t curPcOffset = 0;
835 {
836 uint32_t scriptIdx = 0;
837 scriptPcIter.reset();
838 scriptPcIter.readNext(&scriptIdx, &curPcOffset);
839 script = nativeToBytecodeScriptList_[scriptIdx];
840 }
841
842 // Read out nativeDeltas and pcDeltas and verify.
843 JitcodeRegionEntry::DeltaIterator deltaIter = entry.deltaIterator();
844 while (deltaIter.hasMore()) {
845 uint32_t nativeDelta = 0;
846 int32_t pcDelta = 0;
847 deltaIter.readNext(&nativeDelta, &pcDelta);
848
849 curNativeOffset += nativeDelta;
850 curPcOffset = uint32_t(int32_t(curPcOffset) + pcDelta);
851
852 // Ensure that nativeOffset still falls within jitcode after delta.
853 MOZ_ASSERT(curNativeOffset <= code->instructionsSize());
854
855 // Ensure that pcOffset still falls within bytecode after delta.
856 MOZ_ASSERT(curPcOffset < script->length());
857 }
858 }
859 #endif // DEBUG
860 }
861
862 bool
generateCompactTrackedOptimizationsMap(JSContext * cx,JitCode * code,IonTrackedTypeVector * allTypes)863 CodeGeneratorShared::generateCompactTrackedOptimizationsMap(JSContext* cx, JitCode* code,
864 IonTrackedTypeVector* allTypes)
865 {
866 MOZ_ASSERT(trackedOptimizationsMap_ == nullptr);
867 MOZ_ASSERT(trackedOptimizationsMapSize_ == 0);
868 MOZ_ASSERT(trackedOptimizationsRegionTableOffset_ == 0);
869 MOZ_ASSERT(trackedOptimizationsTypesTableOffset_ == 0);
870 MOZ_ASSERT(trackedOptimizationsAttemptsTableOffset_ == 0);
871
872 if (trackedOptimizations_.empty())
873 return true;
874
875 UniqueTrackedOptimizations unique(cx);
876 if (!unique.init())
877 return false;
878
879 // Iterate through all entries to deduplicate their optimization attempts.
880 for (size_t i = 0; i < trackedOptimizations_.length(); i++) {
881 NativeToTrackedOptimizations& entry = trackedOptimizations_[i];
882 if (!unique.add(entry.optimizations))
883 return false;
884 }
885
886 // Sort the unique optimization attempts by frequency to stabilize the
887 // attempts' indices in the compact table we will write later.
888 if (!unique.sortByFrequency(cx))
889 return false;
890
891 // Write out the ranges and the table.
892 CompactBufferWriter writer;
893 uint32_t numRegions;
894 uint32_t regionTableOffset;
895 uint32_t typesTableOffset;
896 uint32_t attemptsTableOffset;
897 if (!WriteIonTrackedOptimizationsTable(cx, writer,
898 trackedOptimizations_.begin(),
899 trackedOptimizations_.end(),
900 unique, &numRegions,
901 ®ionTableOffset, &typesTableOffset,
902 &attemptsTableOffset, allTypes))
903 {
904 return false;
905 }
906
907 MOZ_ASSERT(regionTableOffset > 0);
908 MOZ_ASSERT(typesTableOffset > 0);
909 MOZ_ASSERT(attemptsTableOffset > 0);
910 MOZ_ASSERT(typesTableOffset > regionTableOffset);
911 MOZ_ASSERT(attemptsTableOffset > typesTableOffset);
912
913 // Copy over the table out of the writer's buffer.
914 uint8_t* data = cx->runtime()->pod_malloc<uint8_t>(writer.length());
915 if (!data)
916 return false;
917
918 memcpy(data, writer.buffer(), writer.length());
919 trackedOptimizationsMap_ = data;
920 trackedOptimizationsMapSize_ = writer.length();
921 trackedOptimizationsRegionTableOffset_ = regionTableOffset;
922 trackedOptimizationsTypesTableOffset_ = typesTableOffset;
923 trackedOptimizationsAttemptsTableOffset_ = attemptsTableOffset;
924
925 verifyCompactTrackedOptimizationsMap(code, numRegions, unique, allTypes);
926
927 JitSpew(JitSpew_OptimizationTracking,
928 "== Compact Native To Optimizations Map [%p-%p] size %u",
929 data, data + trackedOptimizationsMapSize_, trackedOptimizationsMapSize_);
930 JitSpew(JitSpew_OptimizationTracking,
931 " with type list of length %" PRIuSIZE ", size %" PRIuSIZE,
932 allTypes->length(), allTypes->length() * sizeof(IonTrackedTypeWithAddendum));
933
934 return true;
935 }
936
937 #ifdef DEBUG
938 class ReadTempAttemptsVectorOp : public JS::ForEachTrackedOptimizationAttemptOp
939 {
940 TempOptimizationAttemptsVector* attempts_;
941 bool oom_;
942
943 public:
ReadTempAttemptsVectorOp(TempOptimizationAttemptsVector * attempts)944 explicit ReadTempAttemptsVectorOp(TempOptimizationAttemptsVector* attempts)
945 : attempts_(attempts), oom_(false)
946 { }
947
oom()948 bool oom() {
949 return oom_;
950 }
951
operator ()(JS::TrackedStrategy strategy,JS::TrackedOutcome outcome)952 void operator()(JS::TrackedStrategy strategy, JS::TrackedOutcome outcome) override {
953 if (!attempts_->append(OptimizationAttempt(strategy, outcome)))
954 oom_ = true;
955 }
956 };
957
958 struct ReadTempTypeInfoVectorOp : public IonTrackedOptimizationsTypeInfo::ForEachOp
959 {
960 TempAllocator& alloc_;
961 TempOptimizationTypeInfoVector* types_;
962 TempTypeList accTypes_;
963 bool oom_;
964
965 public:
ReadTempTypeInfoVectorOpjs::jit::ReadTempTypeInfoVectorOp966 ReadTempTypeInfoVectorOp(TempAllocator& alloc, TempOptimizationTypeInfoVector* types)
967 : alloc_(alloc),
968 types_(types),
969 accTypes_(alloc),
970 oom_(false)
971 { }
972
oomjs::jit::ReadTempTypeInfoVectorOp973 bool oom() {
974 return oom_;
975 }
976
readTypejs::jit::ReadTempTypeInfoVectorOp977 void readType(const IonTrackedTypeWithAddendum& tracked) override {
978 if (!accTypes_.append(tracked.type))
979 oom_ = true;
980 }
981
operator ()js::jit::ReadTempTypeInfoVectorOp982 void operator()(JS::TrackedTypeSite site, MIRType mirType) override {
983 OptimizationTypeInfo ty(alloc_, site, mirType);
984 for (uint32_t i = 0; i < accTypes_.length(); i++) {
985 if (!ty.trackType(accTypes_[i]))
986 oom_ = true;
987 }
988 if (!types_->append(mozilla::Move(ty)))
989 oom_ = true;
990 accTypes_.clear();
991 }
992 };
993 #endif // DEBUG
994
995 void
verifyCompactTrackedOptimizationsMap(JitCode * code,uint32_t numRegions,const UniqueTrackedOptimizations & unique,const IonTrackedTypeVector * allTypes)996 CodeGeneratorShared::verifyCompactTrackedOptimizationsMap(JitCode* code, uint32_t numRegions,
997 const UniqueTrackedOptimizations& unique,
998 const IonTrackedTypeVector* allTypes)
999 {
1000 #ifdef DEBUG
1001 MOZ_ASSERT(trackedOptimizationsMap_ != nullptr);
1002 MOZ_ASSERT(trackedOptimizationsMapSize_ > 0);
1003 MOZ_ASSERT(trackedOptimizationsRegionTableOffset_ > 0);
1004 MOZ_ASSERT(trackedOptimizationsTypesTableOffset_ > 0);
1005 MOZ_ASSERT(trackedOptimizationsAttemptsTableOffset_ > 0);
1006
1007 // Table pointers must all be 4-byte aligned.
1008 const uint8_t* regionTableAddr = trackedOptimizationsMap_ +
1009 trackedOptimizationsRegionTableOffset_;
1010 const uint8_t* typesTableAddr = trackedOptimizationsMap_ +
1011 trackedOptimizationsTypesTableOffset_;
1012 const uint8_t* attemptsTableAddr = trackedOptimizationsMap_ +
1013 trackedOptimizationsAttemptsTableOffset_;
1014 MOZ_ASSERT(uintptr_t(regionTableAddr) % sizeof(uint32_t) == 0);
1015 MOZ_ASSERT(uintptr_t(typesTableAddr) % sizeof(uint32_t) == 0);
1016 MOZ_ASSERT(uintptr_t(attemptsTableAddr) % sizeof(uint32_t) == 0);
1017
1018 // Assert that the number of entries matches up for the tables.
1019 const IonTrackedOptimizationsRegionTable* regionTable =
1020 (const IonTrackedOptimizationsRegionTable*) regionTableAddr;
1021 MOZ_ASSERT(regionTable->numEntries() == numRegions);
1022 const IonTrackedOptimizationsTypesTable* typesTable =
1023 (const IonTrackedOptimizationsTypesTable*) typesTableAddr;
1024 MOZ_ASSERT(typesTable->numEntries() == unique.count());
1025 const IonTrackedOptimizationsAttemptsTable* attemptsTable =
1026 (const IonTrackedOptimizationsAttemptsTable*) attemptsTableAddr;
1027 MOZ_ASSERT(attemptsTable->numEntries() == unique.count());
1028
1029 // Verify each region.
1030 uint32_t trackedIdx = 0;
1031 for (uint32_t regionIdx = 0; regionIdx < regionTable->numEntries(); regionIdx++) {
1032 // Check reverse offsets are within bounds.
1033 MOZ_ASSERT(regionTable->entryOffset(regionIdx) <= trackedOptimizationsRegionTableOffset_);
1034 MOZ_ASSERT_IF(regionIdx > 0, regionTable->entryOffset(regionIdx) <
1035 regionTable->entryOffset(regionIdx - 1));
1036
1037 IonTrackedOptimizationsRegion region = regionTable->entry(regionIdx);
1038
1039 // Check the region range is covered by jitcode.
1040 MOZ_ASSERT(region.startOffset() <= code->instructionsSize());
1041 MOZ_ASSERT(region.endOffset() <= code->instructionsSize());
1042
1043 IonTrackedOptimizationsRegion::RangeIterator iter = region.ranges();
1044 while (iter.more()) {
1045 // Assert that the offsets are correctly decoded from the delta.
1046 uint32_t startOffset, endOffset;
1047 uint8_t index;
1048 iter.readNext(&startOffset, &endOffset, &index);
1049 NativeToTrackedOptimizations& entry = trackedOptimizations_[trackedIdx++];
1050 MOZ_ASSERT(startOffset == entry.startOffset.offset());
1051 MOZ_ASSERT(endOffset == entry.endOffset.offset());
1052 MOZ_ASSERT(index == unique.indexOf(entry.optimizations));
1053
1054 // Assert that the type info and attempts vectors are correctly
1055 // decoded. This is disabled for now if the types table might
1056 // contain nursery pointers, in which case the types might not
1057 // match, see bug 1175761.
1058 if (!code->runtimeFromMainThread()->gc.storeBuffer.cancelIonCompilations()) {
1059 IonTrackedOptimizationsTypeInfo typeInfo = typesTable->entry(index);
1060 TempOptimizationTypeInfoVector tvec(alloc());
1061 ReadTempTypeInfoVectorOp top(alloc(), &tvec);
1062 typeInfo.forEach(top, allTypes);
1063 MOZ_ASSERT_IF(!top.oom(), entry.optimizations->matchTypes(tvec));
1064 }
1065
1066 IonTrackedOptimizationsAttempts attempts = attemptsTable->entry(index);
1067 TempOptimizationAttemptsVector avec(alloc());
1068 ReadTempAttemptsVectorOp aop(&avec);
1069 attempts.forEach(aop);
1070 MOZ_ASSERT_IF(!aop.oom(), entry.optimizations->matchAttempts(avec));
1071 }
1072 }
1073 #endif
1074 }
1075
1076 void
markSafepoint(LInstruction * ins)1077 CodeGeneratorShared::markSafepoint(LInstruction* ins)
1078 {
1079 markSafepointAt(masm.currentOffset(), ins);
1080 }
1081
1082 void
markSafepointAt(uint32_t offset,LInstruction * ins)1083 CodeGeneratorShared::markSafepointAt(uint32_t offset, LInstruction* ins)
1084 {
1085 MOZ_ASSERT_IF(!safepointIndices_.empty() && !masm.oom(),
1086 offset - safepointIndices_.back().displacement() >= sizeof(uint32_t));
1087 masm.propagateOOM(safepointIndices_.append(SafepointIndex(offset, ins->safepoint())));
1088 }
1089
1090 void
ensureOsiSpace()1091 CodeGeneratorShared::ensureOsiSpace()
1092 {
1093 // For a refresher, an invalidation point is of the form:
1094 // 1: call <target>
1095 // 2: ...
1096 // 3: <osipoint>
1097 //
1098 // The four bytes *before* instruction 2 are overwritten with an offset.
1099 // Callers must ensure that the instruction itself has enough bytes to
1100 // support this.
1101 //
1102 // The bytes *at* instruction 3 are overwritten with an invalidation jump.
1103 // jump. These bytes may be in a completely different IR sequence, but
1104 // represent the join point of the call out of the function.
1105 //
1106 // At points where we want to ensure that invalidation won't corrupt an
1107 // important instruction, we make sure to pad with nops.
1108 if (masm.currentOffset() - lastOsiPointOffset_ < Assembler::PatchWrite_NearCallSize()) {
1109 int32_t paddingSize = Assembler::PatchWrite_NearCallSize();
1110 paddingSize -= masm.currentOffset() - lastOsiPointOffset_;
1111 for (int32_t i = 0; i < paddingSize; ++i)
1112 masm.nop();
1113 }
1114 MOZ_ASSERT_IF(!masm.oom(),
1115 masm.currentOffset() - lastOsiPointOffset_ >= Assembler::PatchWrite_NearCallSize());
1116 lastOsiPointOffset_ = masm.currentOffset();
1117 }
1118
1119 uint32_t
markOsiPoint(LOsiPoint * ins)1120 CodeGeneratorShared::markOsiPoint(LOsiPoint* ins)
1121 {
1122 encode(ins->snapshot());
1123 ensureOsiSpace();
1124
1125 uint32_t offset = masm.currentOffset();
1126 SnapshotOffset so = ins->snapshot()->snapshotOffset();
1127 masm.propagateOOM(osiIndices_.append(OsiIndex(offset, so)));
1128
1129 return offset;
1130 }
1131
1132 #ifdef CHECK_OSIPOINT_REGISTERS
1133 template <class Op>
1134 static void
HandleRegisterDump(Op op,MacroAssembler & masm,LiveRegisterSet liveRegs,Register activation,Register scratch)1135 HandleRegisterDump(Op op, MacroAssembler& masm, LiveRegisterSet liveRegs, Register activation,
1136 Register scratch)
1137 {
1138 const size_t baseOffset = JitActivation::offsetOfRegs();
1139
1140 // Handle live GPRs.
1141 for (GeneralRegisterIterator iter(liveRegs.gprs()); iter.more(); ++iter) {
1142 Register reg = *iter;
1143 Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
1144
1145 if (reg == activation) {
1146 // To use the original value of the activation register (that's
1147 // now on top of the stack), we need the scratch register.
1148 masm.push(scratch);
1149 masm.loadPtr(Address(masm.getStackPointer(), sizeof(uintptr_t)), scratch);
1150 op(scratch, dump);
1151 masm.pop(scratch);
1152 } else {
1153 op(reg, dump);
1154 }
1155 }
1156
1157 // Handle live FPRs.
1158 for (FloatRegisterIterator iter(liveRegs.fpus()); iter.more(); ++iter) {
1159 FloatRegister reg = *iter;
1160 Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
1161 op(reg, dump);
1162 }
1163 }
1164
1165 class StoreOp
1166 {
1167 MacroAssembler& masm;
1168
1169 public:
StoreOp(MacroAssembler & masm)1170 explicit StoreOp(MacroAssembler& masm)
1171 : masm(masm)
1172 {}
1173
operator ()(Register reg,Address dump)1174 void operator()(Register reg, Address dump) {
1175 masm.storePtr(reg, dump);
1176 }
operator ()(FloatRegister reg,Address dump)1177 void operator()(FloatRegister reg, Address dump) {
1178 if (reg.isDouble())
1179 masm.storeDouble(reg, dump);
1180 else if (reg.isSingle())
1181 masm.storeFloat32(reg, dump);
1182 #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
1183 else if (reg.isSimd128())
1184 masm.storeUnalignedSimd128Float(reg, dump);
1185 #endif
1186 else
1187 MOZ_CRASH("Unexpected register type.");
1188 }
1189 };
1190
1191 static void
StoreAllLiveRegs(MacroAssembler & masm,LiveRegisterSet liveRegs)1192 StoreAllLiveRegs(MacroAssembler& masm, LiveRegisterSet liveRegs)
1193 {
1194 // Store a copy of all live registers before performing the call.
1195 // When we reach the OsiPoint, we can use this to check nothing
1196 // modified them in the meantime.
1197
1198 // Load pointer to the JitActivation in a scratch register.
1199 AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
1200 Register scratch = allRegs.takeAny();
1201 masm.push(scratch);
1202 masm.loadJitActivation(scratch);
1203
1204 Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
1205 masm.add32(Imm32(1), checkRegs);
1206
1207 StoreOp op(masm);
1208 HandleRegisterDump<StoreOp>(op, masm, liveRegs, scratch, allRegs.getAny());
1209
1210 masm.pop(scratch);
1211 }
1212
1213 class VerifyOp
1214 {
1215 MacroAssembler& masm;
1216 Label* failure_;
1217
1218 public:
VerifyOp(MacroAssembler & masm,Label * failure)1219 VerifyOp(MacroAssembler& masm, Label* failure)
1220 : masm(masm), failure_(failure)
1221 {}
1222
operator ()(Register reg,Address dump)1223 void operator()(Register reg, Address dump) {
1224 masm.branchPtr(Assembler::NotEqual, dump, reg, failure_);
1225 }
operator ()(FloatRegister reg,Address dump)1226 void operator()(FloatRegister reg, Address dump) {
1227 FloatRegister scratch;
1228 if (reg.isDouble()) {
1229 scratch = ScratchDoubleReg;
1230 masm.loadDouble(dump, scratch);
1231 masm.branchDouble(Assembler::DoubleNotEqual, scratch, reg, failure_);
1232 } else if (reg.isSingle()) {
1233 scratch = ScratchFloat32Reg;
1234 masm.loadFloat32(dump, scratch);
1235 masm.branchFloat(Assembler::DoubleNotEqual, scratch, reg, failure_);
1236 }
1237
1238 // :TODO: (Bug 1133745) Add support to verify SIMD registers.
1239 }
1240 };
1241
1242 void
verifyOsiPointRegs(LSafepoint * safepoint)1243 CodeGeneratorShared::verifyOsiPointRegs(LSafepoint* safepoint)
1244 {
1245 // Ensure the live registers stored by callVM did not change between
1246 // the call and this OsiPoint. Try-catch relies on this invariant.
1247
1248 // Load pointer to the JitActivation in a scratch register.
1249 AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
1250 Register scratch = allRegs.takeAny();
1251 masm.push(scratch);
1252 masm.loadJitActivation(scratch);
1253
1254 // If we should not check registers (because the instruction did not call
1255 // into the VM, or a GC happened), we're done.
1256 Label failure, done;
1257 Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
1258 masm.branch32(Assembler::Equal, checkRegs, Imm32(0), &done);
1259
1260 // Having more than one VM function call made in one visit function at
1261 // runtime is a sec-ciritcal error, because if we conservatively assume that
1262 // one of the function call can re-enter Ion, then the invalidation process
1263 // will potentially add a call at a random location, by patching the code
1264 // before the return address.
1265 masm.branch32(Assembler::NotEqual, checkRegs, Imm32(1), &failure);
1266
1267 // Set checkRegs to 0, so that we don't try to verify registers after we
1268 // return from this script to the caller.
1269 masm.store32(Imm32(0), checkRegs);
1270
1271 // Ignore clobbered registers. Some instructions (like LValueToInt32) modify
1272 // temps after calling into the VM. This is fine because no other
1273 // instructions (including this OsiPoint) will depend on them. Also
1274 // backtracking can also use the same register for an input and an output.
1275 // These are marked as clobbered and shouldn't get checked.
1276 LiveRegisterSet liveRegs;
1277 liveRegs.set() = RegisterSet::Intersect(safepoint->liveRegs().set(),
1278 RegisterSet::Not(safepoint->clobberedRegs().set()));
1279
1280 VerifyOp op(masm, &failure);
1281 HandleRegisterDump<VerifyOp>(op, masm, liveRegs, scratch, allRegs.getAny());
1282
1283 masm.jump(&done);
1284
1285 // Do not profile the callWithABI that occurs below. This is to avoid a
1286 // rare corner case that occurs when profiling interacts with itself:
1287 //
1288 // When slow profiling assertions are turned on, FunctionBoundary ops
1289 // (which update the profiler pseudo-stack) may emit a callVM, which
1290 // forces them to have an osi point associated with them. The
1291 // FunctionBoundary for inline function entry is added to the caller's
1292 // graph with a PC from the caller's code, but during codegen it modifies
1293 // SPS instrumentation to add the callee as the current top-most script.
1294 // When codegen gets to the OSIPoint, and the callWithABI below is
1295 // emitted, the codegen thinks that the current frame is the callee, but
1296 // the PC it's using from the OSIPoint refers to the caller. This causes
1297 // the profiler instrumentation of the callWithABI below to ASSERT, since
1298 // the script and pc are mismatched. To avoid this, we simply omit
1299 // instrumentation for these callWithABIs.
1300
1301 // Any live register captured by a safepoint (other than temp registers)
1302 // must remain unchanged between the call and the OsiPoint instruction.
1303 masm.bind(&failure);
1304 masm.assumeUnreachable("Modified registers between VM call and OsiPoint");
1305
1306 masm.bind(&done);
1307 masm.pop(scratch);
1308 }
1309
1310 bool
shouldVerifyOsiPointRegs(LSafepoint * safepoint)1311 CodeGeneratorShared::shouldVerifyOsiPointRegs(LSafepoint* safepoint)
1312 {
1313 if (!checkOsiPointRegisters)
1314 return false;
1315
1316 if (safepoint->liveRegs().emptyGeneral() && safepoint->liveRegs().emptyFloat())
1317 return false; // No registers to check.
1318
1319 return true;
1320 }
1321
1322 void
resetOsiPointRegs(LSafepoint * safepoint)1323 CodeGeneratorShared::resetOsiPointRegs(LSafepoint* safepoint)
1324 {
1325 if (!shouldVerifyOsiPointRegs(safepoint))
1326 return;
1327
1328 // Set checkRegs to 0. If we perform a VM call, the instruction
1329 // will set it to 1.
1330 AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
1331 Register scratch = allRegs.takeAny();
1332 masm.push(scratch);
1333 masm.loadJitActivation(scratch);
1334 Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
1335 masm.store32(Imm32(0), checkRegs);
1336 masm.pop(scratch);
1337 }
1338 #endif
1339
1340 // Before doing any call to Cpp, you should ensure that volatile
1341 // registers are evicted by the register allocator.
1342 void
callVM(const VMFunction & fun,LInstruction * ins,const Register * dynStack)1343 CodeGeneratorShared::callVM(const VMFunction& fun, LInstruction* ins, const Register* dynStack)
1344 {
1345 // If we're calling a function with an out parameter type of double, make
1346 // sure we have an FPU.
1347 MOZ_ASSERT_IF(fun.outParam == Type_Double, GetJitContext()->runtime->jitSupportsFloatingPoint());
1348
1349 #ifdef DEBUG
1350 if (ins->mirRaw()) {
1351 MOZ_ASSERT(ins->mirRaw()->isInstruction());
1352 MInstruction* mir = ins->mirRaw()->toInstruction();
1353 MOZ_ASSERT_IF(mir->needsResumePoint(), mir->resumePoint());
1354 }
1355 #endif
1356
1357 // Stack is:
1358 // ... frame ...
1359 // [args]
1360 #ifdef DEBUG
1361 MOZ_ASSERT(pushedArgs_ == fun.explicitArgs);
1362 pushedArgs_ = 0;
1363 #endif
1364
1365 // Get the wrapper of the VM function.
1366 JitCode* wrapper = gen->jitRuntime()->getVMWrapper(fun);
1367 if (!wrapper) {
1368 masm.setOOM();
1369 return;
1370 }
1371
1372 #ifdef CHECK_OSIPOINT_REGISTERS
1373 if (shouldVerifyOsiPointRegs(ins->safepoint()))
1374 StoreAllLiveRegs(masm, ins->safepoint()->liveRegs());
1375 #endif
1376
1377 // Push an exit frame descriptor. If |dynStack| is a valid pointer to a
1378 // register, then its value is added to the value of the |framePushed()| to
1379 // fill the frame descriptor.
1380 if (dynStack) {
1381 masm.addPtr(Imm32(masm.framePushed()), *dynStack);
1382 masm.makeFrameDescriptor(*dynStack, JitFrame_IonJS, ExitFrameLayout::Size());
1383 masm.Push(*dynStack); // descriptor
1384 } else {
1385 masm.pushStaticFrameDescriptor(JitFrame_IonJS, ExitFrameLayout::Size());
1386 }
1387
1388 // Call the wrapper function. The wrapper is in charge to unwind the stack
1389 // when returning from the call. Failures are handled with exceptions based
1390 // on the return value of the C functions. To guard the outcome of the
1391 // returned value, use another LIR instruction.
1392 uint32_t callOffset = masm.callJit(wrapper);
1393 markSafepointAt(callOffset, ins);
1394
1395 // Remove rest of the frame left on the stack. We remove the return address
1396 // which is implicitly poped when returning.
1397 int framePop = sizeof(ExitFrameLayout) - sizeof(void*);
1398
1399 // Pop arguments from framePushed.
1400 masm.implicitPop(fun.explicitStackSlots() * sizeof(void*) + framePop);
1401 // Stack is:
1402 // ... frame ...
1403 }
1404
1405 class OutOfLineTruncateSlow : public OutOfLineCodeBase<CodeGeneratorShared>
1406 {
1407 FloatRegister src_;
1408 Register dest_;
1409 bool widenFloatToDouble_;
1410
1411 public:
OutOfLineTruncateSlow(FloatRegister src,Register dest,bool widenFloatToDouble=false)1412 OutOfLineTruncateSlow(FloatRegister src, Register dest, bool widenFloatToDouble = false)
1413 : src_(src), dest_(dest), widenFloatToDouble_(widenFloatToDouble)
1414 { }
1415
accept(CodeGeneratorShared * codegen)1416 void accept(CodeGeneratorShared* codegen) {
1417 codegen->visitOutOfLineTruncateSlow(this);
1418 }
src() const1419 FloatRegister src() const {
1420 return src_;
1421 }
dest() const1422 Register dest() const {
1423 return dest_;
1424 }
widenFloatToDouble() const1425 bool widenFloatToDouble() const {
1426 return widenFloatToDouble_;
1427 }
1428
1429 };
1430
1431 OutOfLineCode*
oolTruncateDouble(FloatRegister src,Register dest,MInstruction * mir)1432 CodeGeneratorShared::oolTruncateDouble(FloatRegister src, Register dest, MInstruction* mir)
1433 {
1434 OutOfLineTruncateSlow* ool = new(alloc()) OutOfLineTruncateSlow(src, dest);
1435 addOutOfLineCode(ool, mir);
1436 return ool;
1437 }
1438
1439 void
emitTruncateDouble(FloatRegister src,Register dest,MInstruction * mir)1440 CodeGeneratorShared::emitTruncateDouble(FloatRegister src, Register dest, MInstruction* mir)
1441 {
1442 OutOfLineCode* ool = oolTruncateDouble(src, dest, mir);
1443
1444 masm.branchTruncateDoubleMaybeModUint32(src, dest, ool->entry());
1445 masm.bind(ool->rejoin());
1446 }
1447
1448 void
emitTruncateFloat32(FloatRegister src,Register dest,MInstruction * mir)1449 CodeGeneratorShared::emitTruncateFloat32(FloatRegister src, Register dest, MInstruction* mir)
1450 {
1451 OutOfLineTruncateSlow* ool = new(alloc()) OutOfLineTruncateSlow(src, dest, true);
1452 addOutOfLineCode(ool, mir);
1453
1454 masm.branchTruncateFloat32MaybeModUint32(src, dest, ool->entry());
1455 masm.bind(ool->rejoin());
1456 }
1457
1458 void
visitOutOfLineTruncateSlow(OutOfLineTruncateSlow * ool)1459 CodeGeneratorShared::visitOutOfLineTruncateSlow(OutOfLineTruncateSlow* ool)
1460 {
1461 FloatRegister src = ool->src();
1462 Register dest = ool->dest();
1463
1464 saveVolatile(dest);
1465 masm.outOfLineTruncateSlow(src, dest, ool->widenFloatToDouble(), gen->compilingWasm());
1466 restoreVolatile(dest);
1467
1468 masm.jump(ool->rejoin());
1469 }
1470
1471 bool
omitOverRecursedCheck() const1472 CodeGeneratorShared::omitOverRecursedCheck() const
1473 {
1474 // If the current function makes no calls (which means it isn't recursive)
1475 // and it uses only a small amount of stack space, it doesn't need a
1476 // stack overflow check. Note that the actual number here is somewhat
1477 // arbitrary, and codegen actually uses small bounded amounts of
1478 // additional stack space in some cases too.
1479 return frameSize() < 64 && !gen->performsCall();
1480 }
1481
1482 void
emitWasmCallBase(LWasmCallBase * ins)1483 CodeGeneratorShared::emitWasmCallBase(LWasmCallBase* ins)
1484 {
1485 MWasmCall* mir = ins->mir();
1486
1487 if (mir->spIncrement())
1488 masm.freeStack(mir->spIncrement());
1489
1490 MOZ_ASSERT((sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment == 0);
1491 static_assert(WasmStackAlignment >= ABIStackAlignment &&
1492 WasmStackAlignment % ABIStackAlignment == 0,
1493 "The wasm stack alignment should subsume the ABI-required alignment");
1494
1495 #ifdef DEBUG
1496 Label ok;
1497 masm.branchTestStackPtr(Assembler::Zero, Imm32(WasmStackAlignment - 1), &ok);
1498 masm.breakpoint();
1499 masm.bind(&ok);
1500 #endif
1501
1502 // Save the caller's TLS register in a reserved stack slot (below the
1503 // call's stack arguments) for retrieval after the call.
1504 if (mir->saveTls())
1505 masm.storePtr(WasmTlsReg, Address(masm.getStackPointer(), mir->tlsStackOffset()));
1506
1507 const wasm::CallSiteDesc& desc = mir->desc();
1508 const wasm::CalleeDesc& callee = mir->callee();
1509 switch (callee.which()) {
1510 case wasm::CalleeDesc::Func:
1511 masm.call(desc, callee.funcIndex());
1512 break;
1513 case wasm::CalleeDesc::Import:
1514 masm.wasmCallImport(desc, callee);
1515 break;
1516 case wasm::CalleeDesc::WasmTable:
1517 case wasm::CalleeDesc::AsmJSTable:
1518 masm.wasmCallIndirect(desc, callee);
1519 break;
1520 case wasm::CalleeDesc::Builtin:
1521 masm.call(callee.builtin());
1522 break;
1523 case wasm::CalleeDesc::BuiltinInstanceMethod:
1524 masm.wasmCallBuiltinInstanceMethod(mir->instanceArg(), callee.builtin());
1525 break;
1526 }
1527
1528 // After return, restore the caller's TLS and pinned registers.
1529 if (mir->saveTls()) {
1530 masm.loadPtr(Address(masm.getStackPointer(), mir->tlsStackOffset()), WasmTlsReg);
1531 masm.loadWasmPinnedRegsFromTls();
1532 }
1533
1534 if (mir->spIncrement())
1535 masm.reserveStack(mir->spIncrement());
1536 }
1537
1538 void
emitPreBarrier(Register base,const LAllocation * index,int32_t offsetAdjustment)1539 CodeGeneratorShared::emitPreBarrier(Register base, const LAllocation* index, int32_t offsetAdjustment)
1540 {
1541 if (index->isConstant()) {
1542 Address address(base, ToInt32(index) * sizeof(Value) + offsetAdjustment);
1543 masm.patchableCallPreBarrier(address, MIRType::Value);
1544 } else {
1545 BaseIndex address(base, ToRegister(index), TimesEight, offsetAdjustment);
1546 masm.patchableCallPreBarrier(address, MIRType::Value);
1547 }
1548 }
1549
1550 void
emitPreBarrier(Address address)1551 CodeGeneratorShared::emitPreBarrier(Address address)
1552 {
1553 masm.patchableCallPreBarrier(address, MIRType::Value);
1554 }
1555
1556 Label*
labelForBackedgeWithImplicitCheck(MBasicBlock * mir)1557 CodeGeneratorShared::labelForBackedgeWithImplicitCheck(MBasicBlock* mir)
1558 {
1559 // If this is a loop backedge to a loop header with an implicit interrupt
1560 // check, use a patchable jump. Skip this search if compiling without a
1561 // script for wasm, as there will be no interrupt check instruction.
1562 // Due to critical edge unsplitting there may no longer be unique loop
1563 // backedges, so just look for any edge going to an earlier block in RPO.
1564 if (!gen->compilingWasm() && mir->isLoopHeader() && mir->id() <= current->mir()->id()) {
1565 for (LInstructionIterator iter = mir->lir()->begin(); iter != mir->lir()->end(); iter++) {
1566 if (iter->isMoveGroup()) {
1567 // Continue searching for an interrupt check.
1568 } else {
1569 // The interrupt check should be the first instruction in the
1570 // loop header other than move groups.
1571 MOZ_ASSERT(iter->isInterruptCheck());
1572 if (iter->toInterruptCheck()->implicit())
1573 return iter->toInterruptCheck()->oolEntry();
1574 return nullptr;
1575 }
1576 }
1577 }
1578
1579 return nullptr;
1580 }
1581
1582 void
jumpToBlock(MBasicBlock * mir)1583 CodeGeneratorShared::jumpToBlock(MBasicBlock* mir)
1584 {
1585 // Skip past trivial blocks.
1586 mir = skipTrivialBlocks(mir);
1587
1588 // No jump necessary if we can fall through to the next block.
1589 if (isNextBlock(mir->lir()))
1590 return;
1591
1592 if (Label* oolEntry = labelForBackedgeWithImplicitCheck(mir)) {
1593 // Note: the backedge is initially a jump to the next instruction.
1594 // It will be patched to the target block's label during link().
1595 RepatchLabel rejoin;
1596 CodeOffsetJump backedge = masm.backedgeJump(&rejoin, mir->lir()->label());
1597 masm.bind(&rejoin);
1598
1599 masm.propagateOOM(patchableBackedges_.append(PatchableBackedgeInfo(backedge, mir->lir()->label(), oolEntry)));
1600 } else {
1601 masm.jump(mir->lir()->label());
1602 }
1603 }
1604
1605 Label*
getJumpLabelForBranch(MBasicBlock * block)1606 CodeGeneratorShared::getJumpLabelForBranch(MBasicBlock* block)
1607 {
1608 // Skip past trivial blocks.
1609 block = skipTrivialBlocks(block);
1610
1611 if (!labelForBackedgeWithImplicitCheck(block))
1612 return block->lir()->label();
1613
1614 // We need to use a patchable jump for this backedge, but want to treat
1615 // this as a normal label target to simplify codegen. Efficiency isn't so
1616 // important here as these tests are extremely unlikely to be used in loop
1617 // backedges, so emit inline code for the patchable jump. Heap allocating
1618 // the label allows it to be used by out of line blocks.
1619 Label* res = alloc().lifoAlloc()->newInfallible<Label>();
1620 Label after;
1621 masm.jump(&after);
1622 masm.bind(res);
1623 jumpToBlock(block);
1624 masm.bind(&after);
1625 return res;
1626 }
1627
1628 // This function is not used for MIPS/MIPS64. MIPS has branchToBlock.
1629 #if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64)
1630 void
jumpToBlock(MBasicBlock * mir,Assembler::Condition cond)1631 CodeGeneratorShared::jumpToBlock(MBasicBlock* mir, Assembler::Condition cond)
1632 {
1633 // Skip past trivial blocks.
1634 mir = skipTrivialBlocks(mir);
1635
1636 if (Label* oolEntry = labelForBackedgeWithImplicitCheck(mir)) {
1637 // Note: the backedge is initially a jump to the next instruction.
1638 // It will be patched to the target block's label during link().
1639 RepatchLabel rejoin;
1640 CodeOffsetJump backedge = masm.jumpWithPatch(&rejoin, cond, mir->lir()->label());
1641 masm.bind(&rejoin);
1642
1643 masm.propagateOOM(patchableBackedges_.append(PatchableBackedgeInfo(backedge, mir->lir()->label(), oolEntry)));
1644 } else {
1645 masm.j(cond, mir->lir()->label());
1646 }
1647 }
1648 #endif
1649
1650 MOZ_MUST_USE bool
addCacheLocations(const CacheLocationList & locs,size_t * numLocs,size_t * curIndex)1651 CodeGeneratorShared::addCacheLocations(const CacheLocationList& locs, size_t* numLocs,
1652 size_t* curIndex)
1653 {
1654 size_t firstIndex = runtimeData_.length();
1655 size_t numLocations = 0;
1656 for (CacheLocationList::iterator iter = locs.begin(); iter != locs.end(); iter++) {
1657 // allocateData() ensures that sizeof(CacheLocation) is word-aligned.
1658 // If this changes, we will need to pad to ensure alignment.
1659 if (!allocateData(sizeof(CacheLocation), curIndex))
1660 return false;
1661 new (&runtimeData_[*curIndex]) CacheLocation(iter->pc, iter->script);
1662 numLocations++;
1663 }
1664 MOZ_ASSERT(numLocations != 0);
1665 *numLocs = numLocations;
1666 *curIndex = firstIndex;
1667 return true;
1668 }
1669
1670 ReciprocalMulConstants
computeDivisionConstants(uint32_t d,int maxLog)1671 CodeGeneratorShared::computeDivisionConstants(uint32_t d, int maxLog) {
1672 MOZ_ASSERT(maxLog >= 2 && maxLog <= 32);
1673 // In what follows, 0 < d < 2^maxLog and d is not a power of 2.
1674 MOZ_ASSERT(d < (uint64_t(1) << maxLog) && (d & (d - 1)) != 0);
1675
1676 // Speeding up division by non power-of-2 constants is possible by
1677 // calculating, during compilation, a value M such that high-order
1678 // bits of M*n correspond to the result of the division of n by d.
1679 // No value of M can serve this purpose for arbitrarily big values
1680 // of n but, for optimizing integer division, we're just concerned
1681 // with values of n whose absolute value is bounded (by fitting in
1682 // an integer type, say). With this in mind, we'll find a constant
1683 // M as above that works for -2^maxLog <= n < 2^maxLog; maxLog can
1684 // then be 31 for signed division or 32 for unsigned division.
1685 //
1686 // The original presentation of this technique appears in Hacker's
1687 // Delight, a book by Henry S. Warren, Jr.. A proof of correctness
1688 // for our version follows; we'll denote maxLog by L in the proof,
1689 // for conciseness.
1690 //
1691 // Formally, for |d| < 2^L, we'll compute two magic values M and s
1692 // in the ranges 0 <= M < 2^(L+1) and 0 <= s <= L such that
1693 // (M * n) >> (32 + s) = floor(n/d) if 0 <= n < 2^L
1694 // (M * n) >> (32 + s) = ceil(n/d) - 1 if -2^L <= n < 0.
1695 //
1696 // Define p = 32 + s, M = ceil(2^p/d), and assume that s satisfies
1697 // M - 2^p/d <= 2^(p-L)/d. (1)
1698 // (Observe that p = CeilLog32(d) + L satisfies this, as the right
1699 // side of (1) is at least one in this case). Then,
1700 //
1701 // a) If p <= CeilLog32(d) + L, then M < 2^(L+1) - 1.
1702 // Proof: Indeed, M is monotone in p and, for p equal to the above
1703 // value, the bounds 2^L > d >= 2^(p-L-1) + 1 readily imply that
1704 // 2^p / d < 2^p/(d - 1) * (d - 1)/d
1705 // <= 2^(L+1) * (1 - 1/d) < 2^(L+1) - 2.
1706 // The claim follows by applying the ceiling function.
1707 //
1708 // b) For any 0 <= n < 2^L, floor(Mn/2^p) = floor(n/d).
1709 // Proof: Put x = floor(Mn/2^p); it's the unique integer for which
1710 // Mn/2^p - 1 < x <= Mn/2^p. (2)
1711 // Using M >= 2^p/d on the LHS and (1) on the RHS, we get
1712 // n/d - 1 < x <= n/d + n/(2^L d) < n/d + 1/d.
1713 // Since x is an integer, it's not in the interval (n/d, (n+1)/d),
1714 // and so n/d - 1 < x <= n/d, which implies x = floor(n/d).
1715 //
1716 // c) For any -2^L <= n < 0, floor(Mn/2^p) + 1 = ceil(n/d).
1717 // Proof: The proof is similar. Equation (2) holds as above. Using
1718 // M > 2^p/d (d isn't a power of 2) on the RHS and (1) on the LHS,
1719 // n/d + n/(2^L d) - 1 < x < n/d.
1720 // Using n >= -2^L and summing 1,
1721 // n/d - 1/d < x + 1 < n/d + 1.
1722 // Since x + 1 is an integer, this implies n/d <= x + 1 < n/d + 1.
1723 // In other words, x + 1 = ceil(n/d).
1724 //
1725 // Condition (1) isn't necessary for the existence of M and s with
1726 // the properties above. Hacker's Delight provides a slightly less
1727 // restrictive condition when d >= 196611, at the cost of a 3-page
1728 // proof of correctness, for the case L = 31.
1729 //
1730 // Note that, since d*M - 2^p = d - (2^p)%d, (1) can be written as
1731 // 2^(p-L) >= d - (2^p)%d.
1732 // In order to avoid overflow in the (2^p) % d calculation, we can
1733 // compute it as (2^p-1) % d + 1, where 2^p-1 can then be computed
1734 // without overflow as UINT64_MAX >> (64-p).
1735
1736 // We now compute the least p >= 32 with the property above...
1737 int32_t p = 32;
1738 while ((uint64_t(1) << (p-maxLog)) + (UINT64_MAX >> (64-p)) % d + 1 < d)
1739 p++;
1740
1741 // ...and the corresponding M. For either the signed (L=31) or the
1742 // unsigned (L=32) case, this value can be too large (cf. item a).
1743 // Codegen can still multiply by M by multiplying by (M - 2^L) and
1744 // adjusting the value afterwards, if this is the case.
1745 ReciprocalMulConstants rmc;
1746 rmc.multiplier = (UINT64_MAX >> (64-p))/d + 1;
1747 rmc.shiftAmount = p - 32;
1748
1749 return rmc;
1750 }
1751
1752 #ifdef JS_TRACE_LOGGING
1753
1754 void
emitTracelogScript(bool isStart)1755 CodeGeneratorShared::emitTracelogScript(bool isStart)
1756 {
1757 if (!TraceLogTextIdEnabled(TraceLogger_Scripts))
1758 return;
1759
1760 Label done;
1761
1762 AllocatableRegisterSet regs(RegisterSet::Volatile());
1763 Register logger = regs.takeAnyGeneral();
1764 Register script = regs.takeAnyGeneral();
1765
1766 masm.Push(logger);
1767
1768 CodeOffset patchLogger = masm.movWithPatch(ImmPtr(nullptr), logger);
1769 masm.propagateOOM(patchableTraceLoggers_.append(patchLogger));
1770
1771 masm.branchTest32(Assembler::Zero, logger, logger, &done);
1772
1773 Address enabledAddress(logger, TraceLoggerThread::offsetOfEnabled());
1774 masm.branch32(Assembler::Equal, enabledAddress, Imm32(0), &done);
1775
1776 masm.Push(script);
1777
1778 CodeOffset patchScript = masm.movWithPatch(ImmWord(0), script);
1779 masm.propagateOOM(patchableTLScripts_.append(patchScript));
1780
1781 if (isStart)
1782 masm.tracelogStartId(logger, script);
1783 else
1784 masm.tracelogStopId(logger, script);
1785
1786 masm.Pop(script);
1787
1788 masm.bind(&done);
1789
1790 masm.Pop(logger);
1791 }
1792
1793 void
emitTracelogTree(bool isStart,uint32_t textId)1794 CodeGeneratorShared::emitTracelogTree(bool isStart, uint32_t textId)
1795 {
1796 if (!TraceLogTextIdEnabled(textId))
1797 return;
1798
1799 Label done;
1800 AllocatableRegisterSet regs(RegisterSet::Volatile());
1801 Register logger = regs.takeAnyGeneral();
1802
1803 masm.Push(logger);
1804
1805 CodeOffset patchLocation = masm.movWithPatch(ImmPtr(nullptr), logger);
1806 masm.propagateOOM(patchableTraceLoggers_.append(patchLocation));
1807
1808 masm.branchTest32(Assembler::Zero, logger, logger, &done);
1809
1810 Address enabledAddress(logger, TraceLoggerThread::offsetOfEnabled());
1811 masm.branch32(Assembler::Equal, enabledAddress, Imm32(0), &done);
1812
1813 if (isStart)
1814 masm.tracelogStartId(logger, textId);
1815 else
1816 masm.tracelogStopId(logger, textId);
1817
1818 masm.bind(&done);
1819
1820 masm.Pop(logger);
1821 }
1822
1823 void
emitTracelogTree(bool isStart,const char * text,TraceLoggerTextId enabledTextId)1824 CodeGeneratorShared::emitTracelogTree(bool isStart, const char* text,
1825 TraceLoggerTextId enabledTextId)
1826 {
1827 if (!TraceLogTextIdEnabled(enabledTextId))
1828 return;
1829
1830 Label done;
1831
1832 AllocatableRegisterSet regs(RegisterSet::Volatile());
1833 Register loggerReg = regs.takeAnyGeneral();
1834 Register eventReg = regs.takeAnyGeneral();
1835
1836 masm.Push(loggerReg);
1837
1838 CodeOffset patchLocation = masm.movWithPatch(ImmPtr(nullptr), loggerReg);
1839 masm.propagateOOM(patchableTraceLoggers_.append(patchLocation));
1840
1841 masm.branchTest32(Assembler::Zero, loggerReg, loggerReg, &done);
1842
1843 Address enabledAddress(loggerReg, TraceLoggerThread::offsetOfEnabled());
1844 masm.branch32(Assembler::Equal, enabledAddress, Imm32(0), &done);
1845
1846 masm.Push(eventReg);
1847
1848 PatchableTLEvent patchEvent(masm.movWithPatch(ImmWord(0), eventReg), text);
1849 masm.propagateOOM(patchableTLEvents_.append(Move(patchEvent)));
1850
1851 if (isStart)
1852 masm.tracelogStartId(loggerReg, eventReg);
1853 else
1854 masm.tracelogStopId(loggerReg, eventReg);
1855
1856 masm.Pop(eventReg);
1857
1858 masm.bind(&done);
1859
1860 masm.Pop(loggerReg);
1861 }
1862 #endif
1863
1864 } // namespace jit
1865 } // namespace js
1866