1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "jit/Bailouts.h"
8 #include "jit/BaselineFrame.h"
9 #include "jit/CalleeToken.h"
10 #include "jit/JitFrames.h"
11 #include "jit/JitRuntime.h"
12 #ifdef JS_ION_PERF
13 # include "jit/PerfSpewer.h"
14 #endif
15 #include "jit/VMFunctions.h"
16 #include "jit/x64/SharedICRegisters-x64.h"
17 #include "vm/JitActivation.h" // js::jit::JitActivation
18 #include "vm/JSContext.h"
19
20 #include "jit/MacroAssembler-inl.h"
21
22 using namespace js;
23 using namespace js::jit;
24
25 using mozilla::IsPowerOfTwo;
26
27 // This struct reflects the contents of the stack entry.
28 // Given a `CommonFrameLayout* frame`:
29 // - `frame->prevType()` should be `FrameType::CppToJSJit`.
30 // - Then EnterJITStackEntry starts at:
31 // (uint8_t*)frame + frame->headerSize() + frame->prevFrameLocalSize()
32 struct EnterJITStackEntry {
33 void* result;
34
35 #if defined(_WIN64)
36 struct XMM {
37 using XMM128 = char[16];
38 XMM128 xmm6;
39 XMM128 xmm7;
40 XMM128 xmm8;
41 XMM128 xmm9;
42 XMM128 xmm10;
43 XMM128 xmm11;
44 XMM128 xmm12;
45 XMM128 xmm13;
46 XMM128 xmm14;
47 XMM128 xmm15;
48 } xmm;
49
50 // 16-byte aligment for xmm registers above.
51 uint64_t xmmPadding;
52
53 void* rsi;
54 void* rdi;
55 #endif
56
57 void* r15;
58 void* r14;
59 void* r13;
60 void* r12;
61 void* rbx;
62 void* rbp;
63
64 // Pushed by CALL.
65 void* rip;
66 };
67
68 // All registers to save and restore. This includes the stack pointer, since we
69 // use the ability to reference register values on the stack by index.
70 static const LiveRegisterSet AllRegs =
71 LiveRegisterSet(GeneralRegisterSet(Registers::AllMask),
72 FloatRegisterSet(FloatRegisters::AllMask));
73
74 // Generates a trampoline for calling Jit compiled code from a C++ function.
75 // The trampoline use the EnterJitCode signature, with the standard x64 fastcall
76 // calling convention.
generateEnterJIT(JSContext * cx,MacroAssembler & masm)77 void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
78 AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
79
80 enterJITOffset_ = startTrampolineCode(masm);
81
82 masm.assertStackAlignment(ABIStackAlignment,
83 -int32_t(sizeof(uintptr_t)) /* return address */);
84
85 const Register reg_code = IntArgReg0;
86 const Register reg_argc = IntArgReg1;
87 const Register reg_argv = IntArgReg2;
88 static_assert(OsrFrameReg == IntArgReg3);
89
90 #if defined(_WIN64)
91 const Address token = Address(rbp, 16 + ShadowStackSpace);
92 const Operand scopeChain = Operand(rbp, 24 + ShadowStackSpace);
93 const Operand numStackValuesAddr = Operand(rbp, 32 + ShadowStackSpace);
94 const Operand result = Operand(rbp, 40 + ShadowStackSpace);
95 #else
96 const Register token = IntArgReg4;
97 const Register scopeChain = IntArgReg5;
98 const Operand numStackValuesAddr = Operand(rbp, 16 + ShadowStackSpace);
99 const Operand result = Operand(rbp, 24 + ShadowStackSpace);
100 #endif
101
102 // Note: the stack pushes below must match the fields in EnterJITStackEntry.
103
104 // Save old stack frame pointer, set new stack frame pointer.
105 masm.push(rbp);
106 masm.mov(rsp, rbp);
107
108 // Save non-volatile registers. These must be saved by the trampoline, rather
109 // than by the JIT'd code, because they are scanned by the conservative
110 // scanner.
111 masm.push(rbx);
112 masm.push(r12);
113 masm.push(r13);
114 masm.push(r14);
115 masm.push(r15);
116 #if defined(_WIN64)
117 masm.push(rdi);
118 masm.push(rsi);
119
120 // 16-byte aligment for vmovdqa
121 masm.subq(Imm32(sizeof(EnterJITStackEntry::XMM) + 8), rsp);
122
123 masm.vmovdqa(xmm6, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm6)));
124 masm.vmovdqa(xmm7, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm7)));
125 masm.vmovdqa(xmm8, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm8)));
126 masm.vmovdqa(xmm9, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm9)));
127 masm.vmovdqa(xmm10, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm10)));
128 masm.vmovdqa(xmm11, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm11)));
129 masm.vmovdqa(xmm12, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm12)));
130 masm.vmovdqa(xmm13, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm13)));
131 masm.vmovdqa(xmm14, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm14)));
132 masm.vmovdqa(xmm15, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm15)));
133 #endif
134
135 // Save arguments passed in registers needed after function call.
136 masm.push(result);
137
138 // End of pushes reflected in EnterJITStackEntry, i.e. EnterJITStackEntry
139 // starts at this rsp.
140 // Remember stack depth without padding and arguments, the frame descriptor
141 // will record the number of bytes pushed after this.
142 masm.mov(rsp, r14);
143
144 // Remember number of bytes occupied by argument vector
145 masm.mov(reg_argc, r13);
146
147 // if we are constructing, that also needs to include newTarget
148 {
149 Label noNewTarget;
150 masm.branchTest32(Assembler::Zero, token,
151 Imm32(CalleeToken_FunctionConstructing), &noNewTarget);
152
153 masm.addq(Imm32(1), r13);
154
155 masm.bind(&noNewTarget);
156 }
157
158 masm.shll(Imm32(3), r13); // r13 = argc * sizeof(Value)
159 static_assert(sizeof(Value) == 1 << 3, "Constant is baked in assembly code");
160
161 // Guarantee stack alignment of Jit frames.
162 //
163 // This code compensates for the offset created by the copy of the vector of
164 // arguments, such that the jit frame will be aligned once the return
165 // address is pushed on the stack.
166 //
167 // In the computation of the offset, we omit the size of the JitFrameLayout
168 // which is pushed on the stack, as the JitFrameLayout size is a multiple of
169 // the JitStackAlignment.
170 masm.mov(rsp, r12);
171 masm.subq(r13, r12);
172 static_assert(
173 sizeof(JitFrameLayout) % JitStackAlignment == 0,
174 "No need to consider the JitFrameLayout for aligning the stack");
175 masm.andl(Imm32(JitStackAlignment - 1), r12);
176 masm.subq(r12, rsp);
177
178 /***************************************************************
179 Loop over argv vector, push arguments onto stack in reverse order
180 ***************************************************************/
181
182 // r13 still stores the number of bytes in the argument vector.
183 masm.addq(reg_argv, r13); // r13 points above last argument or newTarget
184
185 // while r13 > rdx, push arguments.
186 {
187 Label header, footer;
188 masm.bind(&header);
189
190 masm.cmpPtr(r13, reg_argv);
191 masm.j(AssemblerX86Shared::BelowOrEqual, &footer);
192
193 masm.subq(Imm32(8), r13);
194 masm.push(Operand(r13, 0));
195 masm.jmp(&header);
196
197 masm.bind(&footer);
198 }
199
200 // Create the frame descriptor.
201 masm.subq(rsp, r14);
202 masm.makeFrameDescriptor(r14, FrameType::CppToJSJit, JitFrameLayout::Size());
203
204 // Push the number of actual arguments. |result| is used to store the
205 // actual number of arguments without adding an extra argument to the enter
206 // JIT.
207 masm.movq(result, reg_argc);
208 masm.unboxInt32(Operand(reg_argc, 0), reg_argc);
209 masm.push(reg_argc);
210
211 // Push the callee token.
212 masm.push(token);
213
214 // Push the descriptor.
215 masm.push(r14);
216
217 CodeLabel returnLabel;
218 CodeLabel oomReturnLabel;
219 {
220 // Handle Interpreter -> Baseline OSR.
221 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
222 regs.takeUnchecked(OsrFrameReg);
223 regs.take(rbp);
224 regs.take(reg_code);
225
226 // Ensure that |scratch| does not end up being JSReturnOperand.
227 // Do takeUnchecked because on Win64/x64, reg_code (IntArgReg0) and
228 // JSReturnOperand are the same (rcx). See bug 849398.
229 regs.takeUnchecked(JSReturnOperand);
230 Register scratch = regs.takeAny();
231
232 Label notOsr;
233 masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr);
234
235 Register numStackValues = regs.takeAny();
236 masm.movq(numStackValuesAddr, numStackValues);
237
238 // Push return address
239 masm.mov(&returnLabel, scratch);
240 masm.push(scratch);
241
242 // Push previous frame pointer.
243 masm.push(rbp);
244
245 // Reserve frame.
246 Register framePtr = rbp;
247 masm.subPtr(Imm32(BaselineFrame::Size()), rsp);
248
249 masm.touchFrameValues(numStackValues, scratch, framePtr);
250 masm.mov(rsp, framePtr);
251
252 // Reserve space for locals and stack values.
253 Register valuesSize = regs.takeAny();
254 masm.mov(numStackValues, valuesSize);
255 masm.shll(Imm32(3), valuesSize);
256 masm.subPtr(valuesSize, rsp);
257
258 // Enter exit frame.
259 masm.addPtr(
260 Imm32(BaselineFrame::Size() + BaselineFrame::FramePointerOffset),
261 valuesSize);
262 masm.makeFrameDescriptor(valuesSize, FrameType::BaselineJS,
263 ExitFrameLayout::Size());
264 masm.push(valuesSize);
265 masm.push(Imm32(0)); // Fake return address.
266 // No GC things to mark, push a bare token.
267 masm.loadJSContext(scratch);
268 masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
269
270 regs.add(valuesSize);
271
272 masm.push(framePtr);
273 masm.push(reg_code);
274
275 using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
276 uint32_t numStackValues);
277 masm.setupUnalignedABICall(scratch);
278 masm.passABIArg(framePtr); // BaselineFrame
279 masm.passABIArg(OsrFrameReg); // InterpreterFrame
280 masm.passABIArg(numStackValues);
281 masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
282 MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
283
284 masm.pop(reg_code);
285 masm.pop(framePtr);
286
287 MOZ_ASSERT(reg_code != ReturnReg);
288
289 Label error;
290 masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), rsp);
291 masm.addPtr(Imm32(BaselineFrame::Size()), framePtr);
292 masm.branchIfFalseBool(ReturnReg, &error);
293
294 // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
295 // if profiler instrumentation is enabled.
296 {
297 Label skipProfilingInstrumentation;
298 Register realFramePtr = numStackValues;
299 AbsoluteAddress addressOfEnabled(
300 cx->runtime()->geckoProfiler().addressOfEnabled());
301 masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
302 &skipProfilingInstrumentation);
303 masm.lea(Operand(framePtr, sizeof(void*)), realFramePtr);
304 masm.profilerEnterFrame(realFramePtr, scratch);
305 masm.bind(&skipProfilingInstrumentation);
306 }
307
308 masm.jump(reg_code);
309
310 // OOM: load error value, discard return address and previous frame
311 // pointer and return.
312 masm.bind(&error);
313 masm.mov(framePtr, rsp);
314 masm.addPtr(Imm32(2 * sizeof(uintptr_t)), rsp);
315 masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
316 masm.mov(&oomReturnLabel, scratch);
317 masm.jump(scratch);
318
319 masm.bind(¬Osr);
320 masm.movq(scopeChain, R1.scratchReg());
321 }
322
323 // The call will push the return address on the stack, thus we check that
324 // the stack would be aligned once the call is complete.
325 masm.assertStackAlignment(JitStackAlignment, sizeof(uintptr_t));
326
327 // Call function.
328 masm.callJitNoProfiler(reg_code);
329
330 {
331 // Interpreter -> Baseline OSR will return here.
332 masm.bind(&returnLabel);
333 masm.addCodeLabel(returnLabel);
334 masm.bind(&oomReturnLabel);
335 masm.addCodeLabel(oomReturnLabel);
336 }
337
338 // Pop arguments and padding from stack.
339 masm.pop(r14); // Pop and decode descriptor.
340 masm.shrq(Imm32(FRAMESIZE_SHIFT), r14);
341 masm.pop(r12); // Discard calleeToken.
342 masm.pop(r12); // Discard numActualArgs.
343 masm.addq(r14, rsp); // Remove arguments.
344
345 /*****************************************************************
346 Place return value where it belongs, pop all saved registers
347 *****************************************************************/
348 masm.pop(r12); // vp
349 masm.storeValue(JSReturnOperand, Operand(r12, 0));
350
351 // Restore non-volatile registers.
352 #if defined(_WIN64)
353 masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm6)), xmm6);
354 masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm7)), xmm7);
355 masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm8)), xmm8);
356 masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm9)), xmm9);
357 masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm10)), xmm10);
358 masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm11)), xmm11);
359 masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm12)), xmm12);
360 masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm13)), xmm13);
361 masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm14)), xmm14);
362 masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm15)), xmm15);
363
364 masm.addq(Imm32(sizeof(EnterJITStackEntry::XMM) + 8), rsp);
365
366 masm.pop(rsi);
367 masm.pop(rdi);
368 #endif
369 masm.pop(r15);
370 masm.pop(r14);
371 masm.pop(r13);
372 masm.pop(r12);
373 masm.pop(rbx);
374
375 // Restore frame pointer and return.
376 masm.pop(rbp);
377 masm.ret();
378 }
379
380 // static
381 mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
getCppEntryRegisters(JitFrameLayout * frameStackAddress)382 JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
383 if (frameStackAddress->prevType() != FrameType::CppToJSJit) {
384 // This is not a CppToJSJit frame, there are no C++ registers here.
385 return mozilla::Nothing{};
386 }
387
388 // The entry is (frame size stored in descriptor) bytes past the header.
389 MOZ_ASSERT(frameStackAddress->headerSize() == JitFrameLayout::Size());
390 const size_t offsetToCppEntry =
391 JitFrameLayout::Size() + frameStackAddress->prevFrameLocalSize();
392 EnterJITStackEntry* enterJITStackEntry =
393 reinterpret_cast<EnterJITStackEntry*>(
394 reinterpret_cast<uint8_t*>(frameStackAddress) + offsetToCppEntry);
395
396 // Extract native function call registers.
397 ::JS::ProfilingFrameIterator::RegisterState registerState;
398 registerState.fp = enterJITStackEntry->rbp;
399 registerState.pc = enterJITStackEntry->rip;
400 // sp should be inside the caller's frame, so set sp to the value of the stack
401 // pointer before the call to the EnterJit trampoline.
402 registerState.sp = &enterJITStackEntry->rip + 1;
403 // No lr in this world.
404 registerState.lr = nullptr;
405 return mozilla::Some(registerState);
406 }
407
408 // Push AllRegs in a way that is compatible with RegisterDump, regardless of
409 // what PushRegsInMask might do to reduce the set size.
DumpAllRegs(MacroAssembler & masm)410 static void DumpAllRegs(MacroAssembler& masm) {
411 #ifdef ENABLE_WASM_SIMD
412 masm.PushRegsInMask(AllRegs);
413 #else
414 // When SIMD isn't supported, PushRegsInMask reduces the set of float
415 // registers to be double-sized, while the RegisterDump expects each of
416 // the float registers to have the maximal possible size
417 // (Simd128DataSize). To work around this, we just spill the double
418 // registers by hand here, using the register dump offset directly.
419 for (GeneralRegisterBackwardIterator iter(AllRegs.gprs()); iter.more();
420 ++iter) {
421 masm.Push(*iter);
422 }
423
424 masm.reserveStack(sizeof(RegisterDump::FPUArray));
425 for (FloatRegisterBackwardIterator iter(AllRegs.fpus()); iter.more();
426 ++iter) {
427 FloatRegister reg = *iter;
428 Address spillAddress(StackPointer, reg.getRegisterDumpOffsetInBytes());
429 masm.storeDouble(reg, spillAddress);
430 }
431 #endif
432 }
433
generateInvalidator(MacroAssembler & masm,Label * bailoutTail)434 void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
435 AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
436
437 // See explanatory comment in x86's JitRuntime::generateInvalidator.
438
439 invalidatorOffset_ = startTrampolineCode(masm);
440
441 // Push registers such that we can access them from [base + code].
442 DumpAllRegs(masm);
443
444 masm.movq(rsp, rax); // Argument to jit::InvalidationBailout.
445
446 // Make space for InvalidationBailout's frameSize outparam.
447 masm.reserveStack(sizeof(size_t));
448 masm.movq(rsp, rbx);
449
450 // Make space for InvalidationBailout's bailoutInfo outparam.
451 masm.reserveStack(sizeof(void*));
452 masm.movq(rsp, r9);
453
454 using Fn = bool (*)(InvalidationBailoutStack * sp, size_t * frameSizeOut,
455 BaselineBailoutInfo * *info);
456 masm.setupUnalignedABICall(rdx);
457 masm.passABIArg(rax);
458 masm.passABIArg(rbx);
459 masm.passABIArg(r9);
460 masm.callWithABI<Fn, InvalidationBailout>(
461 MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
462
463 masm.pop(r9); // Get the bailoutInfo outparam.
464 masm.pop(rbx); // Get the frameSize outparam.
465
466 // Pop the machine state and the dead frame.
467 masm.lea(Operand(rsp, rbx, TimesOne, sizeof(InvalidationBailoutStack)), rsp);
468
469 // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9.
470 masm.jmp(bailoutTail);
471 }
472
generateArgumentsRectifier(MacroAssembler & masm,ArgumentsRectifierKind kind)473 void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
474 ArgumentsRectifierKind kind) {
475 // Do not erase the frame pointer in this function.
476
477 AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
478
479 switch (kind) {
480 case ArgumentsRectifierKind::Normal:
481 argumentsRectifierOffset_ = startTrampolineCode(masm);
482 break;
483 case ArgumentsRectifierKind::TrialInlining:
484 trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
485 break;
486 }
487
488 // Caller:
489 // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- rsp
490
491 // Add |this|, in the counter of known arguments.
492 masm.loadPtr(Address(rsp, RectifierFrameLayout::offsetOfNumActualArgs()), r8);
493 masm.addl(Imm32(1), r8);
494
495 // Load |nformals| into %rcx.
496 masm.loadPtr(Address(rsp, RectifierFrameLayout::offsetOfCalleeToken()), rax);
497 masm.mov(rax, rcx);
498 masm.andq(Imm32(uint32_t(CalleeTokenMask)), rcx);
499 masm.load32(Operand(rcx, JSFunction::offsetOfFlagsAndArgCount()), rcx);
500 masm.rshift32(Imm32(JSFunction::ArgCountShift), rcx);
501
502 // Stash another copy in r11, since we are going to do destructive operations
503 // on rcx
504 masm.mov(rcx, r11);
505
506 static_assert(
507 CalleeToken_FunctionConstructing == 1,
508 "Ensure that we can use the constructing bit to count the value");
509 masm.mov(rax, rdx);
510 masm.andq(Imm32(uint32_t(CalleeToken_FunctionConstructing)), rdx);
511
512 // Including |this|, and |new.target|, there are (|nformals| + 1 +
513 // isConstructing) arguments to push to the stack. Then we push a
514 // JitFrameLayout. We compute the padding expressed in the number of extra
515 // |undefined| values to push on the stack.
516 static_assert(
517 sizeof(JitFrameLayout) % JitStackAlignment == 0,
518 "No need to consider the JitFrameLayout for aligning the stack");
519 static_assert(
520 JitStackAlignment % sizeof(Value) == 0,
521 "Ensure that we can pad the stack by pushing extra UndefinedValue");
522 static_assert(IsPowerOfTwo(JitStackValueAlignment),
523 "must have power of two for masm.andl to do its job");
524
525 masm.addl(
526 Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */),
527 rcx);
528 masm.addl(rdx, rcx);
529 masm.andl(Imm32(~(JitStackValueAlignment - 1)), rcx);
530
531 // Load the number of |undefined|s to push into %rcx.
532 masm.subq(r8, rcx);
533
534 // Caller:
535 // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- rsp <- r9
536 // '------ #r8 -------'
537 //
538 // Rectifier frame:
539 // [undef] [undef] [undef] [arg2] [arg1] [this] [ [argc] [callee]
540 // [descr] [raddr] ]
541 // '------- #rcx --------' '------ #r8 -------'
542
543 // Copy the number of actual arguments into rdx. Use lea to subtract 1 for
544 // |this|.
545 masm.lea(Operand(r8, -1), rdx);
546
547 masm.moveValue(UndefinedValue(), ValueOperand(r10));
548
549 masm.movq(rsp, r9); // Save %rsp.
550
551 // Push undefined. (including the padding)
552 {
553 Label undefLoopTop;
554 masm.bind(&undefLoopTop);
555
556 masm.push(r10);
557 masm.subl(Imm32(1), rcx);
558 masm.j(Assembler::NonZero, &undefLoopTop);
559 }
560
561 // Get the topmost argument.
562 static_assert(sizeof(Value) == 8, "TimesEight is used to skip arguments");
563
564 // | - sizeof(Value)| is used to put rcx such that we can read the last
565 // argument, and not the value which is after.
566 BaseIndex b(r9, r8, TimesEight, sizeof(RectifierFrameLayout) - sizeof(Value));
567 masm.lea(Operand(b), rcx);
568
569 // Copy & Push arguments, |nargs| + 1 times (to include |this|).
570 {
571 Label copyLoopTop;
572
573 masm.bind(©LoopTop);
574 masm.push(Operand(rcx, 0x0));
575 masm.subq(Imm32(sizeof(Value)), rcx);
576 masm.subl(Imm32(1), r8);
577 masm.j(Assembler::NonZero, ©LoopTop);
578 }
579
580 // if constructing, copy newTarget
581 {
582 Label notConstructing;
583
584 masm.branchTest32(Assembler::Zero, rax,
585 Imm32(CalleeToken_FunctionConstructing),
586 ¬Constructing);
587
588 // thisFrame[numFormals] = prevFrame[argc]
589 ValueOperand newTarget(r10);
590
591 // +1 for |this|. We want vp[argc], so don't subtract 1
592 BaseIndex newTargetSrc(r9, rdx, TimesEight,
593 sizeof(RectifierFrameLayout) + sizeof(Value));
594 masm.loadValue(newTargetSrc, newTarget);
595
596 // Again, 1 for |this|
597 BaseIndex newTargetDest(rsp, r11, TimesEight, sizeof(Value));
598 masm.storeValue(newTarget, newTargetDest);
599
600 masm.bind(¬Constructing);
601 }
602
603 // Caller:
604 // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- r9
605 //
606 //
607 // Rectifier frame:
608 // [undef] [undef] [undef] [arg2] [arg1] [this] <- rsp [ [argc] [callee]
609 // [descr] [raddr] ]
610 //
611
612 // Construct descriptor.
613 masm.subq(rsp, r9);
614 masm.makeFrameDescriptor(r9, FrameType::Rectifier, JitFrameLayout::Size());
615
616 // Construct JitFrameLayout.
617 masm.push(rdx); // numActualArgs
618 masm.push(rax); // callee token
619 masm.push(r9); // descriptor
620
621 // Call the target function.
622 masm.andq(Imm32(uint32_t(CalleeTokenMask)), rax);
623 switch (kind) {
624 case ArgumentsRectifierKind::Normal:
625 masm.loadJitCodeRaw(rax, rax);
626 argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(rax);
627 break;
628 case ArgumentsRectifierKind::TrialInlining:
629 Label noBaselineScript, done;
630 masm.loadBaselineJitCodeRaw(rax, rbx, &noBaselineScript);
631 masm.callJitNoProfiler(rbx);
632 masm.jump(&done);
633
634 // See BaselineCacheIRCompiler::emitCallInlinedFunction.
635 masm.bind(&noBaselineScript);
636 masm.loadJitCodeRaw(rax, rax);
637 masm.callJitNoProfiler(rax);
638 masm.bind(&done);
639 break;
640 }
641
642 // Remove the rectifier frame.
643 masm.pop(r9); // r9 <- descriptor with FrameType.
644 masm.shrq(Imm32(FRAMESIZE_SHIFT), r9);
645 masm.pop(r11); // Discard calleeToken.
646 masm.pop(r11); // Discard numActualArgs.
647 masm.addq(r9, rsp); // Discard pushed arguments.
648
649 masm.ret();
650 }
651
PushBailoutFrame(MacroAssembler & masm,Register spArg)652 static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
653 // Push registers such that we can access them from [base + code].
654 DumpAllRegs(masm);
655
656 // Get the stack pointer into a register, pre-alignment.
657 masm.movq(rsp, spArg);
658 }
659
GenerateBailoutThunk(MacroAssembler & masm,uint32_t frameClass,Label * bailoutTail)660 static void GenerateBailoutThunk(MacroAssembler& masm, uint32_t frameClass,
661 Label* bailoutTail) {
662 PushBailoutFrame(masm, r8);
663
664 // Make space for Bailout's bailoutInfo outparam.
665 masm.reserveStack(sizeof(void*));
666 masm.movq(rsp, r9);
667
668 // Call the bailout function.
669 using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
670 masm.setupUnalignedABICall(rax);
671 masm.passABIArg(r8);
672 masm.passABIArg(r9);
673 masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
674 CheckUnsafeCallWithABI::DontCheckOther);
675
676 masm.pop(r9); // Get the bailoutInfo outparam.
677
678 // Stack is:
679 // [frame]
680 // snapshotOffset
681 // frameSize
682 // [bailoutFrame]
683 //
684 // Remove both the bailout frame and the topmost Ion frame's stack.
685 static const uint32_t BailoutDataSize = sizeof(RegisterDump);
686 masm.addq(Imm32(BailoutDataSize), rsp);
687 masm.pop(rcx);
688 masm.lea(Operand(rsp, rcx, TimesOne, sizeof(void*)), rsp);
689
690 // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9.
691 masm.jmp(bailoutTail);
692 }
693
generateBailoutTable(MacroAssembler & masm,Label * bailoutTail,uint32_t frameClass)694 JitRuntime::BailoutTable JitRuntime::generateBailoutTable(MacroAssembler& masm,
695 Label* bailoutTail,
696 uint32_t frameClass) {
697 MOZ_CRASH("x64 does not use bailout tables");
698 }
699
generateBailoutHandler(MacroAssembler & masm,Label * bailoutTail)700 void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
701 Label* bailoutTail) {
702 AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
703
704 bailoutHandlerOffset_ = startTrampolineCode(masm);
705
706 GenerateBailoutThunk(masm, NO_FRAME_SIZE_CLASS_ID, bailoutTail);
707 }
708
generateVMWrapper(JSContext * cx,MacroAssembler & masm,const VMFunctionData & f,DynFn nativeFun,uint32_t * wrapperOffset)709 bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
710 const VMFunctionData& f, DynFn nativeFun,
711 uint32_t* wrapperOffset) {
712 AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
713
714 *wrapperOffset = startTrampolineCode(masm);
715
716 // Avoid conflicts with argument registers while discarding the result after
717 // the function call.
718 AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
719
720 static_assert(
721 (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
722 "Wrapper register set must be a superset of Volatile register set");
723
724 // The context is the first argument.
725 Register cxreg = IntArgReg0;
726 regs.take(cxreg);
727
728 // Stack is:
729 // ... frame ...
730 // +12 [args]
731 // +8 descriptor
732 // +0 returnAddress
733 //
734 // We're aligned to an exit frame, so link it up.
735 masm.loadJSContext(cxreg);
736 masm.enterExitFrame(cxreg, regs.getAny(), &f);
737
738 // Save the current stack pointer as the base for copying arguments.
739 Register argsBase = InvalidReg;
740 if (f.explicitArgs) {
741 argsBase = r10;
742 regs.take(argsBase);
743 masm.lea(Operand(rsp, ExitFrameLayout::SizeWithFooter()), argsBase);
744 }
745
746 // Reserve space for the outparameter.
747 Register outReg = InvalidReg;
748 switch (f.outParam) {
749 case Type_Value:
750 outReg = regs.takeAny();
751 masm.reserveStack(sizeof(Value));
752 masm.movq(esp, outReg);
753 break;
754
755 case Type_Handle:
756 outReg = regs.takeAny();
757 masm.PushEmptyRooted(f.outParamRootType);
758 masm.movq(esp, outReg);
759 break;
760
761 case Type_Int32:
762 case Type_Bool:
763 case Type_Pointer:
764 outReg = regs.takeAny();
765 masm.reserveStack(sizeof(uintptr_t));
766 masm.movq(esp, outReg);
767 break;
768
769 case Type_Double:
770 outReg = regs.takeAny();
771 masm.reserveStack(sizeof(double));
772 masm.movq(esp, outReg);
773 break;
774
775 default:
776 MOZ_ASSERT(f.outParam == Type_Void);
777 break;
778 }
779
780 if (!generateTLEnterVM(masm, f)) {
781 return false;
782 }
783
784 masm.setupUnalignedABICall(regs.getAny());
785 masm.passABIArg(cxreg);
786
787 size_t argDisp = 0;
788
789 // Copy arguments.
790 for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
791 switch (f.argProperties(explicitArg)) {
792 case VMFunctionData::WordByValue:
793 if (f.argPassedInFloatReg(explicitArg)) {
794 masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE);
795 } else {
796 masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
797 }
798 argDisp += sizeof(void*);
799 break;
800 case VMFunctionData::WordByRef:
801 masm.passABIArg(
802 MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
803 MoveOp::GENERAL);
804 argDisp += sizeof(void*);
805 break;
806 case VMFunctionData::DoubleByValue:
807 case VMFunctionData::DoubleByRef:
808 MOZ_CRASH("NYI: x64 callVM should not be used with 128bits values.");
809 }
810 }
811
812 // Copy the implicit outparam, if any.
813 if (outReg != InvalidReg) {
814 masm.passABIArg(outReg);
815 }
816
817 masm.callWithABI(nativeFun, MoveOp::GENERAL,
818 CheckUnsafeCallWithABI::DontCheckHasExitFrame);
819
820 if (!generateTLExitVM(masm, f)) {
821 return false;
822 }
823
824 // Test for failure.
825 switch (f.failType()) {
826 case Type_Cell:
827 masm.branchTestPtr(Assembler::Zero, rax, rax, masm.failureLabel());
828 break;
829 case Type_Bool:
830 masm.testb(rax, rax);
831 masm.j(Assembler::Zero, masm.failureLabel());
832 break;
833 case Type_Void:
834 break;
835 default:
836 MOZ_CRASH("unknown failure kind");
837 }
838
839 // Load the outparam and free any allocated stack.
840 switch (f.outParam) {
841 case Type_Handle:
842 masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
843 break;
844
845 case Type_Value:
846 masm.loadValue(Address(esp, 0), JSReturnOperand);
847 masm.freeStack(sizeof(Value));
848 break;
849
850 case Type_Int32:
851 masm.load32(Address(esp, 0), ReturnReg);
852 masm.freeStack(sizeof(uintptr_t));
853 break;
854
855 case Type_Bool:
856 masm.load8ZeroExtend(Address(esp, 0), ReturnReg);
857 masm.freeStack(sizeof(uintptr_t));
858 break;
859
860 case Type_Double:
861 masm.loadDouble(Address(esp, 0), ReturnDoubleReg);
862 masm.freeStack(sizeof(double));
863 break;
864
865 case Type_Pointer:
866 masm.loadPtr(Address(esp, 0), ReturnReg);
867 masm.freeStack(sizeof(uintptr_t));
868 break;
869
870 default:
871 MOZ_ASSERT(f.outParam == Type_Void);
872 break;
873 }
874
875 // Until C++ code is instrumented against Spectre, prevent speculative
876 // execution from returning any private data.
877 if (f.returnsData() && JitOptions.spectreJitToCxxCalls) {
878 masm.speculationBarrier();
879 }
880
881 masm.leaveExitFrame();
882 masm.retn(Imm32(sizeof(ExitFrameLayout) +
883 f.explicitStackSlots() * sizeof(void*) +
884 f.extraValuesToPop * sizeof(Value)));
885
886 return true;
887 }
888
generatePreBarrier(JSContext * cx,MacroAssembler & masm,MIRType type)889 uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
890 MIRType type) {
891 AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
892
893 uint32_t offset = startTrampolineCode(masm);
894
895 static_assert(PreBarrierReg == rdx);
896 Register temp1 = rax;
897 Register temp2 = rbx;
898 Register temp3 = rcx;
899 masm.push(temp1);
900 masm.push(temp2);
901 masm.push(temp3);
902
903 Label noBarrier;
904 masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
905 &noBarrier);
906
907 // Call into C++ to mark this GC thing.
908 masm.pop(temp3);
909 masm.pop(temp2);
910 masm.pop(temp1);
911
912 LiveRegisterSet regs =
913 LiveRegisterSet(GeneralRegisterSet(Registers::VolatileMask),
914 FloatRegisterSet(FloatRegisters::VolatileMask));
915 masm.PushRegsInMask(regs);
916
917 masm.mov(ImmPtr(cx->runtime()), rcx);
918
919 masm.setupUnalignedABICall(rax);
920 masm.passABIArg(rcx);
921 masm.passABIArg(rdx);
922 masm.callWithABI(JitPreWriteBarrier(type));
923
924 masm.PopRegsInMask(regs);
925 masm.ret();
926
927 masm.bind(&noBarrier);
928 masm.pop(temp3);
929 masm.pop(temp2);
930 masm.pop(temp1);
931 masm.ret();
932
933 return offset;
934 }
935
generateExceptionTailStub(MacroAssembler & masm,Label * profilerExitTail)936 void JitRuntime::generateExceptionTailStub(MacroAssembler& masm,
937 Label* profilerExitTail) {
938 AutoCreatedBy acb(masm, "JitRuntime::generateExceptionTailStub");
939
940 exceptionTailOffset_ = startTrampolineCode(masm);
941
942 masm.bind(masm.failureLabel());
943 masm.handleFailureWithHandlerTail(profilerExitTail);
944 }
945
generateBailoutTailStub(MacroAssembler & masm,Label * bailoutTail)946 void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
947 Label* bailoutTail) {
948 AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
949
950 bailoutTailOffset_ = startTrampolineCode(masm);
951 masm.bind(bailoutTail);
952
953 masm.generateBailoutTail(rdx, r9);
954 }
955
generateProfilerExitFrameTailStub(MacroAssembler & masm,Label * profilerExitTail)956 void JitRuntime::generateProfilerExitFrameTailStub(MacroAssembler& masm,
957 Label* profilerExitTail) {
958 AutoCreatedBy acb(masm, "JitRuntime::generateProfilerExitFrameTailStub");
959
960 profilerExitFrameTailOffset_ = startTrampolineCode(masm);
961 masm.bind(profilerExitTail);
962
963 Register scratch1 = r8;
964 Register scratch2 = r9;
965 Register scratch3 = r10;
966 Register scratch4 = r11;
967
968 //
969 // The code generated below expects that the current stack pointer points
970 // to an Ion or Baseline frame, at the state it would be immediately
971 // before a ret(). Thus, after this stub's business is done, it executes
972 // a ret() and returns directly to the caller script, on behalf of the
973 // callee script that jumped to this code.
974 //
975 // Thus the expected stack is:
976 //
977 // StackPointer ----+
978 // v
979 // ..., ActualArgc, CalleeToken, Descriptor, ReturnAddr
980 // MEM-HI MEM-LOW
981 //
982 //
983 // The generated jitcode is responsible for overwriting the
984 // jitActivation->lastProfilingFrame field with a pointer to the previous
985 // Ion or Baseline jit-frame that was pushed before this one. It is also
986 // responsible for overwriting jitActivation->lastProfilingCallSite with
987 // the return address into that frame. The frame could either be an
988 // immediate "caller" frame, or it could be a frame in a previous
989 // JitActivation (if the current frame was entered from C++, and the C++
990 // was entered by some caller jit-frame further down the stack).
991 //
992 // So this jitcode is responsible for "walking up" the jit stack, finding
993 // the previous Ion or Baseline JS frame, and storing its address and the
994 // return address into the appropriate fields on the current jitActivation.
995 //
996 // There are a fixed number of different path types that can lead to the
997 // current frame, which is either a baseline or ion frame:
998 //
999 // <Baseline-Or-Ion>
1000 // ^
1001 // |
1002 // ^--- Ion
1003 // |
1004 // ^--- Baseline Stub <---- Baseline
1005 // |
1006 // ^--- Argument Rectifier
1007 // | ^
1008 // | |
1009 // | ^--- Ion
1010 // | |
1011 // | ^--- Baseline Stub <---- Baseline
1012 // |
1013 // ^--- Entry Frame (From C++)
1014 //
1015 Register actReg = scratch4;
1016 masm.loadJSContext(actReg);
1017 masm.loadPtr(Address(actReg, offsetof(JSContext, profilingActivation_)),
1018 actReg);
1019
1020 Address lastProfilingFrame(actReg,
1021 JitActivation::offsetOfLastProfilingFrame());
1022 Address lastProfilingCallSite(actReg,
1023 JitActivation::offsetOfLastProfilingCallSite());
1024
1025 #ifdef DEBUG
1026 // Ensure that frame we are exiting is current lastProfilingFrame
1027 {
1028 masm.loadPtr(lastProfilingFrame, scratch1);
1029 Label checkOk;
1030 masm.branchPtr(Assembler::Equal, scratch1, ImmWord(0), &checkOk);
1031 masm.branchPtr(Assembler::Equal, StackPointer, scratch1, &checkOk);
1032 masm.assumeUnreachable(
1033 "Mismatch between stored lastProfilingFrame and current stack "
1034 "pointer.");
1035 masm.bind(&checkOk);
1036 }
1037 #endif
1038
1039 // Load the frame descriptor into |scratch1|, figure out what to do depending
1040 // on its type.
1041 masm.loadPtr(Address(StackPointer, JitFrameLayout::offsetOfDescriptor()),
1042 scratch1);
1043
1044 // Going into the conditionals, we will have:
1045 // FrameDescriptor.size in scratch1
1046 // FrameDescriptor.type in scratch2
1047 masm.movePtr(scratch1, scratch2);
1048 masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch1);
1049 masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch2);
1050
1051 // Handling of each case is dependent on FrameDescriptor.type
1052 Label handle_IonJS;
1053 Label handle_BaselineStub;
1054 Label handle_Rectifier;
1055 Label handle_IonICCall;
1056 Label handle_Entry;
1057 Label end;
1058
1059 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::IonJS),
1060 &handle_IonJS);
1061 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::BaselineJS),
1062 &handle_IonJS);
1063 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::BaselineStub),
1064 &handle_BaselineStub);
1065 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::Rectifier),
1066 &handle_Rectifier);
1067 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::IonICCall),
1068 &handle_IonICCall);
1069 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::CppToJSJit),
1070 &handle_Entry);
1071
1072 // The WasmToJSJit is just another kind of entry
1073 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::WasmToJSJit),
1074 &handle_Entry);
1075
1076 masm.assumeUnreachable(
1077 "Invalid caller frame type when exiting from Ion frame.");
1078
1079 //
1080 // FrameType::IonJS
1081 //
1082 // Stack layout:
1083 // ...
1084 // Ion-Descriptor
1085 // Prev-FP ---> Ion-ReturnAddr
1086 // ... previous frame data ... |- Descriptor.Size
1087 // ... arguments ... |
1088 // ActualArgc |
1089 // CalleeToken |- JitFrameLayout::Size()
1090 // Descriptor |
1091 // FP -----> ReturnAddr |
1092 //
1093 masm.bind(&handle_IonJS);
1094 {
1095 // returning directly to an IonJS frame. Store return addr to frame
1096 // in lastProfilingCallSite.
1097 masm.loadPtr(Address(StackPointer, JitFrameLayout::offsetOfReturnAddress()),
1098 scratch2);
1099 masm.storePtr(scratch2, lastProfilingCallSite);
1100
1101 // Store return frame in lastProfilingFrame.
1102 // scratch2 := StackPointer + Descriptor.size*1 + JitFrameLayout::Size();
1103 masm.lea(Operand(StackPointer, scratch1, TimesOne, JitFrameLayout::Size()),
1104 scratch2);
1105 masm.storePtr(scratch2, lastProfilingFrame);
1106 masm.ret();
1107 }
1108
1109 //
1110 // FrameType::BaselineStub
1111 //
1112 // Look past the stub and store the frame pointer to
1113 // the baselineJS frame prior to it.
1114 //
1115 // Stack layout:
1116 // ...
1117 // BL-Descriptor
1118 // Prev-FP ---> BL-ReturnAddr
1119 // +-----> BL-PrevFramePointer
1120 // | ... BL-FrameData ...
1121 // | BLStub-Descriptor
1122 // | BLStub-ReturnAddr
1123 // | BLStub-StubPointer |
1124 // +------ BLStub-SavedFramePointer |- Descriptor.Size
1125 // ... arguments ... |
1126 // ActualArgc |
1127 // CalleeToken |- JitFrameLayout::Size()
1128 // Descriptor |
1129 // FP -----> ReturnAddr |
1130 //
1131 // We take advantage of the fact that the stub frame saves the frame
1132 // pointer pointing to the baseline frame, so a bunch of calculation can
1133 // be avoided.
1134 //
1135 masm.bind(&handle_BaselineStub);
1136 {
1137 BaseIndex stubFrameReturnAddr(
1138 StackPointer, scratch1, TimesOne,
1139 JitFrameLayout::Size() +
1140 BaselineStubFrameLayout::offsetOfReturnAddress());
1141 masm.loadPtr(stubFrameReturnAddr, scratch2);
1142 masm.storePtr(scratch2, lastProfilingCallSite);
1143
1144 BaseIndex stubFrameSavedFramePtr(
1145 StackPointer, scratch1, TimesOne,
1146 JitFrameLayout::Size() - (2 * sizeof(void*)));
1147 masm.loadPtr(stubFrameSavedFramePtr, scratch2);
1148 masm.addPtr(Imm32(sizeof(void*)), scratch2); // Skip past BL-PrevFramePtr
1149 masm.storePtr(scratch2, lastProfilingFrame);
1150 masm.ret();
1151 }
1152
1153 //
1154 // FrameType::Rectifier
1155 //
1156 // The rectifier frame can be preceded by either an IonJS, a BaselineStub,
1157 // or a CppToJSJit/WasmToJSJit frame.
1158 //
1159 // Stack layout if caller of rectifier was Ion or CppToJSJit/WasmToJSJit:
1160 //
1161 // Ion-Descriptor
1162 // Ion-ReturnAddr
1163 // ... ion frame data ... |- Rect-Descriptor.Size
1164 // < COMMON LAYOUT >
1165 //
1166 // Stack layout if caller of rectifier was Baseline:
1167 //
1168 // BL-Descriptor
1169 // Prev-FP ---> BL-ReturnAddr
1170 // +-----> BL-SavedFramePointer
1171 // | ... baseline frame data ...
1172 // | BLStub-Descriptor
1173 // | BLStub-ReturnAddr
1174 // | BLStub-StubPointer |
1175 // +------ BLStub-SavedFramePointer |- Rect-Descriptor.Size
1176 // ... args to rectifier ... |
1177 // < COMMON LAYOUT >
1178 //
1179 // Common stack layout:
1180 //
1181 // ActualArgc |
1182 // CalleeToken |- IonRectitiferFrameLayout::Size()
1183 // Rect-Descriptor |
1184 // Rect-ReturnAddr |
1185 // ... rectifier data & args ... |- Descriptor.Size
1186 // ActualArgc |
1187 // CalleeToken |- JitFrameLayout::Size()
1188 // Descriptor |
1189 // FP -----> ReturnAddr |
1190 //
1191 masm.bind(&handle_Rectifier);
1192 {
1193 // scratch2 := StackPointer + Descriptor.size + JitFrameLayout::Size()
1194 masm.lea(Operand(StackPointer, scratch1, TimesOne, JitFrameLayout::Size()),
1195 scratch2);
1196 masm.loadPtr(Address(scratch2, RectifierFrameLayout::offsetOfDescriptor()),
1197 scratch3);
1198 masm.movePtr(scratch3, scratch1);
1199 masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch3);
1200 masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch1);
1201
1202 // Now |scratch1| contains Rect-Descriptor.Size
1203 // and |scratch2| points to Rectifier frame
1204 // and |scratch3| contains Rect-Descriptor.Type
1205
1206 masm.assertRectifierFrameParentType(scratch3);
1207
1208 // Check for either Ion or something else frame.
1209 Label notIonFrame;
1210 masm.branch32(Assembler::NotEqual, scratch3, Imm32(FrameType::IonJS),
1211 ¬IonFrame);
1212
1213 // Handle Rectifier <- IonJS
1214 // scratch3 := RectFrame[ReturnAddr]
1215 masm.loadPtr(
1216 Address(scratch2, RectifierFrameLayout::offsetOfReturnAddress()),
1217 scratch3);
1218 masm.storePtr(scratch3, lastProfilingCallSite);
1219
1220 // scratch3 := RectFrame + Rect-Descriptor.Size +
1221 // RectifierFrameLayout::Size()
1222 masm.lea(
1223 Operand(scratch2, scratch1, TimesOne, RectifierFrameLayout::Size()),
1224 scratch3);
1225 masm.storePtr(scratch3, lastProfilingFrame);
1226 masm.ret();
1227
1228 masm.bind(¬IonFrame);
1229
1230 // Check for either BaselineStub or a CppToJSJit/WasmToJSJit entry
1231 // frame.
1232 masm.branch32(Assembler::NotEqual, scratch3, Imm32(FrameType::BaselineStub),
1233 &handle_Entry);
1234
1235 // Handle Rectifier <- BaselineStub <- BaselineJS
1236 BaseIndex stubFrameReturnAddr(
1237 scratch2, scratch1, TimesOne,
1238 RectifierFrameLayout::Size() +
1239 BaselineStubFrameLayout::offsetOfReturnAddress());
1240 masm.loadPtr(stubFrameReturnAddr, scratch3);
1241 masm.storePtr(scratch3, lastProfilingCallSite);
1242
1243 BaseIndex stubFrameSavedFramePtr(
1244 scratch2, scratch1, TimesOne,
1245 RectifierFrameLayout::Size() - (2 * sizeof(void*)));
1246 masm.loadPtr(stubFrameSavedFramePtr, scratch3);
1247 masm.addPtr(Imm32(sizeof(void*)), scratch3);
1248 masm.storePtr(scratch3, lastProfilingFrame);
1249 masm.ret();
1250 }
1251
1252 // FrameType::IonICCall
1253 //
1254 // The caller is always an IonJS frame.
1255 //
1256 // Ion-Descriptor
1257 // Ion-ReturnAddr
1258 // ... ion frame data ... |- CallFrame-Descriptor.Size
1259 // StubCode |
1260 // ICCallFrame-Descriptor |- IonICCallFrameLayout::Size()
1261 // ICCallFrame-ReturnAddr |
1262 // ... call frame data & args ... |- Descriptor.Size
1263 // ActualArgc |
1264 // CalleeToken |- JitFrameLayout::Size()
1265 // Descriptor |
1266 // FP -----> ReturnAddr |
1267 masm.bind(&handle_IonICCall);
1268 {
1269 // scratch2 := StackPointer + Descriptor.size + JitFrameLayout::Size()
1270 masm.lea(Operand(StackPointer, scratch1, TimesOne, JitFrameLayout::Size()),
1271 scratch2);
1272
1273 // scratch3 := ICCallFrame-Descriptor.Size
1274 masm.loadPtr(Address(scratch2, IonICCallFrameLayout::offsetOfDescriptor()),
1275 scratch3);
1276 #ifdef DEBUG
1277 // Assert previous frame is an IonJS frame.
1278 masm.movePtr(scratch3, scratch1);
1279 masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch1);
1280 {
1281 Label checkOk;
1282 masm.branch32(Assembler::Equal, scratch1, Imm32(FrameType::IonJS),
1283 &checkOk);
1284 masm.assumeUnreachable("IonICCall frame must be preceded by IonJS frame");
1285 masm.bind(&checkOk);
1286 }
1287 #endif
1288 masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch3);
1289
1290 // lastProfilingCallSite := ICCallFrame-ReturnAddr
1291 masm.loadPtr(
1292 Address(scratch2, IonICCallFrameLayout::offsetOfReturnAddress()),
1293 scratch1);
1294 masm.storePtr(scratch1, lastProfilingCallSite);
1295
1296 // lastProfilingFrame := ICCallFrame + ICCallFrame-Descriptor.Size +
1297 // IonICCallFrameLayout::Size()
1298 masm.lea(
1299 Operand(scratch2, scratch3, TimesOne, IonICCallFrameLayout::Size()),
1300 scratch1);
1301 masm.storePtr(scratch1, lastProfilingFrame);
1302 masm.ret();
1303 }
1304
1305 //
1306 // FrameType::CppToJSJit / FrameType::WasmToJSJit
1307 //
1308 // If at an entry frame, store null into both fields.
1309 // A fast-path wasm->jit transition frame is an entry frame from the point
1310 // of view of the JIT.
1311 //
1312 masm.bind(&handle_Entry);
1313 {
1314 masm.movePtr(ImmPtr(nullptr), scratch1);
1315 masm.storePtr(scratch1, lastProfilingCallSite);
1316 masm.storePtr(scratch1, lastProfilingFrame);
1317 masm.ret();
1318 }
1319 }
1320