1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "jit/Bailouts.h"
8 #include "jit/BaselineFrame.h"
9 #include "jit/CalleeToken.h"
10 #include "jit/JitFrames.h"
11 #include "jit/JitRuntime.h"
12 #ifdef JS_ION_PERF
13 # include "jit/PerfSpewer.h"
14 #endif
15 #include "jit/arm64/SharedICHelpers-arm64.h"
16 #include "jit/VMFunctions.h"
17 #include "vm/JitActivation.h" // js::jit::JitActivation
18 #include "vm/JSContext.h"
19
20 #include "jit/MacroAssembler-inl.h"
21
22 using namespace js;
23 using namespace js::jit;
24
25 /* This method generates a trampoline on ARM64 for a c++ function with
26 * the following signature:
27 * bool blah(void* code, int argc, Value* argv,
28 * JSObject* scopeChain, Value* vp)
29 * ...using standard AArch64 calling convention
30 */
generateEnterJIT(JSContext * cx,MacroAssembler & masm)31 void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
32 AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
33
34 enterJITOffset_ = startTrampolineCode(masm);
35
36 const Register reg_code = IntArgReg0; // EnterJitData::jitcode.
37 const Register reg_argc = IntArgReg1; // EnterJitData::maxArgc.
38 const Register reg_argv = IntArgReg2; // EnterJitData::maxArgv.
39 const Register reg_osrFrame = IntArgReg3; // EnterJitData::osrFrame.
40 const Register reg_callee = IntArgReg4; // EnterJitData::calleeToken.
41 const Register reg_scope = IntArgReg5; // EnterJitData::scopeChain.
42 const Register reg_osrNStack =
43 IntArgReg6; // EnterJitData::osrNumStackValues.
44 const Register reg_vp = IntArgReg7; // Address of EnterJitData::result.
45
46 static_assert(OsrFrameReg == IntArgReg3);
47
48 // During the pushes below, use the normal stack pointer.
49 masm.SetStackPointer64(sp);
50
51 // Save old frame pointer and return address; set new frame pointer.
52 masm.push(r29, r30);
53 masm.moveStackPtrTo(r29);
54
55 // Save callee-save integer registers.
56 // Also save x7 (reg_vp) and x30 (lr), for use later.
57 masm.push(r19, r20, r21, r22);
58 masm.push(r23, r24, r25, r26);
59 masm.push(r27, r28, r7, r30);
60
61 // Save callee-save floating-point registers.
62 // AArch64 ABI specifies that only the lower 64 bits must be saved.
63 masm.push(d8, d9, d10, d11);
64 masm.push(d12, d13, d14, d15);
65
66 #ifdef DEBUG
67 // Emit stack canaries.
68 masm.movePtr(ImmWord(0xdeadd00d), r23);
69 masm.movePtr(ImmWord(0xdeadd11d), r24);
70 masm.push(r23, r24);
71 #endif
72
73 // Common code below attempts to push single registers at a time,
74 // which breaks the stack pointer's 16-byte alignment requirement.
75 // Note that movePtr() is invalid because StackPointer is treated as xzr.
76 //
77 // FIXME: After testing, this entire function should be rewritten to not
78 // use the PseudoStackPointer: since the amount of data pushed is
79 // precalculated, we can just allocate the whole frame header at once and
80 // index off sp. This will save a significant number of instructions where
81 // Push() updates sp.
82 masm.Mov(PseudoStackPointer64, sp);
83 masm.SetStackPointer64(PseudoStackPointer64);
84
85 // Save the stack pointer at this point for Baseline OSR.
86 masm.moveStackPtrTo(BaselineFrameReg);
87 // Remember stack depth without padding and arguments.
88 masm.moveStackPtrTo(r19);
89
90 // If constructing, include newTarget in argument vector.
91 {
92 Label noNewTarget;
93 Imm32 constructingToken(CalleeToken_FunctionConstructing);
94 masm.branchTest32(Assembler::Zero, reg_callee, constructingToken,
95 &noNewTarget);
96 masm.add32(Imm32(1), reg_argc);
97 masm.bind(&noNewTarget);
98 }
99
100 // JitFrameLayout is as follows (higher is higher in memory):
101 // N*8 - [ JS argument vector ] (base 16-byte aligned)
102 // 8 - numActualArgs
103 // 8 - calleeToken (16-byte aligned)
104 // 8 - frameDescriptor
105 // 8 - returnAddress (16-byte aligned, pushed by callee)
106
107 // Touch frame incrementally (a requirement for Windows).
108 //
109 // Use already saved callee-save registers r20 and r21 as temps.
110 //
111 // This has to be done outside the ScratchRegisterScope, as the temps are
112 // under demand inside the touchFrameValues call.
113
114 // Give sp 16-byte alignment and sync stack pointers.
115 masm.andToStackPtr(Imm32(~0xf));
116 // We needn't worry about the Gecko Profiler mark because touchFrameValues
117 // touches in large increments.
118 masm.touchFrameValues(reg_argc, r20, r21);
119 // Restore stack pointer, preserved above.
120 masm.moveToStackPtr(r19);
121
122 // Push the argument vector onto the stack.
123 // WARNING: destructively modifies reg_argv
124 {
125 vixl::UseScratchRegisterScope temps(&masm.asVIXL());
126
127 const ARMRegister tmp_argc = temps.AcquireX();
128 const ARMRegister tmp_sp = temps.AcquireX();
129
130 Label noArguments;
131 Label loopHead;
132
133 masm.movePtr(reg_argc, tmp_argc.asUnsized());
134
135 // sp -= 8
136 // Since we're using PostIndex Str below, this is necessary to avoid
137 // overwriting the Gecko Profiler mark pushed above.
138 masm.subFromStackPtr(Imm32(8));
139
140 // sp -= 8 * argc
141 masm.Sub(PseudoStackPointer64, PseudoStackPointer64,
142 Operand(tmp_argc, vixl::SXTX, 3));
143
144 // Give sp 16-byte alignment and sync stack pointers.
145 masm.andToStackPtr(Imm32(~0xf));
146 masm.moveStackPtrTo(tmp_sp.asUnsized());
147
148 masm.branchTestPtr(Assembler::Zero, reg_argc, reg_argc, &noArguments);
149
150 // Begin argument-pushing loop.
151 // This could be optimized using Ldp and Stp.
152 {
153 masm.bind(&loopHead);
154
155 // Load an argument from argv, then increment argv by 8.
156 masm.Ldr(x24, MemOperand(ARMRegister(reg_argv, 64), Operand(8),
157 vixl::PostIndex));
158
159 // Store the argument to tmp_sp, then increment tmp_sp by 8.
160 masm.Str(x24, MemOperand(tmp_sp, Operand(8), vixl::PostIndex));
161
162 // Decrement tmp_argc and set the condition codes for the new value.
163 masm.Subs(tmp_argc, tmp_argc, Operand(1));
164
165 // Branch if arguments remain.
166 masm.B(&loopHead, vixl::Condition::NonZero);
167 }
168
169 masm.bind(&noArguments);
170 }
171 masm.checkStackAlignment();
172
173 // Calculate the number of bytes pushed so far.
174 masm.subStackPtrFrom(r19);
175
176 // Create the frame descriptor.
177 masm.makeFrameDescriptor(r19, FrameType::CppToJSJit, JitFrameLayout::Size());
178
179 // Push the number of actual arguments and the calleeToken.
180 // The result address is used to store the actual number of arguments
181 // without adding an argument to EnterJIT.
182 {
183 vixl::UseScratchRegisterScope temps(&masm.asVIXL());
184 MOZ_ASSERT(temps.IsAvailable(ScratchReg64)); // ip0
185 temps.Exclude(ScratchReg64);
186 masm.unboxInt32(Address(reg_vp, 0x0), ScratchReg64.asUnsized());
187 masm.push(ScratchReg64.asUnsized(), reg_callee);
188 }
189 masm.checkStackAlignment();
190
191 // Push the descriptor.
192 masm.Push(r19);
193
194 Label osrReturnPoint;
195 {
196 // Check for Interpreter -> Baseline OSR.
197 Label notOsr;
198 masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr);
199
200 // Push return address and previous frame pointer.
201 {
202 vixl::UseScratchRegisterScope temps(&masm.asVIXL());
203 MOZ_ASSERT(temps.IsAvailable(ScratchReg2_64)); // ip1
204 temps.Exclude(ScratchReg2_64);
205
206 masm.Adr(ScratchReg2_64, &osrReturnPoint);
207 masm.push(ScratchReg2, BaselineFrameReg);
208
209 // Reserve frame.
210 masm.subFromStackPtr(Imm32(BaselineFrame::Size()));
211
212 masm.touchFrameValues(reg_osrNStack, ScratchReg2, BaselineFrameReg);
213 }
214 masm.moveStackPtrTo(BaselineFrameReg);
215
216 // Reserve space for locals and stack values.
217 masm.Lsl(w19, ARMRegister(reg_osrNStack, 32),
218 3); // w19 = num_stack_values * sizeof(Value).
219 masm.subFromStackPtr(r19);
220
221 // Enter exit frame.
222 masm.addPtr(
223 Imm32(BaselineFrame::Size() + BaselineFrame::FramePointerOffset), r19);
224 masm.makeFrameDescriptor(r19, FrameType::BaselineJS,
225 ExitFrameLayout::Size());
226 masm.asVIXL().Push(x19, xzr); // Push xzr for a fake return address.
227 // No GC things to mark: push a bare token.
228 masm.loadJSContext(r19);
229 masm.enterFakeExitFrame(r19, r19, ExitFrameType::Bare);
230
231 masm.push(BaselineFrameReg, reg_code);
232
233 // Initialize the frame, including filling in the slots.
234 using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
235 uint32_t numStackValues);
236 masm.setupUnalignedABICall(r19);
237 masm.passABIArg(BaselineFrameReg); // BaselineFrame.
238 masm.passABIArg(reg_osrFrame); // InterpreterFrame.
239 masm.passABIArg(reg_osrNStack);
240 masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
241 MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
242
243 masm.pop(r19, BaselineFrameReg);
244 MOZ_ASSERT(r19 != ReturnReg);
245
246 masm.addToStackPtr(Imm32(ExitFrameLayout::SizeWithFooter()));
247 masm.addPtr(Imm32(BaselineFrame::Size()), BaselineFrameReg);
248
249 Label error;
250 masm.branchIfFalseBool(ReturnReg, &error);
251
252 masm.jump(r19);
253
254 // OOM: load error value, discard return address and previous frame
255 // pointer, and return.
256 masm.bind(&error);
257 masm.Add(masm.GetStackPointer64(), BaselineFrameReg64,
258 Operand(2 * sizeof(uintptr_t)));
259 masm.syncStackPtr();
260 masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
261 masm.B(&osrReturnPoint);
262
263 masm.bind(¬Osr);
264 masm.movePtr(reg_scope, R1_);
265 }
266
267 // Call function.
268 // Since AArch64 doesn't have the pc register available, the callee must push
269 // lr.
270 masm.callJitNoProfiler(reg_code);
271
272 // Interpreter -> Baseline OSR will return here.
273 masm.bind(&osrReturnPoint);
274
275 masm.Pop(r19); // Pop frame descriptor.
276 masm.pop(r24, r23); // Discard calleeToken, numActualArgs.
277
278 // Discard arguments and the stack alignment padding.
279 masm.Add(masm.GetStackPointer64(), masm.GetStackPointer64(),
280 Operand(x19, vixl::LSR, FRAMESIZE_SHIFT));
281
282 masm.syncStackPtr();
283 masm.SetStackPointer64(sp);
284
285 #ifdef DEBUG
286 // Check that canaries placed on function entry are still present.
287 masm.pop(r24, r23);
288 Label x23OK, x24OK;
289
290 masm.branchPtr(Assembler::Equal, r23, ImmWord(0xdeadd00d), &x23OK);
291 masm.breakpoint();
292 masm.bind(&x23OK);
293
294 masm.branchPtr(Assembler::Equal, r24, ImmWord(0xdeadd11d), &x24OK);
295 masm.breakpoint();
296 masm.bind(&x24OK);
297 #endif
298
299 // Restore callee-save floating-point registers.
300 masm.pop(d15, d14, d13, d12);
301 masm.pop(d11, d10, d9, d8);
302
303 // Restore callee-save integer registers.
304 // Also restore x7 (reg_vp) and x30 (lr).
305 masm.pop(r30, r7, r28, r27);
306 masm.pop(r26, r25, r24, r23);
307 masm.pop(r22, r21, r20, r19);
308
309 // Store return value (in JSReturnReg = x2 to just-popped reg_vp).
310 masm.storeValue(JSReturnOperand, Address(reg_vp, 0));
311
312 // Restore old frame pointer.
313 masm.pop(r30, r29);
314
315 // Return using the value popped into x30.
316 masm.abiret();
317
318 // Reset stack pointer.
319 masm.SetStackPointer64(PseudoStackPointer64);
320 }
321
322 // static
323 mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
getCppEntryRegisters(JitFrameLayout * frameStackAddress)324 JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
325 // Not supported, or not implemented yet.
326 // TODO: Implement along with the corresponding stack-walker changes, in
327 // coordination with the Gecko Profiler, see bug 1635987 and follow-ups.
328 return mozilla::Nothing{};
329 }
330
PushRegisterDump(MacroAssembler & masm)331 static void PushRegisterDump(MacroAssembler& masm) {
332 const LiveRegisterSet First28GeneralRegisters = LiveRegisterSet(
333 GeneralRegisterSet(Registers::AllMask &
334 ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28)),
335 FloatRegisterSet(FloatRegisters::NoneMask));
336
337 const LiveRegisterSet AllFloatRegisters =
338 LiveRegisterSet(GeneralRegisterSet(Registers::NoneMask),
339 FloatRegisterSet(FloatRegisters::AllMask));
340
341 // Push all general-purpose registers.
342 //
343 // The ARM64 ABI does not treat SP as a normal register that can
344 // be pushed. So pushing happens in two phases.
345 //
346 // Registers are pushed in reverse order of code.
347 //
348 // See block comment in MacroAssembler.h for further required invariants.
349
350 // First, push the last four registers, passing zero for sp.
351 // Zero is pushed for x28 and x31: the pseudo-SP and SP, respectively.
352 masm.asVIXL().Push(xzr, x30, x29, xzr);
353
354 // Second, push the first 28 registers that serve no special purpose.
355 masm.PushRegsInMask(First28GeneralRegisters);
356
357 // Finally, push all floating-point registers, completing the RegisterDump.
358 masm.PushRegsInMask(AllFloatRegisters);
359 }
360
generateInvalidator(MacroAssembler & masm,Label * bailoutTail)361 void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
362 AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
363
364 invalidatorOffset_ = startTrampolineCode(masm);
365
366 // The InvalidationBailoutStack saved in r0 must be:
367 // - osiPointReturnAddress_
368 // - ionScript_ (pushed by CodeGeneratorARM64::generateInvalidateEpilogue())
369 // - regs_ (pushed here)
370 // - fpregs_ (pushed here) [=r0]
371 PushRegisterDump(masm);
372 masm.moveStackPtrTo(r0);
373
374 masm.Sub(x1, masm.GetStackPointer64(), Operand(sizeof(size_t)));
375 masm.Sub(x2, masm.GetStackPointer64(),
376 Operand(sizeof(size_t) + sizeof(void*)));
377 masm.moveToStackPtr(r2);
378
379 using Fn = bool (*)(InvalidationBailoutStack * sp, size_t * frameSizeOut,
380 BaselineBailoutInfo * *info);
381 masm.setupUnalignedABICall(r10);
382 masm.passABIArg(r0);
383 masm.passABIArg(r1);
384 masm.passABIArg(r2);
385
386 masm.callWithABI<Fn, InvalidationBailout>(
387 MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
388
389 masm.pop(r2, r1);
390
391 masm.Add(masm.GetStackPointer64(), masm.GetStackPointer64(), x1);
392 masm.Add(masm.GetStackPointer64(), masm.GetStackPointer64(),
393 Operand(sizeof(InvalidationBailoutStack)));
394 masm.syncStackPtr();
395
396 masm.jump(bailoutTail);
397 }
398
generateArgumentsRectifier(MacroAssembler & masm,ArgumentsRectifierKind kind)399 void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
400 ArgumentsRectifierKind kind) {
401 AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
402
403 switch (kind) {
404 case ArgumentsRectifierKind::Normal:
405 argumentsRectifierOffset_ = startTrampolineCode(masm);
406 break;
407 case ArgumentsRectifierKind::TrialInlining:
408 trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
409 break;
410 }
411
412 // Save the return address for later.
413 masm.push(lr);
414
415 // Load the information that the rectifier needs from the stack.
416 masm.Ldr(w0, MemOperand(masm.GetStackPointer64(),
417 RectifierFrameLayout::offsetOfNumActualArgs()));
418 masm.Ldr(x1, MemOperand(masm.GetStackPointer64(),
419 RectifierFrameLayout::offsetOfCalleeToken()));
420
421 // Extract a JSFunction pointer from the callee token and keep the
422 // intermediary to avoid later recalculation.
423 masm.And(x5, x1, Operand(CalleeTokenMask));
424
425 // Get the arguments from the function object.
426 masm.Ldr(ARMRegister(x6.code(), 32),
427 MemOperand(x5, JSFunction::offsetOfFlagsAndArgCount()));
428 masm.Lsr(x6, x6, JSFunction::ArgCountShift);
429
430 static_assert(CalleeToken_FunctionConstructing == 0x1,
431 "Constructing must be low-order bit");
432 masm.And(x4, x1, Operand(CalleeToken_FunctionConstructing));
433 masm.Add(x7, x6, x4);
434
435 // Copy the number of actual arguments into r8.
436 masm.mov(r0, r8);
437
438 // Calculate the position that our arguments are at before sp gets modified.
439 masm.Add(x3, masm.GetStackPointer64(), Operand(x8, vixl::LSL, 3));
440 masm.Add(x3, x3, Operand(sizeof(RectifierFrameLayout)));
441
442 // Pad to a multiple of 16 bytes. This neglects the |this| value,
443 // which will also be pushed, because the rest of the frame will
444 // round off that value. See pushes of |argc|, |callee| and |desc| below.
445 Label noPadding;
446 masm.Tbnz(x7, 0, &noPadding);
447 masm.asVIXL().Push(xzr);
448 masm.Add(x7, x7, Operand(1));
449 masm.bind(&noPadding);
450
451 {
452 Label notConstructing;
453 masm.Cbz(x4, ¬Constructing);
454
455 // new.target lives at the end of the pushed args
456 // NB: The arg vector holder starts at the beginning of the last arg,
457 // add a value to get to argv[argc]
458 masm.loadPtr(Address(r3, sizeof(Value)), r4);
459 masm.Push(r4);
460
461 masm.bind(¬Constructing);
462 }
463
464 // Calculate the number of undefineds that need to be pushed.
465 masm.Sub(w2, w6, w8);
466
467 // Put an undefined in a register so it can be pushed.
468 masm.moveValue(UndefinedValue(), ValueOperand(r4));
469
470 // Push undefined N times.
471 {
472 Label undefLoopTop;
473 masm.bind(&undefLoopTop);
474 masm.Push(r4);
475 masm.Subs(w2, w2, Operand(1));
476 masm.B(&undefLoopTop, Assembler::NonZero);
477 }
478
479 // Arguments copy loop. Copy for x8 >= 0 to include |this|.
480 {
481 Label copyLoopTop;
482 masm.bind(©LoopTop);
483 masm.Ldr(x4, MemOperand(x3, -sizeof(Value), vixl::PostIndex));
484 masm.Push(r4);
485 masm.Subs(x8, x8, Operand(1));
486 masm.B(©LoopTop, Assembler::NotSigned);
487 }
488
489 // Fix up the size of the stack frame. +1 accounts for |this|.
490 masm.Add(x6, x7, Operand(1));
491 masm.Lsl(x6, x6, 3);
492
493 // Make that into a frame descriptor.
494 masm.makeFrameDescriptor(r6, FrameType::Rectifier, JitFrameLayout::Size());
495
496 masm.push(r0, // Number of actual arguments.
497 r1, // Callee token.
498 r6); // Frame descriptor.
499
500 // Call the target function.
501 switch (kind) {
502 case ArgumentsRectifierKind::Normal:
503 masm.loadJitCodeRaw(r5, r3);
504 argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(r3);
505 break;
506 case ArgumentsRectifierKind::TrialInlining:
507 Label noBaselineScript, done;
508 masm.loadBaselineJitCodeRaw(r5, r3, &noBaselineScript);
509 masm.callJitNoProfiler(r3);
510 masm.jump(&done);
511
512 // See BaselineCacheIRCompiler::emitCallInlinedFunction.
513 masm.bind(&noBaselineScript);
514 masm.loadJitCodeRaw(r5, r3);
515 masm.callJitNoProfiler(r3);
516 masm.bind(&done);
517 break;
518 }
519
520 // Clean up!
521 // Get the size of the stack frame, and clean up the later fixed frame.
522 masm.Ldr(x4, MemOperand(masm.GetStackPointer64(), 24, vixl::PostIndex));
523
524 // Now that the size of the stack frame sans the fixed frame has been loaded,
525 // add that onto the stack pointer.
526 masm.Add(masm.GetStackPointer64(), masm.GetStackPointer64(),
527 Operand(x4, vixl::LSR, FRAMESIZE_SHIFT));
528
529 // Pop the return address from earlier and branch.
530 masm.ret();
531 }
532
PushBailoutFrame(MacroAssembler & masm,Register spArg)533 static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
534 // This assumes no SIMD registers, as JS does not support SIMD.
535
536 // The stack saved in spArg must be (higher entries have higher memory
537 // addresses):
538 // - snapshotOffset_
539 // - frameSize_
540 // - regs_
541 // - fpregs_ (spArg + 0)
542 PushRegisterDump(masm);
543 masm.moveStackPtrTo(spArg);
544 }
545
GenerateBailoutThunk(MacroAssembler & masm,Label * bailoutTail)546 static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
547 PushBailoutFrame(masm, r0);
548
549 // SP % 8 == 4
550 // STEP 1c: Call the bailout function, giving a pointer to the
551 // structure we just blitted onto the stack.
552 // Make space for the BaselineBailoutInfo* outparam.
553 masm.reserveStack(sizeof(void*));
554 masm.moveStackPtrTo(r1);
555
556 using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
557 masm.setupUnalignedABICall(r2);
558 masm.passABIArg(r0);
559 masm.passABIArg(r1);
560 masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
561 CheckUnsafeCallWithABI::DontCheckOther);
562
563 // Get the bailoutInfo outparam.
564 masm.pop(r2);
565
566 // Stack is:
567 // [frame]
568 // snapshotOffset
569 // frameSize
570 // [bailoutFrame]
571 //
572 // We want to remove both the bailout frame and the topmost Ion frame's stack.
573
574 // Remove the bailoutFrame.
575 static const uint32_t BailoutDataSize = sizeof(RegisterDump);
576 masm.addToStackPtr(Imm32(BailoutDataSize));
577
578 // Pop the frame, snapshotOffset, and frameSize.
579 vixl::UseScratchRegisterScope temps(&masm.asVIXL());
580 const ARMRegister scratch64 = temps.AcquireX();
581 masm.Ldr(scratch64, MemOperand(masm.GetStackPointer64(), 0x0));
582 masm.addPtr(Imm32(2 * sizeof(void*)), scratch64.asUnsized());
583 masm.addToStackPtr(scratch64.asUnsized());
584
585 // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
586 masm.jump(bailoutTail);
587 }
588
generateBailoutTable(MacroAssembler & masm,Label * bailoutTail,uint32_t frameClass)589 JitRuntime::BailoutTable JitRuntime::generateBailoutTable(MacroAssembler& masm,
590 Label* bailoutTail,
591 uint32_t frameClass) {
592 MOZ_CRASH("arm64 does not use bailout tables");
593 }
594
generateBailoutHandler(MacroAssembler & masm,Label * bailoutTail)595 void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
596 Label* bailoutTail) {
597 AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
598
599 bailoutHandlerOffset_ = startTrampolineCode(masm);
600
601 GenerateBailoutThunk(masm, bailoutTail);
602 }
603
generateVMWrapper(JSContext * cx,MacroAssembler & masm,const VMFunctionData & f,DynFn nativeFun,uint32_t * wrapperOffset)604 bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
605 const VMFunctionData& f, DynFn nativeFun,
606 uint32_t* wrapperOffset) {
607 AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
608
609 *wrapperOffset = startTrampolineCode(masm);
610
611 // Avoid conflicts with argument registers while discarding the result after
612 // the function call.
613 AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
614
615 static_assert(
616 (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
617 "Wrapper register set must be a superset of the Volatile register set.");
618
619 // Unlike on other platforms, it is the responsibility of the VM *callee* to
620 // push the return address, while the caller must ensure that the address
621 // is stored in lr on entry. This allows the VM wrapper to work with both
622 // direct calls and tail calls.
623 masm.push(lr);
624
625 // First argument is the JSContext.
626 Register reg_cx = IntArgReg0;
627 regs.take(reg_cx);
628
629 // Stack is:
630 // ... frame ...
631 // +12 [args]
632 // +8 descriptor
633 // +0 returnAddress (pushed by this function, caller sets as lr)
634 //
635 // We're aligned to an exit frame, so link it up.
636 masm.loadJSContext(reg_cx);
637 masm.enterExitFrame(reg_cx, regs.getAny(), &f);
638
639 // Save the current stack pointer as the base for copying arguments.
640 Register argsBase = InvalidReg;
641 if (f.explicitArgs) {
642 // argsBase can't be an argument register. Bad things would happen if
643 // the MoveResolver didn't throw an assertion failure first.
644 argsBase = r8;
645 regs.take(argsBase);
646 masm.Add(ARMRegister(argsBase, 64), masm.GetStackPointer64(),
647 Operand(ExitFrameLayout::SizeWithFooter()));
648 }
649
650 // Reserve space for any outparameter.
651 Register outReg = InvalidReg;
652 switch (f.outParam) {
653 case Type_Value:
654 outReg = regs.takeAny();
655 masm.reserveStack(sizeof(Value));
656 masm.moveStackPtrTo(outReg);
657 break;
658
659 case Type_Handle:
660 outReg = regs.takeAny();
661 masm.PushEmptyRooted(f.outParamRootType);
662 masm.moveStackPtrTo(outReg);
663 break;
664
665 case Type_Int32:
666 case Type_Bool:
667 outReg = regs.takeAny();
668 masm.reserveStack(sizeof(int64_t));
669 masm.moveStackPtrTo(outReg);
670 break;
671
672 case Type_Double:
673 outReg = regs.takeAny();
674 masm.reserveStack(sizeof(double));
675 masm.moveStackPtrTo(outReg);
676 break;
677
678 case Type_Pointer:
679 outReg = regs.takeAny();
680 masm.reserveStack(sizeof(uintptr_t));
681 masm.moveStackPtrTo(outReg);
682 break;
683
684 default:
685 MOZ_ASSERT(f.outParam == Type_Void);
686 break;
687 }
688
689 if (!generateTLEnterVM(masm, f)) {
690 return false;
691 }
692
693 masm.setupUnalignedABICall(regs.getAny());
694 masm.passABIArg(reg_cx);
695
696 size_t argDisp = 0;
697
698 // Copy arguments.
699 for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
700 switch (f.argProperties(explicitArg)) {
701 case VMFunctionData::WordByValue:
702 masm.passABIArg(MoveOperand(argsBase, argDisp),
703 (f.argPassedInFloatReg(explicitArg) ? MoveOp::DOUBLE
704 : MoveOp::GENERAL));
705 argDisp += sizeof(void*);
706 break;
707
708 case VMFunctionData::WordByRef:
709 masm.passABIArg(
710 MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
711 MoveOp::GENERAL);
712 argDisp += sizeof(void*);
713 break;
714
715 case VMFunctionData::DoubleByValue:
716 case VMFunctionData::DoubleByRef:
717 MOZ_CRASH("NYI: AArch64 callVM should not be used with 128bit values.");
718 }
719 }
720
721 // Copy the semi-implicit outparam, if any.
722 // It is not a C++-abi outparam, which would get passed in the
723 // outparam register, but a real parameter to the function, which
724 // was stack-allocated above.
725 if (outReg != InvalidReg) {
726 masm.passABIArg(outReg);
727 }
728
729 masm.callWithABI(nativeFun, MoveOp::GENERAL,
730 CheckUnsafeCallWithABI::DontCheckHasExitFrame);
731
732 if (!generateTLExitVM(masm, f)) {
733 return false;
734 }
735
736 // SP is used to transfer stack across call boundaries.
737 masm.initPseudoStackPtr();
738
739 // Test for failure.
740 switch (f.failType()) {
741 case Type_Cell:
742 masm.branchTestPtr(Assembler::Zero, r0, r0, masm.failureLabel());
743 break;
744 case Type_Bool:
745 masm.branchIfFalseBool(r0, masm.failureLabel());
746 break;
747 case Type_Void:
748 break;
749 default:
750 MOZ_CRASH("unknown failure kind");
751 }
752
753 // Load the outparam and free any allocated stack.
754 switch (f.outParam) {
755 case Type_Value:
756 masm.Ldr(ARMRegister(JSReturnReg, 64),
757 MemOperand(masm.GetStackPointer64()));
758 masm.freeStack(sizeof(Value));
759 break;
760
761 case Type_Handle:
762 masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
763 break;
764
765 case Type_Int32:
766 masm.Ldr(ARMRegister(ReturnReg, 32),
767 MemOperand(masm.GetStackPointer64()));
768 masm.freeStack(sizeof(int64_t));
769 break;
770
771 case Type_Bool:
772 masm.Ldrb(ARMRegister(ReturnReg, 32),
773 MemOperand(masm.GetStackPointer64()));
774 masm.freeStack(sizeof(int64_t));
775 break;
776
777 case Type_Double:
778 masm.Ldr(ARMFPRegister(ReturnDoubleReg, 64),
779 MemOperand(masm.GetStackPointer64()));
780 masm.freeStack(sizeof(double));
781 break;
782
783 case Type_Pointer:
784 masm.Ldr(ARMRegister(ReturnReg, 64),
785 MemOperand(masm.GetStackPointer64()));
786 masm.freeStack(sizeof(uintptr_t));
787 break;
788
789 default:
790 MOZ_ASSERT(f.outParam == Type_Void);
791 break;
792 }
793
794 // Until C++ code is instrumented against Spectre, prevent speculative
795 // execution from returning any private data.
796 if (f.returnsData() && JitOptions.spectreJitToCxxCalls) {
797 masm.speculationBarrier();
798 }
799
800 masm.leaveExitFrame();
801 masm.retn(Imm32(sizeof(ExitFrameLayout) +
802 f.explicitStackSlots() * sizeof(void*) +
803 f.extraValuesToPop * sizeof(Value)));
804
805 return true;
806 }
807
generatePreBarrier(JSContext * cx,MacroAssembler & masm,MIRType type)808 uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
809 MIRType type) {
810 AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
811
812 uint32_t offset = startTrampolineCode(masm);
813
814 static_assert(PreBarrierReg == r1);
815 Register temp1 = r2;
816 Register temp2 = r3;
817 Register temp3 = r4;
818 masm.push(temp1);
819 masm.push(temp2);
820 masm.push(temp3);
821
822 Label noBarrier;
823 masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
824 &noBarrier);
825
826 // Call into C++ to mark this GC thing.
827 masm.pop(temp3);
828 masm.pop(temp2);
829 masm.pop(temp1);
830
831 LiveRegisterSet regs =
832 LiveRegisterSet(GeneralRegisterSet(Registers::VolatileMask),
833 FloatRegisterSet(FloatRegisters::VolatileMask));
834
835 // Also preserve the return address.
836 regs.add(lr);
837
838 masm.PushRegsInMask(regs);
839
840 masm.movePtr(ImmPtr(cx->runtime()), r3);
841
842 masm.setupUnalignedABICall(r0);
843 masm.passABIArg(r3);
844 masm.passABIArg(PreBarrierReg);
845 masm.callWithABI(JitPreWriteBarrier(type));
846
847 // Pop the volatile regs and restore LR.
848 masm.PopRegsInMask(regs);
849 masm.abiret();
850
851 masm.bind(&noBarrier);
852 masm.pop(temp3);
853 masm.pop(temp2);
854 masm.pop(temp1);
855 masm.abiret();
856
857 return offset;
858 }
859
generateExceptionTailStub(MacroAssembler & masm,Label * profilerExitTail)860 void JitRuntime::generateExceptionTailStub(MacroAssembler& masm,
861 Label* profilerExitTail) {
862 AutoCreatedBy acb(masm, "JitRuntime::generateExceptionTailStub");
863
864 exceptionTailOffset_ = startTrampolineCode(masm);
865
866 masm.bind(masm.failureLabel());
867 masm.handleFailureWithHandlerTail(profilerExitTail);
868 }
869
generateBailoutTailStub(MacroAssembler & masm,Label * bailoutTail)870 void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
871 Label* bailoutTail) {
872 AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
873
874 bailoutTailOffset_ = startTrampolineCode(masm);
875 masm.bind(bailoutTail);
876
877 masm.generateBailoutTail(r1, r2);
878 }
879
generateProfilerExitFrameTailStub(MacroAssembler & masm,Label * profilerExitTail)880 void JitRuntime::generateProfilerExitFrameTailStub(MacroAssembler& masm,
881 Label* profilerExitTail) {
882 AutoCreatedBy acb(masm, "JitRuntime::generateProfilerExitFrameTailStub");
883
884 profilerExitFrameTailOffset_ = startTrampolineCode(masm);
885 masm.bind(profilerExitTail);
886
887 Register scratch1 = r8;
888 Register scratch2 = r9;
889 Register scratch3 = r10;
890 Register scratch4 = r11;
891
892 //
893 // The code generated below expects that the current stack pointer points
894 // to an Ion or Baseline frame, at the state it would be immediately
895 // before a ret(). Thus, after this stub's business is done, it executes
896 // a ret() and returns directly to the caller script, on behalf of the
897 // callee script that jumped to this code.
898 //
899 // Thus the expected stack is:
900 //
901 // StackPointer ----+
902 // v
903 // ..., ActualArgc, CalleeToken, Descriptor, ReturnAddr
904 // MEM-HI MEM-LOW
905 //
906 //
907 // The generated jitcode is responsible for overwriting the
908 // jitActivation->lastProfilingFrame field with a pointer to the previous
909 // Ion or Baseline jit-frame that was pushed before this one. It is also
910 // responsible for overwriting jitActivation->lastProfilingCallSite with
911 // the return address into that frame. The frame could either be an
912 // immediate "caller" frame, or it could be a frame in a previous
913 // JitActivation (if the current frame was entered from C++, and the C++
914 // was entered by some caller jit-frame further down the stack).
915 //
916 // So this jitcode is responsible for "walking up" the jit stack, finding
917 // the previous Ion or Baseline JS frame, and storing its address and the
918 // return address into the appropriate fields on the current jitActivation.
919 //
920 // There are a fixed number of different path types that can lead to the
921 // current frame, which is either a baseline or ion frame:
922 //
923 // <Baseline-Or-Ion>
924 // ^
925 // |
926 // ^--- Ion
927 // |
928 // ^--- Baseline Stub <---- Baseline
929 // |
930 // ^--- Argument Rectifier
931 // | ^
932 // | |
933 // | ^--- Ion
934 // | |
935 // | ^--- Baseline Stub <---- Baseline
936 // |
937 // ^--- Entry Frame (From C++)
938 //
939 Register actReg = scratch4;
940 masm.loadJSContext(actReg);
941 masm.loadPtr(Address(actReg, offsetof(JSContext, profilingActivation_)),
942 actReg);
943
944 Address lastProfilingFrame(actReg,
945 JitActivation::offsetOfLastProfilingFrame());
946 Address lastProfilingCallSite(actReg,
947 JitActivation::offsetOfLastProfilingCallSite());
948
949 #ifdef DEBUG
950 // Ensure that frame we are exiting is current lastProfilingFrame
951 {
952 masm.loadPtr(lastProfilingFrame, scratch1);
953 Label checkOk;
954 masm.branchPtr(Assembler::Equal, scratch1, ImmWord(0), &checkOk);
955 masm.branchStackPtr(Assembler::Equal, scratch1, &checkOk);
956 masm.assumeUnreachable(
957 "Mismatch between stored lastProfilingFrame and current stack "
958 "pointer.");
959 masm.bind(&checkOk);
960 }
961 #endif
962
963 // Load the frame descriptor into |scratch1|, figure out what to do depending
964 // on its type.
965 masm.loadPtr(
966 Address(masm.getStackPointer(), JitFrameLayout::offsetOfDescriptor()),
967 scratch1);
968
969 // Going into the conditionals, we will have:
970 // FrameDescriptor.size in scratch1
971 // FrameDescriptor.type in scratch2
972 masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch1, scratch2);
973 masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch1);
974
975 // Handling of each case is dependent on FrameDescriptor.type
976 Label handle_IonJS;
977 Label handle_BaselineStub;
978 Label handle_Rectifier;
979 Label handle_IonICCall;
980 Label handle_Entry;
981 Label end;
982
983 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::IonJS),
984 &handle_IonJS);
985 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::BaselineJS),
986 &handle_IonJS);
987 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::BaselineStub),
988 &handle_BaselineStub);
989 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::Rectifier),
990 &handle_Rectifier);
991 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::IonICCall),
992 &handle_IonICCall);
993 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::CppToJSJit),
994 &handle_Entry);
995
996 // The WasmToJSJit is just another kind of entry.
997 masm.branch32(Assembler::Equal, scratch2, Imm32(FrameType::WasmToJSJit),
998 &handle_Entry);
999
1000 masm.assumeUnreachable(
1001 "Invalid caller frame type when exiting from Ion frame.");
1002
1003 //
1004 // FrameType::IonJS
1005 //
1006 // Stack layout:
1007 // ...
1008 // Ion-Descriptor
1009 // Prev-FP ---> Ion-ReturnAddr
1010 // ... previous frame data ... |- Descriptor.Size
1011 // ... arguments ... |
1012 // ActualArgc |
1013 // CalleeToken |- JitFrameLayout::Size()
1014 // Descriptor |
1015 // FP -----> ReturnAddr |
1016 //
1017 masm.bind(&handle_IonJS);
1018 {
1019 // |scratch1| contains Descriptor.size
1020
1021 // returning directly to an IonJS frame. Store return addr to frame
1022 // in lastProfilingCallSite.
1023 masm.loadPtr(Address(masm.getStackPointer(),
1024 JitFrameLayout::offsetOfReturnAddress()),
1025 scratch2);
1026 masm.storePtr(scratch2, lastProfilingCallSite);
1027
1028 // Store return frame in lastProfilingFrame.
1029 // scratch2 := masm.getStackPointer() + Descriptor.size*1 +
1030 // JitFrameLayout::Size();
1031 masm.Add(ARMRegister(scratch2, 64), masm.GetStackPointer64(),
1032 ARMRegister(scratch1, 64));
1033 masm.syncStackPtr();
1034 masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2, scratch2);
1035 masm.storePtr(scratch2, lastProfilingFrame);
1036 masm.ret();
1037 }
1038
1039 //
1040 // FrameType::BaselineStub
1041 //
1042 // Look past the stub and store the frame pointer to
1043 // the baselineJS frame prior to it.
1044 //
1045 // Stack layout:
1046 // ...
1047 // BL-Descriptor
1048 // Prev-FP ---> BL-ReturnAddr
1049 // +-----> BL-PrevFramePointer
1050 // | ... BL-FrameData ...
1051 // | BLStub-Descriptor
1052 // | BLStub-ReturnAddr
1053 // | BLStub-StubPointer |
1054 // +------ BLStub-SavedFramePointer |- Descriptor.Size
1055 // ... arguments ... |
1056 // ActualArgc |
1057 // CalleeToken |- JitFrameLayout::Size()
1058 // Descriptor |
1059 // FP -----> ReturnAddr |
1060 //
1061 // We take advantage of the fact that the stub frame saves the frame
1062 // pointer pointing to the baseline frame, so a bunch of calculation can
1063 // be avoided.
1064 //
1065 masm.bind(&handle_BaselineStub);
1066 {
1067 masm.Add(ARMRegister(scratch3, 64), masm.GetStackPointer64(),
1068 ARMRegister(scratch1, 64));
1069 masm.syncStackPtr();
1070 Address stubFrameReturnAddr(
1071 scratch3, JitFrameLayout::Size() +
1072 BaselineStubFrameLayout::offsetOfReturnAddress());
1073 masm.loadPtr(stubFrameReturnAddr, scratch2);
1074 masm.storePtr(scratch2, lastProfilingCallSite);
1075
1076 Address stubFrameSavedFramePtr(
1077 scratch3, JitFrameLayout::Size() - (2 * sizeof(void*)));
1078 masm.loadPtr(stubFrameSavedFramePtr, scratch2);
1079 masm.addPtr(Imm32(sizeof(void*)), scratch2); // Skip past BL-PrevFramePtr.
1080 masm.storePtr(scratch2, lastProfilingFrame);
1081 masm.ret();
1082 }
1083
1084 //
1085 // FrameType::Rectifier
1086 //
1087 // The rectifier frame can be preceded by either an IonJS, a BaselineStub,
1088 // or a CppToJSJit/WasmToJSJit frame.
1089 //
1090 // Stack layout if caller of rectifier was Ion or CppToJSJit/WasmToJSJit:
1091 //
1092 // Ion-Descriptor
1093 // Ion-ReturnAddr
1094 // ... ion frame data ... |- Rect-Descriptor.Size
1095 // < COMMON LAYOUT >
1096 //
1097 // Stack layout if caller of rectifier was Baseline:
1098 //
1099 // BL-Descriptor
1100 // Prev-FP ---> BL-ReturnAddr
1101 // +-----> BL-SavedFramePointer
1102 // | ... baseline frame data ...
1103 // | BLStub-Descriptor
1104 // | BLStub-ReturnAddr
1105 // | BLStub-StubPointer |
1106 // +------ BLStub-SavedFramePointer |- Rect-Descriptor.Size
1107 // ... args to rectifier ... |
1108 // < COMMON LAYOUT >
1109 //
1110 // Common stack layout:
1111 //
1112 // ActualArgc |
1113 // CalleeToken |- IonRectitiferFrameLayout::Size()
1114 // Rect-Descriptor |
1115 // Rect-ReturnAddr |
1116 // ... rectifier data & args ... |- Descriptor.Size
1117 // ActualArgc |
1118 // CalleeToken |- JitFrameLayout::Size()
1119 // Descriptor |
1120 // FP -----> ReturnAddr |
1121 //
1122 masm.bind(&handle_Rectifier);
1123 {
1124 // scratch2 := StackPointer + Descriptor.size*1 + JitFrameLayout::Size();
1125 masm.Add(ARMRegister(scratch2, 64), masm.GetStackPointer64(),
1126 ARMRegister(scratch1, 64));
1127 masm.syncStackPtr();
1128 masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2);
1129 masm.loadPtr(Address(scratch2, RectifierFrameLayout::offsetOfDescriptor()),
1130 scratch3);
1131 masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch3, scratch1);
1132 masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch3);
1133
1134 // Now |scratch1| contains Rect-Descriptor.Size
1135 // and |scratch2| points to Rectifier frame
1136 // and |scratch3| contains Rect-Descriptor.Type
1137
1138 masm.assertRectifierFrameParentType(scratch3);
1139
1140 // Check for either Ion or BaselineStub frame.
1141 Label notIonFrame;
1142 masm.branch32(Assembler::NotEqual, scratch3, Imm32(FrameType::IonJS),
1143 ¬IonFrame);
1144
1145 // Handle Rectifier <- IonJS
1146 // scratch3 := RectFrame[ReturnAddr]
1147 masm.loadPtr(
1148 Address(scratch2, RectifierFrameLayout::offsetOfReturnAddress()),
1149 scratch3);
1150 masm.storePtr(scratch3, lastProfilingCallSite);
1151
1152 // scratch3 := RectFrame + Rect-Descriptor.Size +
1153 // RectifierFrameLayout::Size()
1154 masm.addPtr(scratch2, scratch1, scratch3);
1155 masm.addPtr(Imm32(RectifierFrameLayout::Size()), scratch3);
1156 masm.storePtr(scratch3, lastProfilingFrame);
1157 masm.ret();
1158
1159 masm.bind(¬IonFrame);
1160
1161 // Check for either BaselineStub or a CppToJSJit/WasmToJSJit entry
1162 // frame.
1163 masm.branch32(Assembler::NotEqual, scratch3, Imm32(FrameType::BaselineStub),
1164 &handle_Entry);
1165
1166 // Handle Rectifier <- BaselineStub <- BaselineJS
1167 masm.addPtr(scratch2, scratch1, scratch3);
1168 Address stubFrameReturnAddr(
1169 scratch3, RectifierFrameLayout::Size() +
1170 BaselineStubFrameLayout::offsetOfReturnAddress());
1171 masm.loadPtr(stubFrameReturnAddr, scratch2);
1172 masm.storePtr(scratch2, lastProfilingCallSite);
1173
1174 Address stubFrameSavedFramePtr(
1175 scratch3, RectifierFrameLayout::Size() - (2 * sizeof(void*)));
1176 masm.loadPtr(stubFrameSavedFramePtr, scratch2);
1177 masm.addPtr(Imm32(sizeof(void*)), scratch2);
1178 masm.storePtr(scratch2, lastProfilingFrame);
1179 masm.ret();
1180 }
1181
1182 // FrameType::IonICCall
1183 //
1184 // The caller is always an IonJS frame.
1185 //
1186 // Ion-Descriptor
1187 // Ion-ReturnAddr
1188 // ... ion frame data ... |- CallFrame-Descriptor.Size
1189 // StubCode |
1190 // ICCallFrame-Descriptor |- IonICCallFrameLayout::Size()
1191 // ICCallFrame-ReturnAddr |
1192 // ... call frame data & args ... |- Descriptor.Size
1193 // ActualArgc |
1194 // CalleeToken |- JitFrameLayout::Size()
1195 // Descriptor |
1196 // FP -----> ReturnAddr |
1197 masm.bind(&handle_IonICCall);
1198 {
1199 // scratch2 := StackPointer + Descriptor.size + JitFrameLayout::Size()
1200 masm.Add(ARMRegister(scratch2, 64), masm.GetStackPointer64(),
1201 ARMRegister(scratch1, 64));
1202 masm.syncStackPtr();
1203 masm.addPtr(Imm32(JitFrameLayout::Size()), scratch2);
1204
1205 // scratch3 := ICCallFrame-Descriptor.Size
1206 masm.loadPtr(Address(scratch2, IonICCallFrameLayout::offsetOfDescriptor()),
1207 scratch3);
1208 #ifdef DEBUG
1209 // Assert previous frame is an IonJS frame.
1210 masm.movePtr(scratch3, scratch1);
1211 masm.and32(Imm32((1 << FRAMETYPE_BITS) - 1), scratch1);
1212 {
1213 Label checkOk;
1214 masm.branch32(Assembler::Equal, scratch1, Imm32(FrameType::IonJS),
1215 &checkOk);
1216 masm.assumeUnreachable("IonICCall frame must be preceded by IonJS frame");
1217 masm.bind(&checkOk);
1218 }
1219 #endif
1220 masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), scratch3);
1221
1222 // lastProfilingCallSite := ICCallFrame-ReturnAddr
1223 masm.loadPtr(
1224 Address(scratch2, IonICCallFrameLayout::offsetOfReturnAddress()),
1225 scratch1);
1226 masm.storePtr(scratch1, lastProfilingCallSite);
1227
1228 // lastProfilingFrame := ICCallFrame + ICCallFrame-Descriptor.Size +
1229 // IonICCallFrameLayout::Size()
1230 masm.addPtr(scratch2, scratch3, scratch1);
1231 masm.addPtr(Imm32(IonICCallFrameLayout::Size()), scratch1);
1232 masm.storePtr(scratch1, lastProfilingFrame);
1233 masm.ret();
1234 }
1235
1236 //
1237 // FrameType::CppToJSJit / FrameType::WasmToJSJit
1238 //
1239 // If at an entry frame, store null into both fields.
1240 // A fast-path wasm->jit transition frame is an entry frame from the point
1241 // of view of the JIT.
1242 //
1243 masm.bind(&handle_Entry);
1244 {
1245 masm.movePtr(ImmPtr(nullptr), scratch1);
1246 masm.storePtr(scratch1, lastProfilingCallSite);
1247 masm.storePtr(scratch1, lastProfilingFrame);
1248 masm.ret();
1249 }
1250 }
1251