1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #ifndef jit_BaselineIC_h
8 #define jit_BaselineIC_h
9
10 #include "mozilla/Assertions.h"
11 #include "mozilla/Attributes.h"
12
13 #include <stddef.h>
14 #include <stdint.h>
15 #include <utility>
16
17 #include "gc/Barrier.h"
18 #include "gc/GC.h"
19 #include "gc/Rooting.h"
20 #include "jit/BaselineICList.h"
21 #include "jit/ICState.h"
22 #include "jit/ICStubSpace.h"
23 #include "jit/JitCode.h"
24 #include "jit/JitOptions.h"
25 #include "jit/Registers.h"
26 #include "jit/RegisterSets.h"
27 #include "jit/shared/Assembler-shared.h"
28 #include "jit/TypeData.h"
29 #include "js/TypeDecls.h"
30 #include "js/Value.h"
31 #include "vm/ArrayObject.h"
32 #include "vm/JSScript.h"
33
34 class JS_PUBLIC_API JSTracer;
35
36 namespace js {
37
38 MOZ_COLD void ReportOutOfMemory(JSContext* cx);
39
40 namespace jit {
41
42 class BaselineFrame;
43 class CacheIRStubInfo;
44 class ICScript;
45 class MacroAssembler;
46
47 enum class TailCallVMFunctionId;
48 enum class VMFunctionId;
49
50 // [SMDOC] JIT Inline Caches (ICs)
51 //
52 // Baseline Inline Caches are polymorphic caches that aggressively
53 // share their stub code.
54 //
55 // Every polymorphic site contains a linked list of stubs which are
56 // specific to that site. These stubs are composed of a |StubData|
57 // structure that stores parametrization information (e.g.
58 // the shape pointer for a shape-check-and-property-get stub), any
59 // dynamic information (e.g. warm-up counters), a pointer to the stub code,
60 // and a pointer to the next stub state in the linked list.
61 //
62 // Every BaselineScript keeps an table of |CacheDescriptor| data
63 // structures, which store the following:
64 // A pointer to the first StubData in the cache.
65 // The bytecode PC of the relevant IC.
66 // The machine-code PC where the call to the stubcode returns.
67 //
68 // A diagram:
69 //
70 // Control flow Pointers
71 // =======# ----. .---->
72 // # | |
73 // #======> \-----/
74 //
75 //
76 // .---------------------------------------.
77 // | .-------------------------. |
78 // | | .----. | |
79 // Baseline | | | | | |
80 // JIT Code 0 ^ 1 ^ 2 ^ | | |
81 // +--------------+ .-->+-----+ +-----+ +-----+ | | |
82 // | | #=|==>| |==>| |==>| FB | | | |
83 // | | # | +-----+ +-----+ +-----+ | | |
84 // | | # | # # # | | |
85 // |==============|==# | # # # | | |
86 // |=== IC =======| | # # # | | |
87 // .->|==============|<===|======#=========#=========# | | |
88 // | | | | | | |
89 // | | | | | | |
90 // | | | | | | |
91 // | | | | v | |
92 // | | | | +---------+ | |
93 // | | | | | Fallback| | |
94 // | | | | | Stub | | |
95 // | | | | | Code | | |
96 // | | | | +---------+ | |
97 // | +--------------+ | | |
98 // | |_______ | +---------+ | |
99 // | | | | Stub |<---/ |
100 // | IC | \--. | Code | |
101 // | Descriptor | | +---------+ |
102 // | Table v | |
103 // | +-----------------+ | +---------+ |
104 // \--| Ins | PC | Stub |----/ | Stub |<-------/
105 // +-----------------+ | Code |
106 // | ... | +---------+
107 // +-----------------+
108 // Shared
109 // Stub Code
110 //
111
112 class ICStub;
113 class ICCacheIRStub;
114 class ICFallbackStub;
115
116 #ifdef JS_JITSPEW
117 void FallbackICSpew(JSContext* cx, ICFallbackStub* stub, const char* fmt, ...)
118 MOZ_FORMAT_PRINTF(3, 4);
119 #else
120 # define FallbackICSpew(...)
121 #endif
122
123 // An entry in the ICScript IC table. There's one ICEntry per IC.
124 class ICEntry {
125 // A pointer to the first IC stub for this instruction.
126 ICStub* firstStub_;
127
128 public:
ICEntry(ICStub * firstStub)129 explicit ICEntry(ICStub* firstStub) : firstStub_(firstStub) {}
130
firstStub()131 ICStub* firstStub() const {
132 MOZ_ASSERT(firstStub_);
133 return firstStub_;
134 }
135
setFirstStub(ICStub * stub)136 void setFirstStub(ICStub* stub) { firstStub_ = stub; }
137
offsetOfFirstStub()138 static constexpr size_t offsetOfFirstStub() {
139 return offsetof(ICEntry, firstStub_);
140 }
141
142 void trace(JSTracer* trc);
143 };
144
145 //
146 // Base class for all IC stubs.
147 //
148 class ICStub {
149 friend class ICFallbackStub;
150
151 protected:
152 // The raw jitcode to call for this stub.
153 uint8_t* stubCode_;
154
155 // Counts the number of times the stub was entered
156 //
157 // See Bug 1494473 comment 6 for a mechanism to handle overflow if overflow
158 // becomes a concern.
159 uint32_t enteredCount_ = 0;
160
161 // Tracks input types for some CacheIR stubs, to help optimize
162 // polymorphic cases. Stored in the base class to make use of
163 // padding bytes.
164 TypeData typeData_;
165
166 // Whether this is an ICFallbackStub or an ICCacheIRStub.
167 bool isFallback_;
168
ICStub(uint8_t * stubCode,bool isFallback)169 ICStub(uint8_t* stubCode, bool isFallback)
170 : stubCode_(stubCode), isFallback_(isFallback) {
171 MOZ_ASSERT(stubCode != nullptr);
172 }
173
174 public:
isFallback()175 inline bool isFallback() const { return isFallback_; }
176
177 inline ICStub* maybeNext() const;
178
toFallbackStub()179 inline const ICFallbackStub* toFallbackStub() const {
180 MOZ_ASSERT(isFallback());
181 return reinterpret_cast<const ICFallbackStub*>(this);
182 }
183
toFallbackStub()184 inline ICFallbackStub* toFallbackStub() {
185 MOZ_ASSERT(isFallback());
186 return reinterpret_cast<ICFallbackStub*>(this);
187 }
188
toCacheIRStub()189 ICCacheIRStub* toCacheIRStub() {
190 MOZ_ASSERT(!isFallback());
191 return reinterpret_cast<ICCacheIRStub*>(this);
192 }
toCacheIRStub()193 const ICCacheIRStub* toCacheIRStub() const {
194 MOZ_ASSERT(!isFallback());
195 return reinterpret_cast<const ICCacheIRStub*>(this);
196 }
197
usesTrampolineCode()198 bool usesTrampolineCode() const {
199 // All fallback code is stored in a single JitCode instance, so we can't
200 // call JitCode::FromExecutable on the raw pointer.
201 return isFallback();
202 }
jitCode()203 JitCode* jitCode() {
204 MOZ_ASSERT(!usesTrampolineCode());
205 return JitCode::FromExecutable(stubCode_);
206 }
207
enteredCount()208 uint32_t enteredCount() const { return enteredCount_; }
incrementEnteredCount()209 inline void incrementEnteredCount() { enteredCount_++; }
resetEnteredCount()210 void resetEnteredCount() { enteredCount_ = 0; }
211
offsetOfStubCode()212 static constexpr size_t offsetOfStubCode() {
213 return offsetof(ICStub, stubCode_);
214 }
offsetOfEnteredCount()215 static constexpr size_t offsetOfEnteredCount() {
216 return offsetof(ICStub, enteredCount_);
217 }
218 };
219
220 class ICFallbackStub final : public ICStub {
221 friend class ICStubConstIterator;
222
223 protected:
224 // The PC offset of this IC's bytecode op within the JSScript.
225 uint32_t pcOffset_;
226
227 // The state of this IC.
228 ICState state_{};
229
230 public:
ICFallbackStub(uint32_t pcOffset,TrampolinePtr stubCode)231 explicit ICFallbackStub(uint32_t pcOffset, TrampolinePtr stubCode)
232 : ICStub(stubCode.value, /* isFallback = */ true), pcOffset_(pcOffset) {}
233
numOptimizedStubs()234 inline size_t numOptimizedStubs() const { return state_.numOptimizedStubs(); }
235
newStubIsFirstStub()236 bool newStubIsFirstStub() const { return state_.newStubIsFirstStub(); }
237
state()238 ICState& state() { return state_; }
239
pcOffset()240 uint32_t pcOffset() const { return pcOffset_; }
pc(JSScript * script)241 jsbytecode* pc(JSScript* script) const {
242 return script->offsetToPC(pcOffset_);
243 }
244
245 // Add a new stub to the IC chain terminated by this fallback stub.
246 inline void addNewStub(ICEntry* icEntry, ICCacheIRStub* stub);
247
248 void discardStubs(JSContext* cx, ICEntry* icEntry);
249
clearUsedByTranspiler()250 void clearUsedByTranspiler() { state_.clearUsedByTranspiler(); }
setUsedByTranspiler()251 void setUsedByTranspiler() { state_.setUsedByTranspiler(); }
252
trialInliningState()253 TrialInliningState trialInliningState() const {
254 return state_.trialInliningState();
255 }
setTrialInliningState(TrialInliningState state)256 void setTrialInliningState(TrialInliningState state) {
257 state_.setTrialInliningState(state);
258 }
259
260 void trackNotAttached();
261
262 void unlinkStub(Zone* zone, ICEntry* icEntry, ICCacheIRStub* prev,
263 ICCacheIRStub* stub);
264 };
265
266 class ICCacheIRStub final : public ICStub {
267 // Pointer to next IC stub.
268 ICStub* next_ = nullptr;
269
270 const CacheIRStubInfo* stubInfo_;
271
272 #ifndef JS_64BIT
273 // Ensure stub data is 8-byte aligned on 32-bit.
274 uintptr_t padding_ = 0;
275 #endif
276
277 public:
ICCacheIRStub(JitCode * stubCode,const CacheIRStubInfo * stubInfo)278 ICCacheIRStub(JitCode* stubCode, const CacheIRStubInfo* stubInfo)
279 : ICStub(stubCode->raw(), /* isFallback = */ false),
280 stubInfo_(stubInfo) {}
281
next()282 ICStub* next() const { return next_; }
setNext(ICStub * stub)283 void setNext(ICStub* stub) { next_ = stub; }
284
stubInfo()285 const CacheIRStubInfo* stubInfo() const { return stubInfo_; }
286 uint8_t* stubDataStart();
287
288 void trace(JSTracer* trc);
289
290 // Optimized stubs get purged on GC. But some stubs can be active on the
291 // stack during GC - specifically the ones that can make calls. To ensure
292 // that these do not get purged, all stubs that can make calls are allocated
293 // in the fallback stub space.
294 bool makesGCCalls() const;
allocatedInFallbackSpace()295 bool allocatedInFallbackSpace() const { return makesGCCalls(); }
296
offsetOfNext()297 static constexpr size_t offsetOfNext() {
298 return offsetof(ICCacheIRStub, next_);
299 }
300
setTypeData(TypeData data)301 void setTypeData(TypeData data) { typeData_ = data; }
typeData()302 TypeData typeData() const { return typeData_; }
303 };
304
305 // Assert stub size is what we expect to catch regressions.
306 #ifdef JS_64BIT
307 static_assert(sizeof(ICFallbackStub) == 3 * sizeof(uintptr_t));
308 static_assert(sizeof(ICCacheIRStub) == 4 * sizeof(uintptr_t));
309 #else
310 static_assert(sizeof(ICFallbackStub) == 5 * sizeof(uintptr_t));
311 static_assert(sizeof(ICCacheIRStub) == 6 * sizeof(uintptr_t));
312 #endif
313
maybeNext()314 inline ICStub* ICStub::maybeNext() const {
315 return isFallback() ? nullptr : toCacheIRStub()->next();
316 }
317
addNewStub(ICEntry * icEntry,ICCacheIRStub * stub)318 inline void ICFallbackStub::addNewStub(ICEntry* icEntry, ICCacheIRStub* stub) {
319 MOZ_ASSERT(stub->next() == nullptr);
320 stub->setNext(icEntry->firstStub());
321 icEntry->setFirstStub(stub);
322 state_.trackAttached();
323 }
324
325 AllocatableGeneralRegisterSet BaselineICAvailableGeneralRegs(size_t numInputs);
326
327 bool ICSupportsPolymorphicTypeData(JSOp op);
328
329 struct IonOsrTempData;
330
331 extern bool DoCallFallback(JSContext* cx, BaselineFrame* frame,
332 ICFallbackStub* stub, uint32_t argc, Value* vp,
333 MutableHandleValue res);
334
335 extern bool DoSpreadCallFallback(JSContext* cx, BaselineFrame* frame,
336 ICFallbackStub* stub, Value* vp,
337 MutableHandleValue res);
338
339 extern bool DoToBoolFallback(JSContext* cx, BaselineFrame* frame,
340 ICFallbackStub* stub, HandleValue arg,
341 MutableHandleValue ret);
342
343 extern bool DoGetElemSuperFallback(JSContext* cx, BaselineFrame* frame,
344 ICFallbackStub* stub, HandleValue lhs,
345 HandleValue rhs, HandleValue receiver,
346 MutableHandleValue res);
347
348 extern bool DoGetElemFallback(JSContext* cx, BaselineFrame* frame,
349 ICFallbackStub* stub, HandleValue lhs,
350 HandleValue rhs, MutableHandleValue res);
351
352 extern bool DoSetElemFallback(JSContext* cx, BaselineFrame* frame,
353 ICFallbackStub* stub, Value* stack,
354 HandleValue objv, HandleValue index,
355 HandleValue rhs);
356
357 extern bool DoInFallback(JSContext* cx, BaselineFrame* frame,
358 ICFallbackStub* stub, HandleValue key,
359 HandleValue objValue, MutableHandleValue res);
360
361 extern bool DoHasOwnFallback(JSContext* cx, BaselineFrame* frame,
362 ICFallbackStub* stub, HandleValue keyValue,
363 HandleValue objValue, MutableHandleValue res);
364
365 extern bool DoCheckPrivateFieldFallback(JSContext* cx, BaselineFrame* frame,
366 ICFallbackStub* stub,
367 HandleValue objValue,
368 HandleValue keyValue,
369 MutableHandleValue res);
370
371 extern bool DoGetNameFallback(JSContext* cx, BaselineFrame* frame,
372 ICFallbackStub* stub, HandleObject envChain,
373 MutableHandleValue res);
374
375 extern bool DoBindNameFallback(JSContext* cx, BaselineFrame* frame,
376 ICFallbackStub* stub, HandleObject envChain,
377 MutableHandleValue res);
378
379 extern bool DoGetIntrinsicFallback(JSContext* cx, BaselineFrame* frame,
380 ICFallbackStub* stub,
381 MutableHandleValue res);
382
383 extern bool DoGetPropFallback(JSContext* cx, BaselineFrame* frame,
384 ICFallbackStub* stub, MutableHandleValue val,
385 MutableHandleValue res);
386
387 extern bool DoGetPropSuperFallback(JSContext* cx, BaselineFrame* frame,
388 ICFallbackStub* stub, HandleValue receiver,
389 MutableHandleValue val,
390 MutableHandleValue res);
391
392 extern bool DoSetPropFallback(JSContext* cx, BaselineFrame* frame,
393 ICFallbackStub* stub, Value* stack,
394 HandleValue lhs, HandleValue rhs);
395
396 extern bool DoGetIteratorFallback(JSContext* cx, BaselineFrame* frame,
397 ICFallbackStub* stub, HandleValue value,
398 MutableHandleValue res);
399
400 extern bool DoOptimizeSpreadCallFallback(JSContext* cx, BaselineFrame* frame,
401 ICFallbackStub* stub,
402 HandleValue value,
403 MutableHandleValue res);
404
405 extern bool DoInstanceOfFallback(JSContext* cx, BaselineFrame* frame,
406 ICFallbackStub* stub, HandleValue lhs,
407 HandleValue rhs, MutableHandleValue res);
408
409 extern bool DoTypeOfFallback(JSContext* cx, BaselineFrame* frame,
410 ICFallbackStub* stub, HandleValue val,
411 MutableHandleValue res);
412
413 extern bool DoToPropertyKeyFallback(JSContext* cx, BaselineFrame* frame,
414 ICFallbackStub* stub, HandleValue val,
415 MutableHandleValue res);
416
417 extern bool DoRestFallback(JSContext* cx, BaselineFrame* frame,
418 ICFallbackStub* stub, MutableHandleValue res);
419
420 extern bool DoUnaryArithFallback(JSContext* cx, BaselineFrame* frame,
421 ICFallbackStub* stub, HandleValue val,
422 MutableHandleValue res);
423
424 extern bool DoBinaryArithFallback(JSContext* cx, BaselineFrame* frame,
425 ICFallbackStub* stub, HandleValue lhs,
426 HandleValue rhs, MutableHandleValue ret);
427
428 extern bool DoNewArrayFallback(JSContext* cx, BaselineFrame* frame,
429 ICFallbackStub* stub, MutableHandleValue res);
430
431 extern bool DoNewObjectFallback(JSContext* cx, BaselineFrame* frame,
432 ICFallbackStub* stub, MutableHandleValue res);
433
434 extern bool DoCompareFallback(JSContext* cx, BaselineFrame* frame,
435 ICFallbackStub* stub, HandleValue lhs,
436 HandleValue rhs, MutableHandleValue ret);
437
438 } // namespace jit
439 } // namespace js
440
441 #endif /* jit_BaselineIC_h */
442