1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "jit/TrialInlining.h"
8
9 #include "jit/BaselineCacheIRCompiler.h"
10 #include "jit/BaselineFrame.h"
11 #include "jit/BaselineIC.h"
12 #include "jit/CacheIRCloner.h"
13 #include "jit/CacheIRHealth.h"
14 #include "jit/CacheIRWriter.h"
15 #include "jit/Ion.h" // TooManyFormalArguments
16
17 #include "vm/BytecodeLocation-inl.h"
18
19 using mozilla::Maybe;
20
21 namespace js {
22 namespace jit {
23
DoTrialInlining(JSContext * cx,BaselineFrame * frame)24 bool DoTrialInlining(JSContext* cx, BaselineFrame* frame) {
25 RootedScript script(cx, frame->script());
26 ICScript* icScript = frame->icScript();
27 bool isRecursive = icScript->depth() > 0;
28
29 #ifdef JS_CACHEIR_SPEW
30 if (cx->spewer().enabled(cx, script, SpewChannel::CacheIRHealthReport)) {
31 for (uint32_t i = 0; i < icScript->numICEntries(); i++) {
32 ICEntry& entry = icScript->icEntry(i);
33 ICFallbackStub* fallbackStub = icScript->fallbackStub(i);
34
35 // If the IC is megamorphic or generic, then we have already
36 // spewed the IC report on transition.
37 if (!(uint8_t(fallbackStub->state().mode()) > 0)) {
38 jit::ICStub* stub = entry.firstStub();
39 bool sawNonZeroCount = false;
40 while (!stub->isFallback()) {
41 uint32_t count = stub->enteredCount();
42 if (count > 0 && sawNonZeroCount) {
43 CacheIRHealth cih;
44 cih.healthReportForIC(cx, &entry, fallbackStub, script,
45 SpewContext::TrialInlining);
46 break;
47 }
48
49 if (count > 0 && !sawNonZeroCount) {
50 sawNonZeroCount = true;
51 }
52
53 stub = stub->toCacheIRStub()->next();
54 }
55 }
56 }
57 }
58 #endif
59
60 if (!script->canIonCompile()) {
61 return true;
62 }
63
64 // Baseline shouldn't attempt trial inlining in scripts that are too large.
65 MOZ_ASSERT_IF(JitOptions.limitScriptSize,
66 script->length() <= JitOptions.ionMaxScriptSize);
67
68 const uint32_t MAX_INLINING_DEPTH = 4;
69 if (icScript->depth() > MAX_INLINING_DEPTH) {
70 return true;
71 }
72
73 InliningRoot* root =
74 isRecursive ? icScript->inliningRoot()
75 : script->jitScript()->getOrCreateInliningRoot(cx, script);
76 if (!root) {
77 return false;
78 }
79
80 JitSpew(JitSpew_WarpTrialInlining,
81 "Trial inlining for %s script %s:%u:%u (%p) (inliningRoot=%p)",
82 (isRecursive ? "inner" : "outer"), script->filename(),
83 script->lineno(), script->column(), frame->script(), root);
84 JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
85
86 TrialInliner inliner(cx, script, icScript, root);
87 return inliner.tryInlining();
88 }
89
cloneSharedPrefix(ICCacheIRStub * stub,const uint8_t * endOfPrefix,CacheIRWriter & writer)90 void TrialInliner::cloneSharedPrefix(ICCacheIRStub* stub,
91 const uint8_t* endOfPrefix,
92 CacheIRWriter& writer) {
93 CacheIRReader reader(stub->stubInfo());
94 CacheIRCloner cloner(stub);
95 while (reader.currentPosition() < endOfPrefix) {
96 CacheOp op = reader.readOp();
97 cloner.cloneOp(op, reader, writer);
98 }
99 }
100
replaceICStub(ICEntry & entry,ICFallbackStub * fallback,CacheIRWriter & writer,CacheKind kind)101 bool TrialInliner::replaceICStub(ICEntry& entry, ICFallbackStub* fallback,
102 CacheIRWriter& writer, CacheKind kind) {
103 MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Candidate);
104
105 fallback->discardStubs(cx(), &entry);
106
107 // Note: AttachBaselineCacheIRStub never throws an exception.
108 ICAttachResult result = AttachBaselineCacheIRStub(cx(), writer, kind, script_,
109 icScript_, fallback);
110 if (result == ICAttachResult::Attached) {
111 MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Inlined);
112 return true;
113 }
114
115 MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Candidate);
116 icScript_->removeInlinedChild(fallback->pcOffset());
117
118 if (result == ICAttachResult::OOM) {
119 ReportOutOfMemory(cx());
120 return false;
121 }
122
123 // We failed to attach a new IC stub due to CacheIR size limits. Disable trial
124 // inlining for this location and return true.
125 MOZ_ASSERT(result == ICAttachResult::TooLarge);
126 fallback->setTrialInliningState(TrialInliningState::Failure);
127 return true;
128 }
129
maybeSingleStub(const ICEntry & entry)130 ICCacheIRStub* TrialInliner::maybeSingleStub(const ICEntry& entry) {
131 // Look for a single non-fallback stub followed by stubs with entered-count 0.
132 // Allow one optimized stub before the fallback stub to support the
133 // CallIRGenerator::emitCalleeGuard optimization where we first try a
134 // GuardSpecificFunction guard before falling back to GuardFunctionHasScript.
135 ICStub* stub = entry.firstStub();
136 if (stub->isFallback()) {
137 return nullptr;
138 }
139 ICStub* next = stub->toCacheIRStub()->next();
140 if (next->enteredCount() != 0) {
141 return nullptr;
142 }
143
144 ICFallbackStub* fallback = nullptr;
145 if (next->isFallback()) {
146 fallback = next->toFallbackStub();
147 } else {
148 ICStub* nextNext = next->toCacheIRStub()->next();
149 if (!nextNext->isFallback() || nextNext->enteredCount() != 0) {
150 return nullptr;
151 }
152 fallback = nextNext->toFallbackStub();
153 }
154
155 if (fallback->trialInliningState() != TrialInliningState::Candidate) {
156 return nullptr;
157 }
158
159 return stub->toCacheIRStub();
160 }
161
FindInlinableOpData(ICCacheIRStub * stub,BytecodeLocation loc)162 Maybe<InlinableOpData> FindInlinableOpData(ICCacheIRStub* stub,
163 BytecodeLocation loc) {
164 if (loc.isInvokeOp()) {
165 Maybe<InlinableCallData> call = FindInlinableCallData(stub);
166 if (call.isSome()) {
167 return call;
168 }
169 }
170 if (loc.isGetPropOp() || loc.isGetElemOp()) {
171 Maybe<InlinableGetterData> getter = FindInlinableGetterData(stub);
172 if (getter.isSome()) {
173 return getter;
174 }
175 }
176 if (loc.isSetPropOp()) {
177 Maybe<InlinableSetterData> setter = FindInlinableSetterData(stub);
178 if (setter.isSome()) {
179 return setter;
180 }
181 }
182 return mozilla::Nothing();
183 }
184
FindInlinableCallData(ICCacheIRStub * stub)185 Maybe<InlinableCallData> FindInlinableCallData(ICCacheIRStub* stub) {
186 Maybe<InlinableCallData> data;
187
188 const CacheIRStubInfo* stubInfo = stub->stubInfo();
189 const uint8_t* stubData = stub->stubDataStart();
190
191 ObjOperandId calleeGuardOperand;
192 CallFlags flags;
193 JSFunction* target = nullptr;
194
195 CacheIRReader reader(stubInfo);
196 while (reader.more()) {
197 const uint8_t* opStart = reader.currentPosition();
198
199 CacheOp op = reader.readOp();
200 CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
201 uint32_t argLength = opInfo.argLength;
202 mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
203
204 switch (op) {
205 case CacheOp::GuardSpecificFunction: {
206 // If we see a guard, remember which operand we are guarding.
207 MOZ_ASSERT(data.isNothing());
208 calleeGuardOperand = reader.objOperandId();
209 uint32_t targetOffset = reader.stubOffset();
210 (void)reader.stubOffset(); // nargsAndFlags
211 uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, targetOffset);
212 target = reinterpret_cast<JSFunction*>(rawTarget);
213 break;
214 }
215 case CacheOp::GuardFunctionScript: {
216 MOZ_ASSERT(data.isNothing());
217 calleeGuardOperand = reader.objOperandId();
218 uint32_t targetOffset = reader.stubOffset();
219 uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, targetOffset);
220 target = reinterpret_cast<BaseScript*>(rawTarget)->function();
221 (void)reader.stubOffset(); // nargsAndFlags
222 break;
223 }
224 case CacheOp::CallScriptedFunction: {
225 // If we see a call, check if `callee` is the previously guarded
226 // operand. If it is, we know the target and can inline.
227 ObjOperandId calleeOperand = reader.objOperandId();
228 mozilla::DebugOnly<Int32OperandId> argcId = reader.int32OperandId();
229 flags = reader.callFlags();
230
231 if (calleeOperand == calleeGuardOperand) {
232 MOZ_ASSERT(static_cast<OperandId&>(argcId).id() == 0);
233 MOZ_ASSERT(data.isNothing());
234 data.emplace();
235 data->endOfSharedPrefix = opStart;
236 }
237 break;
238 }
239 case CacheOp::CallInlinedFunction: {
240 ObjOperandId calleeOperand = reader.objOperandId();
241 mozilla::DebugOnly<Int32OperandId> argcId = reader.int32OperandId();
242 uint32_t icScriptOffset = reader.stubOffset();
243 flags = reader.callFlags();
244
245 if (calleeOperand == calleeGuardOperand) {
246 MOZ_ASSERT(static_cast<OperandId&>(argcId).id() == 0);
247 MOZ_ASSERT(data.isNothing());
248 data.emplace();
249 data->endOfSharedPrefix = opStart;
250 uintptr_t rawICScript =
251 stubInfo->getStubRawWord(stubData, icScriptOffset);
252 data->icScript = reinterpret_cast<ICScript*>(rawICScript);
253 }
254 break;
255 }
256 default:
257 if (!opInfo.transpile) {
258 return mozilla::Nothing();
259 }
260 if (data.isSome()) {
261 MOZ_ASSERT(op == CacheOp::ReturnFromIC);
262 }
263 reader.skip(argLength);
264 break;
265 }
266 MOZ_ASSERT(argStart + argLength == reader.currentPosition());
267 }
268
269 if (data.isSome()) {
270 data->calleeOperand = calleeGuardOperand;
271 data->callFlags = flags;
272 data->target = target;
273 }
274 return data;
275 }
276
FindInlinableGetterData(ICCacheIRStub * stub)277 Maybe<InlinableGetterData> FindInlinableGetterData(ICCacheIRStub* stub) {
278 Maybe<InlinableGetterData> data;
279
280 const CacheIRStubInfo* stubInfo = stub->stubInfo();
281 const uint8_t* stubData = stub->stubDataStart();
282
283 CacheIRReader reader(stubInfo);
284 while (reader.more()) {
285 const uint8_t* opStart = reader.currentPosition();
286
287 CacheOp op = reader.readOp();
288 CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
289 uint32_t argLength = opInfo.argLength;
290 mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
291
292 switch (op) {
293 case CacheOp::CallScriptedGetterResult: {
294 data.emplace();
295 data->receiverOperand = reader.valOperandId();
296
297 uint32_t getterOffset = reader.stubOffset();
298 uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, getterOffset);
299 data->target = reinterpret_cast<JSFunction*>(rawTarget);
300
301 data->sameRealm = reader.readBool();
302 (void)reader.stubOffset(); // nargsAndFlags
303
304 data->endOfSharedPrefix = opStart;
305 break;
306 }
307 case CacheOp::CallInlinedGetterResult: {
308 data.emplace();
309 data->receiverOperand = reader.valOperandId();
310
311 uint32_t getterOffset = reader.stubOffset();
312 uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, getterOffset);
313 data->target = reinterpret_cast<JSFunction*>(rawTarget);
314
315 uint32_t icScriptOffset = reader.stubOffset();
316 uintptr_t rawICScript =
317 stubInfo->getStubRawWord(stubData, icScriptOffset);
318 data->icScript = reinterpret_cast<ICScript*>(rawICScript);
319
320 data->sameRealm = reader.readBool();
321 (void)reader.stubOffset(); // nargsAndFlags
322
323 data->endOfSharedPrefix = opStart;
324 break;
325 }
326 default:
327 if (!opInfo.transpile) {
328 return mozilla::Nothing();
329 }
330 if (data.isSome()) {
331 MOZ_ASSERT(op == CacheOp::ReturnFromIC);
332 }
333 reader.skip(argLength);
334 break;
335 }
336 MOZ_ASSERT(argStart + argLength == reader.currentPosition());
337 }
338
339 return data;
340 }
341
FindInlinableSetterData(ICCacheIRStub * stub)342 Maybe<InlinableSetterData> FindInlinableSetterData(ICCacheIRStub* stub) {
343 Maybe<InlinableSetterData> data;
344
345 const CacheIRStubInfo* stubInfo = stub->stubInfo();
346 const uint8_t* stubData = stub->stubDataStart();
347
348 CacheIRReader reader(stubInfo);
349 while (reader.more()) {
350 const uint8_t* opStart = reader.currentPosition();
351
352 CacheOp op = reader.readOp();
353 CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
354 uint32_t argLength = opInfo.argLength;
355 mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
356
357 switch (op) {
358 case CacheOp::CallScriptedSetter: {
359 data.emplace();
360 data->receiverOperand = reader.objOperandId();
361
362 uint32_t setterOffset = reader.stubOffset();
363 uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, setterOffset);
364 data->target = reinterpret_cast<JSFunction*>(rawTarget);
365
366 data->rhsOperand = reader.valOperandId();
367 data->sameRealm = reader.readBool();
368 (void)reader.stubOffset(); // nargsAndFlags
369
370 data->endOfSharedPrefix = opStart;
371 break;
372 }
373 case CacheOp::CallInlinedSetter: {
374 data.emplace();
375 data->receiverOperand = reader.objOperandId();
376
377 uint32_t setterOffset = reader.stubOffset();
378 uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, setterOffset);
379 data->target = reinterpret_cast<JSFunction*>(rawTarget);
380
381 data->rhsOperand = reader.valOperandId();
382
383 uint32_t icScriptOffset = reader.stubOffset();
384 uintptr_t rawICScript =
385 stubInfo->getStubRawWord(stubData, icScriptOffset);
386 data->icScript = reinterpret_cast<ICScript*>(rawICScript);
387
388 data->sameRealm = reader.readBool();
389 (void)reader.stubOffset(); // nargsAndFlags
390
391 data->endOfSharedPrefix = opStart;
392 break;
393 }
394 default:
395 if (!opInfo.transpile) {
396 return mozilla::Nothing();
397 }
398 if (data.isSome()) {
399 MOZ_ASSERT(op == CacheOp::ReturnFromIC);
400 }
401 reader.skip(argLength);
402 break;
403 }
404 MOZ_ASSERT(argStart + argLength == reader.currentPosition());
405 }
406
407 return data;
408 }
409
410 // Return the number of actual arguments that will be passed to the
411 // target function.
GetCalleeNumActuals(BytecodeLocation loc)412 static uint32_t GetCalleeNumActuals(BytecodeLocation loc) {
413 switch (loc.getOp()) {
414 case JSOp::GetProp:
415 case JSOp::GetElem:
416 // Getters do not pass arguments.
417 return 0;
418
419 case JSOp::SetProp:
420 case JSOp::StrictSetProp:
421 // Setters pass 1 argument.
422 return 1;
423
424 case JSOp::FunCall: {
425 // If FunCall is passed arguments, one of them will become |this|.
426 uint32_t callArgc = loc.getCallArgc();
427 return callArgc > 0 ? callArgc - 1 : 0;
428 }
429
430 case JSOp::Call:
431 case JSOp::CallIgnoresRv:
432 case JSOp::CallIter:
433 case JSOp::New:
434 case JSOp::SuperCall:
435 return loc.getCallArgc();
436
437 default:
438 MOZ_CRASH("Unsupported op");
439 }
440 }
441
442 /*static*/
canInline(JSFunction * target,HandleScript caller,BytecodeLocation loc)443 bool TrialInliner::canInline(JSFunction* target, HandleScript caller,
444 BytecodeLocation loc) {
445 if (!target->hasJitScript()) {
446 JitSpew(JitSpew_WarpTrialInlining, "SKIP: no JIT script");
447 return false;
448 }
449 JSScript* script = target->nonLazyScript();
450 if (!script->jitScript()->hasBaselineScript()) {
451 JitSpew(JitSpew_WarpTrialInlining, "SKIP: no BaselineScript");
452 return false;
453 }
454 if (script->uninlineable()) {
455 JitSpew(JitSpew_WarpTrialInlining, "SKIP: uninlineable flag");
456 return false;
457 }
458 if (!script->canIonCompile()) {
459 JitSpew(JitSpew_WarpTrialInlining, "SKIP: can't ion-compile");
460 return false;
461 }
462 if (script->isDebuggee()) {
463 JitSpew(JitSpew_WarpTrialInlining, "SKIP: is debuggee");
464 return false;
465 }
466 // Don't inline cross-realm calls.
467 if (target->realm() != caller->realm()) {
468 JitSpew(JitSpew_WarpTrialInlining, "SKIP: cross-realm call");
469 return false;
470 }
471
472 uint32_t calleeNumActuals = GetCalleeNumActuals(loc);
473 if (script->needsArgsObj() &&
474 calleeNumActuals > ArgumentsObject::MaxInlinedArgs) {
475 JitSpew(JitSpew_WarpTrialInlining,
476 "SKIP: needs arguments object with %u actual args (maximum %u)",
477 calleeNumActuals, ArgumentsObject::MaxInlinedArgs);
478 return false;
479 }
480
481 if (TooManyFormalArguments(target->nargs())) {
482 JitSpew(JitSpew_WarpTrialInlining, "SKIP: Too many formal arguments: %u",
483 unsigned(target->nargs()));
484 return false;
485 }
486
487 if (TooManyFormalArguments(calleeNumActuals)) {
488 JitSpew(JitSpew_WarpTrialInlining, "SKIP: argc too large: %u",
489 unsigned(loc.getCallArgc()));
490 return false;
491 }
492
493 return true;
494 }
495
shouldInline(JSFunction * target,ICCacheIRStub * stub,BytecodeLocation loc)496 bool TrialInliner::shouldInline(JSFunction* target, ICCacheIRStub* stub,
497 BytecodeLocation loc) {
498 #ifdef JS_JITSPEW
499 BaseScript* baseScript =
500 target->hasBaseScript() ? target->baseScript() : nullptr;
501 JitSpew(JitSpew_WarpTrialInlining,
502 "Inlining candidate JSOp::%s (offset=%u): callee script %s:%u:%u",
503 CodeName(loc.getOp()), loc.bytecodeToOffset(script_),
504 baseScript ? baseScript->filename() : "<not-scripted>",
505 baseScript ? baseScript->lineno() : 0,
506 baseScript ? baseScript->column() : 0);
507 JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
508 #endif
509
510 if (!canInline(target, script_, loc)) {
511 return false;
512 }
513
514 // Don't inline (direct) recursive calls. This still allows recursion if
515 // called through another function (f => g => f).
516 JSScript* targetScript = target->nonLazyScript();
517 if (script_ == targetScript) {
518 JitSpew(JitSpew_WarpTrialInlining, "SKIP: recursion");
519 return false;
520 }
521
522 // Don't inline if the callee has a loop that was hot enough to enter Warp
523 // via OSR. This helps prevent getting stuck in Baseline code for a long time.
524 if (targetScript->jitScript()->hadIonOSR()) {
525 JitSpew(JitSpew_WarpTrialInlining, "SKIP: had OSR");
526 return false;
527 }
528
529 // Ensure the total bytecode size does not exceed ionMaxScriptSize.
530 size_t newTotalSize = root_->totalBytecodeSize() + targetScript->length();
531 if (newTotalSize > JitOptions.ionMaxScriptSize) {
532 JitSpew(JitSpew_WarpTrialInlining, "SKIP: total size too big");
533 return false;
534 }
535
536 uint32_t entryCount = stub->enteredCount();
537 if (entryCount < JitOptions.inliningEntryThreshold) {
538 JitSpew(JitSpew_WarpTrialInlining, "SKIP: Entry count is %u (minimum %u)",
539 unsigned(entryCount), unsigned(JitOptions.inliningEntryThreshold));
540 return false;
541 }
542
543 if (!JitOptions.isSmallFunction(targetScript)) {
544 if (!targetScript->isInlinableLargeFunction()) {
545 JitSpew(JitSpew_WarpTrialInlining, "SKIP: Length is %u (maximum %u)",
546 unsigned(targetScript->length()),
547 unsigned(JitOptions.smallFunctionMaxBytecodeLength));
548 return false;
549 }
550
551 JitSpew(JitSpew_WarpTrialInlining,
552 "INFO: Ignored length (%u) of InlinableLargeFunction",
553 unsigned(targetScript->length()));
554 }
555
556 return true;
557 }
558
createInlinedICScript(JSFunction * target,BytecodeLocation loc)559 ICScript* TrialInliner::createInlinedICScript(JSFunction* target,
560 BytecodeLocation loc) {
561 MOZ_ASSERT(target->hasJitEntry());
562 MOZ_ASSERT(target->hasJitScript());
563
564 JSScript* targetScript = target->baseScript()->asJSScript();
565
566 // We don't have to check for overflow here because we have already
567 // successfully allocated an ICScript with this number of entries
568 // when creating the JitScript for the target function, and we
569 // checked for overflow then.
570 uint32_t fallbackStubsOffset =
571 sizeof(ICScript) + targetScript->numICEntries() * sizeof(ICEntry);
572 uint32_t allocSize = fallbackStubsOffset +
573 targetScript->numICEntries() * sizeof(ICFallbackStub);
574
575 void* raw = cx()->pod_malloc<uint8_t>(allocSize);
576 MOZ_ASSERT(uintptr_t(raw) % alignof(ICScript) == 0);
577 if (!raw) {
578 return nullptr;
579 }
580
581 uint32_t initialWarmUpCount = JitOptions.trialInliningInitialWarmUpCount;
582
583 uint32_t depth = icScript_->depth() + 1;
584 UniquePtr<ICScript> inlinedICScript(new (raw) ICScript(
585 initialWarmUpCount, fallbackStubsOffset, allocSize, depth, root_));
586
587 inlinedICScript->initICEntries(cx(), targetScript);
588
589 uint32_t pcOffset = loc.bytecodeToOffset(script_);
590 ICScript* result = inlinedICScript.get();
591 if (!icScript_->addInlinedChild(cx(), std::move(inlinedICScript), pcOffset)) {
592 return nullptr;
593 }
594 MOZ_ASSERT(result->numICEntries() == targetScript->numICEntries());
595
596 root_->addToTotalBytecodeSize(targetScript->length());
597
598 JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
599 JitSpew(JitSpew_WarpTrialInlining,
600 "SUCCESS: Outer ICScript: %p Inner ICScript: %p", icScript_, result);
601
602 return result;
603 }
604
maybeInlineCall(ICEntry & entry,ICFallbackStub * fallback,BytecodeLocation loc)605 bool TrialInliner::maybeInlineCall(ICEntry& entry, ICFallbackStub* fallback,
606 BytecodeLocation loc) {
607 ICCacheIRStub* stub = maybeSingleStub(entry);
608 if (!stub) {
609 #ifdef JS_JITSPEW
610 if (fallback->numOptimizedStubs() > 1) {
611 JitSpew(JitSpew_WarpTrialInlining,
612 "Inlining candidate JSOp::%s (offset=%u):", CodeName(loc.getOp()),
613 fallback->pcOffset());
614 JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
615 JitSpew(JitSpew_WarpTrialInlining, "SKIP: Polymorphic (%u stubs)",
616 (unsigned)fallback->numOptimizedStubs());
617 }
618 #endif
619 return true;
620 }
621
622 MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
623
624 // Look for a CallScriptedFunction with a known target.
625 Maybe<InlinableCallData> data = FindInlinableCallData(stub);
626 if (data.isNothing()) {
627 return true;
628 }
629
630 MOZ_ASSERT(!data->icScript);
631
632 // Decide whether to inline the target.
633 if (!shouldInline(data->target, stub, loc)) {
634 return true;
635 }
636
637 // We only inline FunCall if we are calling the js::fun_call builtin.
638 MOZ_ASSERT_IF(loc.getOp() == JSOp::FunCall,
639 data->callFlags.getArgFormat() == CallFlags::FunCall);
640
641 ICScript* newICScript = createInlinedICScript(data->target, loc);
642 if (!newICScript) {
643 return false;
644 }
645
646 CacheIRWriter writer(cx());
647 Int32OperandId argcId(writer.setInputOperandId(0));
648 cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
649
650 writer.callInlinedFunction(data->calleeOperand, argcId, newICScript,
651 data->callFlags);
652 writer.returnFromIC();
653
654 return replaceICStub(entry, fallback, writer, CacheKind::Call);
655 }
656
maybeInlineGetter(ICEntry & entry,ICFallbackStub * fallback,BytecodeLocation loc,CacheKind kind)657 bool TrialInliner::maybeInlineGetter(ICEntry& entry, ICFallbackStub* fallback,
658 BytecodeLocation loc, CacheKind kind) {
659 ICCacheIRStub* stub = maybeSingleStub(entry);
660 if (!stub) {
661 return true;
662 }
663
664 MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
665
666 Maybe<InlinableGetterData> data = FindInlinableGetterData(stub);
667 if (data.isNothing()) {
668 return true;
669 }
670
671 MOZ_ASSERT(!data->icScript);
672
673 // Decide whether to inline the target.
674 if (!shouldInline(data->target, stub, loc)) {
675 return true;
676 }
677
678 ICScript* newICScript = createInlinedICScript(data->target, loc);
679 if (!newICScript) {
680 return false;
681 }
682
683 CacheIRWriter writer(cx());
684 ValOperandId valId(writer.setInputOperandId(0));
685 if (kind == CacheKind::GetElem) {
686 // Register the key operand.
687 writer.setInputOperandId(1);
688 }
689 cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
690
691 writer.callInlinedGetterResult(data->receiverOperand, data->target,
692 newICScript, data->sameRealm);
693 writer.returnFromIC();
694
695 return replaceICStub(entry, fallback, writer, kind);
696 }
697
maybeInlineSetter(ICEntry & entry,ICFallbackStub * fallback,BytecodeLocation loc,CacheKind kind)698 bool TrialInliner::maybeInlineSetter(ICEntry& entry, ICFallbackStub* fallback,
699 BytecodeLocation loc, CacheKind kind) {
700 ICCacheIRStub* stub = maybeSingleStub(entry);
701 if (!stub) {
702 return true;
703 }
704
705 MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
706
707 Maybe<InlinableSetterData> data = FindInlinableSetterData(stub);
708 if (data.isNothing()) {
709 return true;
710 }
711
712 MOZ_ASSERT(!data->icScript);
713
714 // Decide whether to inline the target.
715 if (!shouldInline(data->target, stub, loc)) {
716 return true;
717 }
718
719 ICScript* newICScript = createInlinedICScript(data->target, loc);
720 if (!newICScript) {
721 return false;
722 }
723
724 CacheIRWriter writer(cx());
725 ValOperandId objValId(writer.setInputOperandId(0));
726 ValOperandId rhsValId(writer.setInputOperandId(1));
727 cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
728
729 writer.callInlinedSetter(data->receiverOperand, data->target,
730 data->rhsOperand, newICScript, data->sameRealm);
731 writer.returnFromIC();
732
733 return replaceICStub(entry, fallback, writer, kind);
734 }
735
tryInlining()736 bool TrialInliner::tryInlining() {
737 uint32_t numICEntries = icScript_->numICEntries();
738 BytecodeLocation startLoc = script_->location();
739
740 for (uint32_t icIndex = 0; icIndex < numICEntries; icIndex++) {
741 ICEntry& entry = icScript_->icEntry(icIndex);
742 ICFallbackStub* fallback = icScript_->fallbackStub(icIndex);
743 BytecodeLocation loc =
744 startLoc + BytecodeLocationOffset(fallback->pcOffset());
745 JSOp op = loc.getOp();
746 switch (op) {
747 case JSOp::Call:
748 case JSOp::CallIgnoresRv:
749 case JSOp::CallIter:
750 case JSOp::FunCall:
751 case JSOp::New:
752 case JSOp::SuperCall:
753 if (!maybeInlineCall(entry, fallback, loc)) {
754 return false;
755 }
756 break;
757 case JSOp::GetProp:
758 if (!maybeInlineGetter(entry, fallback, loc, CacheKind::GetProp)) {
759 return false;
760 }
761 break;
762 case JSOp::GetElem:
763 if (!maybeInlineGetter(entry, fallback, loc, CacheKind::GetElem)) {
764 return false;
765 }
766 break;
767 case JSOp::SetProp:
768 case JSOp::StrictSetProp:
769 if (!maybeInlineSetter(entry, fallback, loc, CacheKind::SetProp)) {
770 return false;
771 }
772 break;
773 default:
774 break;
775 }
776 }
777
778 return true;
779 }
780
addInlinedScript(UniquePtr<ICScript> icScript)781 bool InliningRoot::addInlinedScript(UniquePtr<ICScript> icScript) {
782 return inlinedScripts_.append(std::move(icScript));
783 }
784
trace(JSTracer * trc)785 void InliningRoot::trace(JSTracer* trc) {
786 TraceEdge(trc, &owningScript_, "inlining-root-owning-script");
787 for (auto& inlinedScript : inlinedScripts_) {
788 inlinedScript->trace(trc);
789 }
790 }
791
purgeOptimizedStubs(Zone * zone)792 void InliningRoot::purgeOptimizedStubs(Zone* zone) {
793 for (auto& inlinedScript : inlinedScripts_) {
794 inlinedScript->purgeOptimizedStubs(zone);
795 }
796 }
797
resetWarmUpCounts(uint32_t count)798 void InliningRoot::resetWarmUpCounts(uint32_t count) {
799 for (auto& inlinedScript : inlinedScripts_) {
800 inlinedScript->resetWarmUpCount(count);
801 }
802 }
803
804 } // namespace jit
805 } // namespace js
806