1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 * vim: set ts=8 sts=4 et sw=4 tw=99:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "jit/CodeGenerator.h"
8
9 #include "mozilla/Assertions.h"
10 #include "mozilla/Attributes.h"
11 #include "mozilla/Casting.h"
12 #include "mozilla/DebugOnly.h"
13 #include "mozilla/EnumeratedArray.h"
14 #include "mozilla/EnumeratedRange.h"
15 #include "mozilla/MathAlgorithms.h"
16 #include "mozilla/ScopeExit.h"
17 #include "mozilla/SizePrintfMacros.h"
18
19 #include "jslibmath.h"
20 #include "jsmath.h"
21 #include "jsnum.h"
22 #include "jsprf.h"
23 #include "jsstr.h"
24
25 #include "builtin/Eval.h"
26 #include "builtin/TypedObject.h"
27 #include "gc/Nursery.h"
28 #include "irregexp/NativeRegExpMacroAssembler.h"
29 #include "jit/AtomicOperations.h"
30 #include "jit/BaselineCompiler.h"
31 #include "jit/IonBuilder.h"
32 #include "jit/IonCaches.h"
33 #include "jit/IonOptimizationLevels.h"
34 #include "jit/JitcodeMap.h"
35 #include "jit/JitSpewer.h"
36 #include "jit/Linker.h"
37 #include "jit/Lowering.h"
38 #include "jit/MIRGenerator.h"
39 #include "jit/MoveEmitter.h"
40 #include "jit/RangeAnalysis.h"
41 #include "jit/SharedICHelpers.h"
42 #include "vm/AsyncFunction.h"
43 #include "vm/MatchPairs.h"
44 #include "vm/RegExpObject.h"
45 #include "vm/RegExpStatics.h"
46 #include "vm/TraceLogging.h"
47 #include "vm/Unicode.h"
48
49 #include "jsboolinlines.h"
50
51 #include "jit/MacroAssembler-inl.h"
52 #include "jit/shared/CodeGenerator-shared-inl.h"
53 #include "jit/shared/Lowering-shared-inl.h"
54 #include "vm/Interpreter-inl.h"
55
56 using namespace js;
57 using namespace js::jit;
58
59 using mozilla::AssertedCast;
60 using mozilla::DebugOnly;
61 using mozilla::FloatingPoint;
62 using mozilla::Maybe;
63 using mozilla::NegativeInfinity;
64 using mozilla::PositiveInfinity;
65 using JS::GenericNaN;
66
67 namespace js {
68 namespace jit {
69
70 // This out-of-line cache is used to do a double dispatch including it-self and
71 // the wrapped IonCache.
72 class OutOfLineUpdateCache :
73 public OutOfLineCodeBase<CodeGenerator>,
74 public IonCacheVisitor
75 {
76 private:
77 LInstruction* lir_;
78 size_t cacheIndex_;
79 RepatchLabel entry_;
80
81 public:
OutOfLineUpdateCache(LInstruction * lir,size_t cacheIndex)82 OutOfLineUpdateCache(LInstruction* lir, size_t cacheIndex)
83 : lir_(lir),
84 cacheIndex_(cacheIndex)
85 { }
86
bind(MacroAssembler * masm)87 void bind(MacroAssembler* masm) {
88 // The binding of the initial jump is done in
89 // CodeGenerator::visitOutOfLineCache.
90 }
91
getCacheIndex() const92 size_t getCacheIndex() const {
93 return cacheIndex_;
94 }
lir() const95 LInstruction* lir() const {
96 return lir_;
97 }
entry()98 RepatchLabel& entry() {
99 return entry_;
100 }
101
accept(CodeGenerator * codegen)102 void accept(CodeGenerator* codegen) {
103 codegen->visitOutOfLineCache(this);
104 }
105
106 // ICs' visit functions delegating the work to the CodeGen visit funtions.
107 #define VISIT_CACHE_FUNCTION(op) \
108 void visit##op##IC(CodeGenerator* codegen) { \
109 CodeGenerator::DataPtr<op##IC> ic(codegen, getCacheIndex()); \
110 codegen->visit##op##IC(this, ic); \
111 }
112
113 IONCACHE_KIND_LIST(VISIT_CACHE_FUNCTION)
114 #undef VISIT_CACHE_FUNCTION
115 };
116
117 // This function is declared here because it needs to instantiate an
118 // OutOfLineUpdateCache, but we want to keep it visible inside the
119 // CodeGeneratorShared such as we can specialize inline caches in function of
120 // the architecture.
121 void
addCache(LInstruction * lir,size_t cacheIndex)122 CodeGeneratorShared::addCache(LInstruction* lir, size_t cacheIndex)
123 {
124 if (cacheIndex == SIZE_MAX) {
125 masm.setOOM();
126 return;
127 }
128
129 DataPtr<IonCache> cache(this, cacheIndex);
130 MInstruction* mir = lir->mirRaw()->toInstruction();
131 if (mir->resumePoint())
132 cache->setScriptedLocation(mir->block()->info().script(),
133 mir->resumePoint()->pc());
134 else
135 cache->setIdempotent();
136
137 OutOfLineUpdateCache* ool = new(alloc()) OutOfLineUpdateCache(lir, cacheIndex);
138 addOutOfLineCode(ool, mir);
139
140 cache->emitInitialJump(masm, ool->entry());
141 masm.bind(ool->rejoin());
142 }
143
144 void
visitOutOfLineCache(OutOfLineUpdateCache * ool)145 CodeGenerator::visitOutOfLineCache(OutOfLineUpdateCache* ool)
146 {
147 DataPtr<IonCache> cache(this, ool->getCacheIndex());
148
149 // Register the location of the OOL path in the IC.
150 cache->setFallbackLabel(masm.labelForPatch());
151 masm.bind(&ool->entry());
152
153 // Dispatch to ICs' accept functions.
154 cache->accept(this, ool);
155 }
156
157 StringObject*
templateObj() const158 MNewStringObject::templateObj() const {
159 return &templateObj_->as<StringObject>();
160 }
161
CodeGenerator(MIRGenerator * gen,LIRGraph * graph,MacroAssembler * masm)162 CodeGenerator::CodeGenerator(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm)
163 : CodeGeneratorSpecific(gen, graph, masm)
164 , ionScriptLabels_(gen->alloc())
165 , scriptCounts_(nullptr)
166 , simdRefreshTemplatesDuringLink_(0)
167 {
168 }
169
~CodeGenerator()170 CodeGenerator::~CodeGenerator()
171 {
172 MOZ_ASSERT_IF(!gen->compilingWasm(), masm.numSymbolicAccesses() == 0);
173 js_delete(scriptCounts_);
174 }
175
176 typedef bool (*StringToNumberFn)(ExclusiveContext*, JSString*, double*);
177 static const VMFunction StringToNumberInfo =
178 FunctionInfo<StringToNumberFn>(StringToNumber, "StringToNumber");
179
180 void
visitValueToInt32(LValueToInt32 * lir)181 CodeGenerator::visitValueToInt32(LValueToInt32* lir)
182 {
183 ValueOperand operand = ToValue(lir, LValueToInt32::Input);
184 Register output = ToRegister(lir->output());
185 FloatRegister temp = ToFloatRegister(lir->tempFloat());
186
187 MDefinition* input;
188 if (lir->mode() == LValueToInt32::NORMAL)
189 input = lir->mirNormal()->input();
190 else
191 input = lir->mirTruncate()->input();
192
193 Label fails;
194 if (lir->mode() == LValueToInt32::TRUNCATE) {
195 OutOfLineCode* oolDouble = oolTruncateDouble(temp, output, lir->mir());
196
197 // We can only handle strings in truncation contexts, like bitwise
198 // operations.
199 Label* stringEntry;
200 Label* stringRejoin;
201 Register stringReg;
202 if (input->mightBeType(MIRType::String)) {
203 stringReg = ToRegister(lir->temp());
204 OutOfLineCode* oolString = oolCallVM(StringToNumberInfo, lir, ArgList(stringReg),
205 StoreFloatRegisterTo(temp));
206 stringEntry = oolString->entry();
207 stringRejoin = oolString->rejoin();
208 } else {
209 stringReg = InvalidReg;
210 stringEntry = nullptr;
211 stringRejoin = nullptr;
212 }
213
214 masm.truncateValueToInt32(operand, input, stringEntry, stringRejoin, oolDouble->entry(),
215 stringReg, temp, output, &fails);
216 masm.bind(oolDouble->rejoin());
217 } else {
218 masm.convertValueToInt32(operand, input, temp, output, &fails,
219 lir->mirNormal()->canBeNegativeZero(),
220 lir->mirNormal()->conversion());
221 }
222
223 bailoutFrom(&fails, lir->snapshot());
224 }
225
226 void
visitValueToDouble(LValueToDouble * lir)227 CodeGenerator::visitValueToDouble(LValueToDouble* lir)
228 {
229 MToDouble* mir = lir->mir();
230 ValueOperand operand = ToValue(lir, LValueToDouble::Input);
231 FloatRegister output = ToFloatRegister(lir->output());
232
233 Register tag = masm.splitTagForTest(operand);
234
235 Label isDouble, isInt32, isBool, isNull, isUndefined, done;
236 bool hasBoolean = false, hasNull = false, hasUndefined = false;
237
238 masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
239 masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
240
241 if (mir->conversion() != MToFPInstruction::NumbersOnly) {
242 masm.branchTestBoolean(Assembler::Equal, tag, &isBool);
243 masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
244 hasBoolean = true;
245 hasUndefined = true;
246 if (mir->conversion() != MToFPInstruction::NonNullNonStringPrimitives) {
247 masm.branchTestNull(Assembler::Equal, tag, &isNull);
248 hasNull = true;
249 }
250 }
251
252 bailout(lir->snapshot());
253
254 if (hasNull) {
255 masm.bind(&isNull);
256 masm.loadConstantDouble(0.0, output);
257 masm.jump(&done);
258 }
259
260 if (hasUndefined) {
261 masm.bind(&isUndefined);
262 masm.loadConstantDouble(GenericNaN(), output);
263 masm.jump(&done);
264 }
265
266 if (hasBoolean) {
267 masm.bind(&isBool);
268 masm.boolValueToDouble(operand, output);
269 masm.jump(&done);
270 }
271
272 masm.bind(&isInt32);
273 masm.int32ValueToDouble(operand, output);
274 masm.jump(&done);
275
276 masm.bind(&isDouble);
277 masm.unboxDouble(operand, output);
278 masm.bind(&done);
279 }
280
281 void
visitValueToFloat32(LValueToFloat32 * lir)282 CodeGenerator::visitValueToFloat32(LValueToFloat32* lir)
283 {
284 MToFloat32* mir = lir->mir();
285 ValueOperand operand = ToValue(lir, LValueToFloat32::Input);
286 FloatRegister output = ToFloatRegister(lir->output());
287
288 Register tag = masm.splitTagForTest(operand);
289
290 Label isDouble, isInt32, isBool, isNull, isUndefined, done;
291 bool hasBoolean = false, hasNull = false, hasUndefined = false;
292
293 masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
294 masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
295
296 if (mir->conversion() != MToFPInstruction::NumbersOnly) {
297 masm.branchTestBoolean(Assembler::Equal, tag, &isBool);
298 masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
299 hasBoolean = true;
300 hasUndefined = true;
301 if (mir->conversion() != MToFPInstruction::NonNullNonStringPrimitives) {
302 masm.branchTestNull(Assembler::Equal, tag, &isNull);
303 hasNull = true;
304 }
305 }
306
307 bailout(lir->snapshot());
308
309 if (hasNull) {
310 masm.bind(&isNull);
311 masm.loadConstantFloat32(0.0f, output);
312 masm.jump(&done);
313 }
314
315 if (hasUndefined) {
316 masm.bind(&isUndefined);
317 masm.loadConstantFloat32(float(GenericNaN()), output);
318 masm.jump(&done);
319 }
320
321 if (hasBoolean) {
322 masm.bind(&isBool);
323 masm.boolValueToFloat32(operand, output);
324 masm.jump(&done);
325 }
326
327 masm.bind(&isInt32);
328 masm.int32ValueToFloat32(operand, output);
329 masm.jump(&done);
330
331 masm.bind(&isDouble);
332 // ARM and MIPS may not have a double register available if we've
333 // allocated output as a float32.
334 #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32)
335 masm.unboxDouble(operand, ScratchDoubleReg);
336 masm.convertDoubleToFloat32(ScratchDoubleReg, output);
337 #else
338 masm.unboxDouble(operand, output);
339 masm.convertDoubleToFloat32(output, output);
340 #endif
341 masm.bind(&done);
342 }
343
344 void
visitInt32ToDouble(LInt32ToDouble * lir)345 CodeGenerator::visitInt32ToDouble(LInt32ToDouble* lir)
346 {
347 masm.convertInt32ToDouble(ToRegister(lir->input()), ToFloatRegister(lir->output()));
348 }
349
350 void
visitFloat32ToDouble(LFloat32ToDouble * lir)351 CodeGenerator::visitFloat32ToDouble(LFloat32ToDouble* lir)
352 {
353 masm.convertFloat32ToDouble(ToFloatRegister(lir->input()), ToFloatRegister(lir->output()));
354 }
355
356 void
visitDoubleToFloat32(LDoubleToFloat32 * lir)357 CodeGenerator::visitDoubleToFloat32(LDoubleToFloat32* lir)
358 {
359 masm.convertDoubleToFloat32(ToFloatRegister(lir->input()), ToFloatRegister(lir->output()));
360 }
361
362 void
visitInt32ToFloat32(LInt32ToFloat32 * lir)363 CodeGenerator::visitInt32ToFloat32(LInt32ToFloat32* lir)
364 {
365 masm.convertInt32ToFloat32(ToRegister(lir->input()), ToFloatRegister(lir->output()));
366 }
367
368 void
visitDoubleToInt32(LDoubleToInt32 * lir)369 CodeGenerator::visitDoubleToInt32(LDoubleToInt32* lir)
370 {
371 Label fail;
372 FloatRegister input = ToFloatRegister(lir->input());
373 Register output = ToRegister(lir->output());
374 masm.convertDoubleToInt32(input, output, &fail, lir->mir()->canBeNegativeZero());
375 bailoutFrom(&fail, lir->snapshot());
376 }
377
378 void
visitFloat32ToInt32(LFloat32ToInt32 * lir)379 CodeGenerator::visitFloat32ToInt32(LFloat32ToInt32* lir)
380 {
381 Label fail;
382 FloatRegister input = ToFloatRegister(lir->input());
383 Register output = ToRegister(lir->output());
384 masm.convertFloat32ToInt32(input, output, &fail, lir->mir()->canBeNegativeZero());
385 bailoutFrom(&fail, lir->snapshot());
386 }
387
388 void
emitOOLTestObject(Register objreg,Label * ifEmulatesUndefined,Label * ifDoesntEmulateUndefined,Register scratch)389 CodeGenerator::emitOOLTestObject(Register objreg,
390 Label* ifEmulatesUndefined,
391 Label* ifDoesntEmulateUndefined,
392 Register scratch)
393 {
394 saveVolatile(scratch);
395 masm.setupUnalignedABICall(scratch);
396 masm.passABIArg(objreg);
397 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::EmulatesUndefined));
398 masm.storeCallBoolResult(scratch);
399 restoreVolatile(scratch);
400
401 masm.branchIfTrueBool(scratch, ifEmulatesUndefined);
402 masm.jump(ifDoesntEmulateUndefined);
403 }
404
405 // Base out-of-line code generator for all tests of the truthiness of an
406 // object, where the object might not be truthy. (Recall that per spec all
407 // objects are truthy, but we implement the JSCLASS_EMULATES_UNDEFINED class
408 // flag to permit objects to look like |undefined| in certain contexts,
409 // including in object truthiness testing.) We check truthiness inline except
410 // when we're testing it on a proxy (or if TI guarantees us that the specified
411 // object will never emulate |undefined|), in which case out-of-line code will
412 // call EmulatesUndefined for a conclusive answer.
413 class OutOfLineTestObject : public OutOfLineCodeBase<CodeGenerator>
414 {
415 Register objreg_;
416 Register scratch_;
417
418 Label* ifEmulatesUndefined_;
419 Label* ifDoesntEmulateUndefined_;
420
421 #ifdef DEBUG
initialized()422 bool initialized() { return ifEmulatesUndefined_ != nullptr; }
423 #endif
424
425 public:
OutOfLineTestObject()426 OutOfLineTestObject()
427 #ifdef DEBUG
428 : ifEmulatesUndefined_(nullptr), ifDoesntEmulateUndefined_(nullptr)
429 #endif
430 { }
431
accept(CodeGenerator * codegen)432 void accept(CodeGenerator* codegen) final override {
433 MOZ_ASSERT(initialized());
434 codegen->emitOOLTestObject(objreg_, ifEmulatesUndefined_, ifDoesntEmulateUndefined_,
435 scratch_);
436 }
437
438 // Specify the register where the object to be tested is found, labels to
439 // jump to if the object is truthy or falsy, and a scratch register for
440 // use in the out-of-line path.
setInputAndTargets(Register objreg,Label * ifEmulatesUndefined,Label * ifDoesntEmulateUndefined,Register scratch)441 void setInputAndTargets(Register objreg, Label* ifEmulatesUndefined, Label* ifDoesntEmulateUndefined,
442 Register scratch)
443 {
444 MOZ_ASSERT(!initialized());
445 MOZ_ASSERT(ifEmulatesUndefined);
446 objreg_ = objreg;
447 scratch_ = scratch;
448 ifEmulatesUndefined_ = ifEmulatesUndefined;
449 ifDoesntEmulateUndefined_ = ifDoesntEmulateUndefined;
450 }
451 };
452
453 // A subclass of OutOfLineTestObject containing two extra labels, for use when
454 // the ifTruthy/ifFalsy labels are needed in inline code as well as out-of-line
455 // code. The user should bind these labels in inline code, and specify them as
456 // targets via setInputAndTargets, as appropriate.
457 class OutOfLineTestObjectWithLabels : public OutOfLineTestObject
458 {
459 Label label1_;
460 Label label2_;
461
462 public:
OutOfLineTestObjectWithLabels()463 OutOfLineTestObjectWithLabels() { }
464
label1()465 Label* label1() { return &label1_; }
label2()466 Label* label2() { return &label2_; }
467 };
468
469 void
testObjectEmulatesUndefinedKernel(Register objreg,Label * ifEmulatesUndefined,Label * ifDoesntEmulateUndefined,Register scratch,OutOfLineTestObject * ool)470 CodeGenerator::testObjectEmulatesUndefinedKernel(Register objreg,
471 Label* ifEmulatesUndefined,
472 Label* ifDoesntEmulateUndefined,
473 Register scratch, OutOfLineTestObject* ool)
474 {
475 ool->setInputAndTargets(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined, scratch);
476
477 // Perform a fast-path check of the object's class flags if the object's
478 // not a proxy. Let out-of-line code handle the slow cases that require
479 // saving registers, making a function call, and restoring registers.
480 masm.branchTestObjectTruthy(false, objreg, scratch, ool->entry(), ifEmulatesUndefined);
481 }
482
483 void
branchTestObjectEmulatesUndefined(Register objreg,Label * ifEmulatesUndefined,Label * ifDoesntEmulateUndefined,Register scratch,OutOfLineTestObject * ool)484 CodeGenerator::branchTestObjectEmulatesUndefined(Register objreg,
485 Label* ifEmulatesUndefined,
486 Label* ifDoesntEmulateUndefined,
487 Register scratch, OutOfLineTestObject* ool)
488 {
489 MOZ_ASSERT(!ifDoesntEmulateUndefined->bound(),
490 "ifDoesntEmulateUndefined will be bound to the fallthrough path");
491
492 testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
493 scratch, ool);
494 masm.bind(ifDoesntEmulateUndefined);
495 }
496
497 void
testObjectEmulatesUndefined(Register objreg,Label * ifEmulatesUndefined,Label * ifDoesntEmulateUndefined,Register scratch,OutOfLineTestObject * ool)498 CodeGenerator::testObjectEmulatesUndefined(Register objreg,
499 Label* ifEmulatesUndefined,
500 Label* ifDoesntEmulateUndefined,
501 Register scratch, OutOfLineTestObject* ool)
502 {
503 testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
504 scratch, ool);
505 masm.jump(ifDoesntEmulateUndefined);
506 }
507
508 void
testValueTruthyKernel(const ValueOperand & value,const LDefinition * scratch1,const LDefinition * scratch2,FloatRegister fr,Label * ifTruthy,Label * ifFalsy,OutOfLineTestObject * ool,MDefinition * valueMIR)509 CodeGenerator::testValueTruthyKernel(const ValueOperand& value,
510 const LDefinition* scratch1, const LDefinition* scratch2,
511 FloatRegister fr,
512 Label* ifTruthy, Label* ifFalsy,
513 OutOfLineTestObject* ool,
514 MDefinition* valueMIR)
515 {
516 // Count the number of possible type tags we might have, so we'll know when
517 // we've checked them all and hence can avoid emitting a tag check for the
518 // last one. In particular, whenever tagCount is 1 that means we've tried
519 // all but one of them already so we know exactly what's left based on the
520 // mightBe* booleans.
521 bool mightBeUndefined = valueMIR->mightBeType(MIRType::Undefined);
522 bool mightBeNull = valueMIR->mightBeType(MIRType::Null);
523 bool mightBeBoolean = valueMIR->mightBeType(MIRType::Boolean);
524 bool mightBeInt32 = valueMIR->mightBeType(MIRType::Int32);
525 bool mightBeObject = valueMIR->mightBeType(MIRType::Object);
526 bool mightBeString = valueMIR->mightBeType(MIRType::String);
527 bool mightBeSymbol = valueMIR->mightBeType(MIRType::Symbol);
528 bool mightBeDouble = valueMIR->mightBeType(MIRType::Double);
529 int tagCount = int(mightBeUndefined) + int(mightBeNull) +
530 int(mightBeBoolean) + int(mightBeInt32) + int(mightBeObject) +
531 int(mightBeString) + int(mightBeSymbol) + int(mightBeDouble);
532
533 MOZ_ASSERT_IF(!valueMIR->emptyResultTypeSet(), tagCount > 0);
534
535 // If we know we're null or undefined, we're definitely falsy, no
536 // need to even check the tag.
537 if (int(mightBeNull) + int(mightBeUndefined) == tagCount) {
538 masm.jump(ifFalsy);
539 return;
540 }
541
542 Register tag = masm.splitTagForTest(value);
543
544 if (mightBeUndefined) {
545 MOZ_ASSERT(tagCount > 1);
546 masm.branchTestUndefined(Assembler::Equal, tag, ifFalsy);
547 --tagCount;
548 }
549
550 if (mightBeNull) {
551 MOZ_ASSERT(tagCount > 1);
552 masm.branchTestNull(Assembler::Equal, tag, ifFalsy);
553 --tagCount;
554 }
555
556 if (mightBeBoolean) {
557 MOZ_ASSERT(tagCount != 0);
558 Label notBoolean;
559 if (tagCount != 1)
560 masm.branchTestBoolean(Assembler::NotEqual, tag, ¬Boolean);
561 masm.branchTestBooleanTruthy(false, value, ifFalsy);
562 if (tagCount != 1)
563 masm.jump(ifTruthy);
564 // Else just fall through to truthiness.
565 masm.bind(¬Boolean);
566 --tagCount;
567 }
568
569 if (mightBeInt32) {
570 MOZ_ASSERT(tagCount != 0);
571 Label notInt32;
572 if (tagCount != 1)
573 masm.branchTestInt32(Assembler::NotEqual, tag, ¬Int32);
574 masm.branchTestInt32Truthy(false, value, ifFalsy);
575 if (tagCount != 1)
576 masm.jump(ifTruthy);
577 // Else just fall through to truthiness.
578 masm.bind(¬Int32);
579 --tagCount;
580 }
581
582 if (mightBeObject) {
583 MOZ_ASSERT(tagCount != 0);
584 if (ool) {
585 Label notObject;
586
587 if (tagCount != 1)
588 masm.branchTestObject(Assembler::NotEqual, tag, ¬Object);
589
590 Register objreg = masm.extractObject(value, ToRegister(scratch1));
591 testObjectEmulatesUndefined(objreg, ifFalsy, ifTruthy, ToRegister(scratch2), ool);
592
593 masm.bind(¬Object);
594 } else {
595 if (tagCount != 1)
596 masm.branchTestObject(Assembler::Equal, tag, ifTruthy);
597 // Else just fall through to truthiness.
598 }
599 --tagCount;
600 } else {
601 MOZ_ASSERT(!ool,
602 "We better not have an unused OOL path, since the code generator will try to "
603 "generate code for it but we never set up its labels, which will cause null "
604 "derefs of those labels.");
605 }
606
607 if (mightBeString) {
608 // Test if a string is non-empty.
609 MOZ_ASSERT(tagCount != 0);
610 Label notString;
611 if (tagCount != 1)
612 masm.branchTestString(Assembler::NotEqual, tag, ¬String);
613 masm.branchTestStringTruthy(false, value, ifFalsy);
614 if (tagCount != 1)
615 masm.jump(ifTruthy);
616 // Else just fall through to truthiness.
617 masm.bind(¬String);
618 --tagCount;
619 }
620
621 if (mightBeSymbol) {
622 // All symbols are truthy.
623 MOZ_ASSERT(tagCount != 0);
624 if (tagCount != 1)
625 masm.branchTestSymbol(Assembler::Equal, tag, ifTruthy);
626 // Else fall through to ifTruthy.
627 --tagCount;
628 }
629
630 if (mightBeDouble) {
631 MOZ_ASSERT(tagCount == 1);
632 // If we reach here the value is a double.
633 masm.unboxDouble(value, fr);
634 masm.branchTestDoubleTruthy(false, fr, ifFalsy);
635 --tagCount;
636 }
637
638 MOZ_ASSERT(tagCount == 0);
639
640 // Fall through for truthy.
641 }
642
643 void
testValueTruthy(const ValueOperand & value,const LDefinition * scratch1,const LDefinition * scratch2,FloatRegister fr,Label * ifTruthy,Label * ifFalsy,OutOfLineTestObject * ool,MDefinition * valueMIR)644 CodeGenerator::testValueTruthy(const ValueOperand& value,
645 const LDefinition* scratch1, const LDefinition* scratch2,
646 FloatRegister fr,
647 Label* ifTruthy, Label* ifFalsy,
648 OutOfLineTestObject* ool,
649 MDefinition* valueMIR)
650 {
651 testValueTruthyKernel(value, scratch1, scratch2, fr, ifTruthy, ifFalsy, ool, valueMIR);
652 masm.jump(ifTruthy);
653 }
654
655 void
visitTestOAndBranch(LTestOAndBranch * lir)656 CodeGenerator::visitTestOAndBranch(LTestOAndBranch* lir)
657 {
658 MIRType inputType = lir->mir()->input()->type();
659 MOZ_ASSERT(inputType == MIRType::ObjectOrNull || lir->mir()->operandMightEmulateUndefined(),
660 "If the object couldn't emulate undefined, this should have been folded.");
661
662 Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
663 Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
664 Register input = ToRegister(lir->input());
665
666 if (lir->mir()->operandMightEmulateUndefined()) {
667 if (inputType == MIRType::ObjectOrNull)
668 masm.branchTestPtr(Assembler::Zero, input, input, falsy);
669
670 OutOfLineTestObject* ool = new(alloc()) OutOfLineTestObject();
671 addOutOfLineCode(ool, lir->mir());
672
673 testObjectEmulatesUndefined(input, falsy, truthy, ToRegister(lir->temp()), ool);
674 } else {
675 MOZ_ASSERT(inputType == MIRType::ObjectOrNull);
676 testZeroEmitBranch(Assembler::NotEqual, input, lir->ifTruthy(), lir->ifFalsy());
677 }
678 }
679
680 void
visitTestVAndBranch(LTestVAndBranch * lir)681 CodeGenerator::visitTestVAndBranch(LTestVAndBranch* lir)
682 {
683 OutOfLineTestObject* ool = nullptr;
684 MDefinition* input = lir->mir()->input();
685 // Unfortunately, it's possible that someone (e.g. phi elimination) switched
686 // out our input after we did cacheOperandMightEmulateUndefined. So we
687 // might think it can emulate undefined _and_ know that it can't be an
688 // object.
689 if (lir->mir()->operandMightEmulateUndefined() && input->mightBeType(MIRType::Object)) {
690 ool = new(alloc()) OutOfLineTestObject();
691 addOutOfLineCode(ool, lir->mir());
692 }
693
694 Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
695 Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
696
697 testValueTruthy(ToValue(lir, LTestVAndBranch::Input),
698 lir->temp1(), lir->temp2(),
699 ToFloatRegister(lir->tempFloat()),
700 truthy, falsy, ool, input);
701 }
702
703 void
visitFunctionDispatch(LFunctionDispatch * lir)704 CodeGenerator::visitFunctionDispatch(LFunctionDispatch* lir)
705 {
706 MFunctionDispatch* mir = lir->mir();
707 Register input = ToRegister(lir->input());
708 Label* lastLabel;
709 size_t casesWithFallback;
710
711 // Determine if the last case is fallback or an ordinary case.
712 if (!mir->hasFallback()) {
713 MOZ_ASSERT(mir->numCases() > 0);
714 casesWithFallback = mir->numCases();
715 lastLabel = skipTrivialBlocks(mir->getCaseBlock(mir->numCases() - 1))->lir()->label();
716 } else {
717 casesWithFallback = mir->numCases() + 1;
718 lastLabel = skipTrivialBlocks(mir->getFallback())->lir()->label();
719 }
720
721 // Compare function pointers, except for the last case.
722 for (size_t i = 0; i < casesWithFallback - 1; i++) {
723 MOZ_ASSERT(i < mir->numCases());
724 LBlock* target = skipTrivialBlocks(mir->getCaseBlock(i))->lir();
725 if (ObjectGroup* funcGroup = mir->getCaseObjectGroup(i)) {
726 masm.branchPtr(Assembler::Equal, Address(input, JSObject::offsetOfGroup()),
727 ImmGCPtr(funcGroup), target->label());
728 } else {
729 JSFunction* func = mir->getCase(i);
730 masm.branchPtr(Assembler::Equal, input, ImmGCPtr(func), target->label());
731 }
732 }
733
734 // Jump to the last case.
735 masm.jump(lastLabel);
736 }
737
738 void
visitObjectGroupDispatch(LObjectGroupDispatch * lir)739 CodeGenerator::visitObjectGroupDispatch(LObjectGroupDispatch* lir)
740 {
741 MObjectGroupDispatch* mir = lir->mir();
742 Register input = ToRegister(lir->input());
743 Register temp = ToRegister(lir->temp());
744
745 // Load the incoming ObjectGroup in temp.
746 masm.loadPtr(Address(input, JSObject::offsetOfGroup()), temp);
747
748 // Compare ObjectGroups.
749 MacroAssembler::BranchGCPtr lastBranch;
750 LBlock* lastBlock = nullptr;
751 InlinePropertyTable* propTable = mir->propTable();
752 for (size_t i = 0; i < mir->numCases(); i++) {
753 JSFunction* func = mir->getCase(i);
754 LBlock* target = skipTrivialBlocks(mir->getCaseBlock(i))->lir();
755
756 DebugOnly<bool> found = false;
757 for (size_t j = 0; j < propTable->numEntries(); j++) {
758 if (propTable->getFunction(j) != func)
759 continue;
760
761 if (lastBranch.isInitialized())
762 lastBranch.emit(masm);
763
764 ObjectGroup* group = propTable->getObjectGroup(j);
765 lastBranch = MacroAssembler::BranchGCPtr(Assembler::Equal, temp, ImmGCPtr(group),
766 target->label());
767 lastBlock = target;
768 found = true;
769 }
770 MOZ_ASSERT(found);
771 }
772
773 // Jump to fallback block if we have an unknown ObjectGroup. If there's no
774 // fallback block, we should have handled all cases.
775
776 if (!mir->hasFallback()) {
777 MOZ_ASSERT(lastBranch.isInitialized());
778 #ifdef DEBUG
779 Label ok;
780 lastBranch.relink(&ok);
781 lastBranch.emit(masm);
782 masm.assumeUnreachable("Unexpected ObjectGroup");
783 masm.bind(&ok);
784 #endif
785 if (!isNextBlock(lastBlock))
786 masm.jump(lastBlock->label());
787 return;
788 }
789
790 LBlock* fallback = skipTrivialBlocks(mir->getFallback())->lir();
791 if (!lastBranch.isInitialized()) {
792 if (!isNextBlock(fallback))
793 masm.jump(fallback->label());
794 return;
795 }
796
797 lastBranch.invertCondition();
798 lastBranch.relink(fallback->label());
799 lastBranch.emit(masm);
800
801 if (!isNextBlock(lastBlock))
802 masm.jump(lastBlock->label());
803 }
804
805 void
visitBooleanToString(LBooleanToString * lir)806 CodeGenerator::visitBooleanToString(LBooleanToString* lir)
807 {
808 Register input = ToRegister(lir->input());
809 Register output = ToRegister(lir->output());
810 const JSAtomState& names = GetJitContext()->runtime->names();
811 Label true_, done;
812
813 masm.branchTest32(Assembler::NonZero, input, input, &true_);
814 masm.movePtr(ImmGCPtr(names.false_), output);
815 masm.jump(&done);
816
817 masm.bind(&true_);
818 masm.movePtr(ImmGCPtr(names.true_), output);
819
820 masm.bind(&done);
821 }
822
823 void
emitIntToString(Register input,Register output,Label * ool)824 CodeGenerator::emitIntToString(Register input, Register output, Label* ool)
825 {
826 masm.branch32(Assembler::AboveOrEqual, input, Imm32(StaticStrings::INT_STATIC_LIMIT), ool);
827
828 // Fast path for small integers.
829 masm.movePtr(ImmPtr(&GetJitContext()->runtime->staticStrings().intStaticTable), output);
830 masm.loadPtr(BaseIndex(output, input, ScalePointer), output);
831 }
832
833 typedef JSFlatString* (*IntToStringFn)(ExclusiveContext*, int);
834 static const VMFunction IntToStringInfo =
835 FunctionInfo<IntToStringFn>(Int32ToString<CanGC>, "Int32ToString");
836
837 void
visitIntToString(LIntToString * lir)838 CodeGenerator::visitIntToString(LIntToString* lir)
839 {
840 Register input = ToRegister(lir->input());
841 Register output = ToRegister(lir->output());
842
843 OutOfLineCode* ool = oolCallVM(IntToStringInfo, lir, ArgList(input),
844 StoreRegisterTo(output));
845
846 emitIntToString(input, output, ool->entry());
847
848 masm.bind(ool->rejoin());
849 }
850
851 typedef JSString* (*DoubleToStringFn)(ExclusiveContext*, double);
852 static const VMFunction DoubleToStringInfo =
853 FunctionInfo<DoubleToStringFn>(NumberToString<CanGC>, "NumberToString");
854
855 void
visitDoubleToString(LDoubleToString * lir)856 CodeGenerator::visitDoubleToString(LDoubleToString* lir)
857 {
858 FloatRegister input = ToFloatRegister(lir->input());
859 Register temp = ToRegister(lir->tempInt());
860 Register output = ToRegister(lir->output());
861
862 OutOfLineCode* ool = oolCallVM(DoubleToStringInfo, lir, ArgList(input),
863 StoreRegisterTo(output));
864
865 // Try double to integer conversion and run integer to string code.
866 masm.convertDoubleToInt32(input, temp, ool->entry(), true);
867 emitIntToString(temp, output, ool->entry());
868
869 masm.bind(ool->rejoin());
870 }
871
872 typedef JSString* (*PrimitiveToStringFn)(JSContext*, HandleValue);
873 static const VMFunction PrimitiveToStringInfo =
874 FunctionInfo<PrimitiveToStringFn>(ToStringSlow, "ToStringSlow");
875
876 void
visitValueToString(LValueToString * lir)877 CodeGenerator::visitValueToString(LValueToString* lir)
878 {
879 ValueOperand input = ToValue(lir, LValueToString::Input);
880 Register output = ToRegister(lir->output());
881
882 OutOfLineCode* ool = oolCallVM(PrimitiveToStringInfo, lir, ArgList(input),
883 StoreRegisterTo(output));
884
885 Label done;
886 Register tag = masm.splitTagForTest(input);
887 const JSAtomState& names = GetJitContext()->runtime->names();
888
889 // String
890 if (lir->mir()->input()->mightBeType(MIRType::String)) {
891 Label notString;
892 masm.branchTestString(Assembler::NotEqual, tag, ¬String);
893 masm.unboxString(input, output);
894 masm.jump(&done);
895 masm.bind(¬String);
896 }
897
898 // Integer
899 if (lir->mir()->input()->mightBeType(MIRType::Int32)) {
900 Label notInteger;
901 masm.branchTestInt32(Assembler::NotEqual, tag, ¬Integer);
902 Register unboxed = ToTempUnboxRegister(lir->tempToUnbox());
903 unboxed = masm.extractInt32(input, unboxed);
904 emitIntToString(unboxed, output, ool->entry());
905 masm.jump(&done);
906 masm.bind(¬Integer);
907 }
908
909 // Double
910 if (lir->mir()->input()->mightBeType(MIRType::Double)) {
911 // Note: no fastpath. Need two extra registers and can only convert doubles
912 // that fit integers and are smaller than StaticStrings::INT_STATIC_LIMIT.
913 masm.branchTestDouble(Assembler::Equal, tag, ool->entry());
914 }
915
916 // Undefined
917 if (lir->mir()->input()->mightBeType(MIRType::Undefined)) {
918 Label notUndefined;
919 masm.branchTestUndefined(Assembler::NotEqual, tag, ¬Undefined);
920 masm.movePtr(ImmGCPtr(names.undefined), output);
921 masm.jump(&done);
922 masm.bind(¬Undefined);
923 }
924
925 // Null
926 if (lir->mir()->input()->mightBeType(MIRType::Null)) {
927 Label notNull;
928 masm.branchTestNull(Assembler::NotEqual, tag, ¬Null);
929 masm.movePtr(ImmGCPtr(names.null), output);
930 masm.jump(&done);
931 masm.bind(¬Null);
932 }
933
934 // Boolean
935 if (lir->mir()->input()->mightBeType(MIRType::Boolean)) {
936 Label notBoolean, true_;
937 masm.branchTestBoolean(Assembler::NotEqual, tag, ¬Boolean);
938 masm.branchTestBooleanTruthy(true, input, &true_);
939 masm.movePtr(ImmGCPtr(names.false_), output);
940 masm.jump(&done);
941 masm.bind(&true_);
942 masm.movePtr(ImmGCPtr(names.true_), output);
943 masm.jump(&done);
944 masm.bind(¬Boolean);
945 }
946
947 // Object
948 if (lir->mir()->input()->mightBeType(MIRType::Object)) {
949 // Bail.
950 MOZ_ASSERT(lir->mir()->fallible());
951 Label bail;
952 masm.branchTestObject(Assembler::Equal, tag, &bail);
953 bailoutFrom(&bail, lir->snapshot());
954 }
955
956 // Symbol
957 if (lir->mir()->input()->mightBeType(MIRType::Symbol))
958 masm.branchTestSymbol(Assembler::Equal, tag, ool->entry());
959
960 #ifdef DEBUG
961 masm.assumeUnreachable("Unexpected type for MValueToString.");
962 #endif
963
964 masm.bind(&done);
965 masm.bind(ool->rejoin());
966 }
967
968 typedef JSObject* (*ToObjectFn)(JSContext*, HandleValue, bool);
969 static const VMFunction ToObjectInfo =
970 FunctionInfo<ToObjectFn>(ToObjectSlow, "ToObjectSlow");
971
972 void
visitValueToObjectOrNull(LValueToObjectOrNull * lir)973 CodeGenerator::visitValueToObjectOrNull(LValueToObjectOrNull* lir)
974 {
975 ValueOperand input = ToValue(lir, LValueToObjectOrNull::Input);
976 Register output = ToRegister(lir->output());
977
978 OutOfLineCode* ool = oolCallVM(ToObjectInfo, lir, ArgList(input, Imm32(0)),
979 StoreRegisterTo(output));
980
981 Label done;
982 masm.branchTestObject(Assembler::Equal, input, &done);
983 masm.branchTestNull(Assembler::NotEqual, input, ool->entry());
984
985 masm.bind(&done);
986 masm.unboxNonDouble(input, output);
987
988 masm.bind(ool->rejoin());
989 }
990
991 typedef JSObject* (*CloneRegExpObjectFn)(JSContext*, JSObject*);
992 static const VMFunction CloneRegExpObjectInfo =
993 FunctionInfo<CloneRegExpObjectFn>(CloneRegExpObject, "CloneRegExpObject");
994
995 void
visitRegExp(LRegExp * lir)996 CodeGenerator::visitRegExp(LRegExp* lir)
997 {
998 pushArg(ImmGCPtr(lir->mir()->source()));
999 callVM(CloneRegExpObjectInfo, lir);
1000 }
1001
1002 // Amount of space to reserve on the stack when executing RegExps inline.
1003 static const size_t RegExpReservedStack = sizeof(irregexp::InputOutputData)
1004 + sizeof(MatchPairs)
1005 + RegExpObject::MaxPairCount * sizeof(MatchPair);
1006
1007 static size_t
RegExpPairsVectorStartOffset(size_t inputOutputDataStartOffset)1008 RegExpPairsVectorStartOffset(size_t inputOutputDataStartOffset)
1009 {
1010 return inputOutputDataStartOffset + sizeof(irregexp::InputOutputData) + sizeof(MatchPairs);
1011 }
1012
1013 static Address
RegExpPairCountAddress(MacroAssembler & masm,size_t inputOutputDataStartOffset)1014 RegExpPairCountAddress(MacroAssembler& masm, size_t inputOutputDataStartOffset)
1015 {
1016 return Address(masm.getStackPointer(), inputOutputDataStartOffset
1017 + sizeof(irregexp::InputOutputData)
1018 + MatchPairs::offsetOfPairCount());
1019 }
1020
1021 // Prepare an InputOutputData and optional MatchPairs which space has been
1022 // allocated for on the stack, and try to execute a RegExp on a string input.
1023 // If the RegExp was successfully executed and matched the input, fallthrough,
1024 // otherwise jump to notFound or failure.
1025 static bool
PrepareAndExecuteRegExp(JSContext * cx,MacroAssembler & masm,Register regexp,Register input,Register lastIndex,Register temp1,Register temp2,Register temp3,size_t inputOutputDataStartOffset,RegExpShared::CompilationMode mode,Label * notFound,Label * failure)1026 PrepareAndExecuteRegExp(JSContext* cx, MacroAssembler& masm, Register regexp, Register input,
1027 Register lastIndex,
1028 Register temp1, Register temp2, Register temp3,
1029 size_t inputOutputDataStartOffset,
1030 RegExpShared::CompilationMode mode,
1031 Label* notFound, Label* failure)
1032 {
1033 size_t matchPairsStartOffset = inputOutputDataStartOffset + sizeof(irregexp::InputOutputData);
1034 size_t pairsVectorStartOffset = RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
1035
1036 Address inputStartAddress(masm.getStackPointer(),
1037 inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, inputStart));
1038 Address inputEndAddress(masm.getStackPointer(),
1039 inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, inputEnd));
1040 Address matchesPointerAddress(masm.getStackPointer(),
1041 inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, matches));
1042 Address startIndexAddress(masm.getStackPointer(),
1043 inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, startIndex));
1044 Address endIndexAddress(masm.getStackPointer(),
1045 inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, endIndex));
1046 Address matchResultAddress(masm.getStackPointer(),
1047 inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, result));
1048
1049 Address pairCountAddress = RegExpPairCountAddress(masm, inputOutputDataStartOffset);
1050 Address pairsPointerAddress(masm.getStackPointer(),
1051 matchPairsStartOffset + MatchPairs::offsetOfPairs());
1052
1053 Address pairsVectorAddress(masm.getStackPointer(), pairsVectorStartOffset);
1054
1055 RegExpStatics* res = cx->global()->getRegExpStatics(cx);
1056 if (!res)
1057 return false;
1058 #ifdef JS_USE_LINK_REGISTER
1059 if (mode != RegExpShared::MatchOnly)
1060 masm.pushReturnAddress();
1061 #endif
1062 if (mode == RegExpShared::Normal) {
1063 // First, fill in a skeletal MatchPairs instance on the stack. This will be
1064 // passed to the OOL stub in the caller if we aren't able to execute the
1065 // RegExp inline, and that stub needs to be able to determine whether the
1066 // execution finished successfully.
1067 masm.store32(Imm32(1), pairCountAddress);
1068 masm.store32(Imm32(-1), pairsVectorAddress);
1069 masm.computeEffectiveAddress(pairsVectorAddress, temp1);
1070 masm.storePtr(temp1, pairsPointerAddress);
1071 }
1072
1073 // Check for a linear input string.
1074 masm.branchIfRopeOrExternal(input, temp1, failure);
1075
1076 // Get the RegExpShared for the RegExp.
1077 masm.loadPtr(Address(regexp, NativeObject::getFixedSlotOffset(RegExpObject::PRIVATE_SLOT)), temp1);
1078 masm.branchPtr(Assembler::Equal, temp1, ImmWord(0), failure);
1079
1080 // ES6 21.2.2.2 step 2.
1081 // See RegExp.cpp ExecuteRegExp for more detail.
1082 {
1083 Label done;
1084
1085 masm.branchTest32(Assembler::Zero, Address(temp1, RegExpShared::offsetOfFlags()),
1086 Imm32(UnicodeFlag), &done);
1087
1088 // If input is latin1, there should not be surrogate pair.
1089 masm.branchLatin1String(input, &done);
1090
1091 // Check if |lastIndex > 0 && lastIndex < input->length()|.
1092 // lastIndex should already have no sign here.
1093 masm.branchTest32(Assembler::Zero, lastIndex, lastIndex, &done);
1094 masm.loadStringLength(input, temp2);
1095 masm.branch32(Assembler::AboveOrEqual, lastIndex, temp2, &done);
1096
1097 // Check if input[lastIndex] is trail surrogate.
1098 masm.loadStringChars(input, temp2);
1099 masm.computeEffectiveAddress(BaseIndex(temp2, lastIndex, TimesTwo), temp3);
1100 masm.load16ZeroExtend(Address(temp3, 0), temp3);
1101
1102 masm.branch32(Assembler::Below, temp3, Imm32(unicode::TrailSurrogateMin), &done);
1103 masm.branch32(Assembler::Above, temp3, Imm32(unicode::TrailSurrogateMax), &done);
1104
1105 // Check if input[lastIndex-1] is lead surrogate.
1106 masm.move32(lastIndex, temp3);
1107 masm.sub32(Imm32(1), temp3);
1108 masm.computeEffectiveAddress(BaseIndex(temp2, temp3, TimesTwo), temp3);
1109 masm.load16ZeroExtend(Address(temp3, 0), temp3);
1110
1111 masm.branch32(Assembler::Below, temp3, Imm32(unicode::LeadSurrogateMin), &done);
1112 masm.branch32(Assembler::Above, temp3, Imm32(unicode::LeadSurrogateMax), &done);
1113
1114 // Move lastIndex to lead surrogate.
1115 masm.subPtr(Imm32(1), lastIndex);
1116
1117 masm.bind(&done);
1118 }
1119
1120 if (mode == RegExpShared::Normal) {
1121 // Don't handle RegExps with excessive parens.
1122 masm.load32(Address(temp1, RegExpShared::offsetOfParenCount()), temp2);
1123 masm.branch32(Assembler::AboveOrEqual, temp2, Imm32(RegExpObject::MaxPairCount), failure);
1124
1125 // Fill in the paren count in the MatchPairs on the stack.
1126 masm.add32(Imm32(1), temp2);
1127 masm.store32(temp2, pairCountAddress);
1128 }
1129
1130 // Load the code pointer for the type of input string we have, and compute
1131 // the input start/end pointers in the InputOutputData.
1132 Register codePointer = temp1;
1133 {
1134 masm.loadStringChars(input, temp2);
1135 masm.storePtr(temp2, inputStartAddress);
1136 masm.loadStringLength(input, temp3);
1137
1138 Label isLatin1, done;
1139 masm.branchLatin1String(input, &isLatin1);
1140 {
1141 masm.lshiftPtr(Imm32(1), temp3);
1142 masm.loadPtr(Address(temp1, RegExpShared::offsetOfTwoByteJitCode(mode)),
1143 codePointer);
1144 }
1145 masm.jump(&done);
1146 {
1147 masm.bind(&isLatin1);
1148 masm.loadPtr(Address(temp1, RegExpShared::offsetOfLatin1JitCode(mode)),
1149 codePointer);
1150 }
1151 masm.bind(&done);
1152
1153 masm.addPtr(temp3, temp2);
1154 masm.storePtr(temp2, inputEndAddress);
1155 }
1156
1157 // Check the RegExpShared has been compiled for this type of input.
1158 masm.branchPtr(Assembler::Equal, codePointer, ImmWord(0), failure);
1159 masm.loadPtr(Address(codePointer, JitCode::offsetOfCode()), codePointer);
1160
1161 // Finish filling in the InputOutputData instance on the stack.
1162 if (mode == RegExpShared::Normal) {
1163 masm.computeEffectiveAddress(Address(masm.getStackPointer(), matchPairsStartOffset), temp2);
1164 masm.storePtr(temp2, matchesPointerAddress);
1165 } else {
1166 // Use InputOutputData.endIndex itself for output.
1167 masm.computeEffectiveAddress(endIndexAddress, temp2);
1168 masm.storePtr(temp2, endIndexAddress);
1169 }
1170 masm.storePtr(lastIndex, startIndexAddress);
1171 masm.store32(Imm32(0), matchResultAddress);
1172
1173 // Save any volatile inputs.
1174 LiveGeneralRegisterSet volatileRegs;
1175 if (lastIndex.volatile_())
1176 volatileRegs.add(lastIndex);
1177 if (input.volatile_())
1178 volatileRegs.add(input);
1179 if (regexp.volatile_())
1180 volatileRegs.add(regexp);
1181
1182 // Execute the RegExp.
1183 masm.computeEffectiveAddress(Address(masm.getStackPointer(), inputOutputDataStartOffset), temp2);
1184 masm.PushRegsInMask(volatileRegs);
1185 masm.setupUnalignedABICall(temp3);
1186 masm.passABIArg(temp2);
1187 masm.callWithABI(codePointer);
1188 masm.PopRegsInMask(volatileRegs);
1189
1190 Label success;
1191 masm.branch32(Assembler::Equal, matchResultAddress,
1192 Imm32(RegExpRunStatus_Success_NotFound), notFound);
1193 masm.branch32(Assembler::Equal, matchResultAddress,
1194 Imm32(RegExpRunStatus_Error), failure);
1195
1196 // Lazily update the RegExpStatics.
1197 masm.movePtr(ImmPtr(res), temp1);
1198
1199 Address pendingInputAddress(temp1, RegExpStatics::offsetOfPendingInput());
1200 Address matchesInputAddress(temp1, RegExpStatics::offsetOfMatchesInput());
1201 Address lazySourceAddress(temp1, RegExpStatics::offsetOfLazySource());
1202 Address lazyIndexAddress(temp1, RegExpStatics::offsetOfLazyIndex());
1203
1204 masm.patchableCallPreBarrier(pendingInputAddress, MIRType::String);
1205 masm.patchableCallPreBarrier(matchesInputAddress, MIRType::String);
1206 masm.patchableCallPreBarrier(lazySourceAddress, MIRType::String);
1207
1208 masm.storePtr(input, pendingInputAddress);
1209 masm.storePtr(input, matchesInputAddress);
1210 masm.storePtr(lastIndex, Address(temp1, RegExpStatics::offsetOfLazyIndex()));
1211 masm.store32(Imm32(1), Address(temp1, RegExpStatics::offsetOfPendingLazyEvaluation()));
1212
1213 masm.loadPtr(Address(regexp, NativeObject::getFixedSlotOffset(RegExpObject::PRIVATE_SLOT)), temp2);
1214 masm.loadPtr(Address(temp2, RegExpShared::offsetOfSource()), temp3);
1215 masm.storePtr(temp3, lazySourceAddress);
1216 masm.load32(Address(temp2, RegExpShared::offsetOfFlags()), temp3);
1217 masm.store32(temp3, Address(temp1, RegExpStatics::offsetOfLazyFlags()));
1218
1219 if (mode == RegExpShared::MatchOnly) {
1220 // endIndex is passed via temp3.
1221 masm.load32(endIndexAddress, temp3);
1222 }
1223
1224 return true;
1225 }
1226
1227 static void
1228 CopyStringChars(MacroAssembler& masm, Register to, Register from, Register len,
1229 Register byteOpScratch, size_t fromWidth, size_t toWidth);
1230
1231 class CreateDependentString
1232 {
1233 Register string_;
1234 Register temp_;
1235 Label* failure_;
1236 enum class FallbackKind : uint8_t {
1237 InlineString,
1238 FatInlineString,
1239 NotInlineString,
1240 Count
1241 };
1242 mozilla::EnumeratedArray<FallbackKind, FallbackKind::Count, Label> fallbacks_, joins_;
1243
1244 public:
1245 // Generate code that creates DependentString.
1246 // Caller should call generateFallback after masm.ret(), to generate
1247 // fallback path.
1248 void generate(MacroAssembler& masm, const JSAtomState& names,
1249 bool latin1, Register string,
1250 Register base, Register temp1, Register temp2,
1251 BaseIndex startIndexAddress, BaseIndex limitIndexAddress,
1252 Label* failure);
1253
1254 // Generate fallback path for creating DependentString.
1255 void generateFallback(MacroAssembler& masm, LiveRegisterSet regsToSave);
1256 };
1257
1258 void
generate(MacroAssembler & masm,const JSAtomState & names,bool latin1,Register string,Register base,Register temp1,Register temp2,BaseIndex startIndexAddress,BaseIndex limitIndexAddress,Label * failure)1259 CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
1260 bool latin1, Register string,
1261 Register base, Register temp1, Register temp2,
1262 BaseIndex startIndexAddress, BaseIndex limitIndexAddress,
1263 Label* failure)
1264 {
1265 string_ = string;
1266 temp_ = temp2;
1267 failure_ = failure;
1268
1269 // Compute the string length.
1270 masm.load32(startIndexAddress, temp2);
1271 masm.load32(limitIndexAddress, temp1);
1272 masm.sub32(temp2, temp1);
1273
1274 Label done, nonEmpty;
1275
1276 // Zero length matches use the empty string.
1277 masm.branchTest32(Assembler::NonZero, temp1, temp1, &nonEmpty);
1278 masm.movePtr(ImmGCPtr(names.empty), string);
1279 masm.jump(&done);
1280
1281 masm.bind(&nonEmpty);
1282
1283 Label notInline;
1284
1285 int32_t maxInlineLength = latin1
1286 ? (int32_t) JSFatInlineString::MAX_LENGTH_LATIN1
1287 : (int32_t) JSFatInlineString::MAX_LENGTH_TWO_BYTE;
1288 masm.branch32(Assembler::Above, temp1, Imm32(maxInlineLength), ¬Inline);
1289
1290 {
1291 // Make a thin or fat inline string.
1292 Label stringAllocated, fatInline;
1293
1294 int32_t maxThinInlineLength = latin1
1295 ? (int32_t) JSThinInlineString::MAX_LENGTH_LATIN1
1296 : (int32_t) JSThinInlineString::MAX_LENGTH_TWO_BYTE;
1297 masm.branch32(Assembler::Above, temp1, Imm32(maxThinInlineLength), &fatInline);
1298
1299 int32_t thinFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_THIN_INLINE_FLAGS;
1300 masm.newGCString(string, temp2, &fallbacks_[FallbackKind::InlineString]);
1301 masm.bind(&joins_[FallbackKind::InlineString]);
1302 masm.store32(Imm32(thinFlags), Address(string, JSString::offsetOfFlags()));
1303 masm.jump(&stringAllocated);
1304
1305 masm.bind(&fatInline);
1306
1307 int32_t fatFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_FAT_INLINE_FLAGS;
1308 masm.newGCFatInlineString(string, temp2, &fallbacks_[FallbackKind::FatInlineString]);
1309 masm.bind(&joins_[FallbackKind::FatInlineString]);
1310 masm.store32(Imm32(fatFlags), Address(string, JSString::offsetOfFlags()));
1311
1312 masm.bind(&stringAllocated);
1313 masm.store32(temp1, Address(string, JSString::offsetOfLength()));
1314
1315 masm.push(string);
1316 masm.push(base);
1317
1318 // Adjust the start index address for the above pushes.
1319 MOZ_ASSERT(startIndexAddress.base == masm.getStackPointer());
1320 BaseIndex newStartIndexAddress = startIndexAddress;
1321 newStartIndexAddress.offset += 2 * sizeof(void*);
1322
1323 // Load chars pointer for the new string.
1324 masm.addPtr(ImmWord(JSInlineString::offsetOfInlineStorage()), string);
1325
1326 // Load the source characters pointer.
1327 masm.loadStringChars(base, base);
1328 masm.load32(newStartIndexAddress, temp2);
1329 if (latin1)
1330 masm.addPtr(temp2, base);
1331 else
1332 masm.computeEffectiveAddress(BaseIndex(base, temp2, TimesTwo), base);
1333
1334 CopyStringChars(masm, string, base, temp1, temp2, latin1 ? 1 : 2, latin1 ? 1 : 2);
1335
1336 // Null-terminate.
1337 if (latin1)
1338 masm.store8(Imm32(0), Address(string, 0));
1339 else
1340 masm.store16(Imm32(0), Address(string, 0));
1341
1342 masm.pop(base);
1343 masm.pop(string);
1344 }
1345
1346 masm.jump(&done);
1347 masm.bind(¬Inline);
1348
1349 {
1350 // Make a dependent string.
1351 int32_t flags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::DEPENDENT_FLAGS;
1352
1353 masm.newGCString(string, temp2, &fallbacks_[FallbackKind::NotInlineString]);
1354 masm.bind(&joins_[FallbackKind::NotInlineString]);
1355 masm.store32(Imm32(flags), Address(string, JSString::offsetOfFlags()));
1356 masm.store32(temp1, Address(string, JSString::offsetOfLength()));
1357
1358 masm.loadPtr(Address(base, JSString::offsetOfNonInlineChars()), temp1);
1359 masm.load32(startIndexAddress, temp2);
1360 if (latin1)
1361 masm.addPtr(temp2, temp1);
1362 else
1363 masm.computeEffectiveAddress(BaseIndex(temp1, temp2, TimesTwo), temp1);
1364 masm.storePtr(temp1, Address(string, JSString::offsetOfNonInlineChars()));
1365 masm.storePtr(base, Address(string, JSDependentString::offsetOfBase()));
1366
1367 // Follow any base pointer if the input is itself a dependent string.
1368 // Watch for undepended strings, which have a base pointer but don't
1369 // actually share their characters with it.
1370 Label noBase;
1371 masm.branchTest32(Assembler::Zero, Address(base, JSString::offsetOfFlags()),
1372 Imm32(JSString::HAS_BASE_BIT), &noBase);
1373 masm.branchTest32(Assembler::NonZero, Address(base, JSString::offsetOfFlags()),
1374 Imm32(JSString::FLAT_BIT), &noBase);
1375 masm.loadPtr(Address(base, JSDependentString::offsetOfBase()), temp1);
1376 masm.storePtr(temp1, Address(string, JSDependentString::offsetOfBase()));
1377 masm.bind(&noBase);
1378 }
1379
1380 masm.bind(&done);
1381 }
1382
1383 static void*
AllocateString(JSContext * cx)1384 AllocateString(JSContext* cx)
1385 {
1386 return js::Allocate<JSString, NoGC>(cx);
1387 }
1388
1389 static void*
AllocateFatInlineString(JSContext * cx)1390 AllocateFatInlineString(JSContext* cx)
1391 {
1392 return js::Allocate<JSFatInlineString, NoGC>(cx);
1393 }
1394
1395 void
generateFallback(MacroAssembler & masm,LiveRegisterSet regsToSave)1396 CreateDependentString::generateFallback(MacroAssembler& masm, LiveRegisterSet regsToSave)
1397 {
1398 regsToSave.take(string_);
1399 regsToSave.take(temp_);
1400 for (FallbackKind kind : mozilla::MakeEnumeratedRange(FallbackKind::Count)) {
1401 masm.bind(&fallbacks_[kind]);
1402
1403 masm.PushRegsInMask(regsToSave);
1404
1405 masm.setupUnalignedABICall(string_);
1406 masm.loadJSContext(string_);
1407 masm.passABIArg(string_);
1408 masm.callWithABI(kind == FallbackKind::FatInlineString
1409 ? JS_FUNC_TO_DATA_PTR(void*, AllocateFatInlineString)
1410 : JS_FUNC_TO_DATA_PTR(void*, AllocateString));
1411 masm.storeCallPointerResult(string_);
1412
1413 masm.PopRegsInMask(regsToSave);
1414
1415 masm.branchPtr(Assembler::Equal, string_, ImmWord(0), failure_);
1416
1417 masm.jump(&joins_[kind]);
1418 }
1419 }
1420
1421 static void*
CreateMatchResultFallbackFunc(JSContext * cx,gc::AllocKind kind,size_t nDynamicSlots)1422 CreateMatchResultFallbackFunc(JSContext* cx, gc::AllocKind kind, size_t nDynamicSlots)
1423 {
1424 return js::Allocate<JSObject, NoGC>(cx, kind, nDynamicSlots, gc::DefaultHeap,
1425 &ArrayObject::class_);
1426 }
1427
1428 static void
CreateMatchResultFallback(MacroAssembler & masm,LiveRegisterSet regsToSave,Register object,Register temp2,Register temp5,ArrayObject * templateObj,Label * fail)1429 CreateMatchResultFallback(MacroAssembler& masm, LiveRegisterSet regsToSave,
1430 Register object, Register temp2, Register temp5,
1431 ArrayObject* templateObj, Label* fail)
1432 {
1433 MOZ_ASSERT(templateObj->group()->clasp() == &ArrayObject::class_);
1434
1435 regsToSave.take(object);
1436 regsToSave.take(temp2);
1437 regsToSave.take(temp5);
1438 masm.PushRegsInMask(regsToSave);
1439
1440 masm.setupUnalignedABICall(object);
1441
1442 masm.loadJSContext(object);
1443 masm.passABIArg(object);
1444 masm.move32(Imm32(int32_t(templateObj->asTenured().getAllocKind())), temp2);
1445 masm.passABIArg(temp2);
1446 masm.move32(Imm32(int32_t(templateObj->as<NativeObject>().numDynamicSlots())), temp5);
1447 masm.passABIArg(temp5);
1448 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, CreateMatchResultFallbackFunc));
1449 masm.storeCallPointerResult(object);
1450
1451 masm.PopRegsInMask(regsToSave);
1452
1453 masm.branchPtr(Assembler::Equal, object, ImmWord(0), fail);
1454
1455 masm.initGCThing(object, temp2, templateObj, true, false);
1456 }
1457
1458 JitCode*
generateRegExpMatcherStub(JSContext * cx)1459 JitCompartment::generateRegExpMatcherStub(JSContext* cx)
1460 {
1461 Register regexp = RegExpMatcherRegExpReg;
1462 Register input = RegExpMatcherStringReg;
1463 Register lastIndex = RegExpMatcherLastIndexReg;
1464 ValueOperand result = JSReturnOperand;
1465
1466 // We are free to clobber all registers, as LRegExpMatcher is a call instruction.
1467 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
1468 regs.take(input);
1469 regs.take(regexp);
1470 regs.take(lastIndex);
1471
1472 // temp5 is used in single byte instructions when creating dependent
1473 // strings, and has restrictions on which register it can be on some
1474 // platforms.
1475 Register temp5;
1476 {
1477 AllocatableGeneralRegisterSet oregs = regs;
1478 do {
1479 temp5 = oregs.takeAny();
1480 } while (!MacroAssembler::canUseInSingleByteInstruction(temp5));
1481 regs.take(temp5);
1482 }
1483
1484 Register temp1 = regs.takeAny();
1485 Register temp2 = regs.takeAny();
1486 Register temp3 = regs.takeAny();
1487
1488 Register maybeTemp4 = InvalidReg;
1489 if (!regs.empty()) {
1490 // There are not enough registers on x86.
1491 maybeTemp4 = regs.takeAny();
1492 }
1493
1494 ArrayObject* templateObject = cx->compartment()->regExps.getOrCreateMatchResultTemplateObject(cx);
1495 if (!templateObject)
1496 return nullptr;
1497
1498 // The template object should have enough space for the maximum number of
1499 // pairs this stub can handle.
1500 MOZ_ASSERT(ObjectElements::VALUES_PER_HEADER + RegExpObject::MaxPairCount ==
1501 gc::GetGCKindSlots(templateObject->asTenured().getAllocKind()));
1502
1503 MacroAssembler masm(cx);
1504
1505 // The InputOutputData is placed above the return address on the stack.
1506 size_t inputOutputDataStartOffset = sizeof(void*);
1507
1508 Label notFound, oolEntry;
1509 if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex,
1510 temp1, temp2, temp5, inputOutputDataStartOffset,
1511 RegExpShared::Normal, ¬Found, &oolEntry))
1512 {
1513 return nullptr;
1514 }
1515
1516 // Construct the result.
1517 Register object = temp1;
1518 Label matchResultFallback, matchResultJoin;
1519 masm.createGCObject(object, temp2, templateObject, gc::DefaultHeap, &matchResultFallback);
1520 masm.bind(&matchResultJoin);
1521
1522 // Initialize slots of result object.
1523 masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2);
1524 masm.storeValue(templateObject->getSlot(0), Address(temp2, 0));
1525 masm.storeValue(templateObject->getSlot(1), Address(temp2, sizeof(Value)));
1526
1527 size_t elementsOffset = NativeObject::offsetOfFixedElements();
1528
1529 #ifdef DEBUG
1530 // Assert the initial value of initializedLength and length to make sure
1531 // restoration on failure case works.
1532 {
1533 Label initLengthOK, lengthOK;
1534 masm.branch32(Assembler::Equal,
1535 Address(object, elementsOffset + ObjectElements::offsetOfInitializedLength()),
1536 Imm32(templateObject->getDenseInitializedLength()),
1537 &initLengthOK);
1538 masm.assumeUnreachable("Initial value of the match object's initializedLength does not match to restoration.");
1539 masm.bind(&initLengthOK);
1540
1541 masm.branch32(Assembler::Equal,
1542 Address(object, elementsOffset + ObjectElements::offsetOfLength()),
1543 Imm32(templateObject->length()),
1544 &lengthOK);
1545 masm.assumeUnreachable("Initial value of The match object's length does not match to restoration.");
1546 masm.bind(&lengthOK);
1547 }
1548 #endif
1549
1550 Register matchIndex = temp2;
1551 masm.move32(Imm32(0), matchIndex);
1552
1553 size_t pairsVectorStartOffset = RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
1554 Address pairsVectorAddress(masm.getStackPointer(), pairsVectorStartOffset);
1555 Address pairCountAddress = RegExpPairCountAddress(masm, inputOutputDataStartOffset);
1556
1557 BaseIndex stringAddress(object, matchIndex, TimesEight, elementsOffset);
1558
1559 JS_STATIC_ASSERT(sizeof(MatchPair) == 8);
1560 BaseIndex stringIndexAddress(masm.getStackPointer(), matchIndex, TimesEight,
1561 pairsVectorStartOffset + offsetof(MatchPair, start));
1562 BaseIndex stringLimitAddress(masm.getStackPointer(), matchIndex, TimesEight,
1563 pairsVectorStartOffset + offsetof(MatchPair, limit));
1564
1565 // Loop to construct the match strings. There are two different loops,
1566 // depending on whether the input is latin1.
1567 CreateDependentString depStr[2];
1568 {
1569 Label isLatin1, done;
1570 masm.branchLatin1String(input, &isLatin1);
1571
1572 Label* failure = &oolEntry;
1573 Register temp4 = (maybeTemp4 == InvalidReg) ? lastIndex : maybeTemp4;
1574
1575 Label failureRestore;
1576 if (maybeTemp4 == InvalidReg) {
1577 failure = &failureRestore;
1578
1579 // Save lastIndex value to temporary space.
1580 masm.store32(lastIndex, Address(object, elementsOffset + ObjectElements::offsetOfLength()));
1581 }
1582
1583 for (int isLatin = 0; isLatin <= 1; isLatin++) {
1584 if (isLatin)
1585 masm.bind(&isLatin1);
1586
1587 Label matchLoop;
1588 masm.bind(&matchLoop);
1589
1590 Label isUndefined, storeDone;
1591 masm.branch32(Assembler::LessThan, stringIndexAddress, Imm32(0), &isUndefined);
1592
1593 depStr[isLatin].generate(masm, cx->names(), isLatin, temp3, input, temp4, temp5,
1594 stringIndexAddress, stringLimitAddress, failure);
1595
1596 masm.storeValue(JSVAL_TYPE_STRING, temp3, stringAddress);
1597
1598 masm.jump(&storeDone);
1599 masm.bind(&isUndefined);
1600
1601 masm.storeValue(UndefinedValue(), stringAddress);
1602 masm.bind(&storeDone);
1603
1604 masm.add32(Imm32(1), matchIndex);
1605 masm.branch32(Assembler::LessThanOrEqual, pairCountAddress, matchIndex, &done);
1606 masm.jump(&matchLoop);
1607 }
1608
1609 if (maybeTemp4 == InvalidReg) {
1610 // Restore lastIndex value from temporary space, both for success
1611 // and failure cases.
1612
1613 masm.load32(Address(object, elementsOffset + ObjectElements::offsetOfLength()), lastIndex);
1614 masm.jump(&done);
1615
1616 masm.bind(&failureRestore);
1617 masm.load32(Address(object, elementsOffset + ObjectElements::offsetOfLength()), lastIndex);
1618
1619 // Restore the match object for failure case.
1620 masm.store32(Imm32(templateObject->getDenseInitializedLength()),
1621 Address(object, elementsOffset + ObjectElements::offsetOfInitializedLength()));
1622 masm.store32(Imm32(templateObject->length()),
1623 Address(object, elementsOffset + ObjectElements::offsetOfLength()));
1624 masm.jump(&oolEntry);
1625 }
1626
1627 masm.bind(&done);
1628 }
1629
1630 // Fill in the rest of the output object.
1631 masm.store32(matchIndex, Address(object, elementsOffset + ObjectElements::offsetOfInitializedLength()));
1632 masm.store32(matchIndex, Address(object, elementsOffset + ObjectElements::offsetOfLength()));
1633
1634 masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2);
1635
1636 MOZ_ASSERT(templateObject->numFixedSlots() == 0);
1637 MOZ_ASSERT(templateObject->lookupPure(cx->names().index)->slot() == 0);
1638 MOZ_ASSERT(templateObject->lookupPure(cx->names().input)->slot() == 1);
1639
1640 masm.load32(pairsVectorAddress, temp3);
1641 masm.storeValue(JSVAL_TYPE_INT32, temp3, Address(temp2, 0));
1642 masm.storeValue(JSVAL_TYPE_STRING, input, Address(temp2, sizeof(Value)));
1643
1644 // All done!
1645 masm.tagValue(JSVAL_TYPE_OBJECT, object, result);
1646 masm.ret();
1647
1648 masm.bind(¬Found);
1649 masm.moveValue(NullValue(), result);
1650 masm.ret();
1651
1652 // Fallback paths for CreateDependentString and createGCObject.
1653 // Need to save all registers in use when they were called.
1654 LiveRegisterSet regsToSave(RegisterSet::Volatile());
1655 regsToSave.addUnchecked(regexp);
1656 regsToSave.addUnchecked(input);
1657 regsToSave.addUnchecked(lastIndex);
1658 regsToSave.addUnchecked(temp1);
1659 regsToSave.addUnchecked(temp2);
1660 regsToSave.addUnchecked(temp3);
1661 if (maybeTemp4 != InvalidReg)
1662 regsToSave.addUnchecked(maybeTemp4);
1663 regsToSave.addUnchecked(temp5);
1664
1665 for (int isLatin = 0; isLatin <= 1; isLatin++)
1666 depStr[isLatin].generateFallback(masm, regsToSave);
1667
1668 masm.bind(&matchResultFallback);
1669 CreateMatchResultFallback(masm, regsToSave, object, temp2, temp5, templateObject, &oolEntry);
1670 masm.jump(&matchResultJoin);
1671
1672 // Use an undefined value to signal to the caller that the OOL stub needs to be called.
1673 masm.bind(&oolEntry);
1674 masm.moveValue(UndefinedValue(), result);
1675 masm.ret();
1676
1677 Linker linker(masm);
1678 AutoFlushICache afc("RegExpMatcherStub");
1679 JitCode* code = linker.newCode<CanGC>(cx, OTHER_CODE);
1680 if (!code)
1681 return nullptr;
1682
1683 #ifdef JS_ION_PERF
1684 writePerfSpewerJitCodeProfile(code, "RegExpMatcherStub");
1685 #endif
1686
1687 if (cx->zone()->needsIncrementalBarrier())
1688 code->togglePreBarriers(true, DontReprotect);
1689
1690 return code;
1691 }
1692
1693 class OutOfLineRegExpMatcher : public OutOfLineCodeBase<CodeGenerator>
1694 {
1695 LRegExpMatcher* lir_;
1696
1697 public:
OutOfLineRegExpMatcher(LRegExpMatcher * lir)1698 explicit OutOfLineRegExpMatcher(LRegExpMatcher* lir)
1699 : lir_(lir)
1700 { }
1701
accept(CodeGenerator * codegen)1702 void accept(CodeGenerator* codegen) {
1703 codegen->visitOutOfLineRegExpMatcher(this);
1704 }
1705
lir() const1706 LRegExpMatcher* lir() const {
1707 return lir_;
1708 }
1709 };
1710
1711 typedef bool (*RegExpMatcherRawFn)(JSContext* cx, HandleObject regexp, HandleString input,
1712 int32_t lastIndex,
1713 MatchPairs* pairs, MutableHandleValue output);
1714 static const VMFunction RegExpMatcherRawInfo =
1715 FunctionInfo<RegExpMatcherRawFn>(RegExpMatcherRaw, "RegExpMatcherRaw");
1716
1717 void
visitOutOfLineRegExpMatcher(OutOfLineRegExpMatcher * ool)1718 CodeGenerator::visitOutOfLineRegExpMatcher(OutOfLineRegExpMatcher* ool)
1719 {
1720 LRegExpMatcher* lir = ool->lir();
1721 Register lastIndex = ToRegister(lir->lastIndex());
1722 Register input = ToRegister(lir->string());
1723 Register regexp = ToRegister(lir->regexp());
1724
1725 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
1726 regs.take(lastIndex);
1727 regs.take(input);
1728 regs.take(regexp);
1729 Register temp = regs.takeAny();
1730
1731 masm.computeEffectiveAddress(Address(masm.getStackPointer(),
1732 sizeof(irregexp::InputOutputData)), temp);
1733
1734 pushArg(temp);
1735 pushArg(lastIndex);
1736 pushArg(input);
1737 pushArg(regexp);
1738
1739 // We are not using oolCallVM because we are in a Call, and that live
1740 // registers are already saved by the the register allocator.
1741 callVM(RegExpMatcherRawInfo, lir);
1742
1743 masm.jump(ool->rejoin());
1744 }
1745
1746 void
visitRegExpMatcher(LRegExpMatcher * lir)1747 CodeGenerator::visitRegExpMatcher(LRegExpMatcher* lir)
1748 {
1749 MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpMatcherRegExpReg);
1750 MOZ_ASSERT(ToRegister(lir->string()) == RegExpMatcherStringReg);
1751 MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpMatcherLastIndexReg);
1752 MOZ_ASSERT(GetValueOutput(lir) == JSReturnOperand);
1753
1754 #if defined(JS_NUNBOX32)
1755 MOZ_ASSERT(RegExpMatcherRegExpReg != JSReturnReg_Type);
1756 MOZ_ASSERT(RegExpMatcherRegExpReg != JSReturnReg_Data);
1757 MOZ_ASSERT(RegExpMatcherStringReg != JSReturnReg_Type);
1758 MOZ_ASSERT(RegExpMatcherStringReg != JSReturnReg_Data);
1759 MOZ_ASSERT(RegExpMatcherLastIndexReg != JSReturnReg_Type);
1760 MOZ_ASSERT(RegExpMatcherLastIndexReg != JSReturnReg_Data);
1761 #elif defined(JS_PUNBOX64)
1762 MOZ_ASSERT(RegExpMatcherRegExpReg != JSReturnReg);
1763 MOZ_ASSERT(RegExpMatcherStringReg != JSReturnReg);
1764 MOZ_ASSERT(RegExpMatcherLastIndexReg != JSReturnReg);
1765 #endif
1766
1767 masm.reserveStack(RegExpReservedStack);
1768
1769 OutOfLineRegExpMatcher* ool = new(alloc()) OutOfLineRegExpMatcher(lir);
1770 addOutOfLineCode(ool, lir->mir());
1771
1772 JitCode* regExpMatcherStub = gen->compartment->jitCompartment()->regExpMatcherStubNoBarrier();
1773 masm.call(regExpMatcherStub);
1774 masm.branchTestUndefined(Assembler::Equal, JSReturnOperand, ool->entry());
1775 masm.bind(ool->rejoin());
1776
1777 masm.freeStack(RegExpReservedStack);
1778 }
1779
1780 static const int32_t RegExpSearcherResultNotFound = -1;
1781 static const int32_t RegExpSearcherResultFailed = -2;
1782
1783 JitCode*
generateRegExpSearcherStub(JSContext * cx)1784 JitCompartment::generateRegExpSearcherStub(JSContext* cx)
1785 {
1786 Register regexp = RegExpTesterRegExpReg;
1787 Register input = RegExpTesterStringReg;
1788 Register lastIndex = RegExpTesterLastIndexReg;
1789 Register result = ReturnReg;
1790
1791 // We are free to clobber all registers, as LRegExpSearcher is a call instruction.
1792 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
1793 regs.take(input);
1794 regs.take(regexp);
1795 regs.take(lastIndex);
1796
1797 Register temp1 = regs.takeAny();
1798 Register temp2 = regs.takeAny();
1799 Register temp3 = regs.takeAny();
1800
1801 MacroAssembler masm(cx);
1802
1803 // The InputOutputData is placed above the return address on the stack.
1804 size_t inputOutputDataStartOffset = sizeof(void*);
1805
1806 Label notFound, oolEntry;
1807 if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex,
1808 temp1, temp2, temp3, inputOutputDataStartOffset,
1809 RegExpShared::Normal, ¬Found, &oolEntry))
1810 {
1811 return nullptr;
1812 }
1813
1814 size_t pairsVectorStartOffset = RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
1815 Address stringIndexAddress(masm.getStackPointer(),
1816 pairsVectorStartOffset + offsetof(MatchPair, start));
1817 Address stringLimitAddress(masm.getStackPointer(),
1818 pairsVectorStartOffset + offsetof(MatchPair, limit));
1819
1820 masm.load32(stringIndexAddress, result);
1821 masm.load32(stringLimitAddress, input);
1822 masm.lshiftPtr(Imm32(15), input);
1823 masm.or32(input, result);
1824 masm.ret();
1825
1826 masm.bind(¬Found);
1827 masm.move32(Imm32(RegExpSearcherResultNotFound), result);
1828 masm.ret();
1829
1830 masm.bind(&oolEntry);
1831 masm.move32(Imm32(RegExpSearcherResultFailed), result);
1832 masm.ret();
1833
1834 Linker linker(masm);
1835 AutoFlushICache afc("RegExpSearcherStub");
1836 JitCode* code = linker.newCode<CanGC>(cx, OTHER_CODE);
1837 if (!code)
1838 return nullptr;
1839
1840 #ifdef JS_ION_PERF
1841 writePerfSpewerJitCodeProfile(code, "RegExpSearcherStub");
1842 #endif
1843
1844 if (cx->zone()->needsIncrementalBarrier())
1845 code->togglePreBarriers(true, DontReprotect);
1846
1847 return code;
1848 }
1849
1850 class OutOfLineRegExpSearcher : public OutOfLineCodeBase<CodeGenerator>
1851 {
1852 LRegExpSearcher* lir_;
1853
1854 public:
OutOfLineRegExpSearcher(LRegExpSearcher * lir)1855 explicit OutOfLineRegExpSearcher(LRegExpSearcher* lir)
1856 : lir_(lir)
1857 { }
1858
accept(CodeGenerator * codegen)1859 void accept(CodeGenerator* codegen) {
1860 codegen->visitOutOfLineRegExpSearcher(this);
1861 }
1862
lir() const1863 LRegExpSearcher* lir() const {
1864 return lir_;
1865 }
1866 };
1867
1868 typedef bool (*RegExpSearcherRawFn)(JSContext* cx, HandleObject regexp, HandleString input,
1869 int32_t lastIndex,
1870 MatchPairs* pairs, int32_t* result);
1871 static const VMFunction RegExpSearcherRawInfo =
1872 FunctionInfo<RegExpSearcherRawFn>(RegExpSearcherRaw, "RegExpSearcherRaw");
1873
1874 void
visitOutOfLineRegExpSearcher(OutOfLineRegExpSearcher * ool)1875 CodeGenerator::visitOutOfLineRegExpSearcher(OutOfLineRegExpSearcher* ool)
1876 {
1877 LRegExpSearcher* lir = ool->lir();
1878 Register lastIndex = ToRegister(lir->lastIndex());
1879 Register input = ToRegister(lir->string());
1880 Register regexp = ToRegister(lir->regexp());
1881
1882 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
1883 regs.take(lastIndex);
1884 regs.take(input);
1885 regs.take(regexp);
1886 Register temp = regs.takeAny();
1887
1888 masm.computeEffectiveAddress(Address(masm.getStackPointer(),
1889 sizeof(irregexp::InputOutputData)), temp);
1890
1891 pushArg(temp);
1892 pushArg(lastIndex);
1893 pushArg(input);
1894 pushArg(regexp);
1895
1896 // We are not using oolCallVM because we are in a Call, and that live
1897 // registers are already saved by the the register allocator.
1898 callVM(RegExpSearcherRawInfo, lir);
1899
1900 masm.jump(ool->rejoin());
1901 }
1902
1903 void
visitRegExpSearcher(LRegExpSearcher * lir)1904 CodeGenerator::visitRegExpSearcher(LRegExpSearcher* lir)
1905 {
1906 MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpTesterRegExpReg);
1907 MOZ_ASSERT(ToRegister(lir->string()) == RegExpTesterStringReg);
1908 MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpTesterLastIndexReg);
1909 MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg);
1910
1911 MOZ_ASSERT(RegExpTesterRegExpReg != ReturnReg);
1912 MOZ_ASSERT(RegExpTesterStringReg != ReturnReg);
1913 MOZ_ASSERT(RegExpTesterLastIndexReg != ReturnReg);
1914
1915 masm.reserveStack(RegExpReservedStack);
1916
1917 OutOfLineRegExpSearcher* ool = new(alloc()) OutOfLineRegExpSearcher(lir);
1918 addOutOfLineCode(ool, lir->mir());
1919
1920 JitCode* regExpSearcherStub = gen->compartment->jitCompartment()->regExpSearcherStubNoBarrier();
1921 masm.call(regExpSearcherStub);
1922 masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpSearcherResultFailed), ool->entry());
1923 masm.bind(ool->rejoin());
1924
1925 masm.freeStack(RegExpReservedStack);
1926 }
1927
1928 static const int32_t RegExpTesterResultNotFound = -1;
1929 static const int32_t RegExpTesterResultFailed = -2;
1930
1931 JitCode*
generateRegExpTesterStub(JSContext * cx)1932 JitCompartment::generateRegExpTesterStub(JSContext* cx)
1933 {
1934 Register regexp = RegExpTesterRegExpReg;
1935 Register input = RegExpTesterStringReg;
1936 Register lastIndex = RegExpTesterLastIndexReg;
1937 Register result = ReturnReg;
1938
1939 MacroAssembler masm(cx);
1940
1941 #ifdef JS_USE_LINK_REGISTER
1942 masm.pushReturnAddress();
1943 #endif
1944
1945 // We are free to clobber all registers, as LRegExpTester is a call instruction.
1946 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
1947 regs.take(input);
1948 regs.take(regexp);
1949 regs.take(lastIndex);
1950
1951 Register temp1 = regs.takeAny();
1952 Register temp2 = regs.takeAny();
1953 Register temp3 = regs.takeAny();
1954
1955 masm.reserveStack(sizeof(irregexp::InputOutputData));
1956
1957 Label notFound, oolEntry;
1958 if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex,
1959 temp1, temp2, temp3, 0,
1960 RegExpShared::MatchOnly, ¬Found, &oolEntry))
1961 {
1962 return nullptr;
1963 }
1964
1965 Label done;
1966
1967 // temp3 contains endIndex.
1968 masm.move32(temp3, result);
1969 masm.jump(&done);
1970
1971 masm.bind(¬Found);
1972 masm.move32(Imm32(RegExpTesterResultNotFound), result);
1973 masm.jump(&done);
1974
1975 masm.bind(&oolEntry);
1976 masm.move32(Imm32(RegExpTesterResultFailed), result);
1977
1978 masm.bind(&done);
1979 masm.freeStack(sizeof(irregexp::InputOutputData));
1980 masm.ret();
1981
1982 Linker linker(masm);
1983 AutoFlushICache afc("RegExpTesterStub");
1984 JitCode* code = linker.newCode<CanGC>(cx, OTHER_CODE);
1985 if (!code)
1986 return nullptr;
1987
1988 #ifdef JS_ION_PERF
1989 writePerfSpewerJitCodeProfile(code, "RegExpTesterStub");
1990 #endif
1991
1992 if (cx->zone()->needsIncrementalBarrier())
1993 code->togglePreBarriers(true, DontReprotect);
1994
1995 return code;
1996 }
1997
1998 class OutOfLineRegExpTester : public OutOfLineCodeBase<CodeGenerator>
1999 {
2000 LRegExpTester* lir_;
2001
2002 public:
OutOfLineRegExpTester(LRegExpTester * lir)2003 explicit OutOfLineRegExpTester(LRegExpTester* lir)
2004 : lir_(lir)
2005 { }
2006
accept(CodeGenerator * codegen)2007 void accept(CodeGenerator* codegen) {
2008 codegen->visitOutOfLineRegExpTester(this);
2009 }
2010
lir() const2011 LRegExpTester* lir() const {
2012 return lir_;
2013 }
2014 };
2015
2016 typedef bool (*RegExpTesterRawFn)(JSContext* cx, HandleObject regexp, HandleString input,
2017 int32_t lastIndex, int32_t* result);
2018 static const VMFunction RegExpTesterRawInfo =
2019 FunctionInfo<RegExpTesterRawFn>(RegExpTesterRaw, "RegExpTesterRaw");
2020
2021 void
visitOutOfLineRegExpTester(OutOfLineRegExpTester * ool)2022 CodeGenerator::visitOutOfLineRegExpTester(OutOfLineRegExpTester* ool)
2023 {
2024 LRegExpTester* lir = ool->lir();
2025 Register lastIndex = ToRegister(lir->lastIndex());
2026 Register input = ToRegister(lir->string());
2027 Register regexp = ToRegister(lir->regexp());
2028
2029 pushArg(lastIndex);
2030 pushArg(input);
2031 pushArg(regexp);
2032
2033 // We are not using oolCallVM because we are in a Call, and that live
2034 // registers are already saved by the the register allocator.
2035 callVM(RegExpTesterRawInfo, lir);
2036
2037 masm.jump(ool->rejoin());
2038 }
2039
2040 void
visitRegExpTester(LRegExpTester * lir)2041 CodeGenerator::visitRegExpTester(LRegExpTester* lir)
2042 {
2043 MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpTesterRegExpReg);
2044 MOZ_ASSERT(ToRegister(lir->string()) == RegExpTesterStringReg);
2045 MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpTesterLastIndexReg);
2046 MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg);
2047
2048 MOZ_ASSERT(RegExpTesterRegExpReg != ReturnReg);
2049 MOZ_ASSERT(RegExpTesterStringReg != ReturnReg);
2050 MOZ_ASSERT(RegExpTesterLastIndexReg != ReturnReg);
2051
2052 OutOfLineRegExpTester* ool = new(alloc()) OutOfLineRegExpTester(lir);
2053 addOutOfLineCode(ool, lir->mir());
2054
2055 JitCode* regExpTesterStub = gen->compartment->jitCompartment()->regExpTesterStubNoBarrier();
2056 masm.call(regExpTesterStub);
2057
2058 masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpTesterResultFailed), ool->entry());
2059 masm.bind(ool->rejoin());
2060 }
2061
2062 class OutOfLineRegExpPrototypeOptimizable : public OutOfLineCodeBase<CodeGenerator>
2063 {
2064 LRegExpPrototypeOptimizable* ins_;
2065
2066 public:
OutOfLineRegExpPrototypeOptimizable(LRegExpPrototypeOptimizable * ins)2067 explicit OutOfLineRegExpPrototypeOptimizable(LRegExpPrototypeOptimizable* ins)
2068 : ins_(ins)
2069 { }
2070
accept(CodeGenerator * codegen)2071 void accept(CodeGenerator* codegen) {
2072 codegen->visitOutOfLineRegExpPrototypeOptimizable(this);
2073 }
ins() const2074 LRegExpPrototypeOptimizable* ins() const {
2075 return ins_;
2076 }
2077 };
2078
2079 void
visitRegExpPrototypeOptimizable(LRegExpPrototypeOptimizable * ins)2080 CodeGenerator::visitRegExpPrototypeOptimizable(LRegExpPrototypeOptimizable* ins)
2081 {
2082 Register object = ToRegister(ins->object());
2083 Register output = ToRegister(ins->output());
2084 Register temp = ToRegister(ins->temp());
2085
2086 OutOfLineRegExpPrototypeOptimizable* ool = new(alloc()) OutOfLineRegExpPrototypeOptimizable(ins);
2087 addOutOfLineCode(ool, ins->mir());
2088
2089 masm.loadJSContext(temp);
2090 masm.loadPtr(Address(temp, JSContext::offsetOfCompartment()), temp);
2091 size_t offset = JSCompartment::offsetOfRegExps() +
2092 RegExpCompartment::offsetOfOptimizableRegExpPrototypeShape();
2093 masm.loadPtr(Address(temp, offset), temp);
2094
2095 masm.loadPtr(Address(object, ShapedObject::offsetOfShape()), output);
2096 masm.branchPtr(Assembler::NotEqual, output, temp, ool->entry());
2097 masm.move32(Imm32(0x1), output);
2098
2099 masm.bind(ool->rejoin());
2100 }
2101
2102 void
visitOutOfLineRegExpPrototypeOptimizable(OutOfLineRegExpPrototypeOptimizable * ool)2103 CodeGenerator::visitOutOfLineRegExpPrototypeOptimizable(OutOfLineRegExpPrototypeOptimizable* ool)
2104 {
2105 LRegExpPrototypeOptimizable* ins = ool->ins();
2106 Register object = ToRegister(ins->object());
2107 Register output = ToRegister(ins->output());
2108
2109 saveVolatile(output);
2110
2111 masm.setupUnalignedABICall(output);
2112 masm.loadJSContext(output);
2113 masm.passABIArg(output);
2114 masm.passABIArg(object);
2115 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, RegExpPrototypeOptimizableRaw));
2116 masm.storeCallBoolResult(output);
2117
2118 restoreVolatile(output);
2119
2120 masm.jump(ool->rejoin());
2121 }
2122
2123 class OutOfLineRegExpInstanceOptimizable : public OutOfLineCodeBase<CodeGenerator>
2124 {
2125 LRegExpInstanceOptimizable* ins_;
2126
2127 public:
OutOfLineRegExpInstanceOptimizable(LRegExpInstanceOptimizable * ins)2128 explicit OutOfLineRegExpInstanceOptimizable(LRegExpInstanceOptimizable* ins)
2129 : ins_(ins)
2130 { }
2131
accept(CodeGenerator * codegen)2132 void accept(CodeGenerator* codegen) {
2133 codegen->visitOutOfLineRegExpInstanceOptimizable(this);
2134 }
ins() const2135 LRegExpInstanceOptimizable* ins() const {
2136 return ins_;
2137 }
2138 };
2139
2140 void
visitRegExpInstanceOptimizable(LRegExpInstanceOptimizable * ins)2141 CodeGenerator::visitRegExpInstanceOptimizable(LRegExpInstanceOptimizable* ins)
2142 {
2143 Register object = ToRegister(ins->object());
2144 Register output = ToRegister(ins->output());
2145 Register temp = ToRegister(ins->temp());
2146
2147 OutOfLineRegExpInstanceOptimizable* ool = new(alloc()) OutOfLineRegExpInstanceOptimizable(ins);
2148 addOutOfLineCode(ool, ins->mir());
2149
2150 masm.loadJSContext(temp);
2151 masm.loadPtr(Address(temp, JSContext::offsetOfCompartment()), temp);
2152 size_t offset = JSCompartment::offsetOfRegExps() +
2153 RegExpCompartment::offsetOfOptimizableRegExpInstanceShape();
2154 masm.loadPtr(Address(temp, offset), temp);
2155
2156 masm.loadPtr(Address(object, ShapedObject::offsetOfShape()), output);
2157 masm.branchPtr(Assembler::NotEqual, output, temp, ool->entry());
2158 masm.move32(Imm32(0x1), output);
2159
2160 masm.bind(ool->rejoin());
2161 }
2162
2163 void
visitOutOfLineRegExpInstanceOptimizable(OutOfLineRegExpInstanceOptimizable * ool)2164 CodeGenerator::visitOutOfLineRegExpInstanceOptimizable(OutOfLineRegExpInstanceOptimizable* ool)
2165 {
2166 LRegExpInstanceOptimizable* ins = ool->ins();
2167 Register object = ToRegister(ins->object());
2168 Register proto = ToRegister(ins->proto());
2169 Register output = ToRegister(ins->output());
2170
2171 saveVolatile(output);
2172
2173 masm.setupUnalignedABICall(output);
2174 masm.loadJSContext(output);
2175 masm.passABIArg(output);
2176 masm.passABIArg(object);
2177 masm.passABIArg(proto);
2178 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, RegExpInstanceOptimizableRaw));
2179 masm.storeCallBoolResult(output);
2180
2181 restoreVolatile(output);
2182
2183 masm.jump(ool->rejoin());
2184 }
2185
2186 static void
FindFirstDollarIndex(MacroAssembler & masm,Register str,Register len,Register chars,Register temp,Register output,bool isLatin1)2187 FindFirstDollarIndex(MacroAssembler& masm, Register str, Register len, Register chars,
2188 Register temp, Register output, bool isLatin1)
2189 {
2190 masm.loadStringChars(str, chars);
2191
2192 masm.move32(Imm32(0), output);
2193
2194 Label start, done;
2195 masm.bind(&start);
2196 if (isLatin1)
2197 masm.load8ZeroExtend(BaseIndex(chars, output, TimesOne), temp);
2198 else
2199 masm.load16ZeroExtend(BaseIndex(chars, output, TimesTwo), temp);
2200
2201 masm.branch32(Assembler::Equal, temp, Imm32('$'), &done);
2202
2203 masm.add32(Imm32(1), output);
2204 masm.branch32(Assembler::NotEqual, output, len, &start);
2205
2206 masm.move32(Imm32(-1), output);
2207
2208 masm.bind(&done);
2209 }
2210
2211 typedef bool (*GetFirstDollarIndexRawFn)(JSContext*, HandleString, int32_t*);
2212 static const VMFunction GetFirstDollarIndexRawInfo =
2213 FunctionInfo<GetFirstDollarIndexRawFn>(GetFirstDollarIndexRaw, "GetFirstDollarIndexRaw");
2214
2215 void
visitGetFirstDollarIndex(LGetFirstDollarIndex * ins)2216 CodeGenerator::visitGetFirstDollarIndex(LGetFirstDollarIndex* ins)
2217 {
2218 Register str = ToRegister(ins->str());
2219 Register output = ToRegister(ins->output());
2220 Register temp0 = ToRegister(ins->temp0());
2221 Register temp1 = ToRegister(ins->temp1());
2222 Register len = ToRegister(ins->temp2());
2223
2224 OutOfLineCode* ool = oolCallVM(GetFirstDollarIndexRawInfo, ins, ArgList(str),
2225 StoreRegisterTo(output));
2226
2227 masm.branchIfRope(str, ool->entry());
2228 masm.loadStringLength(str, len);
2229
2230 Label isLatin1, done;
2231 masm.branchLatin1String(str, &isLatin1);
2232 {
2233 FindFirstDollarIndex(masm, str, len, temp0, temp1, output, /* isLatin1 = */ false);
2234 }
2235 masm.jump(&done);
2236 {
2237 masm.bind(&isLatin1);
2238 FindFirstDollarIndex(masm, str, len, temp0, temp1, output, /* isLatin1 = */ true);
2239 }
2240 masm.bind(&done);
2241 masm.bind(ool->rejoin());
2242 }
2243
2244 typedef JSString* (*StringReplaceFn)(JSContext*, HandleString, HandleString, HandleString);
2245 static const VMFunction StringFlatReplaceInfo =
2246 FunctionInfo<StringReplaceFn>(js::str_flat_replace_string, "str_flat_replace_string");
2247 static const VMFunction StringReplaceInfo =
2248 FunctionInfo<StringReplaceFn>(StringReplace, "StringReplace");
2249
2250 void
visitStringReplace(LStringReplace * lir)2251 CodeGenerator::visitStringReplace(LStringReplace* lir)
2252 {
2253 if (lir->replacement()->isConstant())
2254 pushArg(ImmGCPtr(lir->replacement()->toConstant()->toString()));
2255 else
2256 pushArg(ToRegister(lir->replacement()));
2257
2258 if (lir->pattern()->isConstant())
2259 pushArg(ImmGCPtr(lir->pattern()->toConstant()->toString()));
2260 else
2261 pushArg(ToRegister(lir->pattern()));
2262
2263 if (lir->string()->isConstant())
2264 pushArg(ImmGCPtr(lir->string()->toConstant()->toString()));
2265 else
2266 pushArg(ToRegister(lir->string()));
2267
2268 if (lir->mir()->isFlatReplacement())
2269 callVM(StringFlatReplaceInfo, lir);
2270 else
2271 callVM(StringReplaceInfo, lir);
2272 }
2273
2274 void
emitSharedStub(ICStub::Kind kind,LInstruction * lir)2275 CodeGenerator::emitSharedStub(ICStub::Kind kind, LInstruction* lir)
2276 {
2277 JSScript* script = lir->mirRaw()->block()->info().script();
2278 jsbytecode* pc = lir->mirRaw()->toInstruction()->resumePoint()->pc();
2279
2280 #ifdef JS_USE_LINK_REGISTER
2281 // Some architectures don't push the return address on the stack but
2282 // use the link register. In that case the stack isn't aligned. Push
2283 // to make sure we are aligned.
2284 masm.Push(Imm32(0));
2285 #endif
2286
2287 // Create descriptor signifying end of Ion frame.
2288 uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS,
2289 JitStubFrameLayout::Size());
2290 masm.Push(Imm32(descriptor));
2291
2292 // Call into the stubcode.
2293 CodeOffset patchOffset;
2294 IonICEntry entry(script->pcToOffset(pc), ICEntry::Kind_Op, script);
2295 EmitCallIC(&patchOffset, masm);
2296 entry.setReturnOffset(CodeOffset(masm.currentOffset()));
2297
2298 SharedStub sharedStub(kind, entry, patchOffset);
2299 masm.propagateOOM(sharedStubs_.append(sharedStub));
2300
2301 // Fix up upon return.
2302 uint32_t callOffset = masm.currentOffset();
2303 #ifdef JS_USE_LINK_REGISTER
2304 masm.freeStack(sizeof(intptr_t) * 2);
2305 #else
2306 masm.freeStack(sizeof(intptr_t));
2307 #endif
2308 markSafepointAt(callOffset, lir);
2309 }
2310
2311 void
visitBinarySharedStub(LBinarySharedStub * lir)2312 CodeGenerator::visitBinarySharedStub(LBinarySharedStub* lir)
2313 {
2314 JSOp jsop = JSOp(*lir->mirRaw()->toInstruction()->resumePoint()->pc());
2315 switch (jsop) {
2316 case JSOP_ADD:
2317 case JSOP_SUB:
2318 case JSOP_MUL:
2319 case JSOP_DIV:
2320 case JSOP_MOD:
2321 case JSOP_POW:
2322 emitSharedStub(ICStub::Kind::BinaryArith_Fallback, lir);
2323 break;
2324 case JSOP_LT:
2325 case JSOP_LE:
2326 case JSOP_GT:
2327 case JSOP_GE:
2328 case JSOP_EQ:
2329 case JSOP_NE:
2330 case JSOP_STRICTEQ:
2331 case JSOP_STRICTNE:
2332 emitSharedStub(ICStub::Kind::Compare_Fallback, lir);
2333 break;
2334 default:
2335 MOZ_CRASH("Unsupported jsop in shared stubs.");
2336 }
2337 }
2338
2339 void
visitUnarySharedStub(LUnarySharedStub * lir)2340 CodeGenerator::visitUnarySharedStub(LUnarySharedStub* lir)
2341 {
2342 JSOp jsop = JSOp(*lir->mir()->resumePoint()->pc());
2343 switch (jsop) {
2344 case JSOP_BITNOT:
2345 case JSOP_NEG:
2346 emitSharedStub(ICStub::Kind::UnaryArith_Fallback, lir);
2347 break;
2348 case JSOP_CALLPROP:
2349 case JSOP_GETPROP:
2350 case JSOP_LENGTH:
2351 emitSharedStub(ICStub::Kind::GetProp_Fallback, lir);
2352 break;
2353 default:
2354 MOZ_CRASH("Unsupported jsop in shared stubs.");
2355 }
2356 }
2357
2358 void
visitNullarySharedStub(LNullarySharedStub * lir)2359 CodeGenerator::visitNullarySharedStub(LNullarySharedStub* lir)
2360 {
2361 jsbytecode* pc = lir->mir()->resumePoint()->pc();
2362 JSOp jsop = JSOp(*pc);
2363 switch (jsop) {
2364 case JSOP_NEWARRAY: {
2365 uint32_t length = GET_UINT32(pc);
2366 MOZ_ASSERT(length <= INT32_MAX,
2367 "the bytecode emitter must fail to compile code that would "
2368 "produce JSOP_NEWARRAY with a length exceeding int32_t range");
2369
2370 // Pass length in R0.
2371 masm.move32(Imm32(AssertedCast<int32_t>(length)), R0.scratchReg());
2372 emitSharedStub(ICStub::Kind::NewArray_Fallback, lir);
2373 break;
2374 }
2375 case JSOP_NEWOBJECT:
2376 emitSharedStub(ICStub::Kind::NewObject_Fallback, lir);
2377 break;
2378 case JSOP_NEWINIT: {
2379 JSProtoKey key = JSProtoKey(GET_UINT8(pc));
2380 if (key == JSProto_Array) {
2381 masm.move32(Imm32(0), R0.scratchReg());
2382 emitSharedStub(ICStub::Kind::NewArray_Fallback, lir);
2383 } else {
2384 emitSharedStub(ICStub::Kind::NewObject_Fallback, lir);
2385 }
2386 break;
2387 }
2388 default:
2389 MOZ_CRASH("Unsupported jsop in shared stubs.");
2390 }
2391 }
2392
2393 typedef JSObject* (*LambdaFn)(JSContext*, HandleFunction, HandleObject);
2394 static const VMFunction LambdaInfo = FunctionInfo<LambdaFn>(js::Lambda, "Lambda");
2395
2396 void
visitLambdaForSingleton(LLambdaForSingleton * lir)2397 CodeGenerator::visitLambdaForSingleton(LLambdaForSingleton* lir)
2398 {
2399 pushArg(ToRegister(lir->environmentChain()));
2400 pushArg(ImmGCPtr(lir->mir()->info().fun));
2401 callVM(LambdaInfo, lir);
2402 }
2403
2404 void
visitLambda(LLambda * lir)2405 CodeGenerator::visitLambda(LLambda* lir)
2406 {
2407 Register envChain = ToRegister(lir->environmentChain());
2408 Register output = ToRegister(lir->output());
2409 Register tempReg = ToRegister(lir->temp());
2410 const LambdaFunctionInfo& info = lir->mir()->info();
2411
2412 OutOfLineCode* ool = oolCallVM(LambdaInfo, lir, ArgList(ImmGCPtr(info.fun), envChain),
2413 StoreRegisterTo(output));
2414
2415 MOZ_ASSERT(!info.singletonType);
2416
2417 masm.createGCObject(output, tempReg, info.fun, gc::DefaultHeap, ool->entry());
2418
2419 emitLambdaInit(output, envChain, info);
2420
2421 if (info.flags & JSFunction::EXTENDED) {
2422 MOZ_ASSERT(info.fun->allowSuperProperty() || info.fun->isSelfHostedBuiltin() ||
2423 info.fun->isAsync());
2424 static_assert(FunctionExtended::NUM_EXTENDED_SLOTS == 2, "All slots must be initialized");
2425 masm.storeValue(UndefinedValue(), Address(output, FunctionExtended::offsetOfExtendedSlot(0)));
2426 masm.storeValue(UndefinedValue(), Address(output, FunctionExtended::offsetOfExtendedSlot(1)));
2427 }
2428
2429 masm.bind(ool->rejoin());
2430 }
2431
2432 class OutOfLineLambdaArrow : public OutOfLineCodeBase<CodeGenerator>
2433 {
2434 public:
2435 LLambdaArrow* lir;
2436 Label entryNoPop_;
2437
OutOfLineLambdaArrow(LLambdaArrow * lir)2438 explicit OutOfLineLambdaArrow(LLambdaArrow* lir)
2439 : lir(lir)
2440 { }
2441
accept(CodeGenerator * codegen)2442 void accept(CodeGenerator* codegen) {
2443 codegen->visitOutOfLineLambdaArrow(this);
2444 }
2445
entryNoPop()2446 Label* entryNoPop() {
2447 return &entryNoPop_;
2448 }
2449 };
2450
2451 typedef JSObject* (*LambdaArrowFn)(JSContext*, HandleFunction, HandleObject, HandleValue);
2452 static const VMFunction LambdaArrowInfo =
2453 FunctionInfo<LambdaArrowFn>(js::LambdaArrow, "LambdaArrow");
2454
2455 void
visitOutOfLineLambdaArrow(OutOfLineLambdaArrow * ool)2456 CodeGenerator::visitOutOfLineLambdaArrow(OutOfLineLambdaArrow* ool)
2457 {
2458 Register envChain = ToRegister(ool->lir->environmentChain());
2459 ValueOperand newTarget = ToValue(ool->lir, LLambdaArrow::NewTargetValue);
2460 Register output = ToRegister(ool->lir->output());
2461 const LambdaFunctionInfo& info = ool->lir->mir()->info();
2462
2463 // When we get here, we may need to restore part of the newTarget,
2464 // which has been conscripted into service as a temp register.
2465 masm.pop(newTarget.scratchReg());
2466
2467 masm.bind(ool->entryNoPop());
2468
2469 saveLive(ool->lir);
2470
2471 pushArg(newTarget);
2472 pushArg(envChain);
2473 pushArg(ImmGCPtr(info.fun));
2474
2475 callVM(LambdaArrowInfo, ool->lir);
2476 StoreRegisterTo(output).generate(this);
2477
2478 restoreLiveIgnore(ool->lir, StoreRegisterTo(output).clobbered());
2479
2480 masm.jump(ool->rejoin());
2481 }
2482
2483 void
visitLambdaArrow(LLambdaArrow * lir)2484 CodeGenerator::visitLambdaArrow(LLambdaArrow* lir)
2485 {
2486 Register envChain = ToRegister(lir->environmentChain());
2487 ValueOperand newTarget = ToValue(lir, LLambdaArrow::NewTargetValue);
2488 Register output = ToRegister(lir->output());
2489 const LambdaFunctionInfo& info = lir->mir()->info();
2490
2491 OutOfLineLambdaArrow* ool = new (alloc()) OutOfLineLambdaArrow(lir);
2492 addOutOfLineCode(ool, lir->mir());
2493
2494 MOZ_ASSERT(!info.useSingletonForClone);
2495
2496 if (info.singletonType) {
2497 // If the function has a singleton type, this instruction will only be
2498 // executed once so we don't bother inlining it.
2499 masm.jump(ool->entryNoPop());
2500 masm.bind(ool->rejoin());
2501 return;
2502 }
2503
2504 // There's not enough registers on x86 with the profiler enabled to request
2505 // a temp. Instead, spill part of one of the values, being prepared to
2506 // restore it if necessary on the out of line path.
2507 Register tempReg = newTarget.scratchReg();
2508 masm.push(newTarget.scratchReg());
2509
2510 masm.createGCObject(output, tempReg, info.fun, gc::DefaultHeap, ool->entry());
2511
2512 masm.pop(newTarget.scratchReg());
2513
2514 emitLambdaInit(output, envChain, info);
2515
2516 // Initialize extended slots. Lexical |this| is stored in the first one.
2517 MOZ_ASSERT(info.flags & JSFunction::EXTENDED);
2518 static_assert(FunctionExtended::NUM_EXTENDED_SLOTS == 2, "All slots must be initialized");
2519 static_assert(FunctionExtended::ARROW_NEWTARGET_SLOT == 0,
2520 "|new.target| must be stored in first slot");
2521 masm.storeValue(newTarget, Address(output, FunctionExtended::offsetOfExtendedSlot(0)));
2522 masm.storeValue(UndefinedValue(), Address(output, FunctionExtended::offsetOfExtendedSlot(1)));
2523
2524 masm.bind(ool->rejoin());
2525 }
2526
2527 void
emitLambdaInit(Register output,Register envChain,const LambdaFunctionInfo & info)2528 CodeGenerator::emitLambdaInit(Register output, Register envChain,
2529 const LambdaFunctionInfo& info)
2530 {
2531 // Initialize nargs and flags. We do this with a single uint32 to avoid
2532 // 16-bit writes.
2533 union {
2534 struct S {
2535 uint16_t nargs;
2536 uint16_t flags;
2537 } s;
2538 uint32_t word;
2539 } u;
2540 u.s.nargs = info.nargs;
2541 u.s.flags = info.flags;
2542
2543 MOZ_ASSERT(JSFunction::offsetOfFlags() == JSFunction::offsetOfNargs() + 2);
2544 masm.store32(Imm32(u.word), Address(output, JSFunction::offsetOfNargs()));
2545 masm.storePtr(ImmGCPtr(info.scriptOrLazyScript),
2546 Address(output, JSFunction::offsetOfNativeOrScript()));
2547 masm.storePtr(envChain, Address(output, JSFunction::offsetOfEnvironment()));
2548 masm.storePtr(ImmGCPtr(info.fun->displayAtom()), Address(output, JSFunction::offsetOfAtom()));
2549 }
2550
2551 void
visitOsiPoint(LOsiPoint * lir)2552 CodeGenerator::visitOsiPoint(LOsiPoint* lir)
2553 {
2554 // Note: markOsiPoint ensures enough space exists between the last
2555 // LOsiPoint and this one to patch adjacent call instructions.
2556
2557 MOZ_ASSERT(masm.framePushed() == frameSize());
2558
2559 uint32_t osiCallPointOffset = markOsiPoint(lir);
2560
2561 LSafepoint* safepoint = lir->associatedSafepoint();
2562 MOZ_ASSERT(!safepoint->osiCallPointOffset());
2563 safepoint->setOsiCallPointOffset(osiCallPointOffset);
2564
2565 #ifdef DEBUG
2566 // There should be no movegroups or other instructions between
2567 // an instruction and its OsiPoint. This is necessary because
2568 // we use the OsiPoint's snapshot from within VM calls.
2569 for (LInstructionReverseIterator iter(current->rbegin(lir)); iter != current->rend(); iter++) {
2570 if (*iter == lir)
2571 continue;
2572 MOZ_ASSERT(!iter->isMoveGroup());
2573 MOZ_ASSERT(iter->safepoint() == safepoint);
2574 break;
2575 }
2576 #endif
2577
2578 #ifdef CHECK_OSIPOINT_REGISTERS
2579 if (shouldVerifyOsiPointRegs(safepoint))
2580 verifyOsiPointRegs(safepoint);
2581 #endif
2582 }
2583
2584 void
visitGoto(LGoto * lir)2585 CodeGenerator::visitGoto(LGoto* lir)
2586 {
2587 jumpToBlock(lir->target());
2588 }
2589
2590 // Out-of-line path to execute any move groups between the start of a loop
2591 // header and its interrupt check, then invoke the interrupt handler.
2592 class OutOfLineInterruptCheckImplicit : public OutOfLineCodeBase<CodeGenerator>
2593 {
2594 public:
2595 LBlock* block;
2596 LInterruptCheck* lir;
2597
OutOfLineInterruptCheckImplicit(LBlock * block,LInterruptCheck * lir)2598 OutOfLineInterruptCheckImplicit(LBlock* block, LInterruptCheck* lir)
2599 : block(block), lir(lir)
2600 { }
2601
accept(CodeGenerator * codegen)2602 void accept(CodeGenerator* codegen) {
2603 codegen->visitOutOfLineInterruptCheckImplicit(this);
2604 }
2605 };
2606
2607 typedef bool (*InterruptCheckFn)(JSContext*);
2608 static const VMFunction InterruptCheckInfo =
2609 FunctionInfo<InterruptCheckFn>(InterruptCheck, "InterruptCheck");
2610
2611 void
visitOutOfLineInterruptCheckImplicit(OutOfLineInterruptCheckImplicit * ool)2612 CodeGenerator::visitOutOfLineInterruptCheckImplicit(OutOfLineInterruptCheckImplicit* ool)
2613 {
2614 #ifdef CHECK_OSIPOINT_REGISTERS
2615 // This is path is entered from the patched back-edge of the loop. This
2616 // means that the JitAtivation flags used for checking the validity of the
2617 // OSI points are not reseted by the path generated by generateBody, so we
2618 // have to reset it here.
2619 resetOsiPointRegs(ool->lir->safepoint());
2620 #endif
2621
2622 LInstructionIterator iter = ool->block->begin();
2623 for (; iter != ool->block->end(); iter++) {
2624 if (iter->isMoveGroup()) {
2625 // Replay this move group that preceds the interrupt check at the
2626 // start of the loop header. Any incoming jumps here will be from
2627 // the backedge and will skip over the move group emitted inline.
2628 visitMoveGroup(iter->toMoveGroup());
2629 } else {
2630 break;
2631 }
2632 }
2633 MOZ_ASSERT(*iter == ool->lir);
2634
2635 saveLive(ool->lir);
2636 callVM(InterruptCheckInfo, ool->lir);
2637 restoreLive(ool->lir);
2638 masm.jump(ool->rejoin());
2639 }
2640
2641 void
visitTableSwitch(LTableSwitch * ins)2642 CodeGenerator::visitTableSwitch(LTableSwitch* ins)
2643 {
2644 MTableSwitch* mir = ins->mir();
2645 Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
2646 const LAllocation* temp;
2647
2648 if (mir->getOperand(0)->type() != MIRType::Int32) {
2649 temp = ins->tempInt()->output();
2650
2651 // The input is a double, so try and convert it to an integer.
2652 // If it does not fit in an integer, take the default case.
2653 masm.convertDoubleToInt32(ToFloatRegister(ins->index()), ToRegister(temp), defaultcase, false);
2654 } else {
2655 temp = ins->index();
2656 }
2657
2658 emitTableSwitchDispatch(mir, ToRegister(temp), ToRegisterOrInvalid(ins->tempPointer()));
2659 }
2660
2661 void
visitTableSwitchV(LTableSwitchV * ins)2662 CodeGenerator::visitTableSwitchV(LTableSwitchV* ins)
2663 {
2664 MTableSwitch* mir = ins->mir();
2665 Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
2666
2667 Register index = ToRegister(ins->tempInt());
2668 ValueOperand value = ToValue(ins, LTableSwitchV::InputValue);
2669 Register tag = masm.extractTag(value, index);
2670 masm.branchTestNumber(Assembler::NotEqual, tag, defaultcase);
2671
2672 Label unboxInt, isInt;
2673 masm.branchTestInt32(Assembler::Equal, tag, &unboxInt);
2674 {
2675 FloatRegister floatIndex = ToFloatRegister(ins->tempFloat());
2676 masm.unboxDouble(value, floatIndex);
2677 masm.convertDoubleToInt32(floatIndex, index, defaultcase, false);
2678 masm.jump(&isInt);
2679 }
2680
2681 masm.bind(&unboxInt);
2682 masm.unboxInt32(value, index);
2683
2684 masm.bind(&isInt);
2685
2686 emitTableSwitchDispatch(mir, index, ToRegisterOrInvalid(ins->tempPointer()));
2687 }
2688
2689 typedef JSObject* (*DeepCloneObjectLiteralFn)(JSContext*, HandleObject, NewObjectKind);
2690 static const VMFunction DeepCloneObjectLiteralInfo =
2691 FunctionInfo<DeepCloneObjectLiteralFn>(DeepCloneObjectLiteral, "DeepCloneObjectLiteral");
2692
2693 void
visitCloneLiteral(LCloneLiteral * lir)2694 CodeGenerator::visitCloneLiteral(LCloneLiteral* lir)
2695 {
2696 pushArg(ImmWord(TenuredObject));
2697 pushArg(ToRegister(lir->getObjectLiteral()));
2698 callVM(DeepCloneObjectLiteralInfo, lir);
2699 }
2700
2701 void
visitParameter(LParameter * lir)2702 CodeGenerator::visitParameter(LParameter* lir)
2703 {
2704 }
2705
2706 void
visitCallee(LCallee * lir)2707 CodeGenerator::visitCallee(LCallee* lir)
2708 {
2709 Register callee = ToRegister(lir->output());
2710 Address ptr(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfCalleeToken());
2711
2712 masm.loadFunctionFromCalleeToken(ptr, callee);
2713 }
2714
2715 void
visitIsConstructing(LIsConstructing * lir)2716 CodeGenerator::visitIsConstructing(LIsConstructing* lir)
2717 {
2718 Register output = ToRegister(lir->output());
2719 Address calleeToken(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfCalleeToken());
2720 masm.loadPtr(calleeToken, output);
2721
2722 // We must be inside a function.
2723 MOZ_ASSERT(current->mir()->info().script()->functionNonDelazifying());
2724
2725 // The low bit indicates whether this call is constructing, just clear the
2726 // other bits.
2727 static_assert(CalleeToken_Function == 0x0, "CalleeTokenTag value should match");
2728 static_assert(CalleeToken_FunctionConstructing == 0x1, "CalleeTokenTag value should match");
2729 masm.andPtr(Imm32(0x1), output);
2730 }
2731
2732 void
visitStart(LStart * lir)2733 CodeGenerator::visitStart(LStart* lir)
2734 {
2735 }
2736
2737 void
visitReturn(LReturn * lir)2738 CodeGenerator::visitReturn(LReturn* lir)
2739 {
2740 #if defined(JS_NUNBOX32)
2741 DebugOnly<LAllocation*> type = lir->getOperand(TYPE_INDEX);
2742 DebugOnly<LAllocation*> payload = lir->getOperand(PAYLOAD_INDEX);
2743 MOZ_ASSERT(ToRegister(type) == JSReturnReg_Type);
2744 MOZ_ASSERT(ToRegister(payload) == JSReturnReg_Data);
2745 #elif defined(JS_PUNBOX64)
2746 DebugOnly<LAllocation*> result = lir->getOperand(0);
2747 MOZ_ASSERT(ToRegister(result) == JSReturnReg);
2748 #endif
2749 // Don't emit a jump to the return label if this is the last block.
2750 if (current->mir() != *gen->graph().poBegin())
2751 masm.jump(&returnLabel_);
2752 }
2753
2754 void
visitOsrEntry(LOsrEntry * lir)2755 CodeGenerator::visitOsrEntry(LOsrEntry* lir)
2756 {
2757 Register temp = ToRegister(lir->temp());
2758
2759 // Remember the OSR entry offset into the code buffer.
2760 masm.flushBuffer();
2761 setOsrEntryOffset(masm.size());
2762
2763 #ifdef JS_TRACE_LOGGING
2764 emitTracelogStopEvent(TraceLogger_Baseline);
2765 emitTracelogStartEvent(TraceLogger_IonMonkey);
2766 #endif
2767
2768 // If profiling, save the current frame pointer to a per-thread global field.
2769 if (isProfilerInstrumentationEnabled())
2770 masm.profilerEnterFrame(masm.getStackPointer(), temp);
2771
2772 // Allocate the full frame for this function
2773 // Note we have a new entry here. So we reset MacroAssembler::framePushed()
2774 // to 0, before reserving the stack.
2775 MOZ_ASSERT(masm.framePushed() == frameSize());
2776 masm.setFramePushed(0);
2777
2778 // Ensure that the Ion frames is properly aligned.
2779 masm.assertStackAlignment(JitStackAlignment, 0);
2780
2781 masm.reserveStack(frameSize());
2782 }
2783
2784 void
visitOsrEnvironmentChain(LOsrEnvironmentChain * lir)2785 CodeGenerator::visitOsrEnvironmentChain(LOsrEnvironmentChain* lir)
2786 {
2787 const LAllocation* frame = lir->getOperand(0);
2788 const LDefinition* object = lir->getDef(0);
2789
2790 const ptrdiff_t frameOffset = BaselineFrame::reverseOffsetOfEnvironmentChain();
2791
2792 masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object));
2793 }
2794
2795 void
visitOsrArgumentsObject(LOsrArgumentsObject * lir)2796 CodeGenerator::visitOsrArgumentsObject(LOsrArgumentsObject* lir)
2797 {
2798 const LAllocation* frame = lir->getOperand(0);
2799 const LDefinition* object = lir->getDef(0);
2800
2801 const ptrdiff_t frameOffset = BaselineFrame::reverseOffsetOfArgsObj();
2802
2803 masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object));
2804 }
2805
2806 void
visitOsrValue(LOsrValue * value)2807 CodeGenerator::visitOsrValue(LOsrValue* value)
2808 {
2809 const LAllocation* frame = value->getOperand(0);
2810 const ValueOperand out = ToOutValue(value);
2811
2812 const ptrdiff_t frameOffset = value->mir()->frameOffset();
2813
2814 masm.loadValue(Address(ToRegister(frame), frameOffset), out);
2815 }
2816
2817 void
visitOsrReturnValue(LOsrReturnValue * lir)2818 CodeGenerator::visitOsrReturnValue(LOsrReturnValue* lir)
2819 {
2820 const LAllocation* frame = lir->getOperand(0);
2821 const ValueOperand out = ToOutValue(lir);
2822
2823 Address flags = Address(ToRegister(frame), BaselineFrame::reverseOffsetOfFlags());
2824 Address retval = Address(ToRegister(frame), BaselineFrame::reverseOffsetOfReturnValue());
2825
2826 masm.moveValue(UndefinedValue(), out);
2827
2828 Label done;
2829 masm.branchTest32(Assembler::Zero, flags, Imm32(BaselineFrame::HAS_RVAL), &done);
2830 masm.loadValue(retval, out);
2831 masm.bind(&done);
2832 }
2833
2834 void
visitStackArgT(LStackArgT * lir)2835 CodeGenerator::visitStackArgT(LStackArgT* lir)
2836 {
2837 const LAllocation* arg = lir->getArgument();
2838 MIRType argType = lir->type();
2839 uint32_t argslot = lir->argslot();
2840 MOZ_ASSERT(argslot - 1u < graph.argumentSlotCount());
2841
2842 int32_t stack_offset = StackOffsetOfPassedArg(argslot);
2843 Address dest(masm.getStackPointer(), stack_offset);
2844
2845 if (arg->isFloatReg())
2846 masm.storeDouble(ToFloatRegister(arg), dest);
2847 else if (arg->isRegister())
2848 masm.storeValue(ValueTypeFromMIRType(argType), ToRegister(arg), dest);
2849 else
2850 masm.storeValue(arg->toConstant()->toJSValue(), dest);
2851 }
2852
2853 void
visitStackArgV(LStackArgV * lir)2854 CodeGenerator::visitStackArgV(LStackArgV* lir)
2855 {
2856 ValueOperand val = ToValue(lir, 0);
2857 uint32_t argslot = lir->argslot();
2858 MOZ_ASSERT(argslot - 1u < graph.argumentSlotCount());
2859
2860 int32_t stack_offset = StackOffsetOfPassedArg(argslot);
2861
2862 masm.storeValue(val, Address(masm.getStackPointer(), stack_offset));
2863 }
2864
2865 void
visitMoveGroup(LMoveGroup * group)2866 CodeGenerator::visitMoveGroup(LMoveGroup* group)
2867 {
2868 if (!group->numMoves())
2869 return;
2870
2871 MoveResolver& resolver = masm.moveResolver();
2872
2873 for (size_t i = 0; i < group->numMoves(); i++) {
2874 const LMove& move = group->getMove(i);
2875
2876 LAllocation from = move.from();
2877 LAllocation to = move.to();
2878 LDefinition::Type type = move.type();
2879
2880 // No bogus moves.
2881 MOZ_ASSERT(from != to);
2882 MOZ_ASSERT(!from.isConstant());
2883 MoveOp::Type moveType;
2884 switch (type) {
2885 case LDefinition::OBJECT:
2886 case LDefinition::SLOTS:
2887 #ifdef JS_NUNBOX32
2888 case LDefinition::TYPE:
2889 case LDefinition::PAYLOAD:
2890 #else
2891 case LDefinition::BOX:
2892 #endif
2893 case LDefinition::GENERAL: moveType = MoveOp::GENERAL; break;
2894 case LDefinition::INT32: moveType = MoveOp::INT32; break;
2895 case LDefinition::FLOAT32: moveType = MoveOp::FLOAT32; break;
2896 case LDefinition::DOUBLE: moveType = MoveOp::DOUBLE; break;
2897 case LDefinition::SIMD128INT: moveType = MoveOp::SIMD128INT; break;
2898 case LDefinition::SIMD128FLOAT: moveType = MoveOp::SIMD128FLOAT; break;
2899 default: MOZ_CRASH("Unexpected move type");
2900 }
2901
2902 masm.propagateOOM(resolver.addMove(toMoveOperand(from), toMoveOperand(to), moveType));
2903 }
2904
2905 masm.propagateOOM(resolver.resolve());
2906 if (masm.oom())
2907 return;
2908
2909 MoveEmitter emitter(masm);
2910
2911 #ifdef JS_CODEGEN_X86
2912 if (group->maybeScratchRegister().isGeneralReg())
2913 emitter.setScratchRegister(group->maybeScratchRegister().toGeneralReg()->reg());
2914 else
2915 resolver.sortMemoryToMemoryMoves();
2916 #endif
2917
2918 emitter.emit(resolver);
2919 emitter.finish();
2920 }
2921
2922 void
visitInteger(LInteger * lir)2923 CodeGenerator::visitInteger(LInteger* lir)
2924 {
2925 masm.move32(Imm32(lir->getValue()), ToRegister(lir->output()));
2926 }
2927
2928 void
visitInteger64(LInteger64 * lir)2929 CodeGenerator::visitInteger64(LInteger64* lir)
2930 {
2931 masm.move64(Imm64(lir->getValue()), ToOutRegister64(lir));
2932 }
2933
2934 void
visitPointer(LPointer * lir)2935 CodeGenerator::visitPointer(LPointer* lir)
2936 {
2937 if (lir->kind() == LPointer::GC_THING)
2938 masm.movePtr(ImmGCPtr(lir->gcptr()), ToRegister(lir->output()));
2939 else
2940 masm.movePtr(ImmPtr(lir->ptr()), ToRegister(lir->output()));
2941 }
2942
2943 void
visitKeepAliveObject(LKeepAliveObject * lir)2944 CodeGenerator::visitKeepAliveObject(LKeepAliveObject* lir)
2945 {
2946 // No-op.
2947 }
2948
2949 void
visitSlots(LSlots * lir)2950 CodeGenerator::visitSlots(LSlots* lir)
2951 {
2952 Address slots(ToRegister(lir->object()), NativeObject::offsetOfSlots());
2953 masm.loadPtr(slots, ToRegister(lir->output()));
2954 }
2955
2956 void
visitLoadSlotT(LLoadSlotT * lir)2957 CodeGenerator::visitLoadSlotT(LLoadSlotT* lir)
2958 {
2959 Register base = ToRegister(lir->slots());
2960 int32_t offset = lir->mir()->slot() * sizeof(js::Value);
2961 AnyRegister result = ToAnyRegister(lir->output());
2962
2963 masm.loadUnboxedValue(Address(base, offset), lir->mir()->type(), result);
2964 }
2965
2966 void
visitLoadSlotV(LLoadSlotV * lir)2967 CodeGenerator::visitLoadSlotV(LLoadSlotV* lir)
2968 {
2969 ValueOperand dest = ToOutValue(lir);
2970 Register base = ToRegister(lir->input());
2971 int32_t offset = lir->mir()->slot() * sizeof(js::Value);
2972
2973 masm.loadValue(Address(base, offset), dest);
2974 }
2975
2976 void
visitStoreSlotT(LStoreSlotT * lir)2977 CodeGenerator::visitStoreSlotT(LStoreSlotT* lir)
2978 {
2979 Register base = ToRegister(lir->slots());
2980 int32_t offset = lir->mir()->slot() * sizeof(js::Value);
2981 Address dest(base, offset);
2982
2983 if (lir->mir()->needsBarrier())
2984 emitPreBarrier(dest);
2985
2986 MIRType valueType = lir->mir()->value()->type();
2987
2988 if (valueType == MIRType::ObjectOrNull) {
2989 masm.storeObjectOrNull(ToRegister(lir->value()), dest);
2990 } else {
2991 ConstantOrRegister value;
2992 if (lir->value()->isConstant())
2993 value = ConstantOrRegister(lir->value()->toConstant()->toJSValue());
2994 else
2995 value = TypedOrValueRegister(valueType, ToAnyRegister(lir->value()));
2996 masm.storeUnboxedValue(value, valueType, dest, lir->mir()->slotType());
2997 }
2998 }
2999
3000 void
visitStoreSlotV(LStoreSlotV * lir)3001 CodeGenerator::visitStoreSlotV(LStoreSlotV* lir)
3002 {
3003 Register base = ToRegister(lir->slots());
3004 int32_t offset = lir->mir()->slot() * sizeof(Value);
3005
3006 const ValueOperand value = ToValue(lir, LStoreSlotV::Value);
3007
3008 if (lir->mir()->needsBarrier())
3009 emitPreBarrier(Address(base, offset));
3010
3011 masm.storeValue(value, Address(base, offset));
3012 }
3013
3014 static void
GuardReceiver(MacroAssembler & masm,const ReceiverGuard & guard,Register obj,Register scratch,Label * miss,bool checkNullExpando)3015 GuardReceiver(MacroAssembler& masm, const ReceiverGuard& guard,
3016 Register obj, Register scratch, Label* miss, bool checkNullExpando)
3017 {
3018 if (guard.group) {
3019 masm.branchTestObjGroup(Assembler::NotEqual, obj, guard.group, miss);
3020
3021 Address expandoAddress(obj, UnboxedPlainObject::offsetOfExpando());
3022 if (guard.shape) {
3023 masm.loadPtr(expandoAddress, scratch);
3024 masm.branchPtr(Assembler::Equal, scratch, ImmWord(0), miss);
3025 masm.branchTestObjShape(Assembler::NotEqual, scratch, guard.shape, miss);
3026 } else if (checkNullExpando) {
3027 masm.branchPtr(Assembler::NotEqual, expandoAddress, ImmWord(0), miss);
3028 }
3029 } else {
3030 masm.branchTestObjShape(Assembler::NotEqual, obj, guard.shape, miss);
3031 }
3032 }
3033
3034 void
emitGetPropertyPolymorphic(LInstruction * ins,Register obj,Register scratch,const TypedOrValueRegister & output)3035 CodeGenerator::emitGetPropertyPolymorphic(LInstruction* ins, Register obj, Register scratch,
3036 const TypedOrValueRegister& output)
3037 {
3038 MGetPropertyPolymorphic* mir = ins->mirRaw()->toGetPropertyPolymorphic();
3039
3040 Label done;
3041
3042 for (size_t i = 0; i < mir->numReceivers(); i++) {
3043 ReceiverGuard receiver = mir->receiver(i);
3044
3045 Label next;
3046 masm.comment("GuardReceiver");
3047 GuardReceiver(masm, receiver, obj, scratch, &next, /* checkNullExpando = */ false);
3048
3049 if (receiver.shape) {
3050 masm.comment("loadTypedOrValue");
3051 // If this is an unboxed expando access, GuardReceiver loaded the
3052 // expando object into scratch.
3053 Register target = receiver.group ? scratch : obj;
3054
3055 Shape* shape = mir->shape(i);
3056 if (shape->slot() < shape->numFixedSlots()) {
3057 // Fixed slot.
3058 masm.loadTypedOrValue(Address(target, NativeObject::getFixedSlotOffset(shape->slot())),
3059 output);
3060 } else {
3061 // Dynamic slot.
3062 uint32_t offset = (shape->slot() - shape->numFixedSlots()) * sizeof(js::Value);
3063 masm.loadPtr(Address(target, NativeObject::offsetOfSlots()), scratch);
3064 masm.loadTypedOrValue(Address(scratch, offset), output);
3065 }
3066 } else {
3067 masm.comment("loadUnboxedProperty");
3068 const UnboxedLayout::Property* property =
3069 receiver.group->unboxedLayout().lookup(mir->name());
3070 Address propertyAddr(obj, UnboxedPlainObject::offsetOfData() + property->offset);
3071
3072 masm.loadUnboxedProperty(propertyAddr, property->type, output);
3073 }
3074
3075 if (i == mir->numReceivers() - 1) {
3076 bailoutFrom(&next, ins->snapshot());
3077 } else {
3078 masm.jump(&done);
3079 masm.bind(&next);
3080 }
3081 }
3082
3083 masm.bind(&done);
3084 }
3085
3086 void
visitGetPropertyPolymorphicV(LGetPropertyPolymorphicV * ins)3087 CodeGenerator::visitGetPropertyPolymorphicV(LGetPropertyPolymorphicV* ins)
3088 {
3089 Register obj = ToRegister(ins->obj());
3090 ValueOperand output = GetValueOutput(ins);
3091 emitGetPropertyPolymorphic(ins, obj, output.scratchReg(), output);
3092 }
3093
3094 void
visitGetPropertyPolymorphicT(LGetPropertyPolymorphicT * ins)3095 CodeGenerator::visitGetPropertyPolymorphicT(LGetPropertyPolymorphicT* ins)
3096 {
3097 Register obj = ToRegister(ins->obj());
3098 TypedOrValueRegister output(ins->mir()->type(), ToAnyRegister(ins->output()));
3099 Register temp = (output.type() == MIRType::Double)
3100 ? ToRegister(ins->temp())
3101 : output.typedReg().gpr();
3102 emitGetPropertyPolymorphic(ins, obj, temp, output);
3103 }
3104
3105 template <typename T>
3106 static void
EmitUnboxedPreBarrier(MacroAssembler & masm,T address,JSValueType type)3107 EmitUnboxedPreBarrier(MacroAssembler &masm, T address, JSValueType type)
3108 {
3109 if (type == JSVAL_TYPE_OBJECT)
3110 masm.patchableCallPreBarrier(address, MIRType::Object);
3111 else if (type == JSVAL_TYPE_STRING)
3112 masm.patchableCallPreBarrier(address, MIRType::String);
3113 else
3114 MOZ_ASSERT(!UnboxedTypeNeedsPreBarrier(type));
3115 }
3116
3117 void
emitSetPropertyPolymorphic(LInstruction * ins,Register obj,Register scratch,const ConstantOrRegister & value)3118 CodeGenerator::emitSetPropertyPolymorphic(LInstruction* ins, Register obj, Register scratch,
3119 const ConstantOrRegister& value)
3120 {
3121 MSetPropertyPolymorphic* mir = ins->mirRaw()->toSetPropertyPolymorphic();
3122
3123 Label done;
3124 for (size_t i = 0; i < mir->numReceivers(); i++) {
3125 ReceiverGuard receiver = mir->receiver(i);
3126
3127 Label next;
3128 GuardReceiver(masm, receiver, obj, scratch, &next, /* checkNullExpando = */ false);
3129
3130 if (receiver.shape) {
3131 // If this is an unboxed expando access, GuardReceiver loaded the
3132 // expando object into scratch.
3133 Register target = receiver.group ? scratch : obj;
3134
3135 Shape* shape = mir->shape(i);
3136 if (shape->slot() < shape->numFixedSlots()) {
3137 // Fixed slot.
3138 Address addr(target, NativeObject::getFixedSlotOffset(shape->slot()));
3139 if (mir->needsBarrier())
3140 emitPreBarrier(addr);
3141 masm.storeConstantOrRegister(value, addr);
3142 } else {
3143 // Dynamic slot.
3144 masm.loadPtr(Address(target, NativeObject::offsetOfSlots()), scratch);
3145 Address addr(scratch, (shape->slot() - shape->numFixedSlots()) * sizeof(js::Value));
3146 if (mir->needsBarrier())
3147 emitPreBarrier(addr);
3148 masm.storeConstantOrRegister(value, addr);
3149 }
3150 } else {
3151 const UnboxedLayout::Property* property =
3152 receiver.group->unboxedLayout().lookup(mir->name());
3153 Address propertyAddr(obj, UnboxedPlainObject::offsetOfData() + property->offset);
3154
3155 EmitUnboxedPreBarrier(masm, propertyAddr, property->type);
3156 masm.storeUnboxedProperty(propertyAddr, property->type, value, nullptr);
3157 }
3158
3159 if (i == mir->numReceivers() - 1) {
3160 bailoutFrom(&next, ins->snapshot());
3161 } else {
3162 masm.jump(&done);
3163 masm.bind(&next);
3164 }
3165 }
3166
3167 masm.bind(&done);
3168 }
3169
3170 void
visitSetPropertyPolymorphicV(LSetPropertyPolymorphicV * ins)3171 CodeGenerator::visitSetPropertyPolymorphicV(LSetPropertyPolymorphicV* ins)
3172 {
3173 Register obj = ToRegister(ins->obj());
3174 Register temp = ToRegister(ins->temp());
3175 ValueOperand value = ToValue(ins, LSetPropertyPolymorphicV::Value);
3176 emitSetPropertyPolymorphic(ins, obj, temp, TypedOrValueRegister(value));
3177 }
3178
3179 void
visitSetPropertyPolymorphicT(LSetPropertyPolymorphicT * ins)3180 CodeGenerator::visitSetPropertyPolymorphicT(LSetPropertyPolymorphicT* ins)
3181 {
3182 Register obj = ToRegister(ins->obj());
3183 Register temp = ToRegister(ins->temp());
3184
3185 ConstantOrRegister value;
3186 if (ins->mir()->value()->isConstant())
3187 value = ConstantOrRegister(ins->mir()->value()->toConstant()->toJSValue());
3188 else
3189 value = TypedOrValueRegister(ins->mir()->value()->type(), ToAnyRegister(ins->value()));
3190
3191 emitSetPropertyPolymorphic(ins, obj, temp, value);
3192 }
3193
3194 void
visitElements(LElements * lir)3195 CodeGenerator::visitElements(LElements* lir)
3196 {
3197 Address elements(ToRegister(lir->object()),
3198 lir->mir()->unboxed() ? UnboxedArrayObject::offsetOfElements()
3199 : NativeObject::offsetOfElements());
3200 masm.loadPtr(elements, ToRegister(lir->output()));
3201 }
3202
3203 typedef bool (*ConvertElementsToDoublesFn)(JSContext*, uintptr_t);
3204 static const VMFunction ConvertElementsToDoublesInfo =
3205 FunctionInfo<ConvertElementsToDoublesFn>(ObjectElements::ConvertElementsToDoubles,
3206 "ObjectElements::ConvertElementsToDoubles");
3207
3208 void
visitConvertElementsToDoubles(LConvertElementsToDoubles * lir)3209 CodeGenerator::visitConvertElementsToDoubles(LConvertElementsToDoubles* lir)
3210 {
3211 Register elements = ToRegister(lir->elements());
3212
3213 OutOfLineCode* ool = oolCallVM(ConvertElementsToDoublesInfo, lir,
3214 ArgList(elements), StoreNothing());
3215
3216 Address convertedAddress(elements, ObjectElements::offsetOfFlags());
3217 Imm32 bit(ObjectElements::CONVERT_DOUBLE_ELEMENTS);
3218 masm.branchTest32(Assembler::Zero, convertedAddress, bit, ool->entry());
3219 masm.bind(ool->rejoin());
3220 }
3221
3222 void
visitMaybeToDoubleElement(LMaybeToDoubleElement * lir)3223 CodeGenerator::visitMaybeToDoubleElement(LMaybeToDoubleElement* lir)
3224 {
3225 Register elements = ToRegister(lir->elements());
3226 Register value = ToRegister(lir->value());
3227 ValueOperand out = ToOutValue(lir);
3228
3229 FloatRegister temp = ToFloatRegister(lir->tempFloat());
3230 Label convert, done;
3231
3232 // If the CONVERT_DOUBLE_ELEMENTS flag is set, convert the int32
3233 // value to double. Else, just box it.
3234 masm.branchTest32(Assembler::NonZero,
3235 Address(elements, ObjectElements::offsetOfFlags()),
3236 Imm32(ObjectElements::CONVERT_DOUBLE_ELEMENTS),
3237 &convert);
3238
3239 masm.tagValue(JSVAL_TYPE_INT32, value, out);
3240 masm.jump(&done);
3241
3242 masm.bind(&convert);
3243 masm.convertInt32ToDouble(value, temp);
3244 masm.boxDouble(temp, out);
3245
3246 masm.bind(&done);
3247 }
3248
3249 typedef bool (*CopyElementsForWriteFn)(ExclusiveContext*, NativeObject*);
3250 static const VMFunction CopyElementsForWriteInfo =
3251 FunctionInfo<CopyElementsForWriteFn>(NativeObject::CopyElementsForWrite,
3252 "NativeObject::CopyElementsForWrite");
3253
3254 void
visitMaybeCopyElementsForWrite(LMaybeCopyElementsForWrite * lir)3255 CodeGenerator::visitMaybeCopyElementsForWrite(LMaybeCopyElementsForWrite* lir)
3256 {
3257 Register object = ToRegister(lir->object());
3258 Register temp = ToRegister(lir->temp());
3259
3260 OutOfLineCode* ool = oolCallVM(CopyElementsForWriteInfo, lir,
3261 ArgList(object), StoreNothing());
3262
3263 if (lir->mir()->checkNative()) {
3264 masm.loadObjClass(object, temp);
3265 masm.branchTest32(Assembler::NonZero, Address(temp, Class::offsetOfFlags()),
3266 Imm32(Class::NON_NATIVE), ool->rejoin());
3267 }
3268
3269 masm.loadPtr(Address(object, NativeObject::offsetOfElements()), temp);
3270 masm.branchTest32(Assembler::NonZero,
3271 Address(temp, ObjectElements::offsetOfFlags()),
3272 Imm32(ObjectElements::COPY_ON_WRITE),
3273 ool->entry());
3274 masm.bind(ool->rejoin());
3275 }
3276
3277 void
visitFunctionEnvironment(LFunctionEnvironment * lir)3278 CodeGenerator::visitFunctionEnvironment(LFunctionEnvironment* lir)
3279 {
3280 Address environment(ToRegister(lir->function()), JSFunction::offsetOfEnvironment());
3281 masm.loadPtr(environment, ToRegister(lir->output()));
3282 }
3283
3284 void
visitGuardObjectIdentity(LGuardObjectIdentity * guard)3285 CodeGenerator::visitGuardObjectIdentity(LGuardObjectIdentity* guard)
3286 {
3287 Register input = ToRegister(guard->input());
3288 Register expected = ToRegister(guard->expected());
3289
3290 Assembler::Condition cond =
3291 guard->mir()->bailOnEquality() ? Assembler::Equal : Assembler::NotEqual;
3292 bailoutCmpPtr(cond, input, expected, guard->snapshot());
3293 }
3294
3295 void
visitGuardReceiverPolymorphic(LGuardReceiverPolymorphic * lir)3296 CodeGenerator::visitGuardReceiverPolymorphic(LGuardReceiverPolymorphic* lir)
3297 {
3298 const MGuardReceiverPolymorphic* mir = lir->mir();
3299 Register obj = ToRegister(lir->object());
3300 Register temp = ToRegister(lir->temp());
3301
3302 Label done;
3303
3304 for (size_t i = 0; i < mir->numReceivers(); i++) {
3305 const ReceiverGuard& receiver = mir->receiver(i);
3306
3307 Label next;
3308 GuardReceiver(masm, receiver, obj, temp, &next, /* checkNullExpando = */ true);
3309
3310 if (i == mir->numReceivers() - 1) {
3311 bailoutFrom(&next, lir->snapshot());
3312 } else {
3313 masm.jump(&done);
3314 masm.bind(&next);
3315 }
3316 }
3317
3318 masm.bind(&done);
3319 }
3320
3321 void
visitGuardUnboxedExpando(LGuardUnboxedExpando * lir)3322 CodeGenerator::visitGuardUnboxedExpando(LGuardUnboxedExpando* lir)
3323 {
3324 Label miss;
3325
3326 Register obj = ToRegister(lir->object());
3327 masm.branchPtr(lir->mir()->requireExpando() ? Assembler::Equal : Assembler::NotEqual,
3328 Address(obj, UnboxedPlainObject::offsetOfExpando()), ImmWord(0), &miss);
3329
3330 bailoutFrom(&miss, lir->snapshot());
3331 }
3332
3333 void
visitLoadUnboxedExpando(LLoadUnboxedExpando * lir)3334 CodeGenerator::visitLoadUnboxedExpando(LLoadUnboxedExpando* lir)
3335 {
3336 Register obj = ToRegister(lir->object());
3337 Register result = ToRegister(lir->getDef(0));
3338
3339 masm.loadPtr(Address(obj, UnboxedPlainObject::offsetOfExpando()), result);
3340 }
3341
3342 void
visitTypeBarrierV(LTypeBarrierV * lir)3343 CodeGenerator::visitTypeBarrierV(LTypeBarrierV* lir)
3344 {
3345 ValueOperand operand = ToValue(lir, LTypeBarrierV::Input);
3346 Register scratch = ToTempRegisterOrInvalid(lir->temp());
3347
3348 Label miss;
3349 masm.guardTypeSet(operand, lir->mir()->resultTypeSet(), lir->mir()->barrierKind(), scratch, &miss);
3350 bailoutFrom(&miss, lir->snapshot());
3351 }
3352
3353 void
visitTypeBarrierO(LTypeBarrierO * lir)3354 CodeGenerator::visitTypeBarrierO(LTypeBarrierO* lir)
3355 {
3356 Register obj = ToRegister(lir->object());
3357 Register scratch = ToTempRegisterOrInvalid(lir->temp());
3358 Label miss, ok;
3359
3360 if (lir->mir()->type() == MIRType::ObjectOrNull) {
3361 masm.comment("Object or Null");
3362 Label* nullTarget = lir->mir()->resultTypeSet()->mightBeMIRType(MIRType::Null) ? &ok : &miss;
3363 masm.branchTestPtr(Assembler::Zero, obj, obj, nullTarget);
3364 } else {
3365 MOZ_ASSERT(lir->mir()->type() == MIRType::Object);
3366 MOZ_ASSERT(lir->mir()->barrierKind() != BarrierKind::TypeTagOnly);
3367 }
3368
3369 if (lir->mir()->barrierKind() != BarrierKind::TypeTagOnly) {
3370 masm.comment("Type tag only");
3371 masm.guardObjectType(obj, lir->mir()->resultTypeSet(), scratch, &miss);
3372 }
3373
3374 bailoutFrom(&miss, lir->snapshot());
3375 masm.bind(&ok);
3376 }
3377
3378 void
visitMonitorTypes(LMonitorTypes * lir)3379 CodeGenerator::visitMonitorTypes(LMonitorTypes* lir)
3380 {
3381 ValueOperand operand = ToValue(lir, LMonitorTypes::Input);
3382 Register scratch = ToTempUnboxRegister(lir->temp());
3383
3384 Label matched, miss;
3385 masm.guardTypeSet(operand, lir->mir()->typeSet(), lir->mir()->barrierKind(), scratch, &miss);
3386 bailoutFrom(&miss, lir->snapshot());
3387 }
3388
3389 // Out-of-line path to update the store buffer.
3390 class OutOfLineCallPostWriteBarrier : public OutOfLineCodeBase<CodeGenerator>
3391 {
3392 LInstruction* lir_;
3393 const LAllocation* object_;
3394
3395 public:
OutOfLineCallPostWriteBarrier(LInstruction * lir,const LAllocation * object)3396 OutOfLineCallPostWriteBarrier(LInstruction* lir, const LAllocation* object)
3397 : lir_(lir), object_(object)
3398 { }
3399
accept(CodeGenerator * codegen)3400 void accept(CodeGenerator* codegen) {
3401 codegen->visitOutOfLineCallPostWriteBarrier(this);
3402 }
3403
lir() const3404 LInstruction* lir() const {
3405 return lir_;
3406 }
object() const3407 const LAllocation* object() const {
3408 return object_;
3409 }
3410 };
3411
3412 static void
EmitStoreBufferCheckForConstant(MacroAssembler & masm,JSObject * object,AllocatableGeneralRegisterSet & regs,Label * exit,Label * callVM)3413 EmitStoreBufferCheckForConstant(MacroAssembler& masm, JSObject* object,
3414 AllocatableGeneralRegisterSet& regs, Label* exit, Label* callVM)
3415 {
3416 Register temp = regs.takeAny();
3417
3418 const gc::TenuredCell* cell = &object->asTenured();
3419 gc::Arena* arena = cell->arena();
3420
3421 Register cells = temp;
3422 masm.loadPtr(AbsoluteAddress(&arena->bufferedCells), cells);
3423
3424 size_t index = gc::ArenaCellSet::getCellIndex(cell);
3425 size_t word;
3426 uint32_t mask;
3427 gc::ArenaCellSet::getWordIndexAndMask(index, &word, &mask);
3428 size_t offset = gc::ArenaCellSet::offsetOfBits() + word * sizeof(uint32_t);
3429
3430 masm.branchTest32(Assembler::NonZero, Address(cells, offset), Imm32(mask), exit);
3431
3432 // Check whether this is the sentinel set and if so call the VM to allocate
3433 // one for this arena.
3434 masm.branchPtr(Assembler::Equal, Address(cells, gc::ArenaCellSet::offsetOfArena()),
3435 ImmPtr(nullptr), callVM);
3436
3437 // Add the cell to the set.
3438 masm.or32(Imm32(mask), Address(cells, offset));
3439 masm.jump(exit);
3440
3441 regs.add(temp);
3442 }
3443
3444 static void
EmitPostWriteBarrier(MacroAssembler & masm,Register objreg,JSObject * maybeConstant,bool isGlobal,AllocatableGeneralRegisterSet & regs)3445 EmitPostWriteBarrier(MacroAssembler& masm, Register objreg, JSObject* maybeConstant, bool isGlobal,
3446 AllocatableGeneralRegisterSet& regs)
3447 {
3448 MOZ_ASSERT_IF(isGlobal, maybeConstant);
3449
3450 Label callVM;
3451 Label exit;
3452
3453 // We already have a fast path to check whether a global is in the store
3454 // buffer.
3455 if (!isGlobal && maybeConstant)
3456 EmitStoreBufferCheckForConstant(masm, maybeConstant, regs, &exit, &callVM);
3457
3458 // Call into the VM to barrier the write.
3459 masm.bind(&callVM);
3460
3461 Register runtimereg = regs.takeAny();
3462 masm.mov(ImmPtr(GetJitContext()->runtime), runtimereg);
3463
3464 void (*fun)(JSRuntime*, JSObject*) = isGlobal ? PostGlobalWriteBarrier : PostWriteBarrier;
3465 masm.setupUnalignedABICall(regs.takeAny());
3466 masm.passABIArg(runtimereg);
3467 masm.passABIArg(objreg);
3468 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, fun));
3469
3470 masm.bind(&exit);
3471 }
3472
3473 void
emitPostWriteBarrier(const LAllocation * obj)3474 CodeGenerator::emitPostWriteBarrier(const LAllocation* obj)
3475 {
3476 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
3477
3478 Register objreg;
3479 JSObject* object = nullptr;
3480 bool isGlobal = false;
3481 if (obj->isConstant()) {
3482 object = &obj->toConstant()->toObject();
3483 isGlobal = isGlobalObject(object);
3484 objreg = regs.takeAny();
3485 masm.movePtr(ImmGCPtr(object), objreg);
3486 } else {
3487 objreg = ToRegister(obj);
3488 regs.takeUnchecked(objreg);
3489 }
3490
3491 EmitPostWriteBarrier(masm, objreg, object, isGlobal, regs);
3492 }
3493
3494 void
emitPostWriteBarrier(Register objreg)3495 CodeGenerator::emitPostWriteBarrier(Register objreg)
3496 {
3497 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
3498 regs.takeUnchecked(objreg);
3499 EmitPostWriteBarrier(masm, objreg, nullptr, false, regs);
3500 }
3501
3502 void
visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier * ool)3503 CodeGenerator::visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* ool)
3504 {
3505 saveLiveVolatile(ool->lir());
3506 const LAllocation* obj = ool->object();
3507 emitPostWriteBarrier(obj);
3508 restoreLiveVolatile(ool->lir());
3509
3510 masm.jump(ool->rejoin());
3511 }
3512
3513 void
maybeEmitGlobalBarrierCheck(const LAllocation * maybeGlobal,OutOfLineCode * ool)3514 CodeGenerator::maybeEmitGlobalBarrierCheck(const LAllocation* maybeGlobal, OutOfLineCode* ool)
3515 {
3516 // Check whether an object is a global that we have already barriered before
3517 // calling into the VM.
3518
3519 if (!maybeGlobal->isConstant())
3520 return;
3521
3522 JSObject* obj = &maybeGlobal->toConstant()->toObject();
3523 if (!isGlobalObject(obj))
3524 return;
3525
3526 JSCompartment* comp = obj->compartment();
3527 auto addr = AbsoluteAddress(&comp->globalWriteBarriered);
3528 masm.branch32(Assembler::NotEqual, addr, Imm32(0), ool->rejoin());
3529 }
3530
3531 template <class LPostBarrierType>
3532 void
visitPostWriteBarrierCommonO(LPostBarrierType * lir,OutOfLineCode * ool)3533 CodeGenerator::visitPostWriteBarrierCommonO(LPostBarrierType* lir, OutOfLineCode* ool)
3534 {
3535 addOutOfLineCode(ool, lir->mir());
3536
3537 Register temp = ToTempRegisterOrInvalid(lir->temp());
3538
3539 if (lir->object()->isConstant()) {
3540 // Constant nursery objects cannot appear here, see LIRGenerator::visitPostWriteElementBarrier.
3541 MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject()));
3542 } else {
3543 masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()), temp,
3544 ool->rejoin());
3545 }
3546
3547 maybeEmitGlobalBarrierCheck(lir->object(), ool);
3548
3549 Register valueObj = ToRegister(lir->value());
3550 masm.branchTestPtr(Assembler::Zero, valueObj, valueObj, ool->rejoin());
3551 masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->value()), temp, ool->entry());
3552
3553 masm.bind(ool->rejoin());
3554 }
3555
3556 template <class LPostBarrierType>
3557 void
visitPostWriteBarrierCommonV(LPostBarrierType * lir,OutOfLineCode * ool)3558 CodeGenerator::visitPostWriteBarrierCommonV(LPostBarrierType* lir, OutOfLineCode* ool)
3559 {
3560 addOutOfLineCode(ool, lir->mir());
3561
3562 Register temp = ToTempRegisterOrInvalid(lir->temp());
3563
3564 if (lir->object()->isConstant()) {
3565 // Constant nursery objects cannot appear here, see LIRGenerator::visitPostWriteElementBarrier.
3566 MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject()));
3567 } else {
3568 masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()), temp,
3569 ool->rejoin());
3570 }
3571
3572 maybeEmitGlobalBarrierCheck(lir->object(), ool);
3573
3574 ValueOperand value = ToValue(lir, LPostBarrierType::Input);
3575 masm.branchValueIsNurseryObject(Assembler::Equal, value, temp, ool->entry());
3576
3577 masm.bind(ool->rejoin());
3578 }
3579
3580 void
visitPostWriteBarrierO(LPostWriteBarrierO * lir)3581 CodeGenerator::visitPostWriteBarrierO(LPostWriteBarrierO* lir)
3582 {
3583 auto ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
3584 visitPostWriteBarrierCommonO(lir, ool);
3585 }
3586
3587 void
visitPostWriteBarrierV(LPostWriteBarrierV * lir)3588 CodeGenerator::visitPostWriteBarrierV(LPostWriteBarrierV* lir)
3589 {
3590 auto ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
3591 visitPostWriteBarrierCommonV(lir, ool);
3592 }
3593
3594 // Out-of-line path to update the store buffer.
3595 class OutOfLineCallPostWriteElementBarrier : public OutOfLineCodeBase<CodeGenerator>
3596 {
3597 LInstruction* lir_;
3598 const LAllocation* object_;
3599 const LAllocation* index_;
3600
3601 public:
OutOfLineCallPostWriteElementBarrier(LInstruction * lir,const LAllocation * object,const LAllocation * index)3602 OutOfLineCallPostWriteElementBarrier(LInstruction* lir, const LAllocation* object,
3603 const LAllocation* index)
3604 : lir_(lir),
3605 object_(object),
3606 index_(index)
3607 { }
3608
accept(CodeGenerator * codegen)3609 void accept(CodeGenerator* codegen) {
3610 codegen->visitOutOfLineCallPostWriteElementBarrier(this);
3611 }
3612
lir() const3613 LInstruction* lir() const {
3614 return lir_;
3615 }
3616
object() const3617 const LAllocation* object() const {
3618 return object_;
3619 }
3620
index() const3621 const LAllocation* index() const {
3622 return index_;
3623 }
3624 };
3625
3626 void
visitOutOfLineCallPostWriteElementBarrier(OutOfLineCallPostWriteElementBarrier * ool)3627 CodeGenerator::visitOutOfLineCallPostWriteElementBarrier(OutOfLineCallPostWriteElementBarrier* ool)
3628 {
3629 saveLiveVolatile(ool->lir());
3630
3631 const LAllocation* obj = ool->object();
3632 const LAllocation* index = ool->index();
3633
3634 Register objreg = obj->isConstant() ? InvalidReg : ToRegister(obj);
3635 Register indexreg = ToRegister(index);
3636
3637 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
3638 regs.takeUnchecked(indexreg);
3639
3640 if (obj->isConstant()) {
3641 objreg = regs.takeAny();
3642 masm.movePtr(ImmGCPtr(&obj->toConstant()->toObject()), objreg);
3643 } else {
3644 regs.takeUnchecked(objreg);
3645 }
3646
3647 Register runtimereg = regs.takeAny();
3648 masm.setupUnalignedABICall(runtimereg);
3649 masm.mov(ImmPtr(GetJitContext()->runtime), runtimereg);
3650 masm.passABIArg(runtimereg);
3651 masm.passABIArg(objreg);
3652 masm.passABIArg(indexreg);
3653 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, PostWriteElementBarrier));
3654
3655 restoreLiveVolatile(ool->lir());
3656
3657 masm.jump(ool->rejoin());
3658 }
3659
3660 void
visitPostWriteElementBarrierO(LPostWriteElementBarrierO * lir)3661 CodeGenerator::visitPostWriteElementBarrierO(LPostWriteElementBarrierO* lir)
3662 {
3663 auto ool = new(alloc()) OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
3664 visitPostWriteBarrierCommonO(lir, ool);
3665 }
3666
3667 void
visitPostWriteElementBarrierV(LPostWriteElementBarrierV * lir)3668 CodeGenerator::visitPostWriteElementBarrierV(LPostWriteElementBarrierV* lir)
3669 {
3670 auto ool = new(alloc()) OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
3671 visitPostWriteBarrierCommonV(lir, ool);
3672 }
3673
3674 void
visitCallNative(LCallNative * call)3675 CodeGenerator::visitCallNative(LCallNative* call)
3676 {
3677 WrappedFunction* target = call->getSingleTarget();
3678 MOZ_ASSERT(target);
3679 MOZ_ASSERT(target->isNative());
3680
3681 int callargslot = call->argslot();
3682 int unusedStack = StackOffsetOfPassedArg(callargslot);
3683
3684 // Registers used for callWithABI() argument-passing.
3685 const Register argContextReg = ToRegister(call->getArgContextReg());
3686 const Register argUintNReg = ToRegister(call->getArgUintNReg());
3687 const Register argVpReg = ToRegister(call->getArgVpReg());
3688
3689 // Misc. temporary registers.
3690 const Register tempReg = ToRegister(call->getTempReg());
3691
3692 DebugOnly<uint32_t> initialStack = masm.framePushed();
3693
3694 masm.checkStackAlignment();
3695
3696 // Native functions have the signature:
3697 // bool (*)(JSContext*, unsigned, Value* vp)
3698 // Where vp[0] is space for an outparam, vp[1] is |this|, and vp[2] onward
3699 // are the function arguments.
3700
3701 // Allocate space for the outparam, moving the StackPointer to what will be &vp[1].
3702 masm.adjustStack(unusedStack);
3703
3704 // Push a Value containing the callee object: natives are allowed to access their callee before
3705 // setitng the return value. The StackPointer is moved to &vp[0].
3706 masm.Push(ObjectValue(*target->rawJSFunction()));
3707
3708 // Preload arguments into registers.
3709 masm.loadJSContext(argContextReg);
3710 masm.move32(Imm32(call->numActualArgs()), argUintNReg);
3711 masm.moveStackPtrTo(argVpReg);
3712
3713 masm.Push(argUintNReg);
3714
3715 // Construct native exit frame.
3716 uint32_t safepointOffset = masm.buildFakeExitFrame(tempReg);
3717 masm.enterFakeExitFrameForNative(call->mir()->isConstructing());
3718
3719 markSafepointAt(safepointOffset, call);
3720
3721 emitTracelogStartEvent(TraceLogger_Call);
3722
3723 // Construct and execute call.
3724 masm.setupUnalignedABICall(tempReg);
3725 masm.passABIArg(argContextReg);
3726 masm.passABIArg(argUintNReg);
3727 masm.passABIArg(argVpReg);
3728 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, target->native()));
3729
3730 emitTracelogStopEvent(TraceLogger_Call);
3731
3732 // Test for failure.
3733 masm.branchIfFalseBool(ReturnReg, masm.failureLabel());
3734
3735 // Load the outparam vp[0] into output register(s).
3736 masm.loadValue(Address(masm.getStackPointer(), NativeExitFrameLayout::offsetOfResult()), JSReturnOperand);
3737
3738 // The next instruction is removing the footer of the exit frame, so there
3739 // is no need for leaveFakeExitFrame.
3740
3741 // Move the StackPointer back to its original location, unwinding the native exit frame.
3742 masm.adjustStack(NativeExitFrameLayout::Size() - unusedStack);
3743 MOZ_ASSERT(masm.framePushed() == initialStack);
3744 }
3745
3746 static void
LoadDOMPrivate(MacroAssembler & masm,Register obj,Register priv)3747 LoadDOMPrivate(MacroAssembler& masm, Register obj, Register priv)
3748 {
3749 // Load the value in DOM_OBJECT_SLOT for a native or proxy DOM object. This
3750 // will be in the first slot but may be fixed or non-fixed.
3751 MOZ_ASSERT(obj != priv);
3752
3753 // Check shape->numFixedSlots != 0.
3754 masm.loadPtr(Address(obj, ShapedObject::offsetOfShape()), priv);
3755
3756 Label hasFixedSlots, done;
3757 masm.branchTest32(Assembler::NonZero,
3758 Address(priv, Shape::offsetOfSlotInfo()),
3759 Imm32(Shape::fixedSlotsMask()),
3760 &hasFixedSlots);
3761
3762 masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), priv);
3763 masm.loadPrivate(Address(priv, 0), priv);
3764
3765 masm.jump(&done);
3766 masm.bind(&hasFixedSlots);
3767
3768 masm.loadPrivate(Address(obj, NativeObject::getFixedSlotOffset(0)), priv);
3769
3770 masm.bind(&done);
3771 }
3772
3773 void
visitCallDOMNative(LCallDOMNative * call)3774 CodeGenerator::visitCallDOMNative(LCallDOMNative* call)
3775 {
3776 WrappedFunction* target = call->getSingleTarget();
3777 MOZ_ASSERT(target);
3778 MOZ_ASSERT(target->isNative());
3779 MOZ_ASSERT(target->jitInfo());
3780 MOZ_ASSERT(call->mir()->isCallDOMNative());
3781
3782 int callargslot = call->argslot();
3783 int unusedStack = StackOffsetOfPassedArg(callargslot);
3784
3785 // Registers used for callWithABI() argument-passing.
3786 const Register argJSContext = ToRegister(call->getArgJSContext());
3787 const Register argObj = ToRegister(call->getArgObj());
3788 const Register argPrivate = ToRegister(call->getArgPrivate());
3789 const Register argArgs = ToRegister(call->getArgArgs());
3790
3791 DebugOnly<uint32_t> initialStack = masm.framePushed();
3792
3793 masm.checkStackAlignment();
3794
3795 // DOM methods have the signature:
3796 // bool (*)(JSContext*, HandleObject, void* private, const JSJitMethodCallArgs& args)
3797 // Where args is initialized from an argc and a vp, vp[0] is space for an
3798 // outparam and the callee, vp[1] is |this|, and vp[2] onward are the
3799 // function arguments. Note that args stores the argv, not the vp, and
3800 // argv == vp + 2.
3801
3802 // Nestle the stack up against the pushed arguments, leaving StackPointer at
3803 // &vp[1]
3804 masm.adjustStack(unusedStack);
3805 // argObj is filled with the extracted object, then returned.
3806 Register obj = masm.extractObject(Address(masm.getStackPointer(), 0), argObj);
3807 MOZ_ASSERT(obj == argObj);
3808
3809 // Push a Value containing the callee object: natives are allowed to access their callee before
3810 // setitng the return value. After this the StackPointer points to &vp[0].
3811 masm.Push(ObjectValue(*target->rawJSFunction()));
3812
3813 // Now compute the argv value. Since StackPointer is pointing to &vp[0] and
3814 // argv is &vp[2] we just need to add 2*sizeof(Value) to the current
3815 // StackPointer.
3816 JS_STATIC_ASSERT(JSJitMethodCallArgsTraits::offsetOfArgv == 0);
3817 JS_STATIC_ASSERT(JSJitMethodCallArgsTraits::offsetOfArgc ==
3818 IonDOMMethodExitFrameLayoutTraits::offsetOfArgcFromArgv);
3819 masm.computeEffectiveAddress(Address(masm.getStackPointer(), 2 * sizeof(Value)), argArgs);
3820
3821 LoadDOMPrivate(masm, obj, argPrivate);
3822
3823 // Push argc from the call instruction into what will become the IonExitFrame
3824 masm.Push(Imm32(call->numActualArgs()));
3825
3826 // Push our argv onto the stack
3827 masm.Push(argArgs);
3828 // And store our JSJitMethodCallArgs* in argArgs.
3829 masm.moveStackPtrTo(argArgs);
3830
3831 // Push |this| object for passing HandleObject. We push after argc to
3832 // maintain the same sp-relative location of the object pointer with other
3833 // DOMExitFrames.
3834 masm.Push(argObj);
3835 masm.moveStackPtrTo(argObj);
3836
3837 // Construct native exit frame.
3838 uint32_t safepointOffset = masm.buildFakeExitFrame(argJSContext);
3839 masm.enterFakeExitFrame(IonDOMMethodExitFrameLayoutToken);
3840
3841 markSafepointAt(safepointOffset, call);
3842
3843 // Construct and execute call.
3844 masm.setupUnalignedABICall(argJSContext);
3845
3846 masm.loadJSContext(argJSContext);
3847
3848 masm.passABIArg(argJSContext);
3849 masm.passABIArg(argObj);
3850 masm.passABIArg(argPrivate);
3851 masm.passABIArg(argArgs);
3852 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, target->jitInfo()->method));
3853
3854 if (target->jitInfo()->isInfallible) {
3855 masm.loadValue(Address(masm.getStackPointer(), IonDOMMethodExitFrameLayout::offsetOfResult()),
3856 JSReturnOperand);
3857 } else {
3858 // Test for failure.
3859 masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
3860
3861 // Load the outparam vp[0] into output register(s).
3862 masm.loadValue(Address(masm.getStackPointer(), IonDOMMethodExitFrameLayout::offsetOfResult()),
3863 JSReturnOperand);
3864 }
3865
3866 // The next instruction is removing the footer of the exit frame, so there
3867 // is no need for leaveFakeExitFrame.
3868
3869 // Move the StackPointer back to its original location, unwinding the native exit frame.
3870 masm.adjustStack(IonDOMMethodExitFrameLayout::Size() - unusedStack);
3871 MOZ_ASSERT(masm.framePushed() == initialStack);
3872 }
3873
3874 typedef bool (*GetIntrinsicValueFn)(JSContext* cx, HandlePropertyName, MutableHandleValue);
3875 static const VMFunction GetIntrinsicValueInfo =
3876 FunctionInfo<GetIntrinsicValueFn>(GetIntrinsicValue, "GetIntrinsicValue");
3877
3878 void
visitCallGetIntrinsicValue(LCallGetIntrinsicValue * lir)3879 CodeGenerator::visitCallGetIntrinsicValue(LCallGetIntrinsicValue* lir)
3880 {
3881 pushArg(ImmGCPtr(lir->mir()->name()));
3882 callVM(GetIntrinsicValueInfo, lir);
3883 }
3884
3885 typedef bool (*InvokeFunctionFn)(JSContext*, HandleObject, bool, uint32_t, Value*, MutableHandleValue);
3886 static const VMFunction InvokeFunctionInfo =
3887 FunctionInfo<InvokeFunctionFn>(InvokeFunction, "InvokeFunction");
3888
3889 void
emitCallInvokeFunction(LInstruction * call,Register calleereg,bool constructing,uint32_t argc,uint32_t unusedStack)3890 CodeGenerator::emitCallInvokeFunction(LInstruction* call, Register calleereg,
3891 bool constructing, uint32_t argc, uint32_t unusedStack)
3892 {
3893 // Nestle %esp up to the argument vector.
3894 // Each path must account for framePushed_ separately, for callVM to be valid.
3895 masm.freeStack(unusedStack);
3896
3897 pushArg(masm.getStackPointer()); // argv.
3898 pushArg(Imm32(argc)); // argc.
3899 pushArg(Imm32(constructing)); // constructing.
3900 pushArg(calleereg); // JSFunction*.
3901
3902 callVM(InvokeFunctionInfo, call);
3903
3904 // Un-nestle %esp from the argument vector. No prefix was pushed.
3905 masm.reserveStack(unusedStack);
3906 }
3907
3908 void
visitCallGeneric(LCallGeneric * call)3909 CodeGenerator::visitCallGeneric(LCallGeneric* call)
3910 {
3911 Register calleereg = ToRegister(call->getFunction());
3912 Register objreg = ToRegister(call->getTempObject());
3913 Register nargsreg = ToRegister(call->getNargsReg());
3914 uint32_t unusedStack = StackOffsetOfPassedArg(call->argslot());
3915 Label invoke, thunk, makeCall, end;
3916
3917 // Known-target case is handled by LCallKnown.
3918 MOZ_ASSERT(!call->hasSingleTarget());
3919
3920 // Generate an ArgumentsRectifier.
3921 JitCode* argumentsRectifier = gen->jitRuntime()->getArgumentsRectifier();
3922
3923 masm.checkStackAlignment();
3924
3925 // Guard that calleereg is actually a function object.
3926 masm.loadObjClass(calleereg, nargsreg);
3927 masm.branchPtr(Assembler::NotEqual, nargsreg, ImmPtr(&JSFunction::class_), &invoke);
3928
3929 // Guard that calleereg is an interpreted function with a JSScript.
3930 // If we are constructing, also ensure the callee is a constructor.
3931 if (call->mir()->isConstructing()) {
3932 masm.branchIfNotInterpretedConstructor(calleereg, nargsreg, &invoke);
3933 } else {
3934 masm.branchIfFunctionHasNoScript(calleereg, &invoke);
3935 masm.branchFunctionKind(Assembler::Equal, JSFunction::ClassConstructor, calleereg, objreg, &invoke);
3936 }
3937
3938 // Knowing that calleereg is a non-native function, load the JSScript.
3939 masm.loadPtr(Address(calleereg, JSFunction::offsetOfNativeOrScript()), objreg);
3940
3941 // Load script jitcode.
3942 masm.loadBaselineOrIonRaw(objreg, objreg, &invoke);
3943
3944 // Nestle the StackPointer up to the argument vector.
3945 masm.freeStack(unusedStack);
3946
3947 // Construct the IonFramePrefix.
3948 uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS,
3949 JitFrameLayout::Size());
3950 masm.Push(Imm32(call->numActualArgs()));
3951 masm.PushCalleeToken(calleereg, call->mir()->isConstructing());
3952 masm.Push(Imm32(descriptor));
3953
3954 // Check whether the provided arguments satisfy target argc.
3955 // We cannot have lowered to LCallGeneric with a known target. Assert that we didn't
3956 // add any undefineds in IonBuilder. NB: MCall::numStackArgs includes |this|.
3957 DebugOnly<unsigned> numNonArgsOnStack = 1 + call->isConstructing();
3958 MOZ_ASSERT(call->numActualArgs() == call->mir()->numStackArgs() - numNonArgsOnStack);
3959 masm.load16ZeroExtend(Address(calleereg, JSFunction::offsetOfNargs()), nargsreg);
3960 masm.branch32(Assembler::Above, nargsreg, Imm32(call->numActualArgs()), &thunk);
3961 masm.jump(&makeCall);
3962
3963 // Argument fixed needed. Load the ArgumentsRectifier.
3964 masm.bind(&thunk);
3965 {
3966 MOZ_ASSERT(ArgumentsRectifierReg != objreg);
3967 masm.movePtr(ImmGCPtr(argumentsRectifier), objreg); // Necessary for GC marking.
3968 masm.loadPtr(Address(objreg, JitCode::offsetOfCode()), objreg);
3969 masm.move32(Imm32(call->numActualArgs()), ArgumentsRectifierReg);
3970 }
3971
3972 // Finally call the function in objreg.
3973 masm.bind(&makeCall);
3974 uint32_t callOffset = masm.callJit(objreg);
3975 markSafepointAt(callOffset, call);
3976
3977 // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass.
3978 // The return address has already been removed from the Ion frame.
3979 int prefixGarbage = sizeof(JitFrameLayout) - sizeof(void*);
3980 masm.adjustStack(prefixGarbage - unusedStack);
3981 masm.jump(&end);
3982
3983 // Handle uncompiled or native functions.
3984 masm.bind(&invoke);
3985 emitCallInvokeFunction(call, calleereg, call->isConstructing(), call->numActualArgs(),
3986 unusedStack);
3987
3988 masm.bind(&end);
3989
3990 // If the return value of the constructing function is Primitive,
3991 // replace the return value with the Object from CreateThis.
3992 if (call->mir()->isConstructing()) {
3993 Label notPrimitive;
3994 masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand, ¬Primitive);
3995 masm.loadValue(Address(masm.getStackPointer(), unusedStack), JSReturnOperand);
3996 masm.bind(¬Primitive);
3997 }
3998 }
3999
4000 typedef bool (*InvokeFunctionShuffleFn)(JSContext*, HandleObject, uint32_t, uint32_t, Value*,
4001 MutableHandleValue);
4002 static const VMFunction InvokeFunctionShuffleInfo =
4003 FunctionInfo<InvokeFunctionShuffleFn>(InvokeFunctionShuffleNewTarget,
4004 "InvokeFunctionShuffleNewTarget");
4005 void
emitCallInvokeFunctionShuffleNewTarget(LCallKnown * call,Register calleeReg,uint32_t numFormals,uint32_t unusedStack)4006 CodeGenerator::emitCallInvokeFunctionShuffleNewTarget(LCallKnown* call, Register calleeReg,
4007 uint32_t numFormals, uint32_t unusedStack)
4008 {
4009 masm.freeStack(unusedStack);
4010
4011 pushArg(masm.getStackPointer());
4012 pushArg(Imm32(numFormals));
4013 pushArg(Imm32(call->numActualArgs()));
4014 pushArg(calleeReg);
4015
4016 callVM(InvokeFunctionShuffleInfo, call);
4017
4018 masm.reserveStack(unusedStack);
4019 }
4020
4021 void
visitCallKnown(LCallKnown * call)4022 CodeGenerator::visitCallKnown(LCallKnown* call)
4023 {
4024 Register calleereg = ToRegister(call->getFunction());
4025 Register objreg = ToRegister(call->getTempObject());
4026 uint32_t unusedStack = StackOffsetOfPassedArg(call->argslot());
4027 WrappedFunction* target = call->getSingleTarget();
4028 Label end, uncompiled;
4029
4030 // Native single targets are handled by LCallNative.
4031 MOZ_ASSERT(!target->isNative());
4032 // Missing arguments must have been explicitly appended by the IonBuilder.
4033 DebugOnly<unsigned> numNonArgsOnStack = 1 + call->isConstructing();
4034 MOZ_ASSERT(target->nargs() <= call->mir()->numStackArgs() - numNonArgsOnStack);
4035
4036 MOZ_ASSERT_IF(call->isConstructing(), target->isConstructor());
4037
4038 masm.checkStackAlignment();
4039
4040 if (target->isClassConstructor() && !call->isConstructing()) {
4041 emitCallInvokeFunction(call, calleereg, call->isConstructing(), call->numActualArgs(), unusedStack);
4042 return;
4043 }
4044
4045 MOZ_ASSERT_IF(target->isClassConstructor(), call->isConstructing());
4046
4047 // The calleereg is known to be a non-native function, but might point to
4048 // a LazyScript instead of a JSScript.
4049 masm.branchIfFunctionHasNoScript(calleereg, &uncompiled);
4050
4051 // Knowing that calleereg is a non-native function, load the JSScript.
4052 masm.loadPtr(Address(calleereg, JSFunction::offsetOfNativeOrScript()), objreg);
4053
4054 // Load script jitcode.
4055 if (call->mir()->needsArgCheck())
4056 masm.loadBaselineOrIonRaw(objreg, objreg, &uncompiled);
4057 else
4058 masm.loadBaselineOrIonNoArgCheck(objreg, objreg, &uncompiled);
4059
4060 // Nestle the StackPointer up to the argument vector.
4061 masm.freeStack(unusedStack);
4062
4063 // Construct the IonFramePrefix.
4064 uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS,
4065 JitFrameLayout::Size());
4066 masm.Push(Imm32(call->numActualArgs()));
4067 masm.PushCalleeToken(calleereg, call->mir()->isConstructing());
4068 masm.Push(Imm32(descriptor));
4069
4070 // Finally call the function in objreg.
4071 uint32_t callOffset = masm.callJit(objreg);
4072 markSafepointAt(callOffset, call);
4073
4074 // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass.
4075 // The return address has already been removed from the Ion frame.
4076 int prefixGarbage = sizeof(JitFrameLayout) - sizeof(void*);
4077 masm.adjustStack(prefixGarbage - unusedStack);
4078 masm.jump(&end);
4079
4080 // Handle uncompiled functions.
4081 masm.bind(&uncompiled);
4082 if (call->isConstructing() && target->nargs() > call->numActualArgs())
4083 emitCallInvokeFunctionShuffleNewTarget(call, calleereg, target->nargs(), unusedStack);
4084 else
4085 emitCallInvokeFunction(call, calleereg, call->isConstructing(), call->numActualArgs(), unusedStack);
4086
4087 masm.bind(&end);
4088
4089 // If the return value of the constructing function is Primitive,
4090 // replace the return value with the Object from CreateThis.
4091 if (call->mir()->isConstructing()) {
4092 Label notPrimitive;
4093 masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand, ¬Primitive);
4094 masm.loadValue(Address(masm.getStackPointer(), unusedStack), JSReturnOperand);
4095 masm.bind(¬Primitive);
4096 }
4097 }
4098
4099 template<typename T>
4100 void
emitCallInvokeFunction(T * apply,Register extraStackSize)4101 CodeGenerator::emitCallInvokeFunction(T* apply, Register extraStackSize)
4102 {
4103 Register objreg = ToRegister(apply->getTempObject());
4104 MOZ_ASSERT(objreg != extraStackSize);
4105
4106 // Push the space used by the arguments.
4107 masm.moveStackPtrTo(objreg);
4108 masm.Push(extraStackSize);
4109
4110 pushArg(objreg); // argv.
4111 pushArg(ToRegister(apply->getArgc())); // argc.
4112 pushArg(Imm32(false)); // isConstrucing.
4113 pushArg(ToRegister(apply->getFunction())); // JSFunction*.
4114
4115 // This specialization og callVM restore the extraStackSize after the call.
4116 callVM(InvokeFunctionInfo, apply, &extraStackSize);
4117
4118 masm.Pop(extraStackSize);
4119 }
4120
4121 // Do not bailout after the execution of this function since the stack no longer
4122 // correspond to what is expected by the snapshots.
4123 void
emitAllocateSpaceForApply(Register argcreg,Register extraStackSpace,Label * end)4124 CodeGenerator::emitAllocateSpaceForApply(Register argcreg, Register extraStackSpace, Label* end)
4125 {
4126 // Initialize the loop counter AND Compute the stack usage (if == 0)
4127 masm.movePtr(argcreg, extraStackSpace);
4128
4129 // Align the JitFrameLayout on the JitStackAlignment.
4130 if (JitStackValueAlignment > 1) {
4131 MOZ_ASSERT(frameSize() % JitStackAlignment == 0,
4132 "Stack padding assumes that the frameSize is correct");
4133 MOZ_ASSERT(JitStackValueAlignment == 2);
4134 Label noPaddingNeeded;
4135 // if the number of arguments is odd, then we do not need any padding.
4136 masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded);
4137 masm.addPtr(Imm32(1), extraStackSpace);
4138 masm.bind(&noPaddingNeeded);
4139 }
4140
4141 // Reserve space for copying the arguments.
4142 NativeObject::elementsSizeMustNotOverflow();
4143 masm.lshiftPtr(Imm32(ValueShift), extraStackSpace);
4144 masm.subFromStackPtr(extraStackSpace);
4145
4146 #ifdef DEBUG
4147 // Put a magic value in the space reserved for padding. Note, this code
4148 // cannot be merged with the previous test, as not all architectures can
4149 // write below their stack pointers.
4150 if (JitStackValueAlignment > 1) {
4151 MOZ_ASSERT(JitStackValueAlignment == 2);
4152 Label noPaddingNeeded;
4153 // if the number of arguments is odd, then we do not need any padding.
4154 masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded);
4155 BaseValueIndex dstPtr(masm.getStackPointer(), argcreg);
4156 masm.storeValue(MagicValue(JS_ARG_POISON), dstPtr);
4157 masm.bind(&noPaddingNeeded);
4158 }
4159 #endif
4160
4161 // Skip the copy of arguments if there are none.
4162 masm.branchTestPtr(Assembler::Zero, argcreg, argcreg, end);
4163 }
4164
4165 // Destroys argvIndex and copyreg.
4166 void
emitCopyValuesForApply(Register argvSrcBase,Register argvIndex,Register copyreg,size_t argvSrcOffset,size_t argvDstOffset)4167 CodeGenerator::emitCopyValuesForApply(Register argvSrcBase, Register argvIndex, Register copyreg,
4168 size_t argvSrcOffset, size_t argvDstOffset)
4169 {
4170 Label loop;
4171 masm.bind(&loop);
4172
4173 // As argvIndex is off by 1, and we use the decBranchPtr instruction
4174 // to loop back, we have to substract the size of the word which are
4175 // copied.
4176 BaseValueIndex srcPtr(argvSrcBase, argvIndex, argvSrcOffset - sizeof(void*));
4177 BaseValueIndex dstPtr(masm.getStackPointer(), argvIndex, argvDstOffset - sizeof(void*));
4178 masm.loadPtr(srcPtr, copyreg);
4179 masm.storePtr(copyreg, dstPtr);
4180
4181 // Handle 32 bits architectures.
4182 if (sizeof(Value) == 2 * sizeof(void*)) {
4183 BaseValueIndex srcPtrLow(argvSrcBase, argvIndex, argvSrcOffset - 2 * sizeof(void*));
4184 BaseValueIndex dstPtrLow(masm.getStackPointer(), argvIndex, argvDstOffset - 2 * sizeof(void*));
4185 masm.loadPtr(srcPtrLow, copyreg);
4186 masm.storePtr(copyreg, dstPtrLow);
4187 }
4188
4189 masm.decBranchPtr(Assembler::NonZero, argvIndex, Imm32(1), &loop);
4190 }
4191
4192 void
emitPopArguments(Register extraStackSpace)4193 CodeGenerator::emitPopArguments(Register extraStackSpace)
4194 {
4195 // Pop |this| and Arguments.
4196 masm.freeStack(extraStackSpace);
4197 }
4198
4199 void
emitPushArguments(LApplyArgsGeneric * apply,Register extraStackSpace)4200 CodeGenerator::emitPushArguments(LApplyArgsGeneric* apply, Register extraStackSpace)
4201 {
4202 // Holds the function nargs. Initially the number of args to the caller.
4203 Register argcreg = ToRegister(apply->getArgc());
4204 Register copyreg = ToRegister(apply->getTempObject());
4205
4206 Label end;
4207 emitAllocateSpaceForApply(argcreg, extraStackSpace, &end);
4208
4209 // We are making a copy of the arguments which are above the JitFrameLayout
4210 // of the current Ion frame.
4211 //
4212 // [arg1] [arg0] <- src [this] [JitFrameLayout] [.. frameSize ..] [pad] [arg1] [arg0] <- dst
4213
4214 // Compute the source and destination offsets into the stack.
4215 size_t argvSrcOffset = frameSize() + JitFrameLayout::offsetOfActualArgs();
4216 size_t argvDstOffset = 0;
4217
4218 // Save the extra stack space, and re-use the register as a base.
4219 masm.push(extraStackSpace);
4220 Register argvSrcBase = extraStackSpace;
4221 argvSrcOffset += sizeof(void*);
4222 argvDstOffset += sizeof(void*);
4223
4224 // Save the actual number of register, and re-use the register as an index register.
4225 masm.push(argcreg);
4226 Register argvIndex = argcreg;
4227 argvSrcOffset += sizeof(void*);
4228 argvDstOffset += sizeof(void*);
4229
4230 // srcPtr = (StackPointer + extraStackSpace) + argvSrcOffset
4231 // dstPtr = (StackPointer ) + argvDstOffset
4232 masm.addStackPtrTo(argvSrcBase);
4233
4234 // Copy arguments.
4235 emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, argvSrcOffset, argvDstOffset);
4236
4237 // Restore argcreg and the extra stack space counter.
4238 masm.pop(argcreg);
4239 masm.pop(extraStackSpace);
4240
4241 // Join with all arguments copied and the extra stack usage computed.
4242 masm.bind(&end);
4243
4244 // Push |this|.
4245 masm.addPtr(Imm32(sizeof(Value)), extraStackSpace);
4246 masm.pushValue(ToValue(apply, LApplyArgsGeneric::ThisIndex));
4247 }
4248
4249 void
emitPushArguments(LApplyArrayGeneric * apply,Register extraStackSpace)4250 CodeGenerator::emitPushArguments(LApplyArrayGeneric* apply, Register extraStackSpace)
4251 {
4252 Label noCopy, epilogue;
4253 Register tmpArgc = ToRegister(apply->getTempObject());
4254 Register elementsAndArgc = ToRegister(apply->getElements());
4255
4256 // Invariants guarded in the caller:
4257 // - the array is not too long
4258 // - the array length equals its initialized length
4259
4260 // The array length is our argc for the purposes of allocating space.
4261 Address length(ToRegister(apply->getElements()), ObjectElements::offsetOfLength());
4262 masm.load32(length, tmpArgc);
4263
4264 // Allocate space for the values.
4265 emitAllocateSpaceForApply(tmpArgc, extraStackSpace, &noCopy);
4266
4267 // Copy the values. This code is skipped entirely if there are
4268 // no values.
4269 size_t argvDstOffset = 0;
4270
4271 Register argvSrcBase = elementsAndArgc; // Elements value
4272
4273 masm.push(extraStackSpace);
4274 Register copyreg = extraStackSpace;
4275 argvDstOffset += sizeof(void*);
4276
4277 masm.push(tmpArgc);
4278 Register argvIndex = tmpArgc;
4279 argvDstOffset += sizeof(void*);
4280
4281 // Copy
4282 emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, 0, argvDstOffset);
4283
4284 // Restore.
4285 masm.pop(elementsAndArgc);
4286 masm.pop(extraStackSpace);
4287 masm.jump(&epilogue);
4288
4289 // Clear argc if we skipped the copy step.
4290 masm.bind(&noCopy);
4291 masm.movePtr(ImmPtr(0), elementsAndArgc);
4292
4293 // Join with all arguments copied and the extra stack usage computed.
4294 // Note, "elements" has become "argc".
4295 masm.bind(&epilogue);
4296
4297 // Push |this|.
4298 masm.addPtr(Imm32(sizeof(Value)), extraStackSpace);
4299 masm.pushValue(ToValue(apply, LApplyArgsGeneric::ThisIndex));
4300 }
4301
4302 template<typename T>
4303 void
emitApplyGeneric(T * apply)4304 CodeGenerator::emitApplyGeneric(T* apply)
4305 {
4306 // Holds the function object.
4307 Register calleereg = ToRegister(apply->getFunction());
4308
4309 // Temporary register for modifying the function object.
4310 Register objreg = ToRegister(apply->getTempObject());
4311 Register extraStackSpace = ToRegister(apply->getTempStackCounter());
4312
4313 // Holds the function nargs, computed in the invoker or (for
4314 // ApplyArray) in the argument pusher.
4315 Register argcreg = ToRegister(apply->getArgc());
4316
4317 // Unless already known, guard that calleereg is actually a function object.
4318 if (!apply->hasSingleTarget()) {
4319 masm.loadObjClass(calleereg, objreg);
4320
4321 ImmPtr ptr = ImmPtr(&JSFunction::class_);
4322 bailoutCmpPtr(Assembler::NotEqual, objreg, ptr, apply->snapshot());
4323 }
4324
4325 // Copy the arguments of the current function.
4326 //
4327 // In the case of ApplyArray, also compute argc: the argc register
4328 // and the elements register are the same; argc must not be
4329 // referenced before the call to emitPushArguments() and elements
4330 // must not be referenced after it returns.
4331 //
4332 // objreg is dead across this call.
4333 //
4334 // extraStackSpace is garbage on entry and defined on exit.
4335 emitPushArguments(apply, extraStackSpace);
4336
4337 masm.checkStackAlignment();
4338
4339 // If the function is native, only emit the call to InvokeFunction.
4340 if (apply->hasSingleTarget() && apply->getSingleTarget()->isNative()) {
4341 emitCallInvokeFunction(apply, extraStackSpace);
4342 emitPopArguments(extraStackSpace);
4343 return;
4344 }
4345
4346 Label end, invoke;
4347
4348 // Guard that calleereg is an interpreted function with a JSScript.
4349 masm.branchIfFunctionHasNoScript(calleereg, &invoke);
4350
4351 // Knowing that calleereg is a non-native function, load the JSScript.
4352 masm.loadPtr(Address(calleereg, JSFunction::offsetOfNativeOrScript()), objreg);
4353
4354 // Load script jitcode.
4355 masm.loadBaselineOrIonRaw(objreg, objreg, &invoke);
4356
4357 // Call with an Ion frame or a rectifier frame.
4358 {
4359 // Create the frame descriptor.
4360 unsigned pushed = masm.framePushed();
4361 Register stackSpace = extraStackSpace;
4362 masm.addPtr(Imm32(pushed), stackSpace);
4363 masm.makeFrameDescriptor(stackSpace, JitFrame_IonJS, JitFrameLayout::Size());
4364
4365 masm.Push(argcreg);
4366 masm.Push(calleereg);
4367 masm.Push(stackSpace); // descriptor
4368
4369 Label underflow, rejoin;
4370
4371 // Check whether the provided arguments satisfy target argc.
4372 if (!apply->hasSingleTarget()) {
4373 Register nformals = extraStackSpace;
4374 masm.load16ZeroExtend(Address(calleereg, JSFunction::offsetOfNargs()), nformals);
4375 masm.branch32(Assembler::Below, argcreg, nformals, &underflow);
4376 } else {
4377 masm.branch32(Assembler::Below, argcreg, Imm32(apply->getSingleTarget()->nargs()),
4378 &underflow);
4379 }
4380
4381 // Skip the construction of the rectifier frame because we have no
4382 // underflow.
4383 masm.jump(&rejoin);
4384
4385 // Argument fixup needed. Get ready to call the argumentsRectifier.
4386 {
4387 masm.bind(&underflow);
4388
4389 // Hardcode the address of the argumentsRectifier code.
4390 JitCode* argumentsRectifier = gen->jitRuntime()->getArgumentsRectifier();
4391
4392 MOZ_ASSERT(ArgumentsRectifierReg != objreg);
4393 masm.movePtr(ImmGCPtr(argumentsRectifier), objreg); // Necessary for GC marking.
4394 masm.loadPtr(Address(objreg, JitCode::offsetOfCode()), objreg);
4395 masm.movePtr(argcreg, ArgumentsRectifierReg);
4396 }
4397
4398 masm.bind(&rejoin);
4399
4400 // Finally call the function in objreg, as assigned by one of the paths above.
4401 uint32_t callOffset = masm.callJit(objreg);
4402 markSafepointAt(callOffset, apply);
4403
4404 // Recover the number of arguments from the frame descriptor.
4405 masm.loadPtr(Address(masm.getStackPointer(), 0), stackSpace);
4406 masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), stackSpace);
4407 masm.subPtr(Imm32(pushed), stackSpace);
4408
4409 // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass.
4410 // The return address has already been removed from the Ion frame.
4411 int prefixGarbage = sizeof(JitFrameLayout) - sizeof(void*);
4412 masm.adjustStack(prefixGarbage);
4413 masm.jump(&end);
4414 }
4415
4416 // Handle uncompiled or native functions.
4417 {
4418 masm.bind(&invoke);
4419 emitCallInvokeFunction(apply, extraStackSpace);
4420 }
4421
4422 // Pop arguments and continue.
4423 masm.bind(&end);
4424 emitPopArguments(extraStackSpace);
4425 }
4426
4427 void
visitApplyArgsGeneric(LApplyArgsGeneric * apply)4428 CodeGenerator::visitApplyArgsGeneric(LApplyArgsGeneric* apply)
4429 {
4430 // Limit the number of parameters we can handle to a number that does not risk
4431 // us allocating too much stack, notably on Windows where there is a 4K guard page
4432 // that has to be touched to extend the stack. See bug 1351278. The value "3000"
4433 // is the size of the guard page minus an arbitrary, but large, safety margin.
4434
4435 LSnapshot* snapshot = apply->snapshot();
4436 Register argcreg = ToRegister(apply->getArgc());
4437
4438 uint32_t limit = 3000 / sizeof(Value);
4439 bailoutCmp32(Assembler::Above, argcreg, Imm32(limit), snapshot);
4440
4441 emitApplyGeneric(apply);
4442 }
4443
4444 void
visitApplyArrayGeneric(LApplyArrayGeneric * apply)4445 CodeGenerator::visitApplyArrayGeneric(LApplyArrayGeneric* apply)
4446 {
4447 LSnapshot* snapshot = apply->snapshot();
4448 Register tmp = ToRegister(apply->getTempObject());
4449
4450 Address length(ToRegister(apply->getElements()), ObjectElements::offsetOfLength());
4451 masm.load32(length, tmp);
4452
4453 // See comment in visitApplyArgsGeneric, above.
4454
4455 uint32_t limit = 3000 / sizeof(Value);
4456 bailoutCmp32(Assembler::Above, tmp, Imm32(limit), snapshot);
4457
4458 // Ensure that the array does not contain an uninitialized tail.
4459
4460 Address initializedLength(ToRegister(apply->getElements()),
4461 ObjectElements::offsetOfInitializedLength());
4462 masm.sub32(initializedLength, tmp);
4463 bailoutCmp32(Assembler::NotEqual, tmp, Imm32(0), snapshot);
4464
4465 emitApplyGeneric(apply);
4466 }
4467
4468 typedef bool (*ArraySpliceDenseFn)(JSContext*, HandleObject, uint32_t, uint32_t);
4469 static const VMFunction ArraySpliceDenseInfo =
4470 FunctionInfo<ArraySpliceDenseFn>(ArraySpliceDense, "ArraySpliceDense");
4471
4472 void
visitArraySplice(LArraySplice * lir)4473 CodeGenerator::visitArraySplice(LArraySplice* lir)
4474 {
4475 pushArg(ToRegister(lir->getDeleteCount()));
4476 pushArg(ToRegister(lir->getStart()));
4477 pushArg(ToRegister(lir->getObject()));
4478 callVM(ArraySpliceDenseInfo, lir);
4479 }
4480
4481 void
visitBail(LBail * lir)4482 CodeGenerator::visitBail(LBail* lir)
4483 {
4484 bailout(lir->snapshot());
4485 }
4486
4487 void
visitUnreachable(LUnreachable * lir)4488 CodeGenerator::visitUnreachable(LUnreachable* lir)
4489 {
4490 masm.assumeUnreachable("end-of-block assumed unreachable");
4491 }
4492
4493 void
visitEncodeSnapshot(LEncodeSnapshot * lir)4494 CodeGenerator::visitEncodeSnapshot(LEncodeSnapshot* lir)
4495 {
4496 encode(lir->snapshot());
4497 }
4498
4499 void
visitGetDynamicName(LGetDynamicName * lir)4500 CodeGenerator::visitGetDynamicName(LGetDynamicName* lir)
4501 {
4502 Register envChain = ToRegister(lir->getEnvironmentChain());
4503 Register name = ToRegister(lir->getName());
4504 Register temp1 = ToRegister(lir->temp1());
4505 Register temp2 = ToRegister(lir->temp2());
4506 Register temp3 = ToRegister(lir->temp3());
4507
4508 masm.loadJSContext(temp3);
4509
4510 /* Make space for the outparam. */
4511 masm.adjustStack(-int32_t(sizeof(Value)));
4512 masm.moveStackPtrTo(temp2);
4513
4514 masm.setupUnalignedABICall(temp1);
4515 masm.passABIArg(temp3);
4516 masm.passABIArg(envChain);
4517 masm.passABIArg(name);
4518 masm.passABIArg(temp2);
4519 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, GetDynamicName));
4520
4521 const ValueOperand out = ToOutValue(lir);
4522
4523 masm.loadValue(Address(masm.getStackPointer(), 0), out);
4524 masm.adjustStack(sizeof(Value));
4525
4526 Label undefined;
4527 masm.branchTestUndefined(Assembler::Equal, out, &undefined);
4528 bailoutFrom(&undefined, lir->snapshot());
4529 }
4530
4531 typedef bool (*DirectEvalSFn)(JSContext*, HandleObject, HandleScript, HandleValue,
4532 HandleString, jsbytecode*, MutableHandleValue);
4533 static const VMFunction DirectEvalStringInfo =
4534 FunctionInfo<DirectEvalSFn>(DirectEvalStringFromIon, "DirectEvalStringFromIon");
4535
4536 void
visitCallDirectEval(LCallDirectEval * lir)4537 CodeGenerator::visitCallDirectEval(LCallDirectEval* lir)
4538 {
4539 Register envChain = ToRegister(lir->getEnvironmentChain());
4540 Register string = ToRegister(lir->getString());
4541
4542 pushArg(ImmPtr(lir->mir()->pc()));
4543 pushArg(string);
4544 pushArg(ToValue(lir, LCallDirectEval::NewTarget));
4545 pushArg(ImmGCPtr(current->mir()->info().script()));
4546 pushArg(envChain);
4547
4548 callVM(DirectEvalStringInfo, lir);
4549 }
4550
4551 void
generateArgumentsChecks(bool bailout)4552 CodeGenerator::generateArgumentsChecks(bool bailout)
4553 {
4554 // Registers safe for use before generatePrologue().
4555 static const uint32_t EntryTempMask = Registers::TempMask & ~(1 << OsrFrameReg.code());
4556
4557 // This function can be used the normal way to check the argument types,
4558 // before entering the function and bailout when arguments don't match.
4559 // For debug purpose, this is can also be used to force/check that the
4560 // arguments are correct. Upon fail it will hit a breakpoint.
4561
4562 MIRGraph& mir = gen->graph();
4563 MResumePoint* rp = mir.entryResumePoint();
4564
4565 // No registers are allocated yet, so it's safe to grab anything.
4566 Register temp = GeneralRegisterSet(EntryTempMask).getAny();
4567
4568 const CompileInfo& info = gen->info();
4569
4570 Label miss;
4571 for (uint32_t i = info.startArgSlot(); i < info.endArgSlot(); i++) {
4572 // All initial parameters are guaranteed to be MParameters.
4573 MParameter* param = rp->getOperand(i)->toParameter();
4574 const TypeSet* types = param->resultTypeSet();
4575 if (!types || types->unknown())
4576 continue;
4577
4578 // Calculate the offset on the stack of the argument.
4579 // (i - info.startArgSlot()) - Compute index of arg within arg vector.
4580 // ... * sizeof(Value) - Scale by value size.
4581 // ArgToStackOffset(...) - Compute displacement within arg vector.
4582 int32_t offset = ArgToStackOffset((i - info.startArgSlot()) * sizeof(Value));
4583 masm.guardTypeSet(Address(masm.getStackPointer(), offset), types, BarrierKind::TypeSet, temp, &miss);
4584 }
4585
4586 if (miss.used()) {
4587 if (bailout) {
4588 bailoutFrom(&miss, graph.entrySnapshot());
4589 } else {
4590 Label success;
4591 masm.jump(&success);
4592 masm.bind(&miss);
4593
4594 // Check for cases where the type set guard might have missed due to
4595 // changing object groups.
4596 for (uint32_t i = info.startArgSlot(); i < info.endArgSlot(); i++) {
4597 MParameter* param = rp->getOperand(i)->toParameter();
4598 const TemporaryTypeSet* types = param->resultTypeSet();
4599 if (!types || types->unknown())
4600 continue;
4601
4602 Label skip;
4603 Address addr(masm.getStackPointer(), ArgToStackOffset((i - info.startArgSlot()) * sizeof(Value)));
4604 masm.branchTestObject(Assembler::NotEqual, addr, &skip);
4605 Register obj = masm.extractObject(addr, temp);
4606 masm.guardTypeSetMightBeIncomplete(types, obj, temp, &success);
4607 masm.bind(&skip);
4608 }
4609
4610 masm.assumeUnreachable("Argument check fail.");
4611 masm.bind(&success);
4612 }
4613 }
4614 }
4615
4616 // Out-of-line path to report over-recursed error and fail.
4617 class CheckOverRecursedFailure : public OutOfLineCodeBase<CodeGenerator>
4618 {
4619 LInstruction* lir_;
4620
4621 public:
CheckOverRecursedFailure(LInstruction * lir)4622 explicit CheckOverRecursedFailure(LInstruction* lir)
4623 : lir_(lir)
4624 { }
4625
accept(CodeGenerator * codegen)4626 void accept(CodeGenerator* codegen) {
4627 codegen->visitCheckOverRecursedFailure(this);
4628 }
4629
lir() const4630 LInstruction* lir() const {
4631 return lir_;
4632 }
4633 };
4634
4635 void
visitCheckOverRecursed(LCheckOverRecursed * lir)4636 CodeGenerator::visitCheckOverRecursed(LCheckOverRecursed* lir)
4637 {
4638 // If we don't push anything on the stack, skip the check.
4639 if (omitOverRecursedCheck())
4640 return;
4641
4642 // Ensure that this frame will not cross the stack limit.
4643 // This is a weak check, justified by Ion using the C stack: we must always
4644 // be some distance away from the actual limit, since if the limit is
4645 // crossed, an error must be thrown, which requires more frames.
4646 //
4647 // It must always be possible to trespass past the stack limit.
4648 // Ion may legally place frames very close to the limit. Calling additional
4649 // C functions may then violate the limit without any checking.
4650
4651 // Since Ion frames exist on the C stack, the stack limit may be
4652 // dynamically set by JS_SetThreadStackLimit() and JS_SetNativeStackQuota().
4653 const void* limitAddr = GetJitContext()->runtime->addressOfJitStackLimit();
4654
4655 CheckOverRecursedFailure* ool = new(alloc()) CheckOverRecursedFailure(lir);
4656 addOutOfLineCode(ool, lir->mir());
4657
4658 // Conditional forward (unlikely) branch to failure.
4659 masm.branchStackPtrRhs(Assembler::AboveOrEqual, AbsoluteAddress(limitAddr), ool->entry());
4660 masm.bind(ool->rejoin());
4661 }
4662
4663 typedef bool (*DefVarFn)(JSContext*, HandlePropertyName, unsigned, HandleObject);
4664 static const VMFunction DefVarInfo = FunctionInfo<DefVarFn>(DefVar, "DefVar");
4665
4666 void
visitDefVar(LDefVar * lir)4667 CodeGenerator::visitDefVar(LDefVar* lir)
4668 {
4669 Register envChain = ToRegister(lir->environmentChain());
4670
4671 pushArg(envChain); // JSObject*
4672 pushArg(Imm32(lir->mir()->attrs())); // unsigned
4673 pushArg(ImmGCPtr(lir->mir()->name())); // PropertyName*
4674
4675 callVM(DefVarInfo, lir);
4676 }
4677
4678 typedef bool (*DefLexicalFn)(JSContext*, HandlePropertyName, unsigned);
4679 static const VMFunction DefLexicalInfo =
4680 FunctionInfo<DefLexicalFn>(DefGlobalLexical, "DefGlobalLexical");
4681
4682 void
visitDefLexical(LDefLexical * lir)4683 CodeGenerator::visitDefLexical(LDefLexical* lir)
4684 {
4685 pushArg(Imm32(lir->mir()->attrs())); // unsigned
4686 pushArg(ImmGCPtr(lir->mir()->name())); // PropertyName*
4687
4688 callVM(DefLexicalInfo, lir);
4689 }
4690
4691 typedef bool (*DefFunOperationFn)(JSContext*, HandleScript, HandleObject, HandleFunction);
4692 static const VMFunction DefFunOperationInfo =
4693 FunctionInfo<DefFunOperationFn>(DefFunOperation, "DefFunOperation");
4694
4695 void
visitDefFun(LDefFun * lir)4696 CodeGenerator::visitDefFun(LDefFun* lir)
4697 {
4698 Register envChain = ToRegister(lir->environmentChain());
4699
4700 Register fun = ToRegister(lir->fun());
4701 pushArg(fun);
4702 pushArg(envChain);
4703 pushArg(ImmGCPtr(current->mir()->info().script()));
4704
4705 callVM(DefFunOperationInfo, lir);
4706 }
4707
4708 typedef bool (*CheckOverRecursedFn)(JSContext*);
4709 static const VMFunction CheckOverRecursedInfo =
4710 FunctionInfo<CheckOverRecursedFn>(CheckOverRecursed, "CheckOverRecursed");
4711
4712 void
visitCheckOverRecursedFailure(CheckOverRecursedFailure * ool)4713 CodeGenerator::visitCheckOverRecursedFailure(CheckOverRecursedFailure* ool)
4714 {
4715 // The OOL path is hit if the recursion depth has been exceeded.
4716 // Throw an InternalError for over-recursion.
4717
4718 // LFunctionEnvironment can appear before LCheckOverRecursed, so we have
4719 // to save all live registers to avoid crashes if CheckOverRecursed triggers
4720 // a GC.
4721 saveLive(ool->lir());
4722
4723 callVM(CheckOverRecursedInfo, ool->lir());
4724
4725 restoreLive(ool->lir());
4726 masm.jump(ool->rejoin());
4727 }
4728
4729 IonScriptCounts*
maybeCreateScriptCounts()4730 CodeGenerator::maybeCreateScriptCounts()
4731 {
4732 // If scripts are being profiled, create a new IonScriptCounts for the
4733 // profiling data, which will be attached to the associated JSScript or
4734 // wasm module after code generation finishes.
4735 if (!GetJitContext()->hasProfilingScripts())
4736 return nullptr;
4737
4738 // This test inhibits IonScriptCount creation for wasm code which is
4739 // currently incompatible with wasm codegen for two reasons: (1) wasm code
4740 // must be serializable and script count codegen bakes in absolute
4741 // addresses, (2) wasm code does not have a JSScript with which to associate
4742 // code coverage data.
4743 JSScript* script = gen->info().script();
4744 if (!script)
4745 return nullptr;
4746
4747 UniquePtr<IonScriptCounts> counts(js_new<IonScriptCounts>());
4748 if (!counts || !counts->init(graph.numBlocks()))
4749 return nullptr;
4750
4751 for (size_t i = 0; i < graph.numBlocks(); i++) {
4752 MBasicBlock* block = graph.getBlock(i)->mir();
4753
4754 uint32_t offset = 0;
4755 char* description = nullptr;
4756 if (MResumePoint* resume = block->entryResumePoint()) {
4757 // Find a PC offset in the outermost script to use. If this
4758 // block is from an inlined script, find a location in the
4759 // outer script to associate information about the inlining
4760 // with.
4761 while (resume->caller())
4762 resume = resume->caller();
4763 offset = script->pcToOffset(resume->pc());
4764
4765 if (block->entryResumePoint()->caller()) {
4766 // Get the filename and line number of the inner script.
4767 JSScript* innerScript = block->info().script();
4768 description = (char*) js_calloc(200);
4769 if (description) {
4770 snprintf(description, 200, "%s:%" PRIuSIZE,
4771 innerScript->filename(), innerScript->lineno());
4772 }
4773 }
4774 }
4775
4776 if (!counts->block(i).init(block->id(), offset, description, block->numSuccessors()))
4777 return nullptr;
4778
4779 for (size_t j = 0; j < block->numSuccessors(); j++)
4780 counts->block(i).setSuccessor(j, skipTrivialBlocks(block->getSuccessor(j))->id());
4781 }
4782
4783 scriptCounts_ = counts.release();
4784 return scriptCounts_;
4785 }
4786
4787 // Structure for managing the state tracked for a block by script counters.
4788 struct ScriptCountBlockState
4789 {
4790 IonBlockCounts& block;
4791 MacroAssembler& masm;
4792
4793 Sprinter printer;
4794
4795 public:
ScriptCountBlockStatejs::jit::ScriptCountBlockState4796 ScriptCountBlockState(IonBlockCounts* block, MacroAssembler* masm)
4797 : block(*block), masm(*masm), printer(GetJitContext()->cx, false)
4798 {
4799 }
4800
initjs::jit::ScriptCountBlockState4801 bool init()
4802 {
4803 if (!printer.init())
4804 return false;
4805
4806 // Bump the hit count for the block at the start. This code is not
4807 // included in either the text for the block or the instruction byte
4808 // counts.
4809 masm.inc64(AbsoluteAddress(block.addressOfHitCount()));
4810
4811 // Collect human readable assembly for the code generated in the block.
4812 masm.setPrinter(&printer);
4813
4814 return true;
4815 }
4816
visitInstructionjs::jit::ScriptCountBlockState4817 void visitInstruction(LInstruction* ins)
4818 {
4819 // Prefix stream of assembly instructions with their LIR instruction
4820 // name and any associated high level info.
4821 if (const char* extra = ins->extraName())
4822 printer.printf("[%s:%s]\n", ins->opName(), extra);
4823 else
4824 printer.printf("[%s]\n", ins->opName());
4825 }
4826
~ScriptCountBlockStatejs::jit::ScriptCountBlockState4827 ~ScriptCountBlockState()
4828 {
4829 masm.setPrinter(nullptr);
4830
4831 if (!printer.hadOutOfMemory())
4832 block.setCode(printer.string());
4833 }
4834 };
4835
4836 void
branchIfInvalidated(Register temp,Label * invalidated)4837 CodeGenerator::branchIfInvalidated(Register temp, Label* invalidated)
4838 {
4839 CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), temp);
4840 masm.propagateOOM(ionScriptLabels_.append(label));
4841
4842 // If IonScript::invalidationCount_ != 0, the script has been invalidated.
4843 masm.branch32(Assembler::NotEqual,
4844 Address(temp, IonScript::offsetOfInvalidationCount()),
4845 Imm32(0),
4846 invalidated);
4847 }
4848
4849 void
emitAssertObjectOrStringResult(Register input,MIRType type,const TemporaryTypeSet * typeset)4850 CodeGenerator::emitAssertObjectOrStringResult(Register input, MIRType type, const TemporaryTypeSet* typeset)
4851 {
4852 MOZ_ASSERT(type == MIRType::Object || type == MIRType::ObjectOrNull ||
4853 type == MIRType::String || type == MIRType::Symbol);
4854
4855 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
4856 regs.take(input);
4857
4858 Register temp = regs.takeAny();
4859 masm.push(temp);
4860
4861 // Don't check if the script has been invalidated. In that case invalid
4862 // types are expected (until we reach the OsiPoint and bailout).
4863 Label done;
4864 branchIfInvalidated(temp, &done);
4865
4866 if ((type == MIRType::Object || type == MIRType::ObjectOrNull) &&
4867 typeset && !typeset->unknownObject())
4868 {
4869 // We have a result TypeSet, assert this object is in it.
4870 Label miss, ok;
4871 if (type == MIRType::ObjectOrNull)
4872 masm.branchPtr(Assembler::Equal, input, ImmWord(0), &ok);
4873 if (typeset->getObjectCount() > 0)
4874 masm.guardObjectType(input, typeset, temp, &miss);
4875 else
4876 masm.jump(&miss);
4877 masm.jump(&ok);
4878
4879 masm.bind(&miss);
4880 masm.guardTypeSetMightBeIncomplete(typeset, input, temp, &ok);
4881
4882 masm.assumeUnreachable("MIR instruction returned object with unexpected type");
4883
4884 masm.bind(&ok);
4885 }
4886
4887 // Check that we have a valid GC pointer.
4888 saveVolatile();
4889 masm.setupUnalignedABICall(temp);
4890 masm.loadJSContext(temp);
4891 masm.passABIArg(temp);
4892 masm.passABIArg(input);
4893
4894 void* callee;
4895 switch (type) {
4896 case MIRType::Object:
4897 callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidObjectPtr);
4898 break;
4899 case MIRType::ObjectOrNull:
4900 callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidObjectOrNullPtr);
4901 break;
4902 case MIRType::String:
4903 callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidStringPtr);
4904 break;
4905 case MIRType::Symbol:
4906 callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidSymbolPtr);
4907 break;
4908 default:
4909 MOZ_CRASH();
4910 }
4911
4912 masm.callWithABI(callee);
4913 restoreVolatile();
4914
4915 masm.bind(&done);
4916 masm.pop(temp);
4917 }
4918
4919 void
emitAssertResultV(const ValueOperand input,const TemporaryTypeSet * typeset)4920 CodeGenerator::emitAssertResultV(const ValueOperand input, const TemporaryTypeSet* typeset)
4921 {
4922 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
4923 regs.take(input);
4924
4925 Register temp1 = regs.takeAny();
4926 Register temp2 = regs.takeAny();
4927 masm.push(temp1);
4928 masm.push(temp2);
4929
4930 // Don't check if the script has been invalidated. In that case invalid
4931 // types are expected (until we reach the OsiPoint and bailout).
4932 Label done;
4933 branchIfInvalidated(temp1, &done);
4934
4935 if (typeset && !typeset->unknown()) {
4936 // We have a result TypeSet, assert this value is in it.
4937 Label miss, ok;
4938 masm.guardTypeSet(input, typeset, BarrierKind::TypeSet, temp1, &miss);
4939 masm.jump(&ok);
4940
4941 masm.bind(&miss);
4942
4943 // Check for cases where the type set guard might have missed due to
4944 // changing object groups.
4945 Label realMiss;
4946 masm.branchTestObject(Assembler::NotEqual, input, &realMiss);
4947 Register payload = masm.extractObject(input, temp1);
4948 masm.guardTypeSetMightBeIncomplete(typeset, payload, temp1, &ok);
4949 masm.bind(&realMiss);
4950
4951 masm.assumeUnreachable("MIR instruction returned value with unexpected type");
4952
4953 masm.bind(&ok);
4954 }
4955
4956 // Check that we have a valid GC pointer.
4957 saveVolatile();
4958
4959 masm.pushValue(input);
4960 masm.moveStackPtrTo(temp1);
4961
4962 masm.setupUnalignedABICall(temp2);
4963 masm.loadJSContext(temp2);
4964 masm.passABIArg(temp2);
4965 masm.passABIArg(temp1);
4966 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, AssertValidValue));
4967 masm.popValue(input);
4968 restoreVolatile();
4969
4970 masm.bind(&done);
4971 masm.pop(temp2);
4972 masm.pop(temp1);
4973 }
4974
4975 #ifdef DEBUG
4976 void
emitObjectOrStringResultChecks(LInstruction * lir,MDefinition * mir)4977 CodeGenerator::emitObjectOrStringResultChecks(LInstruction* lir, MDefinition* mir)
4978 {
4979 if (lir->numDefs() == 0)
4980 return;
4981
4982 MOZ_ASSERT(lir->numDefs() == 1);
4983 Register output = ToRegister(lir->getDef(0));
4984
4985 emitAssertObjectOrStringResult(output, mir->type(), mir->resultTypeSet());
4986 }
4987
4988 void
emitValueResultChecks(LInstruction * lir,MDefinition * mir)4989 CodeGenerator::emitValueResultChecks(LInstruction* lir, MDefinition* mir)
4990 {
4991 if (lir->numDefs() == 0)
4992 return;
4993
4994 MOZ_ASSERT(lir->numDefs() == BOX_PIECES);
4995 if (!lir->getDef(0)->output()->isRegister())
4996 return;
4997
4998 ValueOperand output = ToOutValue(lir);
4999
5000 emitAssertResultV(output, mir->resultTypeSet());
5001 }
5002
5003 void
emitDebugResultChecks(LInstruction * ins)5004 CodeGenerator::emitDebugResultChecks(LInstruction* ins)
5005 {
5006 // In debug builds, check that LIR instructions return valid values.
5007
5008 MDefinition* mir = ins->mirRaw();
5009 if (!mir)
5010 return;
5011
5012 switch (mir->type()) {
5013 case MIRType::Object:
5014 case MIRType::ObjectOrNull:
5015 case MIRType::String:
5016 case MIRType::Symbol:
5017 emitObjectOrStringResultChecks(ins, mir);
5018 break;
5019 case MIRType::Value:
5020 emitValueResultChecks(ins, mir);
5021 break;
5022 default:
5023 break;
5024 }
5025 }
5026
5027 void
emitDebugForceBailing(LInstruction * lir)5028 CodeGenerator::emitDebugForceBailing(LInstruction* lir)
5029 {
5030 if (!lir->snapshot())
5031 return;
5032 if (lir->isStart())
5033 return;
5034 if (lir->isOsiPoint())
5035 return;
5036
5037 masm.comment("emitDebugForceBailing");
5038 const void* bailAfterAddr = GetJitContext()->runtime->addressOfIonBailAfter();
5039
5040 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
5041
5042 Label done, notBail, bail;
5043 masm.branch32(Assembler::Equal, AbsoluteAddress(bailAfterAddr), Imm32(0), &done);
5044 {
5045 Register temp = regs.takeAny();
5046
5047 masm.push(temp);
5048 masm.load32(AbsoluteAddress(bailAfterAddr), temp);
5049 masm.sub32(Imm32(1), temp);
5050 masm.store32(temp, AbsoluteAddress(bailAfterAddr));
5051
5052 masm.branch32(Assembler::NotEqual, temp, Imm32(0), ¬Bail);
5053 {
5054 masm.pop(temp);
5055 masm.jump(&bail);
5056 bailoutFrom(&bail, lir->snapshot());
5057 }
5058 masm.bind(¬Bail);
5059 masm.pop(temp);
5060 }
5061 masm.bind(&done);
5062 }
5063 #endif
5064
5065 bool
generateBody()5066 CodeGenerator::generateBody()
5067 {
5068 IonScriptCounts* counts = maybeCreateScriptCounts();
5069
5070 #if defined(JS_ION_PERF)
5071 PerfSpewer* perfSpewer = &perfSpewer_;
5072 if (gen->compilingWasm())
5073 perfSpewer = &gen->perfSpewer();
5074 #endif
5075
5076 for (size_t i = 0; i < graph.numBlocks(); i++) {
5077 current = graph.getBlock(i);
5078
5079 // Don't emit any code for trivial blocks, containing just a goto. Such
5080 // blocks are created to split critical edges, and if we didn't end up
5081 // putting any instructions in them, we can skip them.
5082 if (current->isTrivial())
5083 continue;
5084
5085 #ifdef JS_JITSPEW
5086 const char* filename = nullptr;
5087 size_t lineNumber = 0;
5088 unsigned columnNumber = 0;
5089 if (current->mir()->info().script()) {
5090 filename = current->mir()->info().script()->filename();
5091 if (current->mir()->pc())
5092 lineNumber = PCToLineNumber(current->mir()->info().script(), current->mir()->pc(),
5093 &columnNumber);
5094 } else {
5095 #ifdef DEBUG
5096 lineNumber = current->mir()->lineno();
5097 columnNumber = current->mir()->columnIndex();
5098 #endif
5099 }
5100 JitSpew(JitSpew_Codegen, "# block%" PRIuSIZE " %s:%" PRIuSIZE ":%u%s:",
5101 i, filename ? filename : "?", lineNumber, columnNumber,
5102 current->mir()->isLoopHeader() ? " (loop header)" : "");
5103 #endif
5104
5105 masm.bind(current->label());
5106
5107 mozilla::Maybe<ScriptCountBlockState> blockCounts;
5108 if (counts) {
5109 blockCounts.emplace(&counts->block(i), &masm);
5110 if (!blockCounts->init())
5111 return false;
5112 }
5113
5114 #if defined(JS_ION_PERF)
5115 perfSpewer->startBasicBlock(current->mir(), masm);
5116 #endif
5117
5118 for (LInstructionIterator iter = current->begin(); iter != current->end(); iter++) {
5119 if (!alloc().ensureBallast())
5120 return false;
5121
5122 #ifdef JS_JITSPEW
5123 JitSpewStart(JitSpew_Codegen, "instruction %s", iter->opName());
5124 if (const char* extra = iter->extraName())
5125 JitSpewCont(JitSpew_Codegen, ":%s", extra);
5126 JitSpewFin(JitSpew_Codegen);
5127 #endif
5128
5129 if (counts)
5130 blockCounts->visitInstruction(*iter);
5131
5132 #ifdef CHECK_OSIPOINT_REGISTERS
5133 if (iter->safepoint())
5134 resetOsiPointRegs(iter->safepoint());
5135 #endif
5136
5137 if (iter->mirRaw()) {
5138 // Only add instructions that have a tracked inline script tree.
5139 if (iter->mirRaw()->trackedTree()) {
5140 if (!addNativeToBytecodeEntry(iter->mirRaw()->trackedSite()))
5141 return false;
5142 }
5143
5144 // Track the start native offset of optimizations.
5145 if (iter->mirRaw()->trackedOptimizations()) {
5146 if (!addTrackedOptimizationsEntry(iter->mirRaw()->trackedOptimizations()))
5147 return false;
5148 }
5149 }
5150
5151 #ifdef DEBUG
5152 setElement(*iter); // needed to encode correct snapshot location.
5153 emitDebugForceBailing(*iter);
5154 #endif
5155
5156 iter->accept(this);
5157
5158 // Track the end native offset of optimizations.
5159 if (iter->mirRaw() && iter->mirRaw()->trackedOptimizations())
5160 extendTrackedOptimizationsEntry(iter->mirRaw()->trackedOptimizations());
5161
5162 #ifdef DEBUG
5163 if (!counts)
5164 emitDebugResultChecks(*iter);
5165 #endif
5166 }
5167 if (masm.oom())
5168 return false;
5169
5170 #if defined(JS_ION_PERF)
5171 perfSpewer->endBasicBlock(masm);
5172 #endif
5173 }
5174
5175 return true;
5176 }
5177
5178 // Out-of-line object allocation for LNewArray.
5179 class OutOfLineNewArray : public OutOfLineCodeBase<CodeGenerator>
5180 {
5181 LNewArray* lir_;
5182
5183 public:
OutOfLineNewArray(LNewArray * lir)5184 explicit OutOfLineNewArray(LNewArray* lir)
5185 : lir_(lir)
5186 { }
5187
accept(CodeGenerator * codegen)5188 void accept(CodeGenerator* codegen) {
5189 codegen->visitOutOfLineNewArray(this);
5190 }
5191
lir() const5192 LNewArray* lir() const {
5193 return lir_;
5194 }
5195 };
5196
5197 typedef JSObject* (*NewArrayOperationFn)(JSContext*, HandleScript, jsbytecode*, uint32_t,
5198 NewObjectKind);
5199 static const VMFunction NewArrayOperationInfo =
5200 FunctionInfo<NewArrayOperationFn>(NewArrayOperation, "NewArrayOperation");
5201
5202 static JSObject*
NewArrayWithGroup(JSContext * cx,uint32_t length,HandleObjectGroup group,bool convertDoubleElements)5203 NewArrayWithGroup(JSContext* cx, uint32_t length, HandleObjectGroup group,
5204 bool convertDoubleElements)
5205 {
5206 JSObject* res = NewFullyAllocatedArrayTryUseGroup(cx, group, length);
5207 if (!res)
5208 return nullptr;
5209 if (convertDoubleElements)
5210 res->as<ArrayObject>().setShouldConvertDoubleElements();
5211 return res;
5212 }
5213
5214 typedef JSObject* (*NewArrayWithGroupFn)(JSContext*, uint32_t, HandleObjectGroup, bool);
5215 static const VMFunction NewArrayWithGroupInfo =
5216 FunctionInfo<NewArrayWithGroupFn>(NewArrayWithGroup, "NewArrayWithGroup");
5217
5218 void
visitNewArrayCallVM(LNewArray * lir)5219 CodeGenerator::visitNewArrayCallVM(LNewArray* lir)
5220 {
5221 Register objReg = ToRegister(lir->output());
5222
5223 MOZ_ASSERT(!lir->isCall());
5224 saveLive(lir);
5225
5226 JSObject* templateObject = lir->mir()->templateObject();
5227
5228 if (templateObject) {
5229 pushArg(Imm32(lir->mir()->convertDoubleElements()));
5230 pushArg(ImmGCPtr(templateObject->group()));
5231 pushArg(Imm32(lir->mir()->length()));
5232
5233 callVM(NewArrayWithGroupInfo, lir);
5234 } else {
5235 pushArg(Imm32(GenericObject));
5236 pushArg(Imm32(lir->mir()->length()));
5237 pushArg(ImmPtr(lir->mir()->pc()));
5238 pushArg(ImmGCPtr(lir->mir()->block()->info().script()));
5239
5240 callVM(NewArrayOperationInfo, lir);
5241 }
5242
5243 if (ReturnReg != objReg)
5244 masm.movePtr(ReturnReg, objReg);
5245
5246 restoreLive(lir);
5247 }
5248
5249 typedef JSObject* (*NewDerivedTypedObjectFn)(JSContext*,
5250 HandleObject type,
5251 HandleObject owner,
5252 int32_t offset);
5253 static const VMFunction CreateDerivedTypedObjInfo =
5254 FunctionInfo<NewDerivedTypedObjectFn>(CreateDerivedTypedObj, "CreateDerivedTypedObj");
5255
5256 void
visitNewDerivedTypedObject(LNewDerivedTypedObject * lir)5257 CodeGenerator::visitNewDerivedTypedObject(LNewDerivedTypedObject* lir)
5258 {
5259 pushArg(ToRegister(lir->offset()));
5260 pushArg(ToRegister(lir->owner()));
5261 pushArg(ToRegister(lir->type()));
5262 callVM(CreateDerivedTypedObjInfo, lir);
5263 }
5264
5265 void
visitAtan2D(LAtan2D * lir)5266 CodeGenerator::visitAtan2D(LAtan2D* lir)
5267 {
5268 Register temp = ToRegister(lir->temp());
5269 FloatRegister y = ToFloatRegister(lir->y());
5270 FloatRegister x = ToFloatRegister(lir->x());
5271
5272 masm.setupUnalignedABICall(temp);
5273 masm.passABIArg(y, MoveOp::DOUBLE);
5274 masm.passABIArg(x, MoveOp::DOUBLE);
5275 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ecmaAtan2), MoveOp::DOUBLE);
5276
5277 MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg);
5278 }
5279
5280 void
visitHypot(LHypot * lir)5281 CodeGenerator::visitHypot(LHypot* lir)
5282 {
5283 Register temp = ToRegister(lir->temp());
5284 uint32_t numArgs = lir->numArgs();
5285 masm.setupUnalignedABICall(temp);
5286
5287 for (uint32_t i = 0 ; i < numArgs; ++i)
5288 masm.passABIArg(ToFloatRegister(lir->getOperand(i)), MoveOp::DOUBLE);
5289
5290 switch(numArgs) {
5291 case 2:
5292 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ecmaHypot), MoveOp::DOUBLE);
5293 break;
5294 case 3:
5295 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, hypot3), MoveOp::DOUBLE);
5296 break;
5297 case 4:
5298 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, hypot4), MoveOp::DOUBLE);
5299 break;
5300 default:
5301 MOZ_CRASH("Unexpected number of arguments to hypot function.");
5302 }
5303 MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg);
5304 }
5305
5306 void
visitNewArray(LNewArray * lir)5307 CodeGenerator::visitNewArray(LNewArray* lir)
5308 {
5309 Register objReg = ToRegister(lir->output());
5310 Register tempReg = ToRegister(lir->temp());
5311 JSObject* templateObject = lir->mir()->templateObject();
5312 DebugOnly<uint32_t> length = lir->mir()->length();
5313
5314 MOZ_ASSERT(length <= NativeObject::MAX_DENSE_ELEMENTS_COUNT);
5315
5316 if (lir->mir()->isVMCall()) {
5317 visitNewArrayCallVM(lir);
5318 return;
5319 }
5320
5321 OutOfLineNewArray* ool = new(alloc()) OutOfLineNewArray(lir);
5322 addOutOfLineCode(ool, lir->mir());
5323
5324 masm.createGCObject(objReg, tempReg, templateObject, lir->mir()->initialHeap(),
5325 ool->entry(), /* initContents = */ true,
5326 lir->mir()->convertDoubleElements());
5327
5328 masm.bind(ool->rejoin());
5329 }
5330
5331 void
visitOutOfLineNewArray(OutOfLineNewArray * ool)5332 CodeGenerator::visitOutOfLineNewArray(OutOfLineNewArray* ool)
5333 {
5334 visitNewArrayCallVM(ool->lir());
5335 masm.jump(ool->rejoin());
5336 }
5337
5338 void
visitNewArrayCopyOnWrite(LNewArrayCopyOnWrite * lir)5339 CodeGenerator::visitNewArrayCopyOnWrite(LNewArrayCopyOnWrite* lir)
5340 {
5341 Register objReg = ToRegister(lir->output());
5342 Register tempReg = ToRegister(lir->temp());
5343 ArrayObject* templateObject = lir->mir()->templateObject();
5344 gc::InitialHeap initialHeap = lir->mir()->initialHeap();
5345
5346 // If we have a template object, we can inline call object creation.
5347 OutOfLineCode* ool = oolCallVM(NewArrayCopyOnWriteInfo, lir,
5348 ArgList(ImmGCPtr(templateObject), Imm32(initialHeap)),
5349 StoreRegisterTo(objReg));
5350
5351 masm.createGCObject(objReg, tempReg, templateObject, initialHeap, ool->entry());
5352
5353 masm.bind(ool->rejoin());
5354 }
5355
5356 typedef JSObject* (*ArrayConstructorOneArgFn)(JSContext*, HandleObjectGroup, int32_t length);
5357 static const VMFunction ArrayConstructorOneArgInfo =
5358 FunctionInfo<ArrayConstructorOneArgFn>(ArrayConstructorOneArg, "ArrayConstructorOneArg");
5359
5360 void
visitNewArrayDynamicLength(LNewArrayDynamicLength * lir)5361 CodeGenerator::visitNewArrayDynamicLength(LNewArrayDynamicLength* lir)
5362 {
5363 Register lengthReg = ToRegister(lir->length());
5364 Register objReg = ToRegister(lir->output());
5365 Register tempReg = ToRegister(lir->temp());
5366
5367 JSObject* templateObject = lir->mir()->templateObject();
5368 gc::InitialHeap initialHeap = lir->mir()->initialHeap();
5369
5370 OutOfLineCode* ool = oolCallVM(ArrayConstructorOneArgInfo, lir,
5371 ArgList(ImmGCPtr(templateObject->group()), lengthReg),
5372 StoreRegisterTo(objReg));
5373
5374 bool canInline = true;
5375 size_t inlineLength = 0;
5376 if (templateObject->is<ArrayObject>()) {
5377 if (templateObject->as<ArrayObject>().hasFixedElements()) {
5378 size_t numSlots = gc::GetGCKindSlots(templateObject->asTenured().getAllocKind());
5379 inlineLength = numSlots - ObjectElements::VALUES_PER_HEADER;
5380 } else {
5381 canInline = false;
5382 }
5383 } else {
5384 if (templateObject->as<UnboxedArrayObject>().hasInlineElements()) {
5385 size_t nbytes =
5386 templateObject->tenuredSizeOfThis() - UnboxedArrayObject::offsetOfInlineElements();
5387 inlineLength = nbytes / templateObject->as<UnboxedArrayObject>().elementSize();
5388 } else {
5389 canInline = false;
5390 }
5391 }
5392
5393 if (canInline) {
5394 // Try to do the allocation inline if the template object is big enough
5395 // for the length in lengthReg. If the length is bigger we could still
5396 // use the template object and not allocate the elements, but it's more
5397 // efficient to do a single big allocation than (repeatedly) reallocating
5398 // the array later on when filling it.
5399 masm.branch32(Assembler::Above, lengthReg, Imm32(inlineLength), ool->entry());
5400
5401 masm.createGCObject(objReg, tempReg, templateObject, initialHeap, ool->entry());
5402
5403 size_t lengthOffset = NativeObject::offsetOfFixedElements() + ObjectElements::offsetOfLength();
5404 masm.store32(lengthReg, Address(objReg, lengthOffset));
5405 } else {
5406 masm.jump(ool->entry());
5407 }
5408
5409 masm.bind(ool->rejoin());
5410 }
5411
5412 typedef TypedArrayObject* (*TypedArrayConstructorOneArgFn)(JSContext*, HandleObject, int32_t length);
5413 static const VMFunction TypedArrayConstructorOneArgInfo =
5414 FunctionInfo<TypedArrayConstructorOneArgFn>(TypedArrayCreateWithTemplate,
5415 "TypedArrayCreateWithTemplate");
5416
5417 void
visitNewTypedArray(LNewTypedArray * lir)5418 CodeGenerator::visitNewTypedArray(LNewTypedArray* lir)
5419 {
5420 Register objReg = ToRegister(lir->output());
5421 Register tempReg = ToRegister(lir->temp1());
5422 Register lengthReg = ToRegister(lir->temp2());
5423 LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
5424
5425 JSObject* templateObject = lir->mir()->templateObject();
5426 gc::InitialHeap initialHeap = lir->mir()->initialHeap();
5427
5428 TypedArrayObject* ttemplate = &templateObject->as<TypedArrayObject>();
5429 uint32_t n = ttemplate->length();
5430
5431 OutOfLineCode* ool = oolCallVM(TypedArrayConstructorOneArgInfo, lir,
5432 ArgList(ImmGCPtr(templateObject), Imm32(n)),
5433 StoreRegisterTo(objReg));
5434
5435 masm.createGCObject(objReg, tempReg, templateObject, initialHeap,
5436 ool->entry(), /*initContents*/true, /*convertDoubleElements*/false);
5437
5438 masm.initTypedArraySlots(objReg, tempReg, lengthReg, liveRegs, ool->entry(),
5439 ttemplate, TypedArrayLength::Fixed);
5440
5441 masm.bind(ool->rejoin());
5442 }
5443
5444 void
visitNewTypedArrayDynamicLength(LNewTypedArrayDynamicLength * lir)5445 CodeGenerator::visitNewTypedArrayDynamicLength(LNewTypedArrayDynamicLength* lir)
5446 {
5447 Register lengthReg = ToRegister(lir->length());
5448 Register objReg = ToRegister(lir->output());
5449 Register tempReg = ToRegister(lir->temp());
5450 LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
5451
5452 JSObject* templateObject = lir->mir()->templateObject();
5453 gc::InitialHeap initialHeap = lir->mir()->initialHeap();
5454
5455 TypedArrayObject* ttemplate = &templateObject->as<TypedArrayObject>();
5456
5457 OutOfLineCode* ool = oolCallVM(TypedArrayConstructorOneArgInfo, lir,
5458 ArgList(ImmGCPtr(templateObject), lengthReg),
5459 StoreRegisterTo(objReg));
5460
5461 masm.createGCObject(objReg, tempReg, templateObject, initialHeap,
5462 ool->entry(), /*initContents*/true, /*convertDoubleElements*/false);
5463
5464 masm.initTypedArraySlots(objReg, tempReg, lengthReg, liveRegs, ool->entry(),
5465 ttemplate, TypedArrayLength::Dynamic);
5466
5467 masm.bind(ool->rejoin());
5468 }
5469
5470 // Out-of-line object allocation for JSOP_NEWOBJECT.
5471 class OutOfLineNewObject : public OutOfLineCodeBase<CodeGenerator>
5472 {
5473 LNewObject* lir_;
5474
5475 public:
OutOfLineNewObject(LNewObject * lir)5476 explicit OutOfLineNewObject(LNewObject* lir)
5477 : lir_(lir)
5478 { }
5479
accept(CodeGenerator * codegen)5480 void accept(CodeGenerator* codegen) {
5481 codegen->visitOutOfLineNewObject(this);
5482 }
5483
lir() const5484 LNewObject* lir() const {
5485 return lir_;
5486 }
5487 };
5488
5489 typedef JSObject* (*NewInitObjectWithTemplateFn)(JSContext*, HandleObject);
5490 static const VMFunction NewInitObjectWithTemplateInfo =
5491 FunctionInfo<NewInitObjectWithTemplateFn>(NewObjectOperationWithTemplate,
5492 "NewObjectOperationWithTemplate");
5493
5494 typedef JSObject* (*NewInitObjectFn)(JSContext*, HandleScript, jsbytecode* pc, NewObjectKind);
5495 static const VMFunction NewInitObjectInfo =
5496 FunctionInfo<NewInitObjectFn>(NewObjectOperation, "NewObjectOperation");
5497
5498 typedef PlainObject* (*ObjectCreateWithTemplateFn)(JSContext*, HandlePlainObject);
5499 static const VMFunction ObjectCreateWithTemplateInfo =
5500 FunctionInfo<ObjectCreateWithTemplateFn>(ObjectCreateWithTemplate, "ObjectCreateWithTemplate");
5501
5502 void
visitNewObjectVMCall(LNewObject * lir)5503 CodeGenerator::visitNewObjectVMCall(LNewObject* lir)
5504 {
5505 Register objReg = ToRegister(lir->output());
5506
5507 MOZ_ASSERT(!lir->isCall());
5508 saveLive(lir);
5509
5510 JSObject* templateObject = lir->mir()->templateObject();
5511
5512 // If we're making a new object with a class prototype (that is, an object
5513 // that derives its class from its prototype instead of being
5514 // PlainObject::class_'d) from self-hosted code, we need a different init
5515 // function.
5516 switch (lir->mir()->mode()) {
5517 case MNewObject::ObjectLiteral:
5518 if (templateObject) {
5519 pushArg(ImmGCPtr(templateObject));
5520 callVM(NewInitObjectWithTemplateInfo, lir);
5521 } else {
5522 pushArg(Imm32(GenericObject));
5523 pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
5524 pushArg(ImmGCPtr(lir->mir()->block()->info().script()));
5525 callVM(NewInitObjectInfo, lir);
5526 }
5527 break;
5528 case MNewObject::ObjectCreate:
5529 pushArg(ImmGCPtr(templateObject));
5530 callVM(ObjectCreateWithTemplateInfo, lir);
5531 break;
5532 }
5533
5534 if (ReturnReg != objReg)
5535 masm.movePtr(ReturnReg, objReg);
5536
5537 restoreLive(lir);
5538 }
5539
5540 static bool
ShouldInitFixedSlots(LInstruction * lir,JSObject * obj)5541 ShouldInitFixedSlots(LInstruction* lir, JSObject* obj)
5542 {
5543 if (!obj->isNative())
5544 return true;
5545 NativeObject* templateObj = &obj->as<NativeObject>();
5546
5547 // Look for StoreFixedSlot instructions following an object allocation
5548 // that write to this object before a GC is triggered or this object is
5549 // passed to a VM call. If all fixed slots will be initialized, the
5550 // allocation code doesn't need to set the slots to |undefined|.
5551
5552 uint32_t nfixed = templateObj->numUsedFixedSlots();
5553 if (nfixed == 0)
5554 return false;
5555
5556 // Only optimize if all fixed slots are initially |undefined|, so that we
5557 // can assume incremental pre-barriers are not necessary. See also the
5558 // comment below.
5559 for (uint32_t slot = 0; slot < nfixed; slot++) {
5560 if (!templateObj->getSlot(slot).isUndefined())
5561 return true;
5562 }
5563
5564 // Keep track of the fixed slots that are initialized. initializedSlots is
5565 // a bit mask with a bit for each slot.
5566 MOZ_ASSERT(nfixed <= NativeObject::MAX_FIXED_SLOTS);
5567 static_assert(NativeObject::MAX_FIXED_SLOTS <= 32, "Slot bits must fit in 32 bits");
5568 uint32_t initializedSlots = 0;
5569 uint32_t numInitialized = 0;
5570
5571 MInstruction* allocMir = lir->mirRaw()->toInstruction();
5572 MBasicBlock* block = allocMir->block();
5573
5574 // Skip the allocation instruction.
5575 MInstructionIterator iter = block->begin(allocMir);
5576 MOZ_ASSERT(*iter == allocMir);
5577 iter++;
5578
5579 while (true) {
5580 for (; iter != block->end(); iter++) {
5581 if (iter->isNop() || iter->isConstant() || iter->isPostWriteBarrier()) {
5582 // These instructions won't trigger a GC or read object slots.
5583 continue;
5584 }
5585
5586 if (iter->isStoreFixedSlot()) {
5587 MStoreFixedSlot* store = iter->toStoreFixedSlot();
5588 if (store->object() != allocMir)
5589 return true;
5590
5591 // We may not initialize this object slot on allocation, so the
5592 // pre-barrier could read uninitialized memory. Simply disable
5593 // the barrier for this store: the object was just initialized
5594 // so the barrier is not necessary.
5595 store->setNeedsBarrier(false);
5596
5597 uint32_t slot = store->slot();
5598 MOZ_ASSERT(slot < nfixed);
5599 if ((initializedSlots & (1 << slot)) == 0) {
5600 numInitialized++;
5601 initializedSlots |= (1 << slot);
5602
5603 if (numInitialized == nfixed) {
5604 // All fixed slots will be initialized.
5605 MOZ_ASSERT(mozilla::CountPopulation32(initializedSlots) == nfixed);
5606 return false;
5607 }
5608 }
5609 continue;
5610 }
5611
5612 if (iter->isGoto()) {
5613 block = iter->toGoto()->target();
5614 if (block->numPredecessors() != 1)
5615 return true;
5616 break;
5617 }
5618
5619 // Unhandled instruction, assume it bails or reads object slots.
5620 return true;
5621 }
5622 iter = block->begin();
5623 }
5624
5625 MOZ_CRASH("Shouldn't get here");
5626 }
5627
5628 void
visitNewObject(LNewObject * lir)5629 CodeGenerator::visitNewObject(LNewObject* lir)
5630 {
5631 Register objReg = ToRegister(lir->output());
5632 Register tempReg = ToRegister(lir->temp());
5633 JSObject* templateObject = lir->mir()->templateObject();
5634
5635 if (lir->mir()->isVMCall()) {
5636 visitNewObjectVMCall(lir);
5637 return;
5638 }
5639
5640 OutOfLineNewObject* ool = new(alloc()) OutOfLineNewObject(lir);
5641 addOutOfLineCode(ool, lir->mir());
5642
5643 bool initContents = ShouldInitFixedSlots(lir, templateObject);
5644 masm.createGCObject(objReg, tempReg, templateObject, lir->mir()->initialHeap(), ool->entry(),
5645 initContents);
5646
5647 masm.bind(ool->rejoin());
5648 }
5649
5650 void
visitOutOfLineNewObject(OutOfLineNewObject * ool)5651 CodeGenerator::visitOutOfLineNewObject(OutOfLineNewObject* ool)
5652 {
5653 visitNewObjectVMCall(ool->lir());
5654 masm.jump(ool->rejoin());
5655 }
5656
5657 typedef InlineTypedObject* (*NewTypedObjectFn)(JSContext*, Handle<InlineTypedObject*>, gc::InitialHeap);
5658 static const VMFunction NewTypedObjectInfo =
5659 FunctionInfo<NewTypedObjectFn>(InlineTypedObject::createCopy, "InlineTypedObject::createCopy");
5660
5661 void
visitNewTypedObject(LNewTypedObject * lir)5662 CodeGenerator::visitNewTypedObject(LNewTypedObject* lir)
5663 {
5664 Register object = ToRegister(lir->output());
5665 Register temp = ToRegister(lir->temp());
5666 InlineTypedObject* templateObject = lir->mir()->templateObject();
5667 gc::InitialHeap initialHeap = lir->mir()->initialHeap();
5668
5669 OutOfLineCode* ool = oolCallVM(NewTypedObjectInfo, lir,
5670 ArgList(ImmGCPtr(templateObject), Imm32(initialHeap)),
5671 StoreRegisterTo(object));
5672
5673 masm.createGCObject(object, temp, templateObject, initialHeap, ool->entry());
5674
5675 masm.bind(ool->rejoin());
5676 }
5677
5678 void
visitSimdBox(LSimdBox * lir)5679 CodeGenerator::visitSimdBox(LSimdBox* lir)
5680 {
5681 FloatRegister in = ToFloatRegister(lir->input());
5682 Register object = ToRegister(lir->output());
5683 Register temp = ToRegister(lir->temp());
5684 InlineTypedObject* templateObject = lir->mir()->templateObject();
5685 gc::InitialHeap initialHeap = lir->mir()->initialHeap();
5686 MIRType type = lir->mir()->input()->type();
5687
5688 registerSimdTemplate(lir->mir()->simdType());
5689
5690 MOZ_ASSERT(lir->safepoint()->liveRegs().has(in), "Save the input register across oolCallVM");
5691 OutOfLineCode* ool = oolCallVM(NewTypedObjectInfo, lir,
5692 ArgList(ImmGCPtr(templateObject), Imm32(initialHeap)),
5693 StoreRegisterTo(object));
5694
5695 masm.createGCObject(object, temp, templateObject, initialHeap, ool->entry());
5696 masm.bind(ool->rejoin());
5697
5698 Address objectData(object, InlineTypedObject::offsetOfDataStart());
5699 switch (type) {
5700 case MIRType::Int8x16:
5701 case MIRType::Int16x8:
5702 case MIRType::Int32x4:
5703 case MIRType::Bool8x16:
5704 case MIRType::Bool16x8:
5705 case MIRType::Bool32x4:
5706 masm.storeUnalignedSimd128Int(in, objectData);
5707 break;
5708 case MIRType::Float32x4:
5709 masm.storeUnalignedSimd128Float(in, objectData);
5710 break;
5711 default:
5712 MOZ_CRASH("Unknown SIMD kind when generating code for SimdBox.");
5713 }
5714 }
5715
5716 void
registerSimdTemplate(SimdType simdType)5717 CodeGenerator::registerSimdTemplate(SimdType simdType)
5718 {
5719 simdRefreshTemplatesDuringLink_ |= 1 << uint32_t(simdType);
5720 }
5721
5722 void
captureSimdTemplate(JSContext * cx)5723 CodeGenerator::captureSimdTemplate(JSContext* cx)
5724 {
5725 JitCompartment* jitCompartment = cx->compartment()->jitCompartment();
5726 while (simdRefreshTemplatesDuringLink_) {
5727 uint32_t typeIndex = mozilla::CountTrailingZeroes32(simdRefreshTemplatesDuringLink_);
5728 simdRefreshTemplatesDuringLink_ ^= 1 << typeIndex;
5729 SimdType type = SimdType(typeIndex);
5730
5731 // Note: the weak-reference on the template object should not have been
5732 // garbage collected. It is either registered by IonBuilder, or verified
5733 // before using it in the EagerSimdUnbox phase.
5734 jitCompartment->registerSimdTemplateObjectFor(type);
5735 }
5736 }
5737
5738 void
visitSimdUnbox(LSimdUnbox * lir)5739 CodeGenerator::visitSimdUnbox(LSimdUnbox* lir)
5740 {
5741 Register object = ToRegister(lir->input());
5742 FloatRegister simd = ToFloatRegister(lir->output());
5743 Register temp = ToRegister(lir->temp());
5744 Label bail;
5745
5746 // obj->group()
5747 masm.loadPtr(Address(object, JSObject::offsetOfGroup()), temp);
5748
5749 // Guard that the object has the same representation as the one produced for
5750 // SIMD value-type.
5751 Address clasp(temp, ObjectGroup::offsetOfClasp());
5752 static_assert(!SimdTypeDescr::Opaque, "SIMD objects are transparent");
5753 masm.branchPtr(Assembler::NotEqual, clasp, ImmPtr(&InlineTransparentTypedObject::class_),
5754 &bail);
5755
5756 // obj->type()->typeDescr()
5757 // The previous class pointer comparison implies that the addendumKind is
5758 // Addendum_TypeDescr.
5759 masm.loadPtr(Address(temp, ObjectGroup::offsetOfAddendum()), temp);
5760
5761 // Check for the /Kind/ reserved slot of the TypeDescr. This is an Int32
5762 // Value which is equivalent to the object class check.
5763 static_assert(JS_DESCR_SLOT_KIND < NativeObject::MAX_FIXED_SLOTS, "Load from fixed slots");
5764 Address typeDescrKind(temp, NativeObject::getFixedSlotOffset(JS_DESCR_SLOT_KIND));
5765 masm.assertTestInt32(Assembler::Equal, typeDescrKind,
5766 "MOZ_ASSERT(obj->type()->typeDescr()->getReservedSlot(JS_DESCR_SLOT_KIND).isInt32())");
5767 masm.branch32(Assembler::NotEqual, masm.ToPayload(typeDescrKind), Imm32(js::type::Simd), &bail);
5768
5769 SimdType type = lir->mir()->simdType();
5770
5771 // Check if the SimdTypeDescr /Type/ match the specialization of this
5772 // MSimdUnbox instruction.
5773 static_assert(JS_DESCR_SLOT_TYPE < NativeObject::MAX_FIXED_SLOTS, "Load from fixed slots");
5774 Address typeDescrType(temp, NativeObject::getFixedSlotOffset(JS_DESCR_SLOT_TYPE));
5775 masm.assertTestInt32(Assembler::Equal, typeDescrType,
5776 "MOZ_ASSERT(obj->type()->typeDescr()->getReservedSlot(JS_DESCR_SLOT_TYPE).isInt32())");
5777 masm.branch32(Assembler::NotEqual, masm.ToPayload(typeDescrType), Imm32(int32_t(type)), &bail);
5778
5779 // Load the value from the data of the InlineTypedObject.
5780 Address objectData(object, InlineTypedObject::offsetOfDataStart());
5781 switch (lir->mir()->type()) {
5782 case MIRType::Int8x16:
5783 case MIRType::Int16x8:
5784 case MIRType::Int32x4:
5785 case MIRType::Bool8x16:
5786 case MIRType::Bool16x8:
5787 case MIRType::Bool32x4:
5788 masm.loadUnalignedSimd128Int(objectData, simd);
5789 break;
5790 case MIRType::Float32x4:
5791 masm.loadUnalignedSimd128Float(objectData, simd);
5792 break;
5793 default:
5794 MOZ_CRASH("The impossible happened!");
5795 }
5796
5797 bailoutFrom(&bail, lir->snapshot());
5798 }
5799
5800 typedef js::NamedLambdaObject* (*NewNamedLambdaObjectFn)(JSContext*, HandleFunction, gc::InitialHeap);
5801 static const VMFunction NewNamedLambdaObjectInfo =
5802 FunctionInfo<NewNamedLambdaObjectFn>(NamedLambdaObject::createTemplateObject,
5803 "NamedLambdaObject::createTemplateObject");
5804
5805 void
visitNewNamedLambdaObject(LNewNamedLambdaObject * lir)5806 CodeGenerator::visitNewNamedLambdaObject(LNewNamedLambdaObject* lir)
5807 {
5808 Register objReg = ToRegister(lir->output());
5809 Register tempReg = ToRegister(lir->temp());
5810 EnvironmentObject* templateObj = lir->mir()->templateObj();
5811 const CompileInfo& info = lir->mir()->block()->info();
5812
5813 // If we have a template object, we can inline call object creation.
5814 OutOfLineCode* ool = oolCallVM(NewNamedLambdaObjectInfo, lir,
5815 ArgList(ImmGCPtr(info.funMaybeLazy()), Imm32(gc::DefaultHeap)),
5816 StoreRegisterTo(objReg));
5817
5818 bool initContents = ShouldInitFixedSlots(lir, templateObj);
5819 masm.createGCObject(objReg, tempReg, templateObj, gc::DefaultHeap, ool->entry(),
5820 initContents);
5821
5822 masm.bind(ool->rejoin());
5823 }
5824
5825 typedef JSObject* (*NewCallObjectFn)(JSContext*, HandleShape, HandleObjectGroup);
5826 static const VMFunction NewCallObjectInfo =
5827 FunctionInfo<NewCallObjectFn>(NewCallObject, "NewCallObject");
5828
5829 void
visitNewCallObject(LNewCallObject * lir)5830 CodeGenerator::visitNewCallObject(LNewCallObject* lir)
5831 {
5832 Register objReg = ToRegister(lir->output());
5833 Register tempReg = ToRegister(lir->temp());
5834
5835 CallObject* templateObj = lir->mir()->templateObject();
5836
5837 OutOfLineCode* ool = oolCallVM(NewCallObjectInfo, lir,
5838 ArgList(ImmGCPtr(templateObj->lastProperty()),
5839 ImmGCPtr(templateObj->group())),
5840 StoreRegisterTo(objReg));
5841
5842 // Inline call object creation, using the OOL path only for tricky cases.
5843 bool initContents = ShouldInitFixedSlots(lir, templateObj);
5844 masm.createGCObject(objReg, tempReg, templateObj, gc::DefaultHeap, ool->entry(),
5845 initContents);
5846
5847 masm.bind(ool->rejoin());
5848 }
5849
5850 typedef JSObject* (*NewSingletonCallObjectFn)(JSContext*, HandleShape);
5851 static const VMFunction NewSingletonCallObjectInfo =
5852 FunctionInfo<NewSingletonCallObjectFn>(NewSingletonCallObject, "NewSingletonCallObject");
5853
5854 void
visitNewSingletonCallObject(LNewSingletonCallObject * lir)5855 CodeGenerator::visitNewSingletonCallObject(LNewSingletonCallObject* lir)
5856 {
5857 Register objReg = ToRegister(lir->output());
5858
5859 JSObject* templateObj = lir->mir()->templateObject();
5860
5861 OutOfLineCode* ool;
5862 ool = oolCallVM(NewSingletonCallObjectInfo, lir,
5863 ArgList(ImmGCPtr(templateObj->as<CallObject>().lastProperty())),
5864 StoreRegisterTo(objReg));
5865
5866 // Objects can only be given singleton types in VM calls. We make the call
5867 // out of line to not bloat inline code, even if (naively) this seems like
5868 // extra work.
5869 masm.jump(ool->entry());
5870 masm.bind(ool->rejoin());
5871 }
5872
5873 typedef JSObject* (*NewStringObjectFn)(JSContext*, HandleString);
5874 static const VMFunction NewStringObjectInfo =
5875 FunctionInfo<NewStringObjectFn>(NewStringObject, "NewStringObject");
5876
5877 void
visitNewStringObject(LNewStringObject * lir)5878 CodeGenerator::visitNewStringObject(LNewStringObject* lir)
5879 {
5880 Register input = ToRegister(lir->input());
5881 Register output = ToRegister(lir->output());
5882 Register temp = ToRegister(lir->temp());
5883
5884 StringObject* templateObj = lir->mir()->templateObj();
5885
5886 OutOfLineCode* ool = oolCallVM(NewStringObjectInfo, lir, ArgList(input),
5887 StoreRegisterTo(output));
5888
5889 masm.createGCObject(output, temp, templateObj, gc::DefaultHeap, ool->entry());
5890
5891 masm.loadStringLength(input, temp);
5892
5893 masm.storeValue(JSVAL_TYPE_STRING, input, Address(output, StringObject::offsetOfPrimitiveValue()));
5894 masm.storeValue(JSVAL_TYPE_INT32, temp, Address(output, StringObject::offsetOfLength()));
5895
5896 masm.bind(ool->rejoin());
5897 }
5898
5899 typedef bool(*InitElemFn)(JSContext* cx, jsbytecode* pc, HandleObject obj,
5900 HandleValue id, HandleValue value);
5901 static const VMFunction InitElemInfo =
5902 FunctionInfo<InitElemFn>(InitElemOperation, "InitElemOperation");
5903
5904 void
visitInitElem(LInitElem * lir)5905 CodeGenerator::visitInitElem(LInitElem* lir)
5906 {
5907 Register objReg = ToRegister(lir->getObject());
5908
5909 pushArg(ToValue(lir, LInitElem::ValueIndex));
5910 pushArg(ToValue(lir, LInitElem::IdIndex));
5911 pushArg(objReg);
5912 pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
5913
5914 callVM(InitElemInfo, lir);
5915 }
5916
5917 typedef bool (*InitElemGetterSetterFn)(JSContext*, jsbytecode*, HandleObject, HandleValue,
5918 HandleObject);
5919 static const VMFunction InitElemGetterSetterInfo =
5920 FunctionInfo<InitElemGetterSetterFn>(InitGetterSetterOperation, "InitGetterSetterOperation");
5921
5922 void
visitInitElemGetterSetter(LInitElemGetterSetter * lir)5923 CodeGenerator::visitInitElemGetterSetter(LInitElemGetterSetter* lir)
5924 {
5925 Register obj = ToRegister(lir->object());
5926 Register value = ToRegister(lir->value());
5927
5928 pushArg(value);
5929 pushArg(ToValue(lir, LInitElemGetterSetter::IdIndex));
5930 pushArg(obj);
5931 pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
5932
5933 callVM(InitElemGetterSetterInfo, lir);
5934 }
5935
5936 typedef bool(*MutatePrototypeFn)(JSContext* cx, HandlePlainObject obj, HandleValue value);
5937 static const VMFunction MutatePrototypeInfo =
5938 FunctionInfo<MutatePrototypeFn>(MutatePrototype, "MutatePrototype");
5939
5940 void
visitMutateProto(LMutateProto * lir)5941 CodeGenerator::visitMutateProto(LMutateProto* lir)
5942 {
5943 Register objReg = ToRegister(lir->getObject());
5944
5945 pushArg(ToValue(lir, LMutateProto::ValueIndex));
5946 pushArg(objReg);
5947
5948 callVM(MutatePrototypeInfo, lir);
5949 }
5950
5951 typedef bool(*InitPropFn)(JSContext*, HandleObject, HandlePropertyName, HandleValue, jsbytecode* pc);
5952 static const VMFunction InitPropInfo = FunctionInfo<InitPropFn>(InitProp, "InitProp");
5953
5954 void
visitInitProp(LInitProp * lir)5955 CodeGenerator::visitInitProp(LInitProp* lir)
5956 {
5957 Register objReg = ToRegister(lir->getObject());
5958
5959 pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
5960 pushArg(ToValue(lir, LInitProp::ValueIndex));
5961 pushArg(ImmGCPtr(lir->mir()->propertyName()));
5962 pushArg(objReg);
5963
5964 callVM(InitPropInfo, lir);
5965 }
5966
5967 typedef bool(*InitPropGetterSetterFn)(JSContext*, jsbytecode*, HandleObject, HandlePropertyName,
5968 HandleObject);
5969 static const VMFunction InitPropGetterSetterInfo =
5970 FunctionInfo<InitPropGetterSetterFn>(InitGetterSetterOperation, "InitGetterSetterOperation");
5971
5972 void
visitInitPropGetterSetter(LInitPropGetterSetter * lir)5973 CodeGenerator::visitInitPropGetterSetter(LInitPropGetterSetter* lir)
5974 {
5975 Register obj = ToRegister(lir->object());
5976 Register value = ToRegister(lir->value());
5977
5978 pushArg(value);
5979 pushArg(ImmGCPtr(lir->mir()->name()));
5980 pushArg(obj);
5981 pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
5982
5983 callVM(InitPropGetterSetterInfo, lir);
5984 }
5985
5986 typedef bool (*CreateThisFn)(JSContext* cx, HandleObject callee, HandleObject newTarget, MutableHandleValue rval);
5987 static const VMFunction CreateThisInfoCodeGen = FunctionInfo<CreateThisFn>(CreateThis, "CreateThis");
5988
5989 void
visitCreateThis(LCreateThis * lir)5990 CodeGenerator::visitCreateThis(LCreateThis* lir)
5991 {
5992 const LAllocation* callee = lir->getCallee();
5993 const LAllocation* newTarget = lir->getNewTarget();
5994
5995 if (newTarget->isConstant())
5996 pushArg(ImmGCPtr(&newTarget->toConstant()->toObject()));
5997 else
5998 pushArg(ToRegister(newTarget));
5999
6000 if (callee->isConstant())
6001 pushArg(ImmGCPtr(&callee->toConstant()->toObject()));
6002 else
6003 pushArg(ToRegister(callee));
6004
6005 callVM(CreateThisInfoCodeGen, lir);
6006 }
6007
6008 static JSObject*
CreateThisForFunctionWithProtoWrapper(JSContext * cx,HandleObject callee,HandleObject newTarget,HandleObject proto)6009 CreateThisForFunctionWithProtoWrapper(JSContext* cx, HandleObject callee, HandleObject newTarget,
6010 HandleObject proto)
6011 {
6012 return CreateThisForFunctionWithProto(cx, callee, newTarget, proto);
6013 }
6014
6015 typedef JSObject* (*CreateThisWithProtoFn)(JSContext* cx, HandleObject callee,
6016 HandleObject newTarget, HandleObject proto);
6017 static const VMFunction CreateThisWithProtoInfo =
6018 FunctionInfo<CreateThisWithProtoFn>(CreateThisForFunctionWithProtoWrapper,
6019 "CreateThisForFunctionWithProtoWrapper");
6020
6021 void
visitCreateThisWithProto(LCreateThisWithProto * lir)6022 CodeGenerator::visitCreateThisWithProto(LCreateThisWithProto* lir)
6023 {
6024 const LAllocation* callee = lir->getCallee();
6025 const LAllocation* newTarget = lir->getNewTarget();
6026 const LAllocation* proto = lir->getPrototype();
6027
6028 if (proto->isConstant())
6029 pushArg(ImmGCPtr(&proto->toConstant()->toObject()));
6030 else
6031 pushArg(ToRegister(proto));
6032
6033 if (newTarget->isConstant())
6034 pushArg(ImmGCPtr(&newTarget->toConstant()->toObject()));
6035 else
6036 pushArg(ToRegister(newTarget));
6037
6038 if (callee->isConstant())
6039 pushArg(ImmGCPtr(&callee->toConstant()->toObject()));
6040 else
6041 pushArg(ToRegister(callee));
6042
6043 callVM(CreateThisWithProtoInfo, lir);
6044 }
6045
6046 void
visitCreateThisWithTemplate(LCreateThisWithTemplate * lir)6047 CodeGenerator::visitCreateThisWithTemplate(LCreateThisWithTemplate* lir)
6048 {
6049 JSObject* templateObject = lir->mir()->templateObject();
6050 Register objReg = ToRegister(lir->output());
6051 Register tempReg = ToRegister(lir->temp());
6052
6053 OutOfLineCode* ool = oolCallVM(NewInitObjectWithTemplateInfo, lir,
6054 ArgList(ImmGCPtr(templateObject)),
6055 StoreRegisterTo(objReg));
6056
6057 // Allocate. If the FreeList is empty, call to VM, which may GC.
6058 bool initContents = !templateObject->is<PlainObject>() ||
6059 ShouldInitFixedSlots(lir, &templateObject->as<PlainObject>());
6060 masm.createGCObject(objReg, tempReg, templateObject, lir->mir()->initialHeap(), ool->entry(),
6061 initContents);
6062
6063 masm.bind(ool->rejoin());
6064 }
6065
6066 typedef JSObject* (*NewIonArgumentsObjectFn)(JSContext* cx, JitFrameLayout* frame, HandleObject);
6067 static const VMFunction NewIonArgumentsObjectInfo =
6068 FunctionInfo<NewIonArgumentsObjectFn>((NewIonArgumentsObjectFn) ArgumentsObject::createForIon,
6069 "ArgumentsObject::createForIon");
6070
6071 void
visitCreateArgumentsObject(LCreateArgumentsObject * lir)6072 CodeGenerator::visitCreateArgumentsObject(LCreateArgumentsObject* lir)
6073 {
6074 // This should be getting constructed in the first block only, and not any OSR entry blocks.
6075 MOZ_ASSERT(lir->mir()->block()->id() == 0);
6076
6077 Register callObj = ToRegister(lir->getCallObject());
6078 Register temp = ToRegister(lir->temp0());
6079 Label done;
6080
6081 if (ArgumentsObject* templateObj = lir->mir()->templateObject()) {
6082 Register objTemp = ToRegister(lir->temp1());
6083 Register cxTemp = ToRegister(lir->temp2());
6084
6085 masm.Push(callObj);
6086
6087 // Try to allocate an arguments object. This will leave the reserved
6088 // slots uninitialized, so it's important we don't GC until we
6089 // initialize these slots in ArgumentsObject::finishForIon.
6090 Label failure;
6091 masm.createGCObject(objTemp, temp, templateObj, gc::DefaultHeap, &failure,
6092 /* initContents = */ false);
6093
6094 masm.moveStackPtrTo(temp);
6095 masm.addPtr(Imm32(masm.framePushed()), temp);
6096
6097 masm.setupUnalignedABICall(cxTemp);
6098 masm.loadJSContext(cxTemp);
6099 masm.passABIArg(cxTemp);
6100 masm.passABIArg(temp);
6101 masm.passABIArg(callObj);
6102 masm.passABIArg(objTemp);
6103
6104 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ArgumentsObject::finishForIon));
6105 masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, &failure);
6106
6107 // Discard saved callObj on the stack.
6108 masm.addToStackPtr(Imm32(sizeof(uintptr_t)));
6109 masm.jump(&done);
6110
6111 masm.bind(&failure);
6112 masm.Pop(callObj);
6113 }
6114
6115 masm.moveStackPtrTo(temp);
6116 masm.addPtr(Imm32(frameSize()), temp);
6117
6118 pushArg(callObj);
6119 pushArg(temp);
6120 callVM(NewIonArgumentsObjectInfo, lir);
6121
6122 masm.bind(&done);
6123 }
6124
6125 void
visitGetArgumentsObjectArg(LGetArgumentsObjectArg * lir)6126 CodeGenerator::visitGetArgumentsObjectArg(LGetArgumentsObjectArg* lir)
6127 {
6128 Register temp = ToRegister(lir->getTemp(0));
6129 Register argsObj = ToRegister(lir->getArgsObject());
6130 ValueOperand out = ToOutValue(lir);
6131
6132 masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()), temp);
6133 Address argAddr(temp, ArgumentsData::offsetOfArgs() + lir->mir()->argno() * sizeof(Value));
6134 masm.loadValue(argAddr, out);
6135 #ifdef DEBUG
6136 Label success;
6137 masm.branchTestMagic(Assembler::NotEqual, out, &success);
6138 masm.assumeUnreachable("Result from ArgumentObject shouldn't be JSVAL_TYPE_MAGIC.");
6139 masm.bind(&success);
6140 #endif
6141 }
6142
6143 void
visitSetArgumentsObjectArg(LSetArgumentsObjectArg * lir)6144 CodeGenerator::visitSetArgumentsObjectArg(LSetArgumentsObjectArg* lir)
6145 {
6146 Register temp = ToRegister(lir->getTemp(0));
6147 Register argsObj = ToRegister(lir->getArgsObject());
6148 ValueOperand value = ToValue(lir, LSetArgumentsObjectArg::ValueIndex);
6149
6150 masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()), temp);
6151 Address argAddr(temp, ArgumentsData::offsetOfArgs() + lir->mir()->argno() * sizeof(Value));
6152 emitPreBarrier(argAddr);
6153 #ifdef DEBUG
6154 Label success;
6155 masm.branchTestMagic(Assembler::NotEqual, argAddr, &success);
6156 masm.assumeUnreachable("Result in ArgumentObject shouldn't be JSVAL_TYPE_MAGIC.");
6157 masm.bind(&success);
6158 #endif
6159 masm.storeValue(value, argAddr);
6160 }
6161
6162 void
visitReturnFromCtor(LReturnFromCtor * lir)6163 CodeGenerator::visitReturnFromCtor(LReturnFromCtor* lir)
6164 {
6165 ValueOperand value = ToValue(lir, LReturnFromCtor::ValueIndex);
6166 Register obj = ToRegister(lir->getObject());
6167 Register output = ToRegister(lir->output());
6168
6169 Label valueIsObject, end;
6170
6171 masm.branchTestObject(Assembler::Equal, value, &valueIsObject);
6172
6173 // Value is not an object. Return that other object.
6174 masm.movePtr(obj, output);
6175 masm.jump(&end);
6176
6177 // Value is an object. Return unbox(Value).
6178 masm.bind(&valueIsObject);
6179 Register payload = masm.extractObject(value, output);
6180 if (payload != output)
6181 masm.movePtr(payload, output);
6182
6183 masm.bind(&end);
6184 }
6185
6186 typedef bool (*BoxNonStrictThisFn)(JSContext*, HandleValue, MutableHandleValue);
6187 static const VMFunction BoxNonStrictThisInfo =
6188 FunctionInfo<BoxNonStrictThisFn>(BoxNonStrictThis, "BoxNonStrictThis");
6189
6190 void
visitComputeThis(LComputeThis * lir)6191 CodeGenerator::visitComputeThis(LComputeThis* lir)
6192 {
6193 ValueOperand value = ToValue(lir, LComputeThis::ValueIndex);
6194 ValueOperand output = ToOutValue(lir);
6195
6196 OutOfLineCode* ool = oolCallVM(BoxNonStrictThisInfo, lir, ArgList(value), StoreValueTo(output));
6197
6198 masm.branchTestObject(Assembler::NotEqual, value, ool->entry());
6199 masm.moveValue(value, output);
6200 masm.bind(ool->rejoin());
6201 }
6202
6203 void
visitArrowNewTarget(LArrowNewTarget * lir)6204 CodeGenerator::visitArrowNewTarget(LArrowNewTarget* lir)
6205 {
6206 Register callee = ToRegister(lir->callee());
6207 ValueOperand output = ToOutValue(lir);
6208 masm.loadValue(Address(callee, FunctionExtended::offsetOfArrowNewTargetSlot()), output);
6209 }
6210
6211 void
visitArrayLength(LArrayLength * lir)6212 CodeGenerator::visitArrayLength(LArrayLength* lir)
6213 {
6214 Address length(ToRegister(lir->elements()), ObjectElements::offsetOfLength());
6215 masm.load32(length, ToRegister(lir->output()));
6216 }
6217
6218 void
visitSetArrayLength(LSetArrayLength * lir)6219 CodeGenerator::visitSetArrayLength(LSetArrayLength* lir)
6220 {
6221 Address length(ToRegister(lir->elements()), ObjectElements::offsetOfLength());
6222 RegisterOrInt32Constant newLength = ToRegisterOrInt32Constant(lir->index());
6223
6224 masm.inc32(&newLength);
6225 masm.store32(newLength, length);
6226 // Restore register value if it is used/captured after.
6227 masm.dec32(&newLength);
6228 }
6229
6230 template <class OrderedHashTable>
6231 static void
6232 RangeFront(MacroAssembler&, Register, Register, Register);
6233
6234 template <>
6235 void
RangeFront(MacroAssembler & masm,Register range,Register i,Register front)6236 RangeFront<ValueMap>(MacroAssembler& masm, Register range, Register i, Register front)
6237 {
6238 masm.loadPtr(Address(range, ValueMap::Range::offsetOfHashTable()), front);
6239 masm.loadPtr(Address(front, ValueMap::offsetOfImplData()), front);
6240
6241 static_assert(ValueMap::offsetOfImplDataElement() == 0, "offsetof(Data, element) is 0");
6242 static_assert(ValueMap::sizeofImplData() == 24, "sizeof(Data) is 24");
6243 masm.mulBy3(i, i);
6244 masm.lshiftPtr(Imm32(3), i);
6245 masm.addPtr(i, front);
6246 }
6247
6248 template <>
6249 void
RangeFront(MacroAssembler & masm,Register range,Register i,Register front)6250 RangeFront<ValueSet>(MacroAssembler& masm, Register range, Register i, Register front)
6251 {
6252 masm.loadPtr(Address(range, ValueSet::Range::offsetOfHashTable()), front);
6253 masm.loadPtr(Address(front, ValueSet::offsetOfImplData()), front);
6254
6255 static_assert(ValueSet::offsetOfImplDataElement() == 0, "offsetof(Data, element) is 0");
6256 static_assert(ValueSet::sizeofImplData() == 16, "sizeof(Data) is 16");
6257 masm.lshiftPtr(Imm32(4), i);
6258 masm.addPtr(i, front);
6259 }
6260
6261 template <class OrderedHashTable>
6262 static void
RangePopFront(MacroAssembler & masm,Register range,Register front,Register dataLength,Register temp)6263 RangePopFront(MacroAssembler& masm, Register range, Register front, Register dataLength,
6264 Register temp)
6265 {
6266 Register i = temp;
6267
6268 masm.add32(Imm32(1), Address(range, OrderedHashTable::Range::offsetOfCount()));
6269
6270 masm.load32(Address(range, OrderedHashTable::Range::offsetOfI()), i);
6271 masm.add32(Imm32(1), i);
6272
6273 Label done, seek;
6274 masm.bind(&seek);
6275 masm.branch32(Assembler::AboveOrEqual, i, dataLength, &done);
6276
6277 // We can add sizeof(Data) to |front| to select the next element, because
6278 // |front| and |range.ht.data[i]| point to the same location.
6279 static_assert(OrderedHashTable::offsetOfImplDataElement() == 0, "offsetof(Data, element) is 0");
6280 masm.addPtr(Imm32(OrderedHashTable::sizeofImplData()), front);
6281
6282 masm.branchTestMagic(Assembler::NotEqual, Address(front, OrderedHashTable::offsetOfEntryKey()),
6283 JS_HASH_KEY_EMPTY, &done);
6284
6285 masm.add32(Imm32(1), i);
6286 masm.jump(&seek);
6287
6288 masm.bind(&done);
6289 masm.store32(i, Address(range, OrderedHashTable::Range::offsetOfI()));
6290 }
6291
6292 template <class OrderedHashTable>
6293 static inline void
RangeDestruct(MacroAssembler & masm,Register range,Register temp0,Register temp1)6294 RangeDestruct(MacroAssembler& masm, Register range, Register temp0, Register temp1)
6295 {
6296 Register next = temp0;
6297 Register prevp = temp1;
6298
6299 masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfNext()), next);
6300 masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfPrevP()), prevp);
6301 masm.storePtr(next, Address(prevp, 0));
6302
6303 Label hasNoNext;
6304 masm.branchTestPtr(Assembler::Zero, next, next, &hasNoNext);
6305
6306 masm.storePtr(prevp, Address(next, OrderedHashTable::Range::offsetOfPrevP()));
6307
6308 masm.bind(&hasNoNext);
6309
6310 masm.callFreeStub(range);
6311 }
6312
6313 template <>
6314 void
emitLoadIteratorValues(Register result,Register temp,Register front)6315 CodeGenerator::emitLoadIteratorValues<ValueMap>(Register result, Register temp, Register front)
6316 {
6317 size_t elementsOffset = NativeObject::offsetOfFixedElements();
6318
6319 Address keyAddress(front, ValueMap::Entry::offsetOfKey());
6320 Address valueAddress(front, ValueMap::Entry::offsetOfValue());
6321 Address keyElemAddress(result, elementsOffset);
6322 Address valueElemAddress(result, elementsOffset + sizeof(Value));
6323 masm.patchableCallPreBarrier(keyElemAddress, MIRType::Value);
6324 masm.patchableCallPreBarrier(valueElemAddress, MIRType::Value);
6325 masm.storeValue(keyAddress, keyElemAddress, temp);
6326 masm.storeValue(valueAddress, valueElemAddress, temp);
6327
6328 Label keyIsNotObject, valueIsNotNurseryObject, emitBarrier;
6329 masm.branchTestObject(Assembler::NotEqual, keyAddress, &keyIsNotObject);
6330 masm.branchValueIsNurseryObject(Assembler::Equal, keyAddress, temp, &emitBarrier);
6331 masm.bind(&keyIsNotObject);
6332 masm.branchTestObject(Assembler::NotEqual, valueAddress, &valueIsNotNurseryObject);
6333 masm.branchValueIsNurseryObject(Assembler::NotEqual, valueAddress, temp,
6334 &valueIsNotNurseryObject);
6335 {
6336 masm.bind(&emitBarrier);
6337 saveVolatile(temp);
6338 emitPostWriteBarrier(result);
6339 restoreVolatile(temp);
6340 }
6341 masm.bind(&valueIsNotNurseryObject);
6342 }
6343
6344 template <>
6345 void
emitLoadIteratorValues(Register result,Register temp,Register front)6346 CodeGenerator::emitLoadIteratorValues<ValueSet>(Register result, Register temp, Register front)
6347 {
6348 size_t elementsOffset = NativeObject::offsetOfFixedElements();
6349
6350 Address keyAddress(front, ValueSet::offsetOfEntryKey());
6351 Address keyElemAddress(result, elementsOffset);
6352 masm.patchableCallPreBarrier(keyElemAddress, MIRType::Value);
6353 masm.storeValue(keyAddress, keyElemAddress, temp);
6354
6355 Label keyIsNotObject;
6356 masm.branchTestObject(Assembler::NotEqual, keyAddress, &keyIsNotObject);
6357 masm.branchValueIsNurseryObject(Assembler::NotEqual, keyAddress, temp, &keyIsNotObject);
6358 {
6359 saveVolatile(temp);
6360 emitPostWriteBarrier(result);
6361 restoreVolatile(temp);
6362 }
6363 masm.bind(&keyIsNotObject);
6364 }
6365
6366 template <class IteratorObject, class OrderedHashTable>
6367 void
emitGetNextEntryForIterator(LGetNextEntryForIterator * lir)6368 CodeGenerator::emitGetNextEntryForIterator(LGetNextEntryForIterator* lir)
6369 {
6370 Register iter = ToRegister(lir->iter());
6371 Register result = ToRegister(lir->result());
6372 Register temp = ToRegister(lir->temp0());
6373 Register dataLength = ToRegister(lir->temp1());
6374 Register range = ToRegister(lir->temp2());
6375 Register output = ToRegister(lir->output());
6376
6377 masm.loadPrivate(Address(iter, NativeObject::getFixedSlotOffset(IteratorObject::RangeSlot)),
6378 range);
6379
6380 Label iterAlreadyDone, iterDone, done;
6381 masm.branchTestPtr(Assembler::Zero, range, range, &iterAlreadyDone);
6382
6383 masm.load32(Address(range, OrderedHashTable::Range::offsetOfI()), temp);
6384 masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfHashTable()), dataLength);
6385 masm.load32(Address(dataLength, OrderedHashTable::offsetOfImplDataLength()), dataLength);
6386 masm.branch32(Assembler::AboveOrEqual, temp, dataLength, &iterDone);
6387 {
6388 masm.push(iter);
6389
6390 Register front = iter;
6391 RangeFront<OrderedHashTable>(masm, range, temp, front);
6392
6393 emitLoadIteratorValues<OrderedHashTable>(result, temp, front);
6394
6395 RangePopFront<OrderedHashTable>(masm, range, front, dataLength, temp);
6396
6397 masm.pop(iter);
6398 masm.move32(Imm32(0), output);
6399 }
6400 masm.jump(&done);
6401 {
6402 masm.bind(&iterDone);
6403
6404 RangeDestruct<OrderedHashTable>(masm, range, temp, dataLength);
6405
6406 masm.storeValue(PrivateValue(nullptr),
6407 Address(iter, NativeObject::getFixedSlotOffset(IteratorObject::RangeSlot)));
6408
6409 masm.bind(&iterAlreadyDone);
6410
6411 masm.move32(Imm32(1), output);
6412 }
6413 masm.bind(&done);
6414 }
6415
6416 void
visitGetNextEntryForIterator(LGetNextEntryForIterator * lir)6417 CodeGenerator::visitGetNextEntryForIterator(LGetNextEntryForIterator* lir)
6418 {
6419 if (lir->mir()->mode() == MGetNextEntryForIterator::Map) {
6420 emitGetNextEntryForIterator<MapIteratorObject, ValueMap>(lir);
6421 } else {
6422 MOZ_ASSERT(lir->mir()->mode() == MGetNextEntryForIterator::Set);
6423 emitGetNextEntryForIterator<SetIteratorObject, ValueSet>(lir);
6424 }
6425 }
6426
6427 void
visitTypedArrayLength(LTypedArrayLength * lir)6428 CodeGenerator::visitTypedArrayLength(LTypedArrayLength* lir)
6429 {
6430 Register obj = ToRegister(lir->object());
6431 Register out = ToRegister(lir->output());
6432 masm.unboxInt32(Address(obj, TypedArrayObject::lengthOffset()), out);
6433 }
6434
6435 void
visitTypedArrayElements(LTypedArrayElements * lir)6436 CodeGenerator::visitTypedArrayElements(LTypedArrayElements* lir)
6437 {
6438 Register obj = ToRegister(lir->object());
6439 Register out = ToRegister(lir->output());
6440 masm.loadPtr(Address(obj, TypedArrayObject::dataOffset()), out);
6441 }
6442
6443 void
visitSetDisjointTypedElements(LSetDisjointTypedElements * lir)6444 CodeGenerator::visitSetDisjointTypedElements(LSetDisjointTypedElements* lir)
6445 {
6446 Register target = ToRegister(lir->target());
6447 Register targetOffset = ToRegister(lir->targetOffset());
6448 Register source = ToRegister(lir->source());
6449
6450 Register temp = ToRegister(lir->temp());
6451
6452 masm.setupUnalignedABICall(temp);
6453 masm.passABIArg(target);
6454 masm.passABIArg(targetOffset);
6455 masm.passABIArg(source);
6456 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::SetDisjointTypedElements));
6457 }
6458
6459 void
visitTypedObjectDescr(LTypedObjectDescr * lir)6460 CodeGenerator::visitTypedObjectDescr(LTypedObjectDescr* lir)
6461 {
6462 Register obj = ToRegister(lir->object());
6463 Register out = ToRegister(lir->output());
6464
6465 masm.loadPtr(Address(obj, JSObject::offsetOfGroup()), out);
6466 masm.loadPtr(Address(out, ObjectGroup::offsetOfAddendum()), out);
6467 }
6468
6469 void
visitTypedObjectElements(LTypedObjectElements * lir)6470 CodeGenerator::visitTypedObjectElements(LTypedObjectElements* lir)
6471 {
6472 Register obj = ToRegister(lir->object());
6473 Register out = ToRegister(lir->output());
6474
6475 if (lir->mir()->definitelyOutline()) {
6476 masm.loadPtr(Address(obj, OutlineTypedObject::offsetOfData()), out);
6477 } else {
6478 Label inlineObject, done;
6479 masm.loadObjClass(obj, out);
6480 masm.branchPtr(Assembler::Equal, out, ImmPtr(&InlineOpaqueTypedObject::class_), &inlineObject);
6481 masm.branchPtr(Assembler::Equal, out, ImmPtr(&InlineTransparentTypedObject::class_), &inlineObject);
6482
6483 masm.loadPtr(Address(obj, OutlineTypedObject::offsetOfData()), out);
6484 masm.jump(&done);
6485
6486 masm.bind(&inlineObject);
6487 masm.computeEffectiveAddress(Address(obj, InlineTypedObject::offsetOfDataStart()), out);
6488 masm.bind(&done);
6489 }
6490 }
6491
6492 void
visitSetTypedObjectOffset(LSetTypedObjectOffset * lir)6493 CodeGenerator::visitSetTypedObjectOffset(LSetTypedObjectOffset* lir)
6494 {
6495 Register object = ToRegister(lir->object());
6496 Register offset = ToRegister(lir->offset());
6497 Register temp0 = ToRegister(lir->temp0());
6498 Register temp1 = ToRegister(lir->temp1());
6499
6500 // Compute the base pointer for the typed object's owner.
6501 masm.loadPtr(Address(object, OutlineTypedObject::offsetOfOwner()), temp0);
6502
6503 Label inlineObject, done;
6504 masm.loadObjClass(temp0, temp1);
6505 masm.branchPtr(Assembler::Equal, temp1, ImmPtr(&InlineOpaqueTypedObject::class_), &inlineObject);
6506 masm.branchPtr(Assembler::Equal, temp1, ImmPtr(&InlineTransparentTypedObject::class_), &inlineObject);
6507
6508 masm.loadPrivate(Address(temp0, ArrayBufferObject::offsetOfDataSlot()), temp0);
6509 masm.jump(&done);
6510
6511 masm.bind(&inlineObject);
6512 masm.addPtr(ImmWord(InlineTypedObject::offsetOfDataStart()), temp0);
6513
6514 masm.bind(&done);
6515
6516 // Compute the new data pointer and set it in the object.
6517 masm.addPtr(offset, temp0);
6518 masm.storePtr(temp0, Address(object, OutlineTypedObject::offsetOfData()));
6519 }
6520
6521 void
visitStringLength(LStringLength * lir)6522 CodeGenerator::visitStringLength(LStringLength* lir)
6523 {
6524 Register input = ToRegister(lir->string());
6525 Register output = ToRegister(lir->output());
6526
6527 masm.loadStringLength(input, output);
6528 }
6529
6530 void
visitMinMaxI(LMinMaxI * ins)6531 CodeGenerator::visitMinMaxI(LMinMaxI* ins)
6532 {
6533 Register first = ToRegister(ins->first());
6534 Register output = ToRegister(ins->output());
6535
6536 MOZ_ASSERT(first == output);
6537
6538 Label done;
6539 Assembler::Condition cond = ins->mir()->isMax()
6540 ? Assembler::GreaterThan
6541 : Assembler::LessThan;
6542
6543 if (ins->second()->isConstant()) {
6544 masm.branch32(cond, first, Imm32(ToInt32(ins->second())), &done);
6545 masm.move32(Imm32(ToInt32(ins->second())), output);
6546 } else {
6547 masm.branch32(cond, first, ToRegister(ins->second()), &done);
6548 masm.move32(ToRegister(ins->second()), output);
6549 }
6550
6551 masm.bind(&done);
6552 }
6553
6554 void
visitAbsI(LAbsI * ins)6555 CodeGenerator::visitAbsI(LAbsI* ins)
6556 {
6557 Register input = ToRegister(ins->input());
6558 Label positive;
6559
6560 MOZ_ASSERT(input == ToRegister(ins->output()));
6561 masm.branchTest32(Assembler::NotSigned, input, input, &positive);
6562 masm.neg32(input);
6563 LSnapshot* snapshot = ins->snapshot();
6564 #if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
6565 if (snapshot)
6566 bailoutCmp32(Assembler::Equal, input, Imm32(INT32_MIN), snapshot);
6567 #else
6568 if (snapshot)
6569 bailoutIf(Assembler::Overflow, snapshot);
6570 #endif
6571 masm.bind(&positive);
6572 }
6573
6574 void
visitPowI(LPowI * ins)6575 CodeGenerator::visitPowI(LPowI* ins)
6576 {
6577 FloatRegister value = ToFloatRegister(ins->value());
6578 Register power = ToRegister(ins->power());
6579 Register temp = ToRegister(ins->temp());
6580
6581 MOZ_ASSERT(power != temp);
6582
6583 masm.setupUnalignedABICall(temp);
6584 masm.passABIArg(value, MoveOp::DOUBLE);
6585 masm.passABIArg(power);
6586
6587 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::powi), MoveOp::DOUBLE);
6588 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
6589 }
6590
6591 void
visitPowD(LPowD * ins)6592 CodeGenerator::visitPowD(LPowD* ins)
6593 {
6594 FloatRegister value = ToFloatRegister(ins->value());
6595 FloatRegister power = ToFloatRegister(ins->power());
6596 Register temp = ToRegister(ins->temp());
6597
6598 masm.setupUnalignedABICall(temp);
6599 masm.passABIArg(value, MoveOp::DOUBLE);
6600 masm.passABIArg(power, MoveOp::DOUBLE);
6601 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ecmaPow), MoveOp::DOUBLE);
6602
6603 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
6604 }
6605
6606 void
visitMathFunctionD(LMathFunctionD * ins)6607 CodeGenerator::visitMathFunctionD(LMathFunctionD* ins)
6608 {
6609 Register temp = ToRegister(ins->temp());
6610 FloatRegister input = ToFloatRegister(ins->input());
6611 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
6612
6613 masm.setupUnalignedABICall(temp);
6614
6615 const MathCache* mathCache = ins->mir()->cache();
6616 if (mathCache) {
6617 masm.movePtr(ImmPtr(mathCache), temp);
6618 masm.passABIArg(temp);
6619 }
6620 masm.passABIArg(input, MoveOp::DOUBLE);
6621
6622 # define MAYBE_CACHED(fcn) (mathCache ? (void*)fcn ## _impl : (void*)fcn ## _uncached)
6623
6624 void* funptr = nullptr;
6625 switch (ins->mir()->function()) {
6626 case MMathFunction::Log:
6627 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log));
6628 break;
6629 case MMathFunction::Sin:
6630 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_sin));
6631 break;
6632 case MMathFunction::Cos:
6633 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_cos));
6634 break;
6635 case MMathFunction::Exp:
6636 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_exp));
6637 break;
6638 case MMathFunction::Tan:
6639 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_tan));
6640 break;
6641 case MMathFunction::ATan:
6642 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_atan));
6643 break;
6644 case MMathFunction::ASin:
6645 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_asin));
6646 break;
6647 case MMathFunction::ACos:
6648 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_acos));
6649 break;
6650 case MMathFunction::Log10:
6651 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log10));
6652 break;
6653 case MMathFunction::Log2:
6654 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log2));
6655 break;
6656 case MMathFunction::Log1P:
6657 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log1p));
6658 break;
6659 case MMathFunction::ExpM1:
6660 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_expm1));
6661 break;
6662 case MMathFunction::CosH:
6663 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_cosh));
6664 break;
6665 case MMathFunction::SinH:
6666 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_sinh));
6667 break;
6668 case MMathFunction::TanH:
6669 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_tanh));
6670 break;
6671 case MMathFunction::ACosH:
6672 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_acosh));
6673 break;
6674 case MMathFunction::ASinH:
6675 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_asinh));
6676 break;
6677 case MMathFunction::ATanH:
6678 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_atanh));
6679 break;
6680 case MMathFunction::Sign:
6681 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_sign));
6682 break;
6683 case MMathFunction::Trunc:
6684 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_trunc));
6685 break;
6686 case MMathFunction::Cbrt:
6687 funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_cbrt));
6688 break;
6689 case MMathFunction::Floor:
6690 funptr = JS_FUNC_TO_DATA_PTR(void*, js::math_floor_impl);
6691 break;
6692 case MMathFunction::Ceil:
6693 funptr = JS_FUNC_TO_DATA_PTR(void*, js::math_ceil_impl);
6694 break;
6695 case MMathFunction::Round:
6696 funptr = JS_FUNC_TO_DATA_PTR(void*, js::math_round_impl);
6697 break;
6698 default:
6699 MOZ_CRASH("Unknown math function");
6700 }
6701
6702 # undef MAYBE_CACHED
6703
6704 masm.callWithABI(funptr, MoveOp::DOUBLE);
6705 }
6706
6707 void
visitMathFunctionF(LMathFunctionF * ins)6708 CodeGenerator::visitMathFunctionF(LMathFunctionF* ins)
6709 {
6710 Register temp = ToRegister(ins->temp());
6711 FloatRegister input = ToFloatRegister(ins->input());
6712 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnFloat32Reg);
6713
6714 masm.setupUnalignedABICall(temp);
6715 masm.passABIArg(input, MoveOp::FLOAT32);
6716
6717 void* funptr = nullptr;
6718 switch (ins->mir()->function()) {
6719 case MMathFunction::Floor: funptr = JS_FUNC_TO_DATA_PTR(void*, floorf); break;
6720 case MMathFunction::Round: funptr = JS_FUNC_TO_DATA_PTR(void*, math_roundf_impl); break;
6721 case MMathFunction::Ceil: funptr = JS_FUNC_TO_DATA_PTR(void*, ceilf); break;
6722 default:
6723 MOZ_CRASH("Unknown or unsupported float32 math function");
6724 }
6725
6726 masm.callWithABI(funptr, MoveOp::FLOAT32);
6727 }
6728
6729 void
visitModD(LModD * ins)6730 CodeGenerator::visitModD(LModD* ins)
6731 {
6732 FloatRegister lhs = ToFloatRegister(ins->lhs());
6733 FloatRegister rhs = ToFloatRegister(ins->rhs());
6734 Register temp = ToRegister(ins->temp());
6735
6736 MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
6737
6738 masm.setupUnalignedABICall(temp);
6739 masm.passABIArg(lhs, MoveOp::DOUBLE);
6740 masm.passABIArg(rhs, MoveOp::DOUBLE);
6741
6742 if (gen->compilingWasm())
6743 masm.callWithABI(wasm::SymbolicAddress::ModD, MoveOp::DOUBLE);
6744 else
6745 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, NumberMod), MoveOp::DOUBLE);
6746 }
6747
6748 typedef bool (*BinaryFn)(JSContext*, MutableHandleValue, MutableHandleValue, MutableHandleValue);
6749
6750 static const VMFunction AddInfo = FunctionInfo<BinaryFn>(js::AddValues, "AddValues");
6751 static const VMFunction SubInfo = FunctionInfo<BinaryFn>(js::SubValues, "SubValues");
6752 static const VMFunction MulInfo = FunctionInfo<BinaryFn>(js::MulValues, "MulValues");
6753 static const VMFunction DivInfo = FunctionInfo<BinaryFn>(js::DivValues, "DivValues");
6754 static const VMFunction ModInfo = FunctionInfo<BinaryFn>(js::ModValues, "ModValues");
6755 static const VMFunction UrshInfo = FunctionInfo<BinaryFn>(js::UrshValues, "UrshValues");
6756
6757 void
visitBinaryV(LBinaryV * lir)6758 CodeGenerator::visitBinaryV(LBinaryV* lir)
6759 {
6760 pushArg(ToValue(lir, LBinaryV::RhsInput));
6761 pushArg(ToValue(lir, LBinaryV::LhsInput));
6762
6763 switch (lir->jsop()) {
6764 case JSOP_ADD:
6765 callVM(AddInfo, lir);
6766 break;
6767
6768 case JSOP_SUB:
6769 callVM(SubInfo, lir);
6770 break;
6771
6772 case JSOP_MUL:
6773 callVM(MulInfo, lir);
6774 break;
6775
6776 case JSOP_DIV:
6777 callVM(DivInfo, lir);
6778 break;
6779
6780 case JSOP_MOD:
6781 callVM(ModInfo, lir);
6782 break;
6783
6784 case JSOP_URSH:
6785 callVM(UrshInfo, lir);
6786 break;
6787
6788 default:
6789 MOZ_CRASH("Unexpected binary op");
6790 }
6791 }
6792
6793 typedef bool (*StringCompareFn)(JSContext*, HandleString, HandleString, bool*);
6794 static const VMFunction StringsEqualInfo =
6795 FunctionInfo<StringCompareFn>(jit::StringsEqual<true>, "StringsEqual");
6796 static const VMFunction StringsNotEqualInfo =
6797 FunctionInfo<StringCompareFn>(jit::StringsEqual<false>, "StringsEqual");
6798
6799 void
emitCompareS(LInstruction * lir,JSOp op,Register left,Register right,Register output)6800 CodeGenerator::emitCompareS(LInstruction* lir, JSOp op, Register left, Register right,
6801 Register output)
6802 {
6803 MOZ_ASSERT(lir->isCompareS() || lir->isCompareStrictS());
6804
6805 OutOfLineCode* ool = nullptr;
6806
6807 if (op == JSOP_EQ || op == JSOP_STRICTEQ) {
6808 ool = oolCallVM(StringsEqualInfo, lir, ArgList(left, right), StoreRegisterTo(output));
6809 } else {
6810 MOZ_ASSERT(op == JSOP_NE || op == JSOP_STRICTNE);
6811 ool = oolCallVM(StringsNotEqualInfo, lir, ArgList(left, right), StoreRegisterTo(output));
6812 }
6813
6814 masm.compareStrings(op, left, right, output, ool->entry());
6815
6816 masm.bind(ool->rejoin());
6817 }
6818
6819 void
visitCompareStrictS(LCompareStrictS * lir)6820 CodeGenerator::visitCompareStrictS(LCompareStrictS* lir)
6821 {
6822 JSOp op = lir->mir()->jsop();
6823 MOZ_ASSERT(op == JSOP_STRICTEQ || op == JSOP_STRICTNE);
6824
6825 const ValueOperand leftV = ToValue(lir, LCompareStrictS::Lhs);
6826 Register right = ToRegister(lir->right());
6827 Register output = ToRegister(lir->output());
6828 Register tempToUnbox = ToTempUnboxRegister(lir->tempToUnbox());
6829
6830 Label string, done;
6831
6832 masm.branchTestString(Assembler::Equal, leftV, &string);
6833 masm.move32(Imm32(op == JSOP_STRICTNE), output);
6834 masm.jump(&done);
6835
6836 masm.bind(&string);
6837 Register left = masm.extractString(leftV, tempToUnbox);
6838 emitCompareS(lir, op, left, right, output);
6839
6840 masm.bind(&done);
6841 }
6842
6843 void
visitCompareS(LCompareS * lir)6844 CodeGenerator::visitCompareS(LCompareS* lir)
6845 {
6846 JSOp op = lir->mir()->jsop();
6847 Register left = ToRegister(lir->left());
6848 Register right = ToRegister(lir->right());
6849 Register output = ToRegister(lir->output());
6850
6851 emitCompareS(lir, op, left, right, output);
6852 }
6853
6854 typedef bool (*CompareFn)(JSContext*, MutableHandleValue, MutableHandleValue, bool*);
6855 static const VMFunction EqInfo =
6856 FunctionInfo<CompareFn>(jit::LooselyEqual<true>, "LooselyEqual");
6857 static const VMFunction NeInfo =
6858 FunctionInfo<CompareFn>(jit::LooselyEqual<false>, "LooselyEqual");
6859 static const VMFunction StrictEqInfo =
6860 FunctionInfo<CompareFn>(jit::StrictlyEqual<true>, "StrictlyEqual");
6861 static const VMFunction StrictNeInfo =
6862 FunctionInfo<CompareFn>(jit::StrictlyEqual<false>, "StrictlyEqual");
6863 static const VMFunction LtInfo =
6864 FunctionInfo<CompareFn>(jit::LessThan, "LessThan");
6865 static const VMFunction LeInfo =
6866 FunctionInfo<CompareFn>(jit::LessThanOrEqual, "LessThanOrEqual");
6867 static const VMFunction GtInfo =
6868 FunctionInfo<CompareFn>(jit::GreaterThan, "GreaterThan");
6869 static const VMFunction GeInfo =
6870 FunctionInfo<CompareFn>(jit::GreaterThanOrEqual, "GreaterThanOrEqual");
6871
6872 void
visitCompareVM(LCompareVM * lir)6873 CodeGenerator::visitCompareVM(LCompareVM* lir)
6874 {
6875 pushArg(ToValue(lir, LBinaryV::RhsInput));
6876 pushArg(ToValue(lir, LBinaryV::LhsInput));
6877
6878 switch (lir->mir()->jsop()) {
6879 case JSOP_EQ:
6880 callVM(EqInfo, lir);
6881 break;
6882
6883 case JSOP_NE:
6884 callVM(NeInfo, lir);
6885 break;
6886
6887 case JSOP_STRICTEQ:
6888 callVM(StrictEqInfo, lir);
6889 break;
6890
6891 case JSOP_STRICTNE:
6892 callVM(StrictNeInfo, lir);
6893 break;
6894
6895 case JSOP_LT:
6896 callVM(LtInfo, lir);
6897 break;
6898
6899 case JSOP_LE:
6900 callVM(LeInfo, lir);
6901 break;
6902
6903 case JSOP_GT:
6904 callVM(GtInfo, lir);
6905 break;
6906
6907 case JSOP_GE:
6908 callVM(GeInfo, lir);
6909 break;
6910
6911 default:
6912 MOZ_CRASH("Unexpected compare op");
6913 }
6914 }
6915
6916 void
visitIsNullOrLikeUndefinedV(LIsNullOrLikeUndefinedV * lir)6917 CodeGenerator::visitIsNullOrLikeUndefinedV(LIsNullOrLikeUndefinedV* lir)
6918 {
6919 JSOp op = lir->mir()->jsop();
6920 MCompare::CompareType compareType = lir->mir()->compareType();
6921 MOZ_ASSERT(compareType == MCompare::Compare_Undefined ||
6922 compareType == MCompare::Compare_Null);
6923
6924 const ValueOperand value = ToValue(lir, LIsNullOrLikeUndefinedV::Value);
6925 Register output = ToRegister(lir->output());
6926
6927 if (op == JSOP_EQ || op == JSOP_NE) {
6928 MOZ_ASSERT(lir->mir()->lhs()->type() != MIRType::Object ||
6929 lir->mir()->operandMightEmulateUndefined(),
6930 "Operands which can't emulate undefined should have been folded");
6931
6932 OutOfLineTestObjectWithLabels* ool = nullptr;
6933 Maybe<Label> label1, label2;
6934 Label* nullOrLikeUndefined;
6935 Label* notNullOrLikeUndefined;
6936 if (lir->mir()->operandMightEmulateUndefined()) {
6937 ool = new(alloc()) OutOfLineTestObjectWithLabels();
6938 addOutOfLineCode(ool, lir->mir());
6939 nullOrLikeUndefined = ool->label1();
6940 notNullOrLikeUndefined = ool->label2();
6941 } else {
6942 label1.emplace();
6943 label2.emplace();
6944 nullOrLikeUndefined = label1.ptr();
6945 notNullOrLikeUndefined = label2.ptr();
6946 }
6947
6948 Register tag = masm.splitTagForTest(value);
6949 MDefinition* input = lir->mir()->lhs();
6950 if (input->mightBeType(MIRType::Null))
6951 masm.branchTestNull(Assembler::Equal, tag, nullOrLikeUndefined);
6952 if (input->mightBeType(MIRType::Undefined))
6953 masm.branchTestUndefined(Assembler::Equal, tag, nullOrLikeUndefined);
6954
6955 if (ool) {
6956 // Check whether it's a truthy object or a falsy object that emulates
6957 // undefined.
6958 masm.branchTestObject(Assembler::NotEqual, tag, notNullOrLikeUndefined);
6959
6960 Register objreg = masm.extractObject(value, ToTempUnboxRegister(lir->tempToUnbox()));
6961 branchTestObjectEmulatesUndefined(objreg, nullOrLikeUndefined, notNullOrLikeUndefined,
6962 ToRegister(lir->temp()), ool);
6963 // fall through
6964 }
6965
6966 Label done;
6967
6968 // It's not null or undefined, and if it's an object it doesn't
6969 // emulate undefined, so it's not like undefined.
6970 masm.move32(Imm32(op == JSOP_NE), output);
6971 masm.jump(&done);
6972
6973 masm.bind(nullOrLikeUndefined);
6974 masm.move32(Imm32(op == JSOP_EQ), output);
6975
6976 // Both branches meet here.
6977 masm.bind(&done);
6978 return;
6979 }
6980
6981 MOZ_ASSERT(op == JSOP_STRICTEQ || op == JSOP_STRICTNE);
6982
6983 Assembler::Condition cond = JSOpToCondition(compareType, op);
6984 if (compareType == MCompare::Compare_Null)
6985 masm.testNullSet(cond, value, output);
6986 else
6987 masm.testUndefinedSet(cond, value, output);
6988 }
6989
6990 void
visitIsNullOrLikeUndefinedAndBranchV(LIsNullOrLikeUndefinedAndBranchV * lir)6991 CodeGenerator::visitIsNullOrLikeUndefinedAndBranchV(LIsNullOrLikeUndefinedAndBranchV* lir)
6992 {
6993 JSOp op = lir->cmpMir()->jsop();
6994 MCompare::CompareType compareType = lir->cmpMir()->compareType();
6995 MOZ_ASSERT(compareType == MCompare::Compare_Undefined ||
6996 compareType == MCompare::Compare_Null);
6997
6998 const ValueOperand value = ToValue(lir, LIsNullOrLikeUndefinedAndBranchV::Value);
6999
7000 if (op == JSOP_EQ || op == JSOP_NE) {
7001 MBasicBlock* ifTrue;
7002 MBasicBlock* ifFalse;
7003
7004 if (op == JSOP_EQ) {
7005 ifTrue = lir->ifTrue();
7006 ifFalse = lir->ifFalse();
7007 } else {
7008 // Swap branches.
7009 ifTrue = lir->ifFalse();
7010 ifFalse = lir->ifTrue();
7011 op = JSOP_EQ;
7012 }
7013
7014 MOZ_ASSERT(lir->cmpMir()->lhs()->type() != MIRType::Object ||
7015 lir->cmpMir()->operandMightEmulateUndefined(),
7016 "Operands which can't emulate undefined should have been folded");
7017
7018 OutOfLineTestObject* ool = nullptr;
7019 if (lir->cmpMir()->operandMightEmulateUndefined()) {
7020 ool = new(alloc()) OutOfLineTestObject();
7021 addOutOfLineCode(ool, lir->cmpMir());
7022 }
7023
7024 Register tag = masm.splitTagForTest(value);
7025
7026 Label* ifTrueLabel = getJumpLabelForBranch(ifTrue);
7027 Label* ifFalseLabel = getJumpLabelForBranch(ifFalse);
7028
7029 MDefinition* input = lir->cmpMir()->lhs();
7030 if (input->mightBeType(MIRType::Null))
7031 masm.branchTestNull(Assembler::Equal, tag, ifTrueLabel);
7032 if (input->mightBeType(MIRType::Undefined))
7033 masm.branchTestUndefined(Assembler::Equal, tag, ifTrueLabel);
7034
7035 if (ool) {
7036 masm.branchTestObject(Assembler::NotEqual, tag, ifFalseLabel);
7037
7038 // Objects that emulate undefined are loosely equal to null/undefined.
7039 Register objreg = masm.extractObject(value, ToTempUnboxRegister(lir->tempToUnbox()));
7040 Register scratch = ToRegister(lir->temp());
7041 testObjectEmulatesUndefined(objreg, ifTrueLabel, ifFalseLabel, scratch, ool);
7042 } else {
7043 masm.jump(ifFalseLabel);
7044 }
7045 return;
7046 }
7047
7048 MOZ_ASSERT(op == JSOP_STRICTEQ || op == JSOP_STRICTNE);
7049
7050 Assembler::Condition cond = JSOpToCondition(compareType, op);
7051 if (compareType == MCompare::Compare_Null)
7052 testNullEmitBranch(cond, value, lir->ifTrue(), lir->ifFalse());
7053 else
7054 testUndefinedEmitBranch(cond, value, lir->ifTrue(), lir->ifFalse());
7055 }
7056
7057 void
visitIsNullOrLikeUndefinedT(LIsNullOrLikeUndefinedT * lir)7058 CodeGenerator::visitIsNullOrLikeUndefinedT(LIsNullOrLikeUndefinedT * lir)
7059 {
7060 MOZ_ASSERT(lir->mir()->compareType() == MCompare::Compare_Undefined ||
7061 lir->mir()->compareType() == MCompare::Compare_Null);
7062
7063 MIRType lhsType = lir->mir()->lhs()->type();
7064 MOZ_ASSERT(lhsType == MIRType::Object || lhsType == MIRType::ObjectOrNull);
7065
7066 JSOp op = lir->mir()->jsop();
7067 MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || op == JSOP_EQ || op == JSOP_NE,
7068 "Strict equality should have been folded");
7069
7070 MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || lir->mir()->operandMightEmulateUndefined(),
7071 "If the object couldn't emulate undefined, this should have been folded.");
7072
7073 Register objreg = ToRegister(lir->input());
7074 Register output = ToRegister(lir->output());
7075
7076 if ((op == JSOP_EQ || op == JSOP_NE) && lir->mir()->operandMightEmulateUndefined()) {
7077 OutOfLineTestObjectWithLabels* ool = new(alloc()) OutOfLineTestObjectWithLabels();
7078 addOutOfLineCode(ool, lir->mir());
7079
7080 Label* emulatesUndefined = ool->label1();
7081 Label* doesntEmulateUndefined = ool->label2();
7082
7083 if (lhsType == MIRType::ObjectOrNull)
7084 masm.branchTestPtr(Assembler::Zero, objreg, objreg, emulatesUndefined);
7085
7086 branchTestObjectEmulatesUndefined(objreg, emulatesUndefined, doesntEmulateUndefined,
7087 output, ool);
7088
7089 Label done;
7090
7091 masm.move32(Imm32(op == JSOP_NE), output);
7092 masm.jump(&done);
7093
7094 masm.bind(emulatesUndefined);
7095 masm.move32(Imm32(op == JSOP_EQ), output);
7096 masm.bind(&done);
7097 } else {
7098 MOZ_ASSERT(lhsType == MIRType::ObjectOrNull);
7099
7100 Label isNull, done;
7101
7102 masm.branchTestPtr(Assembler::Zero, objreg, objreg, &isNull);
7103
7104 masm.move32(Imm32(op == JSOP_NE || op == JSOP_STRICTNE), output);
7105 masm.jump(&done);
7106
7107 masm.bind(&isNull);
7108 masm.move32(Imm32(op == JSOP_EQ || op == JSOP_STRICTEQ), output);
7109
7110 masm.bind(&done);
7111 }
7112 }
7113
7114 void
visitIsNullOrLikeUndefinedAndBranchT(LIsNullOrLikeUndefinedAndBranchT * lir)7115 CodeGenerator::visitIsNullOrLikeUndefinedAndBranchT(LIsNullOrLikeUndefinedAndBranchT* lir)
7116 {
7117 DebugOnly<MCompare::CompareType> compareType = lir->cmpMir()->compareType();
7118 MOZ_ASSERT(compareType == MCompare::Compare_Undefined ||
7119 compareType == MCompare::Compare_Null);
7120
7121 MIRType lhsType = lir->cmpMir()->lhs()->type();
7122 MOZ_ASSERT(lhsType == MIRType::Object || lhsType == MIRType::ObjectOrNull);
7123
7124 JSOp op = lir->cmpMir()->jsop();
7125 MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || op == JSOP_EQ || op == JSOP_NE,
7126 "Strict equality should have been folded");
7127
7128 MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || lir->cmpMir()->operandMightEmulateUndefined(),
7129 "If the object couldn't emulate undefined, this should have been folded.");
7130
7131 MBasicBlock* ifTrue;
7132 MBasicBlock* ifFalse;
7133
7134 if (op == JSOP_EQ || op == JSOP_STRICTEQ) {
7135 ifTrue = lir->ifTrue();
7136 ifFalse = lir->ifFalse();
7137 } else {
7138 // Swap branches.
7139 ifTrue = lir->ifFalse();
7140 ifFalse = lir->ifTrue();
7141 }
7142
7143 Register input = ToRegister(lir->getOperand(0));
7144
7145 if ((op == JSOP_EQ || op == JSOP_NE) && lir->cmpMir()->operandMightEmulateUndefined()) {
7146 OutOfLineTestObject* ool = new(alloc()) OutOfLineTestObject();
7147 addOutOfLineCode(ool, lir->cmpMir());
7148
7149 Label* ifTrueLabel = getJumpLabelForBranch(ifTrue);
7150 Label* ifFalseLabel = getJumpLabelForBranch(ifFalse);
7151
7152 if (lhsType == MIRType::ObjectOrNull)
7153 masm.branchTestPtr(Assembler::Zero, input, input, ifTrueLabel);
7154
7155 // Objects that emulate undefined are loosely equal to null/undefined.
7156 Register scratch = ToRegister(lir->temp());
7157 testObjectEmulatesUndefined(input, ifTrueLabel, ifFalseLabel, scratch, ool);
7158 } else {
7159 MOZ_ASSERT(lhsType == MIRType::ObjectOrNull);
7160 testZeroEmitBranch(Assembler::Equal, input, ifTrue, ifFalse);
7161 }
7162 }
7163
7164 typedef JSString* (*ConcatStringsFn)(ExclusiveContext*, HandleString, HandleString);
7165 static const VMFunction ConcatStringsInfo =
7166 FunctionInfo<ConcatStringsFn>(ConcatStrings<CanGC>, "ConcatStrings");
7167
7168 void
emitConcat(LInstruction * lir,Register lhs,Register rhs,Register output)7169 CodeGenerator::emitConcat(LInstruction* lir, Register lhs, Register rhs, Register output)
7170 {
7171 OutOfLineCode* ool = oolCallVM(ConcatStringsInfo, lir, ArgList(lhs, rhs),
7172 StoreRegisterTo(output));
7173
7174 JitCode* stringConcatStub = gen->compartment->jitCompartment()->stringConcatStubNoBarrier();
7175 masm.call(stringConcatStub);
7176 masm.branchTestPtr(Assembler::Zero, output, output, ool->entry());
7177
7178 masm.bind(ool->rejoin());
7179 }
7180
7181 void
visitConcat(LConcat * lir)7182 CodeGenerator::visitConcat(LConcat* lir)
7183 {
7184 Register lhs = ToRegister(lir->lhs());
7185 Register rhs = ToRegister(lir->rhs());
7186
7187 Register output = ToRegister(lir->output());
7188
7189 MOZ_ASSERT(lhs == CallTempReg0);
7190 MOZ_ASSERT(rhs == CallTempReg1);
7191 MOZ_ASSERT(ToRegister(lir->temp1()) == CallTempReg0);
7192 MOZ_ASSERT(ToRegister(lir->temp2()) == CallTempReg1);
7193 MOZ_ASSERT(ToRegister(lir->temp3()) == CallTempReg2);
7194 MOZ_ASSERT(ToRegister(lir->temp4()) == CallTempReg3);
7195 MOZ_ASSERT(ToRegister(lir->temp5()) == CallTempReg4);
7196 MOZ_ASSERT(output == CallTempReg5);
7197
7198 emitConcat(lir, lhs, rhs, output);
7199 }
7200
7201 static void
CopyStringChars(MacroAssembler & masm,Register to,Register from,Register len,Register byteOpScratch,size_t fromWidth,size_t toWidth)7202 CopyStringChars(MacroAssembler& masm, Register to, Register from, Register len,
7203 Register byteOpScratch, size_t fromWidth, size_t toWidth)
7204 {
7205 // Copy |len| char16_t code units from |from| to |to|. Assumes len > 0
7206 // (checked below in debug builds), and when done |to| must point to the
7207 // next available char.
7208
7209 #ifdef DEBUG
7210 Label ok;
7211 masm.branch32(Assembler::GreaterThan, len, Imm32(0), &ok);
7212 masm.assumeUnreachable("Length should be greater than 0.");
7213 masm.bind(&ok);
7214 #endif
7215
7216 MOZ_ASSERT(fromWidth == 1 || fromWidth == 2);
7217 MOZ_ASSERT(toWidth == 1 || toWidth == 2);
7218 MOZ_ASSERT_IF(toWidth == 1, fromWidth == 1);
7219
7220 Label start;
7221 masm.bind(&start);
7222 if (fromWidth == 2)
7223 masm.load16ZeroExtend(Address(from, 0), byteOpScratch);
7224 else
7225 masm.load8ZeroExtend(Address(from, 0), byteOpScratch);
7226 if (toWidth == 2)
7227 masm.store16(byteOpScratch, Address(to, 0));
7228 else
7229 masm.store8(byteOpScratch, Address(to, 0));
7230 masm.addPtr(Imm32(fromWidth), from);
7231 masm.addPtr(Imm32(toWidth), to);
7232 masm.branchSub32(Assembler::NonZero, Imm32(1), len, &start);
7233 }
7234
7235 static void
CopyStringCharsMaybeInflate(MacroAssembler & masm,Register input,Register destChars,Register temp1,Register temp2)7236 CopyStringCharsMaybeInflate(MacroAssembler& masm, Register input, Register destChars,
7237 Register temp1, Register temp2)
7238 {
7239 // destChars is TwoByte and input is a Latin1 or TwoByte string, so we may
7240 // have to inflate.
7241
7242 Label isLatin1, done;
7243 masm.loadStringLength(input, temp1);
7244 masm.branchLatin1String(input, &isLatin1);
7245 {
7246 masm.loadStringChars(input, input);
7247 CopyStringChars(masm, destChars, input, temp1, temp2, sizeof(char16_t), sizeof(char16_t));
7248 masm.jump(&done);
7249 }
7250 masm.bind(&isLatin1);
7251 {
7252 masm.loadStringChars(input, input);
7253 CopyStringChars(masm, destChars, input, temp1, temp2, sizeof(char), sizeof(char16_t));
7254 }
7255 masm.bind(&done);
7256 }
7257
7258 static void
ConcatInlineString(MacroAssembler & masm,Register lhs,Register rhs,Register output,Register temp1,Register temp2,Register temp3,Label * failure,Label * failurePopTemps,bool isTwoByte)7259 ConcatInlineString(MacroAssembler& masm, Register lhs, Register rhs, Register output,
7260 Register temp1, Register temp2, Register temp3,
7261 Label* failure, Label* failurePopTemps, bool isTwoByte)
7262 {
7263 // State: result length in temp2.
7264
7265 // Ensure both strings are linear.
7266 masm.branchIfRope(lhs, failure);
7267 masm.branchIfRope(rhs, failure);
7268
7269 // Allocate a JSThinInlineString or JSFatInlineString.
7270 size_t maxThinInlineLength;
7271 if (isTwoByte)
7272 maxThinInlineLength = JSThinInlineString::MAX_LENGTH_TWO_BYTE;
7273 else
7274 maxThinInlineLength = JSThinInlineString::MAX_LENGTH_LATIN1;
7275
7276 Label isFat, allocDone;
7277 masm.branch32(Assembler::Above, temp2, Imm32(maxThinInlineLength), &isFat);
7278 {
7279 uint32_t flags = JSString::INIT_THIN_INLINE_FLAGS;
7280 if (!isTwoByte)
7281 flags |= JSString::LATIN1_CHARS_BIT;
7282 masm.newGCString(output, temp1, failure);
7283 masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
7284 masm.jump(&allocDone);
7285 }
7286 masm.bind(&isFat);
7287 {
7288 uint32_t flags = JSString::INIT_FAT_INLINE_FLAGS;
7289 if (!isTwoByte)
7290 flags |= JSString::LATIN1_CHARS_BIT;
7291 masm.newGCFatInlineString(output, temp1, failure);
7292 masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
7293 }
7294 masm.bind(&allocDone);
7295
7296 // Store length.
7297 masm.store32(temp2, Address(output, JSString::offsetOfLength()));
7298
7299 // Load chars pointer in temp2.
7300 masm.computeEffectiveAddress(Address(output, JSInlineString::offsetOfInlineStorage()), temp2);
7301
7302 {
7303 // Copy lhs chars. Note that this advances temp2 to point to the next
7304 // char. This also clobbers the lhs register.
7305 if (isTwoByte) {
7306 CopyStringCharsMaybeInflate(masm, lhs, temp2, temp1, temp3);
7307 } else {
7308 masm.loadStringLength(lhs, temp3);
7309 masm.loadStringChars(lhs, lhs);
7310 CopyStringChars(masm, temp2, lhs, temp3, temp1, sizeof(char), sizeof(char));
7311 }
7312
7313 // Copy rhs chars. Clobbers the rhs register.
7314 if (isTwoByte) {
7315 CopyStringCharsMaybeInflate(masm, rhs, temp2, temp1, temp3);
7316 } else {
7317 masm.loadStringLength(rhs, temp3);
7318 masm.loadStringChars(rhs, rhs);
7319 CopyStringChars(masm, temp2, rhs, temp3, temp1, sizeof(char), sizeof(char));
7320 }
7321
7322 // Null-terminate.
7323 if (isTwoByte)
7324 masm.store16(Imm32(0), Address(temp2, 0));
7325 else
7326 masm.store8(Imm32(0), Address(temp2, 0));
7327 }
7328
7329 masm.ret();
7330 }
7331
7332 typedef JSString* (*SubstringKernelFn)(JSContext* cx, HandleString str, int32_t begin, int32_t len);
7333 static const VMFunction SubstringKernelInfo =
7334 FunctionInfo<SubstringKernelFn>(SubstringKernel, "SubstringKernel");
7335
7336 void
visitSubstr(LSubstr * lir)7337 CodeGenerator::visitSubstr(LSubstr* lir)
7338 {
7339 Register string = ToRegister(lir->string());
7340 Register begin = ToRegister(lir->begin());
7341 Register length = ToRegister(lir->length());
7342 Register output = ToRegister(lir->output());
7343 Register temp = ToRegister(lir->temp());
7344 Register temp3 = ToRegister(lir->temp3());
7345
7346 // On x86 there are not enough registers. In that case reuse the string
7347 // register as temporary.
7348 Register temp2 = lir->temp2()->isBogusTemp() ? string : ToRegister(lir->temp2());
7349
7350 Address stringFlags(string, JSString::offsetOfFlags());
7351
7352 Label isLatin1, notInline, nonZero, isInlinedLatin1;
7353
7354 // For every edge case use the C++ variant.
7355 // Note: we also use this upon allocation failure in newGCString and
7356 // newGCFatInlineString. To squeeze out even more performance those failures
7357 // can be handled by allocate in ool code and returning to jit code to fill
7358 // in all data.
7359 OutOfLineCode* ool = oolCallVM(SubstringKernelInfo, lir,
7360 ArgList(string, begin, length),
7361 StoreRegisterTo(output));
7362 Label* slowPath = ool->entry();
7363 Label* done = ool->rejoin();
7364
7365 // Zero length, return emptystring.
7366 masm.branchTest32(Assembler::NonZero, length, length, &nonZero);
7367 const JSAtomState& names = GetJitContext()->runtime->names();
7368 masm.movePtr(ImmGCPtr(names.empty), output);
7369 masm.jump(done);
7370
7371 // Use slow path for ropes.
7372 masm.bind(&nonZero);
7373 masm.branchIfRopeOrExternal(string, temp, slowPath);
7374
7375 // Handle inlined strings by creating a FatInlineString.
7376 masm.branchTest32(Assembler::Zero, stringFlags, Imm32(JSString::INLINE_CHARS_BIT), ¬Inline);
7377 masm.newGCFatInlineString(output, temp, slowPath);
7378 masm.store32(length, Address(output, JSString::offsetOfLength()));
7379 Address stringStorage(string, JSInlineString::offsetOfInlineStorage());
7380 Address outputStorage(output, JSInlineString::offsetOfInlineStorage());
7381
7382 masm.branchLatin1String(string, &isInlinedLatin1);
7383 {
7384 masm.store32(Imm32(JSString::INIT_FAT_INLINE_FLAGS),
7385 Address(output, JSString::offsetOfFlags()));
7386 masm.computeEffectiveAddress(stringStorage, temp);
7387 if (temp2 == string)
7388 masm.push(string);
7389 BaseIndex chars(temp, begin, ScaleFromElemWidth(sizeof(char16_t)));
7390 masm.computeEffectiveAddress(chars, temp2);
7391 masm.computeEffectiveAddress(outputStorage, temp);
7392 CopyStringChars(masm, temp, temp2, length, temp3, sizeof(char16_t), sizeof(char16_t));
7393 masm.load32(Address(output, JSString::offsetOfLength()), length);
7394 masm.store16(Imm32(0), Address(temp, 0));
7395 if (temp2 == string)
7396 masm.pop(string);
7397 masm.jump(done);
7398 }
7399 masm.bind(&isInlinedLatin1);
7400 {
7401 masm.store32(Imm32(JSString::INIT_FAT_INLINE_FLAGS | JSString::LATIN1_CHARS_BIT),
7402 Address(output, JSString::offsetOfFlags()));
7403 if (temp2 == string)
7404 masm.push(string);
7405 masm.computeEffectiveAddress(stringStorage, temp2);
7406 static_assert(sizeof(char) == 1, "begin index shouldn't need scaling");
7407 masm.addPtr(begin, temp2);
7408 masm.computeEffectiveAddress(outputStorage, temp);
7409 CopyStringChars(masm, temp, temp2, length, temp3, sizeof(char), sizeof(char));
7410 masm.load32(Address(output, JSString::offsetOfLength()), length);
7411 masm.store8(Imm32(0), Address(temp, 0));
7412 if (temp2 == string)
7413 masm.pop(string);
7414 masm.jump(done);
7415 }
7416
7417 // Handle other cases with a DependentString.
7418 masm.bind(¬Inline);
7419 masm.newGCString(output, temp, slowPath);
7420 masm.store32(length, Address(output, JSString::offsetOfLength()));
7421 masm.storePtr(string, Address(output, JSDependentString::offsetOfBase()));
7422
7423 masm.branchLatin1String(string, &isLatin1);
7424 {
7425 masm.store32(Imm32(JSString::DEPENDENT_FLAGS), Address(output, JSString::offsetOfFlags()));
7426 masm.loadPtr(Address(string, JSString::offsetOfNonInlineChars()), temp);
7427 BaseIndex chars(temp, begin, ScaleFromElemWidth(sizeof(char16_t)));
7428 masm.computeEffectiveAddress(chars, temp);
7429 masm.storePtr(temp, Address(output, JSString::offsetOfNonInlineChars()));
7430 masm.jump(done);
7431 }
7432 masm.bind(&isLatin1);
7433 {
7434 masm.store32(Imm32(JSString::DEPENDENT_FLAGS | JSString::LATIN1_CHARS_BIT),
7435 Address(output, JSString::offsetOfFlags()));
7436 masm.loadPtr(Address(string, JSString::offsetOfNonInlineChars()), temp);
7437 static_assert(sizeof(char) == 1, "begin index shouldn't need scaling");
7438 masm.addPtr(begin, temp);
7439 masm.storePtr(temp, Address(output, JSString::offsetOfNonInlineChars()));
7440 masm.jump(done);
7441 }
7442
7443 masm.bind(done);
7444 }
7445
7446 JitCode*
generateStringConcatStub(JSContext * cx)7447 JitCompartment::generateStringConcatStub(JSContext* cx)
7448 {
7449 MacroAssembler masm(cx);
7450
7451 Register lhs = CallTempReg0;
7452 Register rhs = CallTempReg1;
7453 Register temp1 = CallTempReg2;
7454 Register temp2 = CallTempReg3;
7455 Register temp3 = CallTempReg4;
7456 Register output = CallTempReg5;
7457
7458 Label failure, failurePopTemps;
7459 #ifdef JS_USE_LINK_REGISTER
7460 masm.pushReturnAddress();
7461 #endif
7462 // If lhs is empty, return rhs.
7463 Label leftEmpty;
7464 masm.loadStringLength(lhs, temp1);
7465 masm.branchTest32(Assembler::Zero, temp1, temp1, &leftEmpty);
7466
7467 // If rhs is empty, return lhs.
7468 Label rightEmpty;
7469 masm.loadStringLength(rhs, temp2);
7470 masm.branchTest32(Assembler::Zero, temp2, temp2, &rightEmpty);
7471
7472 masm.add32(temp1, temp2);
7473
7474 // Check if we can use a JSFatInlineString. The result is a Latin1 string if
7475 // lhs and rhs are both Latin1, so we AND the flags.
7476 Label isFatInlineTwoByte, isFatInlineLatin1;
7477 masm.load32(Address(lhs, JSString::offsetOfFlags()), temp1);
7478 masm.and32(Address(rhs, JSString::offsetOfFlags()), temp1);
7479
7480 Label isLatin1, notInline;
7481 masm.branchTest32(Assembler::NonZero, temp1, Imm32(JSString::LATIN1_CHARS_BIT), &isLatin1);
7482 {
7483 masm.branch32(Assembler::BelowOrEqual, temp2, Imm32(JSFatInlineString::MAX_LENGTH_TWO_BYTE),
7484 &isFatInlineTwoByte);
7485 masm.jump(¬Inline);
7486 }
7487 masm.bind(&isLatin1);
7488 {
7489 masm.branch32(Assembler::BelowOrEqual, temp2, Imm32(JSFatInlineString::MAX_LENGTH_LATIN1),
7490 &isFatInlineLatin1);
7491 }
7492 masm.bind(¬Inline);
7493
7494 // Keep AND'ed flags in temp1.
7495
7496 // Ensure result length <= JSString::MAX_LENGTH.
7497 masm.branch32(Assembler::Above, temp2, Imm32(JSString::MAX_LENGTH), &failure);
7498
7499 // Allocate a new rope.
7500 masm.newGCString(output, temp3, &failure);
7501
7502 // Store rope length and flags. temp1 still holds the result of AND'ing the
7503 // lhs and rhs flags, so we just have to clear the other flags to get our
7504 // rope flags (Latin1 if both lhs and rhs are Latin1).
7505 static_assert(JSString::ROPE_FLAGS == 0, "Rope flags must be 0");
7506 masm.and32(Imm32(JSString::LATIN1_CHARS_BIT), temp1);
7507 masm.store32(temp1, Address(output, JSString::offsetOfFlags()));
7508 masm.store32(temp2, Address(output, JSString::offsetOfLength()));
7509
7510 // Store left and right nodes.
7511 masm.storePtr(lhs, Address(output, JSRope::offsetOfLeft()));
7512 masm.storePtr(rhs, Address(output, JSRope::offsetOfRight()));
7513 masm.ret();
7514
7515 masm.bind(&leftEmpty);
7516 masm.mov(rhs, output);
7517 masm.ret();
7518
7519 masm.bind(&rightEmpty);
7520 masm.mov(lhs, output);
7521 masm.ret();
7522
7523 masm.bind(&isFatInlineTwoByte);
7524 ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3,
7525 &failure, &failurePopTemps, true);
7526
7527 masm.bind(&isFatInlineLatin1);
7528 ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3,
7529 &failure, &failurePopTemps, false);
7530
7531 masm.bind(&failurePopTemps);
7532 masm.pop(temp2);
7533 masm.pop(temp1);
7534
7535 masm.bind(&failure);
7536 masm.movePtr(ImmPtr(nullptr), output);
7537 masm.ret();
7538
7539 Linker linker(masm);
7540 AutoFlushICache afc("StringConcatStub");
7541 JitCode* code = linker.newCode<CanGC>(cx, OTHER_CODE);
7542
7543 #ifdef JS_ION_PERF
7544 writePerfSpewerJitCodeProfile(code, "StringConcatStub");
7545 #endif
7546
7547 return code;
7548 }
7549
7550 JitCode*
generateMallocStub(JSContext * cx)7551 JitRuntime::generateMallocStub(JSContext* cx)
7552 {
7553 const Register regReturn = CallTempReg0;
7554 const Register regNBytes = CallTempReg0;
7555
7556 MacroAssembler masm(cx);
7557
7558 AllocatableRegisterSet regs(RegisterSet::Volatile());
7559 #ifdef JS_USE_LINK_REGISTER
7560 masm.pushReturnAddress();
7561 #endif
7562 regs.takeUnchecked(regNBytes);
7563 LiveRegisterSet save(regs.asLiveSet());
7564 masm.PushRegsInMask(save);
7565
7566 const Register regTemp = regs.takeAnyGeneral();
7567 const Register regRuntime = regTemp;
7568 MOZ_ASSERT(regTemp != regNBytes);
7569
7570 masm.setupUnalignedABICall(regTemp);
7571 masm.movePtr(ImmPtr(cx->runtime()), regRuntime);
7572 masm.passABIArg(regRuntime);
7573 masm.passABIArg(regNBytes);
7574 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, MallocWrapper));
7575 masm.storeCallPointerResult(regReturn);
7576
7577 masm.PopRegsInMask(save);
7578 masm.ret();
7579
7580 Linker linker(masm);
7581 AutoFlushICache afc("MallocStub");
7582 JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE);
7583
7584 #ifdef JS_ION_PERF
7585 writePerfSpewerJitCodeProfile(code, "MallocStub");
7586 #endif
7587
7588 return code;
7589 }
7590
7591 JitCode*
generateFreeStub(JSContext * cx)7592 JitRuntime::generateFreeStub(JSContext* cx)
7593 {
7594 const Register regSlots = CallTempReg0;
7595
7596 MacroAssembler masm(cx);
7597 #ifdef JS_USE_LINK_REGISTER
7598 masm.pushReturnAddress();
7599 #endif
7600 AllocatableRegisterSet regs(RegisterSet::Volatile());
7601 regs.takeUnchecked(regSlots);
7602 LiveRegisterSet save(regs.asLiveSet());
7603 masm.PushRegsInMask(save);
7604
7605 const Register regTemp = regs.takeAnyGeneral();
7606 MOZ_ASSERT(regTemp != regSlots);
7607
7608 masm.setupUnalignedABICall(regTemp);
7609 masm.passABIArg(regSlots);
7610 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js_free));
7611
7612 masm.PopRegsInMask(save);
7613
7614 masm.ret();
7615
7616 Linker linker(masm);
7617 AutoFlushICache afc("FreeStub");
7618 JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE);
7619
7620 #ifdef JS_ION_PERF
7621 writePerfSpewerJitCodeProfile(code, "FreeStub");
7622 #endif
7623
7624 return code;
7625 }
7626
7627
7628 JitCode*
generateLazyLinkStub(JSContext * cx)7629 JitRuntime::generateLazyLinkStub(JSContext* cx)
7630 {
7631 MacroAssembler masm(cx);
7632 #ifdef JS_USE_LINK_REGISTER
7633 masm.pushReturnAddress();
7634 #endif
7635
7636 AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
7637 Register temp0 = regs.takeAny();
7638
7639 masm.enterFakeExitFrame(LazyLinkExitFrameLayoutToken);
7640 masm.PushStubCode();
7641
7642 masm.setupUnalignedABICall(temp0);
7643 masm.loadJSContext(temp0);
7644 masm.passABIArg(temp0);
7645 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, LazyLinkTopActivation));
7646
7647 masm.leaveExitFrame(/* stub code */ sizeof(JitCode*));
7648
7649 #ifdef JS_USE_LINK_REGISTER
7650 // Restore the return address such that the emitPrologue function of the
7651 // CodeGenerator can push it back on the stack with pushReturnAddress.
7652 masm.popReturnAddress();
7653 #endif
7654 masm.jump(ReturnReg);
7655
7656 Linker linker(masm);
7657 AutoFlushICache afc("LazyLinkStub");
7658 JitCode* code = linker.newCode<NoGC>(cx, OTHER_CODE);
7659
7660 #ifdef JS_ION_PERF
7661 writePerfSpewerJitCodeProfile(code, "LazyLinkStub");
7662 #endif
7663 return code;
7664 }
7665
7666 bool
generateTLEventVM(JSContext * cx,MacroAssembler & masm,const VMFunction & f,bool enter)7667 JitRuntime::generateTLEventVM(JSContext* cx, MacroAssembler& masm, const VMFunction& f,
7668 bool enter)
7669 {
7670 #ifdef JS_TRACE_LOGGING
7671 TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime());
7672
7673 bool vmEventEnabled = TraceLogTextIdEnabled(TraceLogger_VM);
7674 bool vmSpecificEventEnabled = TraceLogTextIdEnabled(TraceLogger_VMSpecific);
7675
7676 if (vmEventEnabled || vmSpecificEventEnabled) {
7677 AllocatableRegisterSet regs(RegisterSet::Volatile());
7678 Register loggerReg = regs.takeAnyGeneral();
7679 masm.Push(loggerReg);
7680 masm.movePtr(ImmPtr(logger), loggerReg);
7681
7682 if (vmEventEnabled) {
7683 if (enter)
7684 masm.tracelogStartId(loggerReg, TraceLogger_VM, /* force = */ true);
7685 else
7686 masm.tracelogStopId(loggerReg, TraceLogger_VM, /* force = */ true);
7687 }
7688 if (vmSpecificEventEnabled) {
7689 TraceLoggerEvent event(logger, f.name());
7690 if (!event.hasPayload())
7691 return false;
7692
7693 if (enter)
7694 masm.tracelogStartId(loggerReg, event.payload()->textId(), /* force = */ true);
7695 else
7696 masm.tracelogStopId(loggerReg, event.payload()->textId(), /* force = */ true);
7697 }
7698
7699 masm.Pop(loggerReg);
7700 }
7701 #endif
7702
7703 return true;
7704 }
7705
7706 typedef bool (*CharCodeAtFn)(JSContext*, HandleString, int32_t, uint32_t*);
7707 static const VMFunction CharCodeAtInfo =
7708 FunctionInfo<CharCodeAtFn>(jit::CharCodeAt, "CharCodeAt");
7709
7710 void
visitCharCodeAt(LCharCodeAt * lir)7711 CodeGenerator::visitCharCodeAt(LCharCodeAt* lir)
7712 {
7713 Register str = ToRegister(lir->str());
7714 Register index = ToRegister(lir->index());
7715 Register output = ToRegister(lir->output());
7716
7717 OutOfLineCode* ool = oolCallVM(CharCodeAtInfo, lir, ArgList(str, index), StoreRegisterTo(output));
7718
7719 masm.branchIfRope(str, ool->entry());
7720 masm.loadStringChar(str, index, output);
7721
7722 masm.bind(ool->rejoin());
7723 }
7724
7725 typedef JSFlatString* (*StringFromCharCodeFn)(JSContext*, int32_t);
7726 static const VMFunction StringFromCharCodeInfo =
7727 FunctionInfo<StringFromCharCodeFn>(jit::StringFromCharCode, "StringFromCharCode");
7728
7729 void
visitFromCharCode(LFromCharCode * lir)7730 CodeGenerator::visitFromCharCode(LFromCharCode* lir)
7731 {
7732 Register code = ToRegister(lir->code());
7733 Register output = ToRegister(lir->output());
7734
7735 OutOfLineCode* ool = oolCallVM(StringFromCharCodeInfo, lir, ArgList(code), StoreRegisterTo(output));
7736
7737 // OOL path if code >= UNIT_STATIC_LIMIT.
7738 masm.branch32(Assembler::AboveOrEqual, code, Imm32(StaticStrings::UNIT_STATIC_LIMIT),
7739 ool->entry());
7740
7741 masm.movePtr(ImmPtr(&GetJitContext()->runtime->staticStrings().unitStaticTable), output);
7742 masm.loadPtr(BaseIndex(output, code, ScalePointer), output);
7743
7744 masm.bind(ool->rejoin());
7745 }
7746
7747 typedef JSString* (*StringFromCodePointFn)(JSContext*, int32_t);
7748 static const VMFunction StringFromCodePointInfo =
7749 FunctionInfo<StringFromCodePointFn>(jit::StringFromCodePoint, "StringFromCodePoint");
7750
7751 void
visitFromCodePoint(LFromCodePoint * lir)7752 CodeGenerator::visitFromCodePoint(LFromCodePoint* lir)
7753 {
7754 Register codePoint = ToRegister(lir->codePoint());
7755 Register output = ToRegister(lir->output());
7756 LSnapshot* snapshot = lir->snapshot();
7757
7758 OutOfLineCode* ool = oolCallVM(StringFromCodePointInfo, lir, ArgList(codePoint),
7759 StoreRegisterTo(output));
7760
7761 // Use a bailout if the input is not a valid code point, because
7762 // MFromCodePoint is movable and it'd be observable when a moved
7763 // fromCodePoint throws an exception before its actual call site.
7764 bailoutCmp32(Assembler::Above, codePoint, Imm32(unicode::NonBMPMax), snapshot);
7765
7766 // OOL path if code point >= UNIT_STATIC_LIMIT.
7767 masm.branch32(Assembler::AboveOrEqual, codePoint, Imm32(StaticStrings::UNIT_STATIC_LIMIT),
7768 ool->entry());
7769
7770 masm.movePtr(ImmPtr(&GetJitContext()->runtime->staticStrings().unitStaticTable), output);
7771 masm.loadPtr(BaseIndex(output, codePoint, ScalePointer), output);
7772
7773 masm.bind(ool->rejoin());
7774 }
7775
7776 void
visitSinCos(LSinCos * lir)7777 CodeGenerator::visitSinCos(LSinCos *lir)
7778 {
7779 Register temp = ToRegister(lir->temp());
7780 Register params = ToRegister(lir->temp2());
7781 FloatRegister input = ToFloatRegister(lir->input());
7782 FloatRegister outputSin = ToFloatRegister(lir->outputSin());
7783 FloatRegister outputCos = ToFloatRegister(lir->outputCos());
7784
7785 masm.reserveStack(sizeof(double) * 2);
7786 masm.movePtr(masm.getStackPointer(), params);
7787
7788 const MathCache* mathCache = lir->mir()->cache();
7789
7790 masm.setupUnalignedABICall(temp);
7791 if (mathCache) {
7792 masm.movePtr(ImmPtr(mathCache), temp);
7793 masm.passABIArg(temp);
7794 }
7795
7796 #define MAYBE_CACHED_(fcn) (mathCache ? (void*)fcn ## _impl : (void*)fcn ## _uncached)
7797
7798 masm.passABIArg(input, MoveOp::DOUBLE);
7799 masm.passABIArg(MoveOperand(params, sizeof(double), MoveOperand::EFFECTIVE_ADDRESS),
7800 MoveOp::GENERAL);
7801 masm.passABIArg(MoveOperand(params, 0, MoveOperand::EFFECTIVE_ADDRESS),
7802 MoveOp::GENERAL);
7803
7804 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED_(js::math_sincos)));
7805 #undef MAYBE_CACHED_
7806
7807 masm.loadDouble(Address(masm.getStackPointer(), 0), outputCos);
7808 masm.loadDouble(Address(masm.getStackPointer(), sizeof(double)), outputSin);
7809 masm.freeStack(sizeof(double) * 2);
7810 }
7811
7812 typedef JSObject* (*StringSplitFn)(JSContext*, HandleObjectGroup, HandleString, HandleString, uint32_t);
7813 static const VMFunction StringSplitInfo =
7814 FunctionInfo<StringSplitFn>(js::str_split_string, "str_split_string");
7815
7816 void
visitStringSplit(LStringSplit * lir)7817 CodeGenerator::visitStringSplit(LStringSplit* lir)
7818 {
7819 pushArg(Imm32(INT32_MAX));
7820 pushArg(ToRegister(lir->separator()));
7821 pushArg(ToRegister(lir->string()));
7822 pushArg(ImmGCPtr(lir->mir()->group()));
7823
7824 callVM(StringSplitInfo, lir);
7825 }
7826
7827 void
visitInitializedLength(LInitializedLength * lir)7828 CodeGenerator::visitInitializedLength(LInitializedLength* lir)
7829 {
7830 Address initLength(ToRegister(lir->elements()), ObjectElements::offsetOfInitializedLength());
7831 masm.load32(initLength, ToRegister(lir->output()));
7832 }
7833
7834 void
visitSetInitializedLength(LSetInitializedLength * lir)7835 CodeGenerator::visitSetInitializedLength(LSetInitializedLength* lir)
7836 {
7837 Address initLength(ToRegister(lir->elements()), ObjectElements::offsetOfInitializedLength());
7838 RegisterOrInt32Constant index = ToRegisterOrInt32Constant(lir->index());
7839
7840 masm.inc32(&index);
7841 masm.store32(index, initLength);
7842 // Restore register value if it is used/captured after.
7843 masm.dec32(&index);
7844 }
7845
7846 void
visitUnboxedArrayLength(LUnboxedArrayLength * lir)7847 CodeGenerator::visitUnboxedArrayLength(LUnboxedArrayLength* lir)
7848 {
7849 Register obj = ToRegister(lir->object());
7850 Register result = ToRegister(lir->output());
7851 masm.load32(Address(obj, UnboxedArrayObject::offsetOfLength()), result);
7852 }
7853
7854 void
visitUnboxedArrayInitializedLength(LUnboxedArrayInitializedLength * lir)7855 CodeGenerator::visitUnboxedArrayInitializedLength(LUnboxedArrayInitializedLength* lir)
7856 {
7857 Register obj = ToRegister(lir->object());
7858 Register result = ToRegister(lir->output());
7859 masm.load32(Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()), result);
7860 masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), result);
7861 }
7862
7863 void
visitIncrementUnboxedArrayInitializedLength(LIncrementUnboxedArrayInitializedLength * lir)7864 CodeGenerator::visitIncrementUnboxedArrayInitializedLength(LIncrementUnboxedArrayInitializedLength* lir)
7865 {
7866 Register obj = ToRegister(lir->object());
7867 masm.add32(Imm32(1), Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()));
7868 }
7869
7870 void
visitSetUnboxedArrayInitializedLength(LSetUnboxedArrayInitializedLength * lir)7871 CodeGenerator::visitSetUnboxedArrayInitializedLength(LSetUnboxedArrayInitializedLength* lir)
7872 {
7873 Register obj = ToRegister(lir->object());
7874 RegisterOrInt32Constant key = ToRegisterOrInt32Constant(lir->length());
7875 Register temp = ToRegister(lir->temp());
7876
7877 Address initLengthAddr(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength());
7878 masm.load32(initLengthAddr, temp);
7879 masm.and32(Imm32(UnboxedArrayObject::CapacityMask), temp);
7880
7881 if (key.isRegister())
7882 masm.or32(key.reg(), temp);
7883 else
7884 masm.or32(Imm32(key.constant()), temp);
7885
7886 masm.store32(temp, initLengthAddr);
7887 }
7888
7889 void
visitNotO(LNotO * lir)7890 CodeGenerator::visitNotO(LNotO* lir)
7891 {
7892 MOZ_ASSERT(lir->mir()->operandMightEmulateUndefined(),
7893 "This should be constant-folded if the object can't emulate undefined.");
7894
7895 OutOfLineTestObjectWithLabels* ool = new(alloc()) OutOfLineTestObjectWithLabels();
7896 addOutOfLineCode(ool, lir->mir());
7897
7898 Label* ifEmulatesUndefined = ool->label1();
7899 Label* ifDoesntEmulateUndefined = ool->label2();
7900
7901 Register objreg = ToRegister(lir->input());
7902 Register output = ToRegister(lir->output());
7903 branchTestObjectEmulatesUndefined(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
7904 output, ool);
7905 // fall through
7906
7907 Label join;
7908
7909 masm.move32(Imm32(0), output);
7910 masm.jump(&join);
7911
7912 masm.bind(ifEmulatesUndefined);
7913 masm.move32(Imm32(1), output);
7914
7915 masm.bind(&join);
7916 }
7917
7918 void
visitNotV(LNotV * lir)7919 CodeGenerator::visitNotV(LNotV* lir)
7920 {
7921 Maybe<Label> ifTruthyLabel, ifFalsyLabel;
7922 Label* ifTruthy;
7923 Label* ifFalsy;
7924
7925 OutOfLineTestObjectWithLabels* ool = nullptr;
7926 MDefinition* operand = lir->mir()->input();
7927 // Unfortunately, it's possible that someone (e.g. phi elimination) switched
7928 // out our operand after we did cacheOperandMightEmulateUndefined. So we
7929 // might think it can emulate undefined _and_ know that it can't be an
7930 // object.
7931 if (lir->mir()->operandMightEmulateUndefined() && operand->mightBeType(MIRType::Object)) {
7932 ool = new(alloc()) OutOfLineTestObjectWithLabels();
7933 addOutOfLineCode(ool, lir->mir());
7934 ifTruthy = ool->label1();
7935 ifFalsy = ool->label2();
7936 } else {
7937 ifTruthyLabel.emplace();
7938 ifFalsyLabel.emplace();
7939 ifTruthy = ifTruthyLabel.ptr();
7940 ifFalsy = ifFalsyLabel.ptr();
7941 }
7942
7943 testValueTruthyKernel(ToValue(lir, LNotV::Input), lir->temp1(), lir->temp2(),
7944 ToFloatRegister(lir->tempFloat()),
7945 ifTruthy, ifFalsy, ool, operand);
7946
7947 Label join;
7948 Register output = ToRegister(lir->output());
7949
7950 // Note that the testValueTruthyKernel call above may choose to fall through
7951 // to ifTruthy instead of branching there.
7952 masm.bind(ifTruthy);
7953 masm.move32(Imm32(0), output);
7954 masm.jump(&join);
7955
7956 masm.bind(ifFalsy);
7957 masm.move32(Imm32(1), output);
7958
7959 // both branches meet here.
7960 masm.bind(&join);
7961 }
7962
7963 void
visitBoundsCheck(LBoundsCheck * lir)7964 CodeGenerator::visitBoundsCheck(LBoundsCheck* lir)
7965 {
7966 const LAllocation* index = lir->index();
7967 const LAllocation* length = lir->length();
7968 LSnapshot* snapshot = lir->snapshot();
7969
7970 if (index->isConstant()) {
7971 // Use uint32 so that the comparison is unsigned.
7972 uint32_t idx = ToInt32(index);
7973 if (length->isConstant()) {
7974 uint32_t len = ToInt32(lir->length());
7975 if (idx < len)
7976 return;
7977 bailout(snapshot);
7978 return;
7979 }
7980
7981 if (length->isRegister())
7982 bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), Imm32(idx), snapshot);
7983 else
7984 bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), Imm32(idx), snapshot);
7985 return;
7986 }
7987
7988 Register indexReg = ToRegister(index);
7989 if (length->isConstant())
7990 bailoutCmp32(Assembler::AboveOrEqual, indexReg, Imm32(ToInt32(length)), snapshot);
7991 else if (length->isRegister())
7992 bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), indexReg, snapshot);
7993 else
7994 bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), indexReg, snapshot);
7995 }
7996
7997 void
visitBoundsCheckRange(LBoundsCheckRange * lir)7998 CodeGenerator::visitBoundsCheckRange(LBoundsCheckRange* lir)
7999 {
8000 int32_t min = lir->mir()->minimum();
8001 int32_t max = lir->mir()->maximum();
8002 MOZ_ASSERT(max >= min);
8003
8004 const LAllocation* length = lir->length();
8005 LSnapshot* snapshot = lir->snapshot();
8006 Register temp = ToRegister(lir->getTemp(0));
8007 if (lir->index()->isConstant()) {
8008 int32_t nmin, nmax;
8009 int32_t index = ToInt32(lir->index());
8010 if (SafeAdd(index, min, &nmin) && SafeAdd(index, max, &nmax) && nmin >= 0) {
8011 if (length->isRegister())
8012 bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), Imm32(nmax), snapshot);
8013 else
8014 bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), Imm32(nmax), snapshot);
8015 return;
8016 }
8017 masm.mov(ImmWord(index), temp);
8018 } else {
8019 masm.mov(ToRegister(lir->index()), temp);
8020 }
8021
8022 // If the minimum and maximum differ then do an underflow check first.
8023 // If the two are the same then doing an unsigned comparison on the
8024 // length will also catch a negative index.
8025 if (min != max) {
8026 if (min != 0) {
8027 Label bail;
8028 masm.branchAdd32(Assembler::Overflow, Imm32(min), temp, &bail);
8029 bailoutFrom(&bail, snapshot);
8030 }
8031
8032 bailoutCmp32(Assembler::LessThan, temp, Imm32(0), snapshot);
8033
8034 if (min != 0) {
8035 int32_t diff;
8036 if (SafeSub(max, min, &diff))
8037 max = diff;
8038 else
8039 masm.sub32(Imm32(min), temp);
8040 }
8041 }
8042
8043 // Compute the maximum possible index. No overflow check is needed when
8044 // max > 0. We can only wraparound to a negative number, which will test as
8045 // larger than all nonnegative numbers in the unsigned comparison, and the
8046 // length is required to be nonnegative (else testing a negative length
8047 // would succeed on any nonnegative index).
8048 if (max != 0) {
8049 if (max < 0) {
8050 Label bail;
8051 masm.branchAdd32(Assembler::Overflow, Imm32(max), temp, &bail);
8052 bailoutFrom(&bail, snapshot);
8053 } else {
8054 masm.add32(Imm32(max), temp);
8055 }
8056 }
8057
8058 if (length->isRegister())
8059 bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), temp, snapshot);
8060 else
8061 bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), temp, snapshot);
8062 }
8063
8064 void
visitBoundsCheckLower(LBoundsCheckLower * lir)8065 CodeGenerator::visitBoundsCheckLower(LBoundsCheckLower* lir)
8066 {
8067 int32_t min = lir->mir()->minimum();
8068 bailoutCmp32(Assembler::LessThan, ToRegister(lir->index()), Imm32(min),
8069 lir->snapshot());
8070 }
8071
8072 class OutOfLineStoreElementHole : public OutOfLineCodeBase<CodeGenerator>
8073 {
8074 LInstruction* ins_;
8075 Label rejoinStore_;
8076
8077 public:
OutOfLineStoreElementHole(LInstruction * ins)8078 explicit OutOfLineStoreElementHole(LInstruction* ins)
8079 : ins_(ins)
8080 {
8081 MOZ_ASSERT(ins->isStoreElementHoleV() || ins->isStoreElementHoleT() ||
8082 ins->isFallibleStoreElementV() || ins->isFallibleStoreElementT());
8083 }
8084
accept(CodeGenerator * codegen)8085 void accept(CodeGenerator* codegen) {
8086 codegen->visitOutOfLineStoreElementHole(this);
8087 }
ins() const8088 LInstruction* ins() const {
8089 return ins_;
8090 }
rejoinStore()8091 Label* rejoinStore() {
8092 return &rejoinStore_;
8093 }
8094 };
8095
8096 void
emitStoreHoleCheck(Register elements,const LAllocation * index,int32_t offsetAdjustment,LSnapshot * snapshot)8097 CodeGenerator::emitStoreHoleCheck(Register elements, const LAllocation* index,
8098 int32_t offsetAdjustment, LSnapshot* snapshot)
8099 {
8100 Label bail;
8101 if (index->isConstant()) {
8102 Address dest(elements, ToInt32(index) * sizeof(js::Value) + offsetAdjustment);
8103 masm.branchTestMagic(Assembler::Equal, dest, &bail);
8104 } else {
8105 BaseIndex dest(elements, ToRegister(index), TimesEight, offsetAdjustment);
8106 masm.branchTestMagic(Assembler::Equal, dest, &bail);
8107 }
8108 bailoutFrom(&bail, snapshot);
8109 }
8110
8111 static ConstantOrRegister
ToConstantOrRegister(const LAllocation * value,MIRType valueType)8112 ToConstantOrRegister(const LAllocation* value, MIRType valueType)
8113 {
8114 if (value->isConstant())
8115 return ConstantOrRegister(value->toConstant()->toJSValue());
8116 return TypedOrValueRegister(valueType, ToAnyRegister(value));
8117 }
8118
8119 void
emitStoreElementTyped(const LAllocation * value,MIRType valueType,MIRType elementType,Register elements,const LAllocation * index,int32_t offsetAdjustment)8120 CodeGenerator::emitStoreElementTyped(const LAllocation* value,
8121 MIRType valueType, MIRType elementType,
8122 Register elements, const LAllocation* index,
8123 int32_t offsetAdjustment)
8124 {
8125 ConstantOrRegister v = ToConstantOrRegister(value, valueType);
8126 if (index->isConstant()) {
8127 Address dest(elements, ToInt32(index) * sizeof(js::Value) + offsetAdjustment);
8128 masm.storeUnboxedValue(v, valueType, dest, elementType);
8129 } else {
8130 BaseIndex dest(elements, ToRegister(index), TimesEight, offsetAdjustment);
8131 masm.storeUnboxedValue(v, valueType, dest, elementType);
8132 }
8133 }
8134
8135 void
visitStoreElementT(LStoreElementT * store)8136 CodeGenerator::visitStoreElementT(LStoreElementT* store)
8137 {
8138 Register elements = ToRegister(store->elements());
8139 const LAllocation* index = store->index();
8140
8141 if (store->mir()->needsBarrier())
8142 emitPreBarrier(elements, index, store->mir()->offsetAdjustment());
8143
8144 if (store->mir()->needsHoleCheck())
8145 emitStoreHoleCheck(elements, index, store->mir()->offsetAdjustment(), store->snapshot());
8146
8147 emitStoreElementTyped(store->value(),
8148 store->mir()->value()->type(), store->mir()->elementType(),
8149 elements, index, store->mir()->offsetAdjustment());
8150 }
8151
8152 void
visitStoreElementV(LStoreElementV * lir)8153 CodeGenerator::visitStoreElementV(LStoreElementV* lir)
8154 {
8155 const ValueOperand value = ToValue(lir, LStoreElementV::Value);
8156 Register elements = ToRegister(lir->elements());
8157 const LAllocation* index = lir->index();
8158
8159 if (lir->mir()->needsBarrier())
8160 emitPreBarrier(elements, index, lir->mir()->offsetAdjustment());
8161
8162 if (lir->mir()->needsHoleCheck())
8163 emitStoreHoleCheck(elements, index, lir->mir()->offsetAdjustment(), lir->snapshot());
8164
8165 if (lir->index()->isConstant()) {
8166 Address dest(elements,
8167 ToInt32(lir->index()) * sizeof(js::Value) + lir->mir()->offsetAdjustment());
8168 masm.storeValue(value, dest);
8169 } else {
8170 BaseIndex dest(elements, ToRegister(lir->index()), TimesEight,
8171 lir->mir()->offsetAdjustment());
8172 masm.storeValue(value, dest);
8173 }
8174 }
8175
8176 template <typename T> void
emitStoreElementHoleT(T * lir)8177 CodeGenerator::emitStoreElementHoleT(T* lir)
8178 {
8179 static_assert(std::is_same<T, LStoreElementHoleT>::value || std::is_same<T, LFallibleStoreElementT>::value,
8180 "emitStoreElementHoleT called with unexpected argument type");
8181
8182 OutOfLineStoreElementHole* ool = new(alloc()) OutOfLineStoreElementHole(lir);
8183 addOutOfLineCode(ool, lir->mir());
8184
8185 Register obj = ToRegister(lir->object());
8186 Register elements = ToRegister(lir->elements());
8187 const LAllocation* index = lir->index();
8188 RegisterOrInt32Constant key = ToRegisterOrInt32Constant(index);
8189
8190 JSValueType unboxedType = lir->mir()->unboxedType();
8191 if (unboxedType == JSVAL_TYPE_MAGIC) {
8192 Address initLength(elements, ObjectElements::offsetOfInitializedLength());
8193 masm.branch32(Assembler::BelowOrEqual, initLength, key, ool->entry());
8194
8195 if (lir->mir()->needsBarrier())
8196 emitPreBarrier(elements, index, 0);
8197
8198 masm.bind(ool->rejoinStore());
8199 emitStoreElementTyped(lir->value(), lir->mir()->value()->type(), lir->mir()->elementType(),
8200 elements, index, 0);
8201 } else {
8202 Register temp = ToRegister(lir->getTemp(0));
8203 Address initLength(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength());
8204 masm.load32(initLength, temp);
8205 masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), temp);
8206 masm.branch32(Assembler::BelowOrEqual, temp, key, ool->entry());
8207
8208 ConstantOrRegister v = ToConstantOrRegister(lir->value(), lir->mir()->value()->type());
8209
8210 if (index->isConstant()) {
8211 Address address(elements, ToInt32(index) * UnboxedTypeSize(unboxedType));
8212 EmitUnboxedPreBarrier(masm, address, unboxedType);
8213
8214 masm.bind(ool->rejoinStore());
8215 masm.storeUnboxedProperty(address, unboxedType, v, nullptr);
8216 } else {
8217 BaseIndex address(elements, ToRegister(index),
8218 ScaleFromElemWidth(UnboxedTypeSize(unboxedType)));
8219 EmitUnboxedPreBarrier(masm, address, unboxedType);
8220
8221 masm.bind(ool->rejoinStore());
8222 masm.storeUnboxedProperty(address, unboxedType, v, nullptr);
8223 }
8224 }
8225
8226 masm.bind(ool->rejoin());
8227 }
8228
8229 void
visitStoreElementHoleT(LStoreElementHoleT * lir)8230 CodeGenerator::visitStoreElementHoleT(LStoreElementHoleT* lir)
8231 {
8232 emitStoreElementHoleT(lir);
8233 }
8234
8235 template <typename T> void
emitStoreElementHoleV(T * lir)8236 CodeGenerator::emitStoreElementHoleV(T* lir)
8237 {
8238 static_assert(std::is_same<T, LStoreElementHoleV>::value || std::is_same<T, LFallibleStoreElementV>::value,
8239 "emitStoreElementHoleV called with unexpected parameter type");
8240
8241 OutOfLineStoreElementHole* ool = new(alloc()) OutOfLineStoreElementHole(lir);
8242 addOutOfLineCode(ool, lir->mir());
8243
8244 Register obj = ToRegister(lir->object());
8245 Register elements = ToRegister(lir->elements());
8246 const LAllocation* index = lir->index();
8247 const ValueOperand value = ToValue(lir, T::Value);
8248 RegisterOrInt32Constant key = ToRegisterOrInt32Constant(index);
8249
8250 JSValueType unboxedType = lir->mir()->unboxedType();
8251 if (unboxedType == JSVAL_TYPE_MAGIC) {
8252 Address initLength(elements, ObjectElements::offsetOfInitializedLength());
8253 masm.branch32(Assembler::BelowOrEqual, initLength, key, ool->entry());
8254
8255 if (lir->mir()->needsBarrier())
8256 emitPreBarrier(elements, index, 0);
8257
8258 masm.bind(ool->rejoinStore());
8259 if (index->isConstant())
8260 masm.storeValue(value, Address(elements, ToInt32(index) * sizeof(js::Value)));
8261 else
8262 masm.storeValue(value, BaseIndex(elements, ToRegister(index), TimesEight));
8263 } else {
8264 Register temp = ToRegister(lir->getTemp(0));
8265 Address initLength(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength());
8266 masm.load32(initLength, temp);
8267 masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), temp);
8268 masm.branch32(Assembler::BelowOrEqual, temp, key, ool->entry());
8269
8270 if (index->isConstant()) {
8271 Address address(elements, ToInt32(index) * UnboxedTypeSize(unboxedType));
8272 EmitUnboxedPreBarrier(masm, address, unboxedType);
8273
8274 masm.bind(ool->rejoinStore());
8275 masm.storeUnboxedProperty(address, unboxedType, ConstantOrRegister(value), nullptr);
8276 } else {
8277 BaseIndex address(elements, ToRegister(index),
8278 ScaleFromElemWidth(UnboxedTypeSize(unboxedType)));
8279 EmitUnboxedPreBarrier(masm, address, unboxedType);
8280
8281 masm.bind(ool->rejoinStore());
8282 masm.storeUnboxedProperty(address, unboxedType, ConstantOrRegister(value), nullptr);
8283 }
8284 }
8285
8286 masm.bind(ool->rejoin());
8287 }
8288
8289 void
visitStoreElementHoleV(LStoreElementHoleV * lir)8290 CodeGenerator::visitStoreElementHoleV(LStoreElementHoleV* lir)
8291 {
8292 emitStoreElementHoleV(lir);
8293 }
8294
8295 typedef bool (*ThrowReadOnlyFn)(JSContext*, int32_t);
8296 static const VMFunction ThrowReadOnlyInfo =
8297 FunctionInfo<ThrowReadOnlyFn>(ThrowReadOnlyError, "ThrowReadOnlyError");
8298
8299 void
visitFallibleStoreElementT(LFallibleStoreElementT * lir)8300 CodeGenerator::visitFallibleStoreElementT(LFallibleStoreElementT* lir)
8301 {
8302 Register elements = ToRegister(lir->elements());
8303
8304 // Handle frozen objects
8305 Label isFrozen;
8306 Address flags(elements, ObjectElements::offsetOfFlags());
8307 if (!lir->mir()->strict()) {
8308 masm.branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::FROZEN), &isFrozen);
8309 } else {
8310 const LAllocation* index = lir->index();
8311 OutOfLineCode* ool;
8312 if (index->isConstant())
8313 ool = oolCallVM(ThrowReadOnlyInfo, lir, ArgList(Imm32(ToInt32(index))), StoreNothing());
8314 else
8315 ool = oolCallVM(ThrowReadOnlyInfo, lir, ArgList(ToRegister(index)), StoreNothing());
8316 masm.branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::FROZEN), ool->entry());
8317 // This OOL code should have thrown an exception, so will never return.
8318 // So, do not bind ool->rejoin() anywhere, so that it implicitly (and without the cost
8319 // of a jump) does a masm.assumeUnreachable().
8320 }
8321
8322 emitStoreElementHoleT(lir);
8323
8324 masm.bind(&isFrozen);
8325 }
8326
8327 void
visitFallibleStoreElementV(LFallibleStoreElementV * lir)8328 CodeGenerator::visitFallibleStoreElementV(LFallibleStoreElementV* lir)
8329 {
8330 Register elements = ToRegister(lir->elements());
8331
8332 // Handle frozen objects
8333 Label isFrozen;
8334 Address flags(elements, ObjectElements::offsetOfFlags());
8335 if (!lir->mir()->strict()) {
8336 masm.branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::FROZEN), &isFrozen);
8337 } else {
8338 const LAllocation* index = lir->index();
8339 OutOfLineCode* ool;
8340 if (index->isConstant())
8341 ool = oolCallVM(ThrowReadOnlyInfo, lir, ArgList(Imm32(ToInt32(index))), StoreNothing());
8342 else
8343 ool = oolCallVM(ThrowReadOnlyInfo, lir, ArgList(ToRegister(index)), StoreNothing());
8344 masm.branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::FROZEN), ool->entry());
8345 // This OOL code should have thrown an exception, so will never return.
8346 // So, do not bind ool->rejoin() anywhere, so that it implicitly (and without the cost
8347 // of a jump) does a masm.assumeUnreachable().
8348 }
8349
8350 emitStoreElementHoleV(lir);
8351
8352 masm.bind(&isFrozen);
8353 }
8354
8355 typedef bool (*SetDenseOrUnboxedArrayElementFn)(JSContext*, HandleObject, int32_t,
8356 HandleValue, bool strict);
8357 static const VMFunction SetDenseOrUnboxedArrayElementInfo =
8358 FunctionInfo<SetDenseOrUnboxedArrayElementFn>(SetDenseOrUnboxedArrayElement,
8359 "SetDenseOrUnboxedArrayElement");
8360
8361 void
visitOutOfLineStoreElementHole(OutOfLineStoreElementHole * ool)8362 CodeGenerator::visitOutOfLineStoreElementHole(OutOfLineStoreElementHole* ool)
8363 {
8364 Register object, elements;
8365 LInstruction* ins = ool->ins();
8366 const LAllocation* index;
8367 MIRType valueType;
8368 ConstantOrRegister value;
8369 JSValueType unboxedType;
8370 LDefinition *temp = nullptr;
8371
8372 if (ins->isStoreElementHoleV()) {
8373 LStoreElementHoleV* store = ins->toStoreElementHoleV();
8374 object = ToRegister(store->object());
8375 elements = ToRegister(store->elements());
8376 index = store->index();
8377 valueType = store->mir()->value()->type();
8378 value = TypedOrValueRegister(ToValue(store, LStoreElementHoleV::Value));
8379 unboxedType = store->mir()->unboxedType();
8380 temp = store->getTemp(0);
8381 } else if (ins->isFallibleStoreElementV()) {
8382 LFallibleStoreElementV* store = ins->toFallibleStoreElementV();
8383 object = ToRegister(store->object());
8384 elements = ToRegister(store->elements());
8385 index = store->index();
8386 valueType = store->mir()->value()->type();
8387 value = TypedOrValueRegister(ToValue(store, LFallibleStoreElementV::Value));
8388 unboxedType = store->mir()->unboxedType();
8389 temp = store->getTemp(0);
8390 } else if (ins->isStoreElementHoleT()) {
8391 LStoreElementHoleT* store = ins->toStoreElementHoleT();
8392 object = ToRegister(store->object());
8393 elements = ToRegister(store->elements());
8394 index = store->index();
8395 valueType = store->mir()->value()->type();
8396 if (store->value()->isConstant())
8397 value = ConstantOrRegister(store->value()->toConstant()->toJSValue());
8398 else
8399 value = TypedOrValueRegister(valueType, ToAnyRegister(store->value()));
8400 unboxedType = store->mir()->unboxedType();
8401 temp = store->getTemp(0);
8402 } else { // ins->isFallibleStoreElementT()
8403 LFallibleStoreElementT* store = ins->toFallibleStoreElementT();
8404 object = ToRegister(store->object());
8405 elements = ToRegister(store->elements());
8406 index = store->index();
8407 valueType = store->mir()->value()->type();
8408 if (store->value()->isConstant())
8409 value = ConstantOrRegister(store->value()->toConstant()->toJSValue());
8410 else
8411 value = TypedOrValueRegister(valueType, ToAnyRegister(store->value()));
8412 unboxedType = store->mir()->unboxedType();
8413 temp = store->getTemp(0);
8414 }
8415
8416 RegisterOrInt32Constant key = ToRegisterOrInt32Constant(index);
8417
8418 // If index == initializedLength, try to bump the initialized length inline.
8419 // If index > initializedLength, call a stub. Note that this relies on the
8420 // condition flags sticking from the incoming branch.
8421 Label callStub;
8422 #if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
8423 // Had to reimplement for MIPS because there are no flags.
8424 if (unboxedType == JSVAL_TYPE_MAGIC) {
8425 Address initLength(elements, ObjectElements::offsetOfInitializedLength());
8426 masm.branch32(Assembler::NotEqual, initLength, key, &callStub);
8427 } else {
8428 Address initLength(object, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength());
8429 masm.load32(initLength, ToRegister(temp));
8430 masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), ToRegister(temp));
8431 masm.branch32(Assembler::NotEqual, ToRegister(temp), key, &callStub);
8432 }
8433 #else
8434 masm.j(Assembler::NotEqual, &callStub);
8435 #endif
8436
8437 if (unboxedType == JSVAL_TYPE_MAGIC) {
8438 // Check array capacity.
8439 masm.branch32(Assembler::BelowOrEqual, Address(elements, ObjectElements::offsetOfCapacity()),
8440 key, &callStub);
8441
8442 // Update initialized length. The capacity guard above ensures this won't overflow,
8443 // due to MAX_DENSE_ELEMENTS_COUNT.
8444 masm.inc32(&key);
8445 masm.store32(key, Address(elements, ObjectElements::offsetOfInitializedLength()));
8446
8447 // Update length if length < initializedLength.
8448 Label dontUpdate;
8449 masm.branch32(Assembler::AboveOrEqual, Address(elements, ObjectElements::offsetOfLength()),
8450 key, &dontUpdate);
8451 masm.store32(key, Address(elements, ObjectElements::offsetOfLength()));
8452 masm.bind(&dontUpdate);
8453
8454 masm.dec32(&key);
8455 } else {
8456 // Check array capacity.
8457 masm.checkUnboxedArrayCapacity(object, key, ToRegister(temp), &callStub);
8458
8459 // Update initialized length.
8460 masm.add32(Imm32(1), Address(object, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()));
8461
8462 // Update length if length < initializedLength.
8463 Address lengthAddr(object, UnboxedArrayObject::offsetOfLength());
8464 Label dontUpdate;
8465 masm.branch32(Assembler::Above, lengthAddr, key, &dontUpdate);
8466 masm.add32(Imm32(1), lengthAddr);
8467 masm.bind(&dontUpdate);
8468 }
8469
8470 if ((ins->isStoreElementHoleT() || ins->isFallibleStoreElementT()) &&
8471 unboxedType == JSVAL_TYPE_MAGIC && valueType != MIRType::Double)
8472 {
8473 // The inline path for StoreElementHoleT and FallibleStoreElementT does not always store
8474 // the type tag, so we do the store on the OOL path. We use MIRType::None for the element
8475 // type so that storeElementTyped will always store the type tag.
8476 if (ins->isStoreElementHoleT()) {
8477 emitStoreElementTyped(ins->toStoreElementHoleT()->value(), valueType, MIRType::None,
8478 elements, index, 0);
8479 masm.jump(ool->rejoin());
8480 } else if (ins->isFallibleStoreElementT()) {
8481 emitStoreElementTyped(ins->toFallibleStoreElementT()->value(), valueType,
8482 MIRType::None, elements, index, 0);
8483 masm.jump(ool->rejoin());
8484 }
8485 } else {
8486 // Jump to the inline path where we will store the value.
8487 masm.jump(ool->rejoinStore());
8488 }
8489
8490 masm.bind(&callStub);
8491 saveLive(ins);
8492
8493 pushArg(Imm32(current->mir()->strict()));
8494 pushArg(value);
8495 if (index->isConstant())
8496 pushArg(Imm32(ToInt32(index)));
8497 else
8498 pushArg(ToRegister(index));
8499 pushArg(object);
8500 callVM(SetDenseOrUnboxedArrayElementInfo, ins);
8501
8502 restoreLive(ins);
8503 masm.jump(ool->rejoin());
8504 }
8505
8506 template <typename T>
8507 static void
StoreUnboxedPointer(MacroAssembler & masm,T address,MIRType type,const LAllocation * value,bool preBarrier)8508 StoreUnboxedPointer(MacroAssembler& masm, T address, MIRType type, const LAllocation* value,
8509 bool preBarrier)
8510 {
8511 if (preBarrier)
8512 masm.patchableCallPreBarrier(address, type);
8513 if (value->isConstant()) {
8514 Value v = value->toConstant()->toJSValue();
8515 if (v.isMarkable()) {
8516 masm.storePtr(ImmGCPtr(v.toMarkablePointer()), address);
8517 } else {
8518 MOZ_ASSERT(v.isNull());
8519 masm.storePtr(ImmWord(0), address);
8520 }
8521 } else {
8522 masm.storePtr(ToRegister(value), address);
8523 }
8524 }
8525
8526 void
visitStoreUnboxedPointer(LStoreUnboxedPointer * lir)8527 CodeGenerator::visitStoreUnboxedPointer(LStoreUnboxedPointer* lir)
8528 {
8529 MIRType type;
8530 int32_t offsetAdjustment;
8531 bool preBarrier;
8532 if (lir->mir()->isStoreUnboxedObjectOrNull()) {
8533 type = MIRType::Object;
8534 offsetAdjustment = lir->mir()->toStoreUnboxedObjectOrNull()->offsetAdjustment();
8535 preBarrier = lir->mir()->toStoreUnboxedObjectOrNull()->preBarrier();
8536 } else if (lir->mir()->isStoreUnboxedString()) {
8537 type = MIRType::String;
8538 offsetAdjustment = lir->mir()->toStoreUnboxedString()->offsetAdjustment();
8539 preBarrier = lir->mir()->toStoreUnboxedString()->preBarrier();
8540 } else {
8541 MOZ_CRASH();
8542 }
8543
8544 Register elements = ToRegister(lir->elements());
8545 const LAllocation* index = lir->index();
8546 const LAllocation* value = lir->value();
8547
8548 if (index->isConstant()) {
8549 Address address(elements, ToInt32(index) * sizeof(uintptr_t) + offsetAdjustment);
8550 StoreUnboxedPointer(masm, address, type, value, preBarrier);
8551 } else {
8552 BaseIndex address(elements, ToRegister(index), ScalePointer, offsetAdjustment);
8553 StoreUnboxedPointer(masm, address, type, value, preBarrier);
8554 }
8555 }
8556
8557 typedef bool (*ConvertUnboxedObjectToNativeFn)(JSContext*, JSObject*);
8558 static const VMFunction ConvertUnboxedPlainObjectToNativeInfo =
8559 FunctionInfo<ConvertUnboxedObjectToNativeFn>(UnboxedPlainObject::convertToNative,
8560 "UnboxedPlainObject::convertToNative");
8561 static const VMFunction ConvertUnboxedArrayObjectToNativeInfo =
8562 FunctionInfo<ConvertUnboxedObjectToNativeFn>(UnboxedArrayObject::convertToNative,
8563 "UnboxedArrayObject::convertToNative");
8564
8565 void
visitConvertUnboxedObjectToNative(LConvertUnboxedObjectToNative * lir)8566 CodeGenerator::visitConvertUnboxedObjectToNative(LConvertUnboxedObjectToNative* lir)
8567 {
8568 Register object = ToRegister(lir->getOperand(0));
8569
8570 OutOfLineCode* ool = oolCallVM(lir->mir()->group()->unboxedLayoutDontCheckGeneration().isArray()
8571 ? ConvertUnboxedArrayObjectToNativeInfo
8572 : ConvertUnboxedPlainObjectToNativeInfo,
8573 lir, ArgList(object), StoreNothing());
8574
8575 masm.branchPtr(Assembler::Equal, Address(object, JSObject::offsetOfGroup()),
8576 ImmGCPtr(lir->mir()->group()), ool->entry());
8577 masm.bind(ool->rejoin());
8578 }
8579
8580 typedef bool (*ArrayPopShiftFn)(JSContext*, HandleObject, MutableHandleValue);
8581 static const VMFunction ArrayPopDenseInfo =
8582 FunctionInfo<ArrayPopShiftFn>(jit::ArrayPopDense, "ArrayPopDense");
8583 static const VMFunction ArrayShiftDenseInfo =
8584 FunctionInfo<ArrayPopShiftFn>(jit::ArrayShiftDense, "ArrayShiftDense");
8585
8586 void
emitArrayPopShift(LInstruction * lir,const MArrayPopShift * mir,Register obj,Register elementsTemp,Register lengthTemp,TypedOrValueRegister out)8587 CodeGenerator::emitArrayPopShift(LInstruction* lir, const MArrayPopShift* mir, Register obj,
8588 Register elementsTemp, Register lengthTemp, TypedOrValueRegister out)
8589 {
8590 OutOfLineCode* ool;
8591
8592 if (mir->mode() == MArrayPopShift::Pop) {
8593 ool = oolCallVM(ArrayPopDenseInfo, lir, ArgList(obj), StoreValueTo(out));
8594 } else {
8595 MOZ_ASSERT(mir->mode() == MArrayPopShift::Shift);
8596 ool = oolCallVM(ArrayShiftDenseInfo, lir, ArgList(obj), StoreValueTo(out));
8597 }
8598
8599 // VM call if a write barrier is necessary.
8600 masm.branchTestNeedsIncrementalBarrier(Assembler::NonZero, ool->entry());
8601
8602 // Load elements and length, and VM call if length != initializedLength.
8603 RegisterOrInt32Constant key = RegisterOrInt32Constant(lengthTemp);
8604 if (mir->unboxedType() == JSVAL_TYPE_MAGIC) {
8605 masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), elementsTemp);
8606 masm.load32(Address(elementsTemp, ObjectElements::offsetOfLength()), lengthTemp);
8607
8608 Address initLength(elementsTemp, ObjectElements::offsetOfInitializedLength());
8609 masm.branch32(Assembler::NotEqual, initLength, key, ool->entry());
8610 } else {
8611 masm.loadPtr(Address(obj, UnboxedArrayObject::offsetOfElements()), elementsTemp);
8612 masm.load32(Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()), lengthTemp);
8613 masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), lengthTemp);
8614
8615 Address lengthAddr(obj, UnboxedArrayObject::offsetOfLength());
8616 masm.branch32(Assembler::NotEqual, lengthAddr, key, ool->entry());
8617 }
8618
8619 // Test for length != 0. On zero length either take a VM call or generate
8620 // an undefined value, depending on whether the call is known to produce
8621 // undefined.
8622 Label done;
8623 if (mir->maybeUndefined()) {
8624 Label notEmpty;
8625 masm.branchTest32(Assembler::NonZero, lengthTemp, lengthTemp, ¬Empty);
8626
8627 // According to the spec we need to set the length 0 (which is already 0).
8628 // This is observable when the array length is made non-writable.
8629 // Handle this case in the OOL. When freezing an unboxed array it is converted
8630 // to an normal array.
8631 if (mir->unboxedType() == JSVAL_TYPE_MAGIC) {
8632 Address elementFlags(elementsTemp, ObjectElements::offsetOfFlags());
8633 Imm32 bit(ObjectElements::NONWRITABLE_ARRAY_LENGTH);
8634 masm.branchTest32(Assembler::NonZero, elementFlags, bit, ool->entry());
8635 }
8636
8637 masm.moveValue(UndefinedValue(), out.valueReg());
8638 masm.jump(&done);
8639 masm.bind(¬Empty);
8640 } else {
8641 masm.branchTest32(Assembler::Zero, lengthTemp, lengthTemp, ool->entry());
8642 }
8643
8644 masm.dec32(&key);
8645
8646 if (mir->mode() == MArrayPopShift::Pop) {
8647 if (mir->unboxedType() == JSVAL_TYPE_MAGIC) {
8648 BaseIndex addr(elementsTemp, lengthTemp, TimesEight);
8649 masm.loadElementTypedOrValue(addr, out, mir->needsHoleCheck(), ool->entry());
8650 } else {
8651 size_t elemSize = UnboxedTypeSize(mir->unboxedType());
8652 BaseIndex addr(elementsTemp, lengthTemp, ScaleFromElemWidth(elemSize));
8653 masm.loadUnboxedProperty(addr, mir->unboxedType(), out);
8654 }
8655 } else {
8656 MOZ_ASSERT(mir->mode() == MArrayPopShift::Shift);
8657 Address addr(elementsTemp, 0);
8658 if (mir->unboxedType() == JSVAL_TYPE_MAGIC)
8659 masm.loadElementTypedOrValue(addr, out, mir->needsHoleCheck(), ool->entry());
8660 else
8661 masm.loadUnboxedProperty(addr, mir->unboxedType(), out);
8662 }
8663
8664 if (mir->unboxedType() == JSVAL_TYPE_MAGIC) {
8665 // Handle the failure case when the array length is non-writable in the
8666 // OOL path. (Unlike in the adding-an-element cases, we can't rely on the
8667 // capacity <= length invariant for such arrays to avoid an explicit
8668 // check.)
8669 Address elementFlags(elementsTemp, ObjectElements::offsetOfFlags());
8670 Imm32 bit(ObjectElements::NONWRITABLE_ARRAY_LENGTH);
8671 masm.branchTest32(Assembler::NonZero, elementFlags, bit, ool->entry());
8672
8673 // Now adjust length and initializedLength.
8674 masm.store32(lengthTemp, Address(elementsTemp, ObjectElements::offsetOfLength()));
8675 masm.store32(lengthTemp, Address(elementsTemp, ObjectElements::offsetOfInitializedLength()));
8676 } else {
8677 // Unboxed arrays always have writable lengths. Adjust length and
8678 // initializedLength.
8679 masm.store32(lengthTemp, Address(obj, UnboxedArrayObject::offsetOfLength()));
8680 masm.add32(Imm32(-1), Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()));
8681 }
8682
8683 if (mir->mode() == MArrayPopShift::Shift) {
8684 // Don't save the temp registers.
8685 LiveRegisterSet temps;
8686 temps.add(elementsTemp);
8687 temps.add(lengthTemp);
8688
8689 saveVolatile(temps);
8690 masm.setupUnalignedABICall(lengthTemp);
8691 masm.passABIArg(obj);
8692 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::ArrayShiftMoveElements));
8693 restoreVolatile(temps);
8694 }
8695
8696 masm.bind(&done);
8697 masm.bind(ool->rejoin());
8698 }
8699
8700 void
visitArrayPopShiftV(LArrayPopShiftV * lir)8701 CodeGenerator::visitArrayPopShiftV(LArrayPopShiftV* lir)
8702 {
8703 Register obj = ToRegister(lir->object());
8704 Register elements = ToRegister(lir->temp0());
8705 Register length = ToRegister(lir->temp1());
8706 TypedOrValueRegister out(ToOutValue(lir));
8707 emitArrayPopShift(lir, lir->mir(), obj, elements, length, out);
8708 }
8709
8710 void
visitArrayPopShiftT(LArrayPopShiftT * lir)8711 CodeGenerator::visitArrayPopShiftT(LArrayPopShiftT* lir)
8712 {
8713 Register obj = ToRegister(lir->object());
8714 Register elements = ToRegister(lir->temp0());
8715 Register length = ToRegister(lir->temp1());
8716 TypedOrValueRegister out(lir->mir()->type(), ToAnyRegister(lir->output()));
8717 emitArrayPopShift(lir, lir->mir(), obj, elements, length, out);
8718 }
8719
8720 typedef bool (*ArrayPushDenseFn)(JSContext*, HandleObject, HandleValue, uint32_t*);
8721 static const VMFunction ArrayPushDenseInfo =
8722 FunctionInfo<ArrayPushDenseFn>(jit::ArrayPushDense, "ArrayPushDense");
8723
8724 void
emitArrayPush(LInstruction * lir,const MArrayPush * mir,Register obj,const ConstantOrRegister & value,Register elementsTemp,Register length)8725 CodeGenerator::emitArrayPush(LInstruction* lir, const MArrayPush* mir, Register obj,
8726 const ConstantOrRegister& value, Register elementsTemp, Register length)
8727 {
8728 OutOfLineCode* ool = oolCallVM(ArrayPushDenseInfo, lir, ArgList(obj, value), StoreRegisterTo(length));
8729
8730 RegisterOrInt32Constant key = RegisterOrInt32Constant(length);
8731 if (mir->unboxedType() == JSVAL_TYPE_MAGIC) {
8732 // Load elements and length.
8733 masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), elementsTemp);
8734 masm.load32(Address(elementsTemp, ObjectElements::offsetOfLength()), length);
8735
8736 // Guard length == initializedLength.
8737 Address initLength(elementsTemp, ObjectElements::offsetOfInitializedLength());
8738 masm.branch32(Assembler::NotEqual, initLength, key, ool->entry());
8739
8740 // Guard length < capacity.
8741 Address capacity(elementsTemp, ObjectElements::offsetOfCapacity());
8742 masm.branch32(Assembler::BelowOrEqual, capacity, key, ool->entry());
8743
8744 // Do the store.
8745 masm.storeConstantOrRegister(value, BaseIndex(elementsTemp, length, TimesEight));
8746 } else {
8747 // Load initialized length.
8748 masm.load32(Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()), length);
8749 masm.and32(Imm32(UnboxedArrayObject::InitializedLengthMask), length);
8750
8751 // Guard length == initializedLength.
8752 Address lengthAddr(obj, UnboxedArrayObject::offsetOfLength());
8753 masm.branch32(Assembler::NotEqual, lengthAddr, key, ool->entry());
8754
8755 // Guard length < capacity.
8756 masm.checkUnboxedArrayCapacity(obj, key, elementsTemp, ool->entry());
8757
8758 // Load elements and do the store.
8759 masm.loadPtr(Address(obj, UnboxedArrayObject::offsetOfElements()), elementsTemp);
8760 size_t elemSize = UnboxedTypeSize(mir->unboxedType());
8761 BaseIndex addr(elementsTemp, length, ScaleFromElemWidth(elemSize));
8762 masm.storeUnboxedProperty(addr, mir->unboxedType(), value, nullptr);
8763 }
8764
8765 masm.inc32(&key);
8766
8767 // Update length and initialized length.
8768 if (mir->unboxedType() == JSVAL_TYPE_MAGIC) {
8769 masm.store32(length, Address(elementsTemp, ObjectElements::offsetOfLength()));
8770 masm.store32(length, Address(elementsTemp, ObjectElements::offsetOfInitializedLength()));
8771 } else {
8772 masm.store32(length, Address(obj, UnboxedArrayObject::offsetOfLength()));
8773 masm.add32(Imm32(1), Address(obj, UnboxedArrayObject::offsetOfCapacityIndexAndInitializedLength()));
8774 }
8775
8776 masm.bind(ool->rejoin());
8777 }
8778
8779 void
visitArrayPushV(LArrayPushV * lir)8780 CodeGenerator::visitArrayPushV(LArrayPushV* lir)
8781 {
8782 Register obj = ToRegister(lir->object());
8783 Register elementsTemp = ToRegister(lir->temp());
8784 Register length = ToRegister(lir->output());
8785 ConstantOrRegister value = TypedOrValueRegister(ToValue(lir, LArrayPushV::Value));
8786 emitArrayPush(lir, lir->mir(), obj, value, elementsTemp, length);
8787 }
8788
8789 void
visitArrayPushT(LArrayPushT * lir)8790 CodeGenerator::visitArrayPushT(LArrayPushT* lir)
8791 {
8792 Register obj = ToRegister(lir->object());
8793 Register elementsTemp = ToRegister(lir->temp());
8794 Register length = ToRegister(lir->output());
8795 ConstantOrRegister value;
8796 if (lir->value()->isConstant())
8797 value = ConstantOrRegister(lir->value()->toConstant()->toJSValue());
8798 else
8799 value = TypedOrValueRegister(lir->mir()->value()->type(), ToAnyRegister(lir->value()));
8800 emitArrayPush(lir, lir->mir(), obj, value, elementsTemp, length);
8801 }
8802
8803 typedef JSObject* (*ArraySliceDenseFn)(JSContext*, HandleObject, int32_t, int32_t, HandleObject);
8804 static const VMFunction ArraySliceDenseInfo =
8805 FunctionInfo<ArraySliceDenseFn>(array_slice_dense, "array_slice_dense");
8806
8807 void
visitArraySlice(LArraySlice * lir)8808 CodeGenerator::visitArraySlice(LArraySlice* lir)
8809 {
8810 Register object = ToRegister(lir->object());
8811 Register begin = ToRegister(lir->begin());
8812 Register end = ToRegister(lir->end());
8813 Register temp1 = ToRegister(lir->temp1());
8814 Register temp2 = ToRegister(lir->temp2());
8815
8816 Label call, fail;
8817
8818 // Try to allocate an object.
8819 masm.createGCObject(temp1, temp2, lir->mir()->templateObj(), lir->mir()->initialHeap(), &fail);
8820
8821 // Fixup the group of the result in case it doesn't match the template object.
8822 masm.loadPtr(Address(object, JSObject::offsetOfGroup()), temp2);
8823 masm.storePtr(temp2, Address(temp1, JSObject::offsetOfGroup()));
8824
8825 masm.jump(&call);
8826 {
8827 masm.bind(&fail);
8828 masm.movePtr(ImmPtr(nullptr), temp1);
8829 }
8830 masm.bind(&call);
8831
8832 pushArg(temp1);
8833 pushArg(end);
8834 pushArg(begin);
8835 pushArg(object);
8836 callVM(ArraySliceDenseInfo, lir);
8837 }
8838
8839 typedef JSString* (*ArrayJoinFn)(JSContext*, HandleObject, HandleString);
8840 static const VMFunction ArrayJoinInfo = FunctionInfo<ArrayJoinFn>(jit::ArrayJoin, "ArrayJoin");
8841
8842 void
visitArrayJoin(LArrayJoin * lir)8843 CodeGenerator::visitArrayJoin(LArrayJoin* lir)
8844 {
8845 pushArg(ToRegister(lir->separator()));
8846 pushArg(ToRegister(lir->array()));
8847
8848 callVM(ArrayJoinInfo, lir);
8849 }
8850
8851 typedef JSObject* (*ValueToIteratorFn)(JSContext*, uint32_t, HandleValue);
8852 static const VMFunction ValueToIteratorInfo =
8853 FunctionInfo<ValueToIteratorFn>(ValueToIterator, "ValueToIterator");
8854
8855 void
visitCallIteratorStartV(LCallIteratorStartV * lir)8856 CodeGenerator::visitCallIteratorStartV(LCallIteratorStartV* lir)
8857 {
8858 pushArg(ToValue(lir, LCallIteratorStartV::Value));
8859 pushArg(Imm32(lir->mir()->flags()));
8860 callVM(ValueToIteratorInfo, lir);
8861 }
8862
8863 typedef JSObject* (*GetIteratorObjectFn)(JSContext*, HandleObject, uint32_t);
8864 static const VMFunction GetIteratorObjectInfo =
8865 FunctionInfo<GetIteratorObjectFn>(GetIteratorObject, "GetIteratorObject");
8866
8867 void
visitCallIteratorStartO(LCallIteratorStartO * lir)8868 CodeGenerator::visitCallIteratorStartO(LCallIteratorStartO* lir)
8869 {
8870 pushArg(Imm32(lir->mir()->flags()));
8871 pushArg(ToRegister(lir->object()));
8872 callVM(GetIteratorObjectInfo, lir);
8873 }
8874
8875 void
branchIfNotEmptyObjectElements(Register obj,Label * target)8876 CodeGenerator::branchIfNotEmptyObjectElements(Register obj, Label* target)
8877 {
8878 Label emptyObj;
8879 masm.branchPtr(Assembler::Equal,
8880 Address(obj, NativeObject::offsetOfElements()),
8881 ImmPtr(js::emptyObjectElements),
8882 &emptyObj);
8883 masm.branchPtr(Assembler::NotEqual,
8884 Address(obj, NativeObject::offsetOfElements()),
8885 ImmPtr(js::emptyObjectElementsShared),
8886 target);
8887 masm.bind(&emptyObj);
8888 }
8889
8890 void
visitIteratorStartO(LIteratorStartO * lir)8891 CodeGenerator::visitIteratorStartO(LIteratorStartO* lir)
8892 {
8893 const Register obj = ToRegister(lir->object());
8894 const Register output = ToRegister(lir->output());
8895
8896 uint32_t flags = lir->mir()->flags();
8897
8898 OutOfLineCode* ool = oolCallVM(GetIteratorObjectInfo, lir,
8899 ArgList(obj, Imm32(flags)), StoreRegisterTo(output));
8900
8901 const Register temp1 = ToRegister(lir->temp1());
8902 const Register temp2 = ToRegister(lir->temp2());
8903 const Register niTemp = ToRegister(lir->temp3()); // Holds the NativeIterator object.
8904
8905 // Iterators other than for-in should use LCallIteratorStart.
8906 MOZ_ASSERT(flags == JSITER_ENUMERATE);
8907
8908 // Fetch the most recent iterator and ensure it's not nullptr.
8909 masm.loadPtr(AbsoluteAddress(gen->compartment->addressOfLastCachedNativeIterator()), output);
8910 masm.branchTestPtr(Assembler::Zero, output, output, ool->entry());
8911
8912 // Load NativeIterator.
8913 masm.loadObjPrivate(output, JSObject::ITER_CLASS_NFIXED_SLOTS, niTemp);
8914
8915 // Ensure the |active| and |unreusable| bits are not set.
8916 masm.branchTest32(Assembler::NonZero, Address(niTemp, offsetof(NativeIterator, flags)),
8917 Imm32(JSITER_ACTIVE|JSITER_UNREUSABLE), ool->entry());
8918
8919 // Load the iterator's receiver guard array.
8920 masm.loadPtr(Address(niTemp, offsetof(NativeIterator, guard_array)), temp2);
8921
8922 // Compare object with the first receiver guard. The last iterator can only
8923 // match for native objects and unboxed objects.
8924 {
8925 Address groupAddr(temp2, offsetof(ReceiverGuard, group));
8926 Address shapeAddr(temp2, offsetof(ReceiverGuard, shape));
8927 Label guardDone, shapeMismatch, noExpando;
8928 masm.loadObjShape(obj, temp1);
8929 masm.branchPtr(Assembler::NotEqual, shapeAddr, temp1, &shapeMismatch);
8930
8931 // Ensure the object does not have any elements. The presence of dense
8932 // elements is not captured by the shape tests above.
8933 branchIfNotEmptyObjectElements(obj, ool->entry());
8934 masm.jump(&guardDone);
8935
8936 masm.bind(&shapeMismatch);
8937 masm.loadObjGroup(obj, temp1);
8938 masm.branchPtr(Assembler::NotEqual, groupAddr, temp1, ool->entry());
8939 masm.loadPtr(Address(obj, UnboxedPlainObject::offsetOfExpando()), temp1);
8940 masm.branchTestPtr(Assembler::Zero, temp1, temp1, &noExpando);
8941 branchIfNotEmptyObjectElements(temp1, ool->entry());
8942 masm.loadObjShape(temp1, temp1);
8943 masm.bind(&noExpando);
8944 masm.branchPtr(Assembler::NotEqual, shapeAddr, temp1, ool->entry());
8945 masm.bind(&guardDone);
8946 }
8947
8948 // Compare shape of object's prototype with the second shape. The prototype
8949 // must be native, as unboxed objects cannot be prototypes (they cannot
8950 // have the delegate flag set). Also check for the absence of dense elements.
8951 Address prototypeShapeAddr(temp2, sizeof(ReceiverGuard) + offsetof(ReceiverGuard, shape));
8952 masm.loadObjProto(obj, temp1);
8953 branchIfNotEmptyObjectElements(temp1, ool->entry());
8954 masm.loadObjShape(temp1, temp1);
8955 masm.branchPtr(Assembler::NotEqual, prototypeShapeAddr, temp1, ool->entry());
8956
8957 // Ensure the object's prototype's prototype is nullptr. The last native
8958 // iterator will always have a prototype chain length of one (i.e. it must
8959 // be a plain object), so we do not need to generate a loop here.
8960 masm.loadObjProto(obj, temp1);
8961 masm.loadObjProto(temp1, temp1);
8962 masm.branchTestPtr(Assembler::NonZero, temp1, temp1, ool->entry());
8963
8964 // Write barrier for stores to the iterator. We only need to take a write
8965 // barrier if NativeIterator::obj is actually going to change.
8966 {
8967 // Bug 867815: Unconditionally take this out- of-line so that we do not
8968 // have to post-barrier the store to NativeIter::obj. This just needs
8969 // JIT support for the Cell* buffer.
8970 Address objAddr(niTemp, offsetof(NativeIterator, obj));
8971 masm.branchPtr(Assembler::NotEqual, objAddr, obj, ool->entry());
8972 }
8973
8974 // Mark iterator as active.
8975 masm.storePtr(obj, Address(niTemp, offsetof(NativeIterator, obj)));
8976 masm.or32(Imm32(JSITER_ACTIVE), Address(niTemp, offsetof(NativeIterator, flags)));
8977
8978 // Chain onto the active iterator stack.
8979 masm.loadPtr(AbsoluteAddress(gen->compartment->addressOfEnumerators()), temp1);
8980
8981 // ni->next = list
8982 masm.storePtr(temp1, Address(niTemp, NativeIterator::offsetOfNext()));
8983
8984 // ni->prev = list->prev
8985 masm.loadPtr(Address(temp1, NativeIterator::offsetOfPrev()), temp2);
8986 masm.storePtr(temp2, Address(niTemp, NativeIterator::offsetOfPrev()));
8987
8988 // list->prev->next = ni
8989 masm.storePtr(niTemp, Address(temp2, NativeIterator::offsetOfNext()));
8990
8991 // list->prev = ni
8992 masm.storePtr(niTemp, Address(temp1, NativeIterator::offsetOfPrev()));
8993
8994 masm.bind(ool->rejoin());
8995 }
8996
8997 static void
LoadNativeIterator(MacroAssembler & masm,Register obj,Register dest,Label * failures)8998 LoadNativeIterator(MacroAssembler& masm, Register obj, Register dest, Label* failures)
8999 {
9000 MOZ_ASSERT(obj != dest);
9001
9002 // Test class.
9003 masm.branchTestObjClass(Assembler::NotEqual, obj, dest, &PropertyIteratorObject::class_, failures);
9004
9005 // Load NativeIterator object.
9006 masm.loadObjPrivate(obj, JSObject::ITER_CLASS_NFIXED_SLOTS, dest);
9007 }
9008
9009 typedef bool (*IteratorMoreFn)(JSContext*, HandleObject, MutableHandleValue);
9010 static const VMFunction IteratorMoreInfo =
9011 FunctionInfo<IteratorMoreFn>(IteratorMore, "IteratorMore");
9012
9013 void
visitIteratorMore(LIteratorMore * lir)9014 CodeGenerator::visitIteratorMore(LIteratorMore* lir)
9015 {
9016 const Register obj = ToRegister(lir->object());
9017 const ValueOperand output = ToOutValue(lir);
9018 const Register temp = ToRegister(lir->temp());
9019
9020 OutOfLineCode* ool = oolCallVM(IteratorMoreInfo, lir, ArgList(obj), StoreValueTo(output));
9021
9022 Register outputScratch = output.scratchReg();
9023 LoadNativeIterator(masm, obj, outputScratch, ool->entry());
9024
9025 masm.branchTest32(Assembler::NonZero, Address(outputScratch, offsetof(NativeIterator, flags)),
9026 Imm32(JSITER_FOREACH), ool->entry());
9027
9028 // If props_cursor < props_end, load the next string and advance the cursor.
9029 // Else, return MagicValue(JS_NO_ITER_VALUE).
9030 Label iterDone;
9031 Address cursorAddr(outputScratch, offsetof(NativeIterator, props_cursor));
9032 Address cursorEndAddr(outputScratch, offsetof(NativeIterator, props_end));
9033 masm.loadPtr(cursorAddr, temp);
9034 masm.branchPtr(Assembler::BelowOrEqual, cursorEndAddr, temp, &iterDone);
9035
9036 // Get next string.
9037 masm.loadPtr(Address(temp, 0), temp);
9038
9039 // Increase the cursor.
9040 masm.addPtr(Imm32(sizeof(JSString*)), cursorAddr);
9041
9042 masm.tagValue(JSVAL_TYPE_STRING, temp, output);
9043 masm.jump(ool->rejoin());
9044
9045 masm.bind(&iterDone);
9046 masm.moveValue(MagicValue(JS_NO_ITER_VALUE), output);
9047
9048 masm.bind(ool->rejoin());
9049 }
9050
9051 void
visitIsNoIterAndBranch(LIsNoIterAndBranch * lir)9052 CodeGenerator::visitIsNoIterAndBranch(LIsNoIterAndBranch* lir)
9053 {
9054 ValueOperand input = ToValue(lir, LIsNoIterAndBranch::Input);
9055 Label* ifTrue = getJumpLabelForBranch(lir->ifTrue());
9056 Label* ifFalse = getJumpLabelForBranch(lir->ifFalse());
9057
9058 masm.branchTestMagic(Assembler::Equal, input, ifTrue);
9059
9060 if (!isNextBlock(lir->ifFalse()->lir()))
9061 masm.jump(ifFalse);
9062 }
9063
9064 typedef bool (*CloseIteratorFn)(JSContext*, HandleObject);
9065 static const VMFunction CloseIteratorInfo =
9066 FunctionInfo<CloseIteratorFn>(CloseIterator, "CloseIterator");
9067
9068 void
visitIteratorEnd(LIteratorEnd * lir)9069 CodeGenerator::visitIteratorEnd(LIteratorEnd* lir)
9070 {
9071 const Register obj = ToRegister(lir->object());
9072 const Register temp1 = ToRegister(lir->temp1());
9073 const Register temp2 = ToRegister(lir->temp2());
9074 const Register temp3 = ToRegister(lir->temp3());
9075
9076 OutOfLineCode* ool = oolCallVM(CloseIteratorInfo, lir, ArgList(obj), StoreNothing());
9077
9078 LoadNativeIterator(masm, obj, temp1, ool->entry());
9079
9080 masm.branchTest32(Assembler::Zero, Address(temp1, offsetof(NativeIterator, flags)),
9081 Imm32(JSITER_ENUMERATE), ool->entry());
9082
9083 // Clear active bit.
9084 masm.and32(Imm32(~JSITER_ACTIVE), Address(temp1, offsetof(NativeIterator, flags)));
9085
9086 // Reset property cursor.
9087 masm.loadPtr(Address(temp1, offsetof(NativeIterator, props_array)), temp2);
9088 masm.storePtr(temp2, Address(temp1, offsetof(NativeIterator, props_cursor)));
9089
9090 // Unlink from the iterator list.
9091 const Register next = temp2;
9092 const Register prev = temp3;
9093 masm.loadPtr(Address(temp1, NativeIterator::offsetOfNext()), next);
9094 masm.loadPtr(Address(temp1, NativeIterator::offsetOfPrev()), prev);
9095 masm.storePtr(prev, Address(next, NativeIterator::offsetOfPrev()));
9096 masm.storePtr(next, Address(prev, NativeIterator::offsetOfNext()));
9097 #ifdef DEBUG
9098 masm.storePtr(ImmPtr(nullptr), Address(temp1, NativeIterator::offsetOfNext()));
9099 masm.storePtr(ImmPtr(nullptr), Address(temp1, NativeIterator::offsetOfPrev()));
9100 #endif
9101
9102 masm.bind(ool->rejoin());
9103 }
9104
9105 void
visitArgumentsLength(LArgumentsLength * lir)9106 CodeGenerator::visitArgumentsLength(LArgumentsLength* lir)
9107 {
9108 // read number of actual arguments from the JS frame.
9109 Register argc = ToRegister(lir->output());
9110 Address ptr(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfNumActualArgs());
9111
9112 masm.loadPtr(ptr, argc);
9113 }
9114
9115 void
visitGetFrameArgument(LGetFrameArgument * lir)9116 CodeGenerator::visitGetFrameArgument(LGetFrameArgument* lir)
9117 {
9118 ValueOperand result = GetValueOutput(lir);
9119 const LAllocation* index = lir->index();
9120 size_t argvOffset = frameSize() + JitFrameLayout::offsetOfActualArgs();
9121
9122 if (index->isConstant()) {
9123 int32_t i = index->toConstant()->toInt32();
9124 Address argPtr(masm.getStackPointer(), sizeof(Value) * i + argvOffset);
9125 masm.loadValue(argPtr, result);
9126 } else {
9127 Register i = ToRegister(index);
9128 BaseValueIndex argPtr(masm.getStackPointer(), i, argvOffset);
9129 masm.loadValue(argPtr, result);
9130 }
9131 }
9132
9133 void
visitSetFrameArgumentT(LSetFrameArgumentT * lir)9134 CodeGenerator::visitSetFrameArgumentT(LSetFrameArgumentT* lir)
9135 {
9136 size_t argOffset = frameSize() + JitFrameLayout::offsetOfActualArgs() +
9137 (sizeof(Value) * lir->mir()->argno());
9138
9139 MIRType type = lir->mir()->value()->type();
9140
9141 if (type == MIRType::Double) {
9142 // Store doubles directly.
9143 FloatRegister input = ToFloatRegister(lir->input());
9144 masm.storeDouble(input, Address(masm.getStackPointer(), argOffset));
9145
9146 } else {
9147 Register input = ToRegister(lir->input());
9148 masm.storeValue(ValueTypeFromMIRType(type), input, Address(masm.getStackPointer(), argOffset));
9149 }
9150 }
9151
9152 void
visitSetFrameArgumentC(LSetFrameArgumentC * lir)9153 CodeGenerator:: visitSetFrameArgumentC(LSetFrameArgumentC* lir)
9154 {
9155 size_t argOffset = frameSize() + JitFrameLayout::offsetOfActualArgs() +
9156 (sizeof(Value) * lir->mir()->argno());
9157 masm.storeValue(lir->val(), Address(masm.getStackPointer(), argOffset));
9158 }
9159
9160 void
visitSetFrameArgumentV(LSetFrameArgumentV * lir)9161 CodeGenerator:: visitSetFrameArgumentV(LSetFrameArgumentV* lir)
9162 {
9163 const ValueOperand val = ToValue(lir, LSetFrameArgumentV::Input);
9164 size_t argOffset = frameSize() + JitFrameLayout::offsetOfActualArgs() +
9165 (sizeof(Value) * lir->mir()->argno());
9166 masm.storeValue(val, Address(masm.getStackPointer(), argOffset));
9167 }
9168
9169 typedef bool (*RunOnceScriptPrologueFn)(JSContext*, HandleScript);
9170 static const VMFunction RunOnceScriptPrologueInfo =
9171 FunctionInfo<RunOnceScriptPrologueFn>(js::RunOnceScriptPrologue, "RunOnceScriptPrologue");
9172
9173 void
visitRunOncePrologue(LRunOncePrologue * lir)9174 CodeGenerator::visitRunOncePrologue(LRunOncePrologue* lir)
9175 {
9176 pushArg(ImmGCPtr(lir->mir()->block()->info().script()));
9177 callVM(RunOnceScriptPrologueInfo, lir);
9178 }
9179
9180 typedef JSObject* (*InitRestParameterFn)(JSContext*, uint32_t, Value*, HandleObject,
9181 HandleObject);
9182 static const VMFunction InitRestParameterInfo =
9183 FunctionInfo<InitRestParameterFn>(InitRestParameter, "InitRestParameter");
9184
9185 void
emitRest(LInstruction * lir,Register array,Register numActuals,Register temp0,Register temp1,unsigned numFormals,JSObject * templateObject,bool saveAndRestore,Register resultreg)9186 CodeGenerator::emitRest(LInstruction* lir, Register array, Register numActuals,
9187 Register temp0, Register temp1, unsigned numFormals,
9188 JSObject* templateObject, bool saveAndRestore, Register resultreg)
9189 {
9190 // Compute actuals() + numFormals.
9191 size_t actualsOffset = frameSize() + JitFrameLayout::offsetOfActualArgs();
9192 masm.moveStackPtrTo(temp1);
9193 masm.addPtr(Imm32(sizeof(Value) * numFormals + actualsOffset), temp1);
9194
9195 // Compute numActuals - numFormals.
9196 Label emptyLength, joinLength;
9197 masm.movePtr(numActuals, temp0);
9198 masm.branch32(Assembler::LessThanOrEqual, temp0, Imm32(numFormals), &emptyLength);
9199 masm.sub32(Imm32(numFormals), temp0);
9200 masm.jump(&joinLength);
9201 {
9202 masm.bind(&emptyLength);
9203 masm.move32(Imm32(0), temp0);
9204 }
9205 masm.bind(&joinLength);
9206
9207 if (saveAndRestore)
9208 saveLive(lir);
9209
9210 pushArg(array);
9211 pushArg(ImmGCPtr(templateObject));
9212 pushArg(temp1);
9213 pushArg(temp0);
9214
9215 callVM(InitRestParameterInfo, lir);
9216
9217 if (saveAndRestore) {
9218 storePointerResultTo(resultreg);
9219 restoreLive(lir);
9220 }
9221 }
9222
9223 void
visitRest(LRest * lir)9224 CodeGenerator::visitRest(LRest* lir)
9225 {
9226 Register numActuals = ToRegister(lir->numActuals());
9227 Register temp0 = ToRegister(lir->getTemp(0));
9228 Register temp1 = ToRegister(lir->getTemp(1));
9229 Register temp2 = ToRegister(lir->getTemp(2));
9230 unsigned numFormals = lir->mir()->numFormals();
9231 ArrayObject* templateObject = lir->mir()->templateObject();
9232
9233 Label joinAlloc, failAlloc;
9234 masm.createGCObject(temp2, temp0, templateObject, gc::DefaultHeap, &failAlloc);
9235 masm.jump(&joinAlloc);
9236 {
9237 masm.bind(&failAlloc);
9238 masm.movePtr(ImmPtr(nullptr), temp2);
9239 }
9240 masm.bind(&joinAlloc);
9241
9242 emitRest(lir, temp2, numActuals, temp0, temp1, numFormals, templateObject, false, ToRegister(lir->output()));
9243 }
9244
9245 bool
generateWasm(wasm::SigIdDesc sigId,wasm::TrapOffset trapOffset,wasm::FuncOffsets * offsets)9246 CodeGenerator::generateWasm(wasm::SigIdDesc sigId, wasm::TrapOffset trapOffset,
9247 wasm::FuncOffsets* offsets)
9248 {
9249 JitSpew(JitSpew_Codegen, "# Emitting wasm code");
9250
9251 wasm::GenerateFunctionPrologue(masm, frameSize(), sigId, offsets);
9252
9253 // Overflow checks are omitted by CodeGenerator in some cases (leaf
9254 // functions with small framePushed). Perform overflow-checking after
9255 // pushing framePushed to catch cases with really large frames.
9256 Label onOverflow;
9257 if (!omitOverRecursedCheck()) {
9258 masm.branchPtr(Assembler::AboveOrEqual,
9259 Address(WasmTlsReg, offsetof(wasm::TlsData, stackLimit)),
9260 masm.getStackPointer(),
9261 &onOverflow);
9262 }
9263
9264 if (!generateBody())
9265 return false;
9266
9267 masm.bind(&returnLabel_);
9268 wasm::GenerateFunctionEpilogue(masm, frameSize(), offsets);
9269
9270 if (!omitOverRecursedCheck()) {
9271 // Since we just overflowed the stack, to be on the safe side, pop the
9272 // stack so that, when the trap exit stub executes, it is a safe
9273 // distance away from the end of the native stack.
9274 wasm::TrapDesc trap(trapOffset, wasm::Trap::StackOverflow, /* framePushed = */ 0);
9275 if (frameSize() > 0) {
9276 masm.bind(&onOverflow);
9277 masm.addToStackPtr(Imm32(frameSize()));
9278 masm.jump(trap);
9279 } else {
9280 masm.bindLater(&onOverflow, trap);
9281 }
9282 }
9283
9284 #if defined(JS_ION_PERF)
9285 // Note the end of the inline code and start of the OOL code.
9286 gen->perfSpewer().noteEndInlineCode(masm);
9287 #endif
9288
9289 if (!generateOutOfLineCode())
9290 return false;
9291
9292 masm.wasmEmitTrapOutOfLineCode();
9293
9294 masm.flush();
9295 if (masm.oom())
9296 return false;
9297
9298 offsets->end = masm.currentOffset();
9299
9300 MOZ_ASSERT(!masm.failureLabel()->used());
9301 MOZ_ASSERT(snapshots_.listSize() == 0);
9302 MOZ_ASSERT(snapshots_.RVATableSize() == 0);
9303 MOZ_ASSERT(recovers_.size() == 0);
9304 MOZ_ASSERT(bailouts_.empty());
9305 MOZ_ASSERT(graph.numConstants() == 0);
9306 MOZ_ASSERT(safepointIndices_.empty());
9307 MOZ_ASSERT(osiIndices_.empty());
9308 MOZ_ASSERT(cacheList_.empty());
9309 MOZ_ASSERT(safepoints_.size() == 0);
9310 MOZ_ASSERT(!scriptCounts_);
9311 return true;
9312 }
9313
9314 bool
generate()9315 CodeGenerator::generate()
9316 {
9317 JitSpew(JitSpew_Codegen, "# Emitting code for script %s:%" PRIuSIZE,
9318 gen->info().script()->filename(),
9319 gen->info().script()->lineno());
9320
9321 // Initialize native code table with an entry to the start of
9322 // top-level script.
9323 InlineScriptTree* tree = gen->info().inlineScriptTree();
9324 jsbytecode* startPC = tree->script()->code();
9325 BytecodeSite* startSite = new(gen->alloc()) BytecodeSite(tree, startPC);
9326 if (!addNativeToBytecodeEntry(startSite))
9327 return false;
9328
9329 if (!snapshots_.init())
9330 return false;
9331
9332 if (!safepoints_.init(gen->alloc()))
9333 return false;
9334
9335 if (!generatePrologue())
9336 return false;
9337
9338 // Before generating any code, we generate type checks for all parameters.
9339 // This comes before deoptTable_, because we can't use deopt tables without
9340 // creating the actual frame.
9341 generateArgumentsChecks();
9342
9343 if (frameClass_ != FrameSizeClass::None()) {
9344 deoptTable_ = gen->jitRuntime()->getBailoutTable(frameClass_);
9345 if (!deoptTable_)
9346 return false;
9347 }
9348
9349 // Skip over the alternative entry to IonScript code.
9350 Label skipPrologue;
9351 masm.jump(&skipPrologue);
9352
9353 // An alternative entry to the IonScript code, which doesn't test the
9354 // arguments.
9355 masm.flushBuffer();
9356 setSkipArgCheckEntryOffset(masm.size());
9357 masm.setFramePushed(0);
9358 if (!generatePrologue())
9359 return false;
9360
9361 masm.bind(&skipPrologue);
9362
9363 #ifdef DEBUG
9364 // Assert that the argument types are correct.
9365 generateArgumentsChecks(/* bailout = */ false);
9366 #endif
9367
9368 // Reset native => bytecode map table with top-level script and startPc.
9369 if (!addNativeToBytecodeEntry(startSite))
9370 return false;
9371
9372 if (!generateBody())
9373 return false;
9374
9375 // Reset native => bytecode map table with top-level script and startPc.
9376 if (!addNativeToBytecodeEntry(startSite))
9377 return false;
9378
9379 if (!generateEpilogue())
9380 return false;
9381
9382 // Reset native => bytecode map table with top-level script and startPc.
9383 if (!addNativeToBytecodeEntry(startSite))
9384 return false;
9385
9386 generateInvalidateEpilogue();
9387 #if defined(JS_ION_PERF)
9388 // Note the end of the inline code and start of the OOL code.
9389 perfSpewer_.noteEndInlineCode(masm);
9390 #endif
9391
9392 // native => bytecode entries for OOL code will be added
9393 // by CodeGeneratorShared::generateOutOfLineCode
9394 if (!generateOutOfLineCode())
9395 return false;
9396
9397 // Add terminal entry.
9398 if (!addNativeToBytecodeEntry(startSite))
9399 return false;
9400
9401 // Dump Native to bytecode entries to spew.
9402 dumpNativeToBytecodeEntries();
9403
9404 return !masm.oom();
9405 }
9406
9407 bool
linkSharedStubs(JSContext * cx)9408 CodeGenerator::linkSharedStubs(JSContext* cx)
9409 {
9410 for (uint32_t i = 0; i < sharedStubs_.length(); i++) {
9411 ICStub *stub = nullptr;
9412
9413 switch (sharedStubs_[i].kind) {
9414 case ICStub::Kind::BinaryArith_Fallback: {
9415 ICBinaryArith_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonMonkey);
9416 stub = stubCompiler.getStub(&stubSpace_);
9417 break;
9418 }
9419 case ICStub::Kind::UnaryArith_Fallback: {
9420 ICUnaryArith_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonMonkey);
9421 stub = stubCompiler.getStub(&stubSpace_);
9422 break;
9423 }
9424 case ICStub::Kind::Compare_Fallback: {
9425 ICCompare_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonMonkey);
9426 stub = stubCompiler.getStub(&stubSpace_);
9427 break;
9428 }
9429 case ICStub::Kind::GetProp_Fallback: {
9430 ICGetProp_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonMonkey);
9431 stub = stubCompiler.getStub(&stubSpace_);
9432 break;
9433 }
9434 case ICStub::Kind::NewArray_Fallback: {
9435 JSScript* script = sharedStubs_[i].entry.script();
9436 jsbytecode* pc = sharedStubs_[i].entry.pc(script);
9437 ObjectGroup* group = ObjectGroup::allocationSiteGroup(cx, script, pc, JSProto_Array);
9438 if (!group)
9439 return false;
9440
9441 ICNewArray_Fallback::Compiler stubCompiler(cx, group, ICStubCompiler::Engine::IonMonkey);
9442 stub = stubCompiler.getStub(&stubSpace_);
9443 break;
9444 }
9445 case ICStub::Kind::NewObject_Fallback: {
9446 ICNewObject_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonMonkey);
9447 stub = stubCompiler.getStub(&stubSpace_);
9448 break;
9449 }
9450 default:
9451 MOZ_CRASH("Unsupported shared stub.");
9452 }
9453
9454 if (!stub)
9455 return false;
9456
9457 sharedStubs_[i].entry.setFirstStub(stub);
9458 }
9459 return true;
9460 }
9461
9462 bool
link(JSContext * cx,CompilerConstraintList * constraints)9463 CodeGenerator::link(JSContext* cx, CompilerConstraintList* constraints)
9464 {
9465 RootedScript script(cx, gen->info().script());
9466 OptimizationLevel optimizationLevel = gen->optimizationInfo().level();
9467
9468 // Capture the SIMD template objects which are used during the
9469 // compilation. This iterates over the template objects, using read-barriers
9470 // to let the GC know that the generated code relies on these template
9471 // objects.
9472 captureSimdTemplate(cx);
9473
9474 // We finished the new IonScript. Invalidate the current active IonScript,
9475 // so we can replace it with this new (probably higher optimized) version.
9476 if (script->hasIonScript()) {
9477 MOZ_ASSERT(script->ionScript()->isRecompiling());
9478 // Do a normal invalidate, except don't cancel offThread compilations,
9479 // since that will cancel this compilation too.
9480 Invalidate(cx, script, /* resetUses */ false, /* cancelOffThread*/ false);
9481 }
9482
9483 if (scriptCounts_ && !script->hasScriptCounts() && !script->initScriptCounts(cx))
9484 return false;
9485
9486 if (!linkSharedStubs(cx))
9487 return false;
9488
9489 // Check to make sure we didn't have a mid-build invalidation. If so, we
9490 // will trickle to jit::Compile() and return Method_Skipped.
9491 uint32_t warmUpCount = script->getWarmUpCount();
9492
9493 // Record constraints. If an error occured, returns false and potentially
9494 // prevent future compilations. Otherwise, if an invalidation occured, then
9495 // skip the current compilation.
9496 RecompileInfo recompileInfo;
9497 bool validRecompiledInfo = false;
9498 if (!FinishCompilation(cx, script, constraints, &recompileInfo, &validRecompiledInfo))
9499 return false;
9500 if (!validRecompiledInfo)
9501 return true;
9502 auto guardRecordedConstraints = mozilla::MakeScopeExit([&] {
9503 // In case of error, invalidate the current recompileInfo.
9504 recompileInfo.compilerOutput(cx->zone()->types)->invalidate();
9505 });
9506
9507 // IonMonkey could have inferred better type information during
9508 // compilation. Since adding the new information to the actual type
9509 // information can reset the usecount, increase it back to what it was
9510 // before.
9511 if (warmUpCount > script->getWarmUpCount())
9512 script->incWarmUpCounter(warmUpCount - script->getWarmUpCount());
9513
9514 uint32_t argumentSlots = (gen->info().nargs() + 1) * sizeof(Value);
9515 uint32_t scriptFrameSize = frameClass_ == FrameSizeClass::None()
9516 ? frameDepth_
9517 : FrameSizeClass::FromDepth(frameDepth_).frameSize();
9518
9519 // We encode safepoints after the OSI-point offsets have been determined.
9520 if (!encodeSafepoints())
9521 return false;
9522
9523 IonScript* ionScript =
9524 IonScript::New(cx, recompileInfo,
9525 graph.totalSlotCount(), argumentSlots, scriptFrameSize,
9526 snapshots_.listSize(), snapshots_.RVATableSize(),
9527 recovers_.size(), bailouts_.length(), graph.numConstants(),
9528 safepointIndices_.length(), osiIndices_.length(),
9529 cacheList_.length(), runtimeData_.length(),
9530 safepoints_.size(), patchableBackedges_.length(),
9531 sharedStubs_.length(), optimizationLevel);
9532 if (!ionScript)
9533 return false;
9534 auto guardIonScript = mozilla::MakeScopeExit([&ionScript] {
9535 // Use js_free instead of IonScript::Destroy: the cache list and
9536 // backedge list are still uninitialized.
9537 js_free(ionScript);
9538 });
9539
9540 // Also, note that creating the code here during an incremental GC will
9541 // trace the code and mark all GC things it refers to. This captures any
9542 // read barriers which were skipped while compiling the script off thread.
9543 Linker linker(masm);
9544 AutoFlushICache afc("IonLink");
9545 JitCode* code = linker.newCode<CanGC>(cx, ION_CODE, !patchableBackedges_.empty());
9546 if (!code)
9547 return false;
9548
9549 // Encode native to bytecode map if profiling is enabled.
9550 if (isProfilerInstrumentationEnabled()) {
9551 // Generate native-to-bytecode main table.
9552 if (!generateCompactNativeToBytecodeMap(cx, code))
9553 return false;
9554
9555 uint8_t* ionTableAddr = ((uint8_t*) nativeToBytecodeMap_) + nativeToBytecodeTableOffset_;
9556 JitcodeIonTable* ionTable = (JitcodeIonTable*) ionTableAddr;
9557
9558 // Construct the IonEntry that will go into the global table.
9559 JitcodeGlobalEntry::IonEntry entry;
9560 if (!ionTable->makeIonEntry(cx, code, nativeToBytecodeScriptListLength_,
9561 nativeToBytecodeScriptList_, entry))
9562 {
9563 js_free(nativeToBytecodeScriptList_);
9564 js_free(nativeToBytecodeMap_);
9565 return false;
9566 }
9567
9568 // nativeToBytecodeScriptList_ is no longer needed.
9569 js_free(nativeToBytecodeScriptList_);
9570
9571 // Generate the tracked optimizations map.
9572 if (isOptimizationTrackingEnabled()) {
9573 // Treat OOMs and failures as if optimization tracking were turned off.
9574 IonTrackedTypeVector* allTypes = cx->new_<IonTrackedTypeVector>();
9575 if (allTypes && generateCompactTrackedOptimizationsMap(cx, code, allTypes)) {
9576 const uint8_t* optsRegionTableAddr = trackedOptimizationsMap_ +
9577 trackedOptimizationsRegionTableOffset_;
9578 const IonTrackedOptimizationsRegionTable* optsRegionTable =
9579 (const IonTrackedOptimizationsRegionTable*) optsRegionTableAddr;
9580 const uint8_t* optsTypesTableAddr = trackedOptimizationsMap_ +
9581 trackedOptimizationsTypesTableOffset_;
9582 const IonTrackedOptimizationsTypesTable* optsTypesTable =
9583 (const IonTrackedOptimizationsTypesTable*) optsTypesTableAddr;
9584 const uint8_t* optsAttemptsTableAddr = trackedOptimizationsMap_ +
9585 trackedOptimizationsAttemptsTableOffset_;
9586 const IonTrackedOptimizationsAttemptsTable* optsAttemptsTable =
9587 (const IonTrackedOptimizationsAttemptsTable*) optsAttemptsTableAddr;
9588 entry.initTrackedOptimizations(optsRegionTable, optsTypesTable, optsAttemptsTable,
9589 allTypes);
9590 } else {
9591 cx->recoverFromOutOfMemory();
9592 }
9593 }
9594
9595 // Add entry to the global table.
9596 JitcodeGlobalTable* globalTable = cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
9597 if (!globalTable->addEntry(entry, cx->runtime())) {
9598 // Memory may have been allocated for the entry.
9599 entry.destroy();
9600 return false;
9601 }
9602
9603 // Mark the jitcode as having a bytecode map.
9604 code->setHasBytecodeMap();
9605 } else {
9606 // Add a dumy jitcodeGlobalTable entry.
9607 JitcodeGlobalEntry::DummyEntry entry;
9608 entry.init(code, code->raw(), code->rawEnd());
9609
9610 // Add entry to the global table.
9611 JitcodeGlobalTable* globalTable = cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
9612 if (!globalTable->addEntry(entry, cx->runtime())) {
9613 // Memory may have been allocated for the entry.
9614 entry.destroy();
9615 return false;
9616 }
9617
9618 // Mark the jitcode as having a bytecode map.
9619 code->setHasBytecodeMap();
9620 }
9621
9622 ionScript->setMethod(code);
9623 ionScript->setSkipArgCheckEntryOffset(getSkipArgCheckEntryOffset());
9624
9625 // If SPS is enabled, mark IonScript as having been instrumented with SPS
9626 if (isProfilerInstrumentationEnabled())
9627 ionScript->setHasProfilingInstrumentation();
9628
9629 script->setIonScript(cx->runtime(), ionScript);
9630
9631 // Adopt fallback shared stubs from the compiler into the ion script.
9632 ionScript->adoptFallbackStubs(&stubSpace_);
9633
9634 Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, invalidateEpilogueData_),
9635 ImmPtr(ionScript),
9636 ImmPtr((void*)-1));
9637
9638 for (size_t i = 0; i < ionScriptLabels_.length(); i++) {
9639 Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, ionScriptLabels_[i]),
9640 ImmPtr(ionScript),
9641 ImmPtr((void*)-1));
9642 }
9643
9644 #ifdef JS_TRACE_LOGGING
9645 bool TLFailed = false;
9646 TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime());
9647
9648 for (uint32_t i = 0; i < patchableTLEvents_.length(); i++) {
9649 // Create an event on the mainthread.
9650 TraceLoggerEvent event(logger, patchableTLEvents_[i].event);
9651 if (!event.hasPayload() || !ionScript->addTraceLoggerEvent(event)) {
9652 TLFailed = true;
9653 break;
9654 }
9655 Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, patchableTLEvents_[i].offset),
9656 ImmPtr((void*) uintptr_t(event.payload()->textId())),
9657 ImmPtr((void*)0));
9658 }
9659
9660 if (!TLFailed && patchableTLScripts_.length() > 0) {
9661 MOZ_ASSERT(TraceLogTextIdEnabled(TraceLogger_Scripts));
9662 TraceLoggerEvent event(logger, TraceLogger_Scripts, script);
9663 if (!event.hasPayload() || !ionScript->addTraceLoggerEvent(event))
9664 TLFailed = true;
9665 if (!TLFailed) {
9666 uint32_t textId = event.payload()->textId();
9667 for (uint32_t i = 0; i < patchableTLScripts_.length(); i++) {
9668 Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, patchableTLScripts_[i]),
9669 ImmPtr((void*) uintptr_t(textId)),
9670 ImmPtr((void*)0));
9671 }
9672 }
9673 }
9674
9675 if (!TLFailed) {
9676 for (uint32_t i = 0; i < patchableTraceLoggers_.length(); i++) {
9677 Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, patchableTraceLoggers_[i]),
9678 ImmPtr(logger),
9679 ImmPtr(nullptr));
9680 }
9681 }
9682 #endif
9683
9684 // Patch shared stub IC loads using IC entries
9685 for (size_t i = 0; i < sharedStubs_.length(); i++) {
9686 CodeOffset label = sharedStubs_[i].label;
9687
9688 IonICEntry& entry = ionScript->sharedStubList()[i];
9689 entry = sharedStubs_[i].entry;
9690 Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, label),
9691 ImmPtr(&entry),
9692 ImmPtr((void*)-1));
9693
9694 MOZ_ASSERT(entry.hasStub());
9695 MOZ_ASSERT(entry.firstStub()->isFallback());
9696
9697 entry.firstStub()->toFallbackStub()->fixupICEntry(&entry);
9698 }
9699
9700 // for generating inline caches during the execution.
9701 if (runtimeData_.length())
9702 ionScript->copyRuntimeData(&runtimeData_[0]);
9703 if (cacheList_.length())
9704 ionScript->copyCacheEntries(&cacheList_[0], masm);
9705
9706 JitSpew(JitSpew_Codegen, "Created IonScript %p (raw %p)",
9707 (void*) ionScript, (void*) code->raw());
9708
9709 ionScript->setInvalidationEpilogueDataOffset(invalidateEpilogueData_.offset());
9710 ionScript->setOsrPc(gen->info().osrPc());
9711 ionScript->setOsrEntryOffset(getOsrEntryOffset());
9712 ionScript->setInvalidationEpilogueOffset(invalidate_.offset());
9713
9714 ionScript->setDeoptTable(deoptTable_);
9715
9716 #if defined(JS_ION_PERF)
9717 if (PerfEnabled())
9718 perfSpewer_.writeProfile(script, code, masm);
9719 #endif
9720
9721 // for marking during GC.
9722 if (safepointIndices_.length())
9723 ionScript->copySafepointIndices(&safepointIndices_[0], masm);
9724 if (safepoints_.size())
9725 ionScript->copySafepoints(&safepoints_);
9726
9727 // for reconvering from an Ion Frame.
9728 if (bailouts_.length())
9729 ionScript->copyBailoutTable(&bailouts_[0]);
9730 if (osiIndices_.length())
9731 ionScript->copyOsiIndices(&osiIndices_[0], masm);
9732 if (snapshots_.listSize())
9733 ionScript->copySnapshots(&snapshots_);
9734 MOZ_ASSERT_IF(snapshots_.listSize(), recovers_.size());
9735 if (recovers_.size())
9736 ionScript->copyRecovers(&recovers_);
9737 if (graph.numConstants()) {
9738 const Value* vp = graph.constantPool();
9739 ionScript->copyConstants(vp);
9740 for (size_t i = 0; i < graph.numConstants(); i++) {
9741 const Value& v = vp[i];
9742 if (v.isObject() && IsInsideNursery(&v.toObject())) {
9743 cx->runtime()->gc.storeBuffer.putWholeCell(script);
9744 break;
9745 }
9746 }
9747 }
9748 if (patchableBackedges_.length() > 0)
9749 ionScript->copyPatchableBackedges(cx, code, patchableBackedges_.begin(), masm);
9750
9751 // The correct state for prebarriers is unknown until the end of compilation,
9752 // since a GC can occur during code generation. All barriers are emitted
9753 // off-by-default, and are toggled on here if necessary.
9754 if (cx->zone()->needsIncrementalBarrier())
9755 ionScript->toggleBarriers(true, DontReprotect);
9756
9757 // Attach any generated script counts to the script.
9758 if (IonScriptCounts* counts = extractScriptCounts())
9759 script->addIonCounts(counts);
9760
9761 guardIonScript.release();
9762 guardRecordedConstraints.release();
9763 return true;
9764 }
9765
9766 // An out-of-line path to convert a boxed int32 to either a float or double.
9767 class OutOfLineUnboxFloatingPoint : public OutOfLineCodeBase<CodeGenerator>
9768 {
9769 LUnboxFloatingPoint* unboxFloatingPoint_;
9770
9771 public:
OutOfLineUnboxFloatingPoint(LUnboxFloatingPoint * unboxFloatingPoint)9772 explicit OutOfLineUnboxFloatingPoint(LUnboxFloatingPoint* unboxFloatingPoint)
9773 : unboxFloatingPoint_(unboxFloatingPoint)
9774 { }
9775
accept(CodeGenerator * codegen)9776 void accept(CodeGenerator* codegen) {
9777 codegen->visitOutOfLineUnboxFloatingPoint(this);
9778 }
9779
unboxFloatingPoint() const9780 LUnboxFloatingPoint* unboxFloatingPoint() const {
9781 return unboxFloatingPoint_;
9782 }
9783 };
9784
9785 void
visitUnboxFloatingPoint(LUnboxFloatingPoint * lir)9786 CodeGenerator::visitUnboxFloatingPoint(LUnboxFloatingPoint* lir)
9787 {
9788 const ValueOperand box = ToValue(lir, LUnboxFloatingPoint::Input);
9789 const LDefinition* result = lir->output();
9790
9791 // Out-of-line path to convert int32 to double or bailout
9792 // if this instruction is fallible.
9793 OutOfLineUnboxFloatingPoint* ool = new(alloc()) OutOfLineUnboxFloatingPoint(lir);
9794 addOutOfLineCode(ool, lir->mir());
9795
9796 FloatRegister resultReg = ToFloatRegister(result);
9797 masm.branchTestDouble(Assembler::NotEqual, box, ool->entry());
9798 masm.unboxDouble(box, resultReg);
9799 if (lir->type() == MIRType::Float32)
9800 masm.convertDoubleToFloat32(resultReg, resultReg);
9801 masm.bind(ool->rejoin());
9802 }
9803
9804 void
visitOutOfLineUnboxFloatingPoint(OutOfLineUnboxFloatingPoint * ool)9805 CodeGenerator::visitOutOfLineUnboxFloatingPoint(OutOfLineUnboxFloatingPoint* ool)
9806 {
9807 LUnboxFloatingPoint* ins = ool->unboxFloatingPoint();
9808 const ValueOperand value = ToValue(ins, LUnboxFloatingPoint::Input);
9809
9810 if (ins->mir()->fallible()) {
9811 Label bail;
9812 masm.branchTestInt32(Assembler::NotEqual, value, &bail);
9813 bailoutFrom(&bail, ins->snapshot());
9814 }
9815 masm.int32ValueToFloatingPoint(value, ToFloatRegister(ins->output()), ins->type());
9816 masm.jump(ool->rejoin());
9817 }
9818
9819 typedef JSObject* (*BindVarFn)(JSContext*, HandleObject);
9820 static const VMFunction BindVarInfo = FunctionInfo<BindVarFn>(jit::BindVar, "BindVar");
9821
9822 void
visitCallBindVar(LCallBindVar * lir)9823 CodeGenerator::visitCallBindVar(LCallBindVar* lir)
9824 {
9825 pushArg(ToRegister(lir->environmentChain()));
9826 callVM(BindVarInfo, lir);
9827 }
9828
9829 typedef bool (*GetPropertyFn)(JSContext*, HandleValue, HandlePropertyName, MutableHandleValue);
9830 static const VMFunction GetPropertyInfo = FunctionInfo<GetPropertyFn>(GetProperty, "GetProperty");
9831
9832 void
visitCallGetProperty(LCallGetProperty * lir)9833 CodeGenerator::visitCallGetProperty(LCallGetProperty* lir)
9834 {
9835 pushArg(ImmGCPtr(lir->mir()->name()));
9836 pushArg(ToValue(lir, LCallGetProperty::Value));
9837
9838 callVM(GetPropertyInfo, lir);
9839 }
9840
9841 typedef bool (*GetOrCallElementFn)(JSContext*, MutableHandleValue, HandleValue, MutableHandleValue);
9842 static const VMFunction GetElementInfo =
9843 FunctionInfo<GetOrCallElementFn>(js::GetElement, "GetElement");
9844 static const VMFunction CallElementInfo =
9845 FunctionInfo<GetOrCallElementFn>(js::CallElement, "CallElement");
9846
9847 void
visitCallGetElement(LCallGetElement * lir)9848 CodeGenerator::visitCallGetElement(LCallGetElement* lir)
9849 {
9850 pushArg(ToValue(lir, LCallGetElement::RhsInput));
9851 pushArg(ToValue(lir, LCallGetElement::LhsInput));
9852
9853 JSOp op = JSOp(*lir->mir()->resumePoint()->pc());
9854
9855 if (op == JSOP_GETELEM) {
9856 callVM(GetElementInfo, lir);
9857 } else {
9858 MOZ_ASSERT(op == JSOP_CALLELEM);
9859 callVM(CallElementInfo, lir);
9860 }
9861 }
9862
9863 typedef bool (*SetObjectElementFn)(JSContext*, HandleObject, HandleValue, HandleValue,
9864 bool strict);
9865 static const VMFunction SetObjectElementInfo =
9866 FunctionInfo<SetObjectElementFn>(SetObjectElement, "SetObjectElement");
9867
9868 void
visitCallSetElement(LCallSetElement * lir)9869 CodeGenerator::visitCallSetElement(LCallSetElement* lir)
9870 {
9871 pushArg(Imm32(lir->mir()->strict()));
9872 pushArg(ToValue(lir, LCallSetElement::Value));
9873 pushArg(ToValue(lir, LCallSetElement::Index));
9874 pushArg(ToRegister(lir->getOperand(0)));
9875 callVM(SetObjectElementInfo, lir);
9876 }
9877
9878 typedef bool (*InitElementArrayFn)(JSContext*, jsbytecode*, HandleObject, uint32_t, HandleValue);
9879 static const VMFunction InitElementArrayInfo =
9880 FunctionInfo<InitElementArrayFn>(js::InitElementArray, "InitElementArray");
9881
9882 void
visitCallInitElementArray(LCallInitElementArray * lir)9883 CodeGenerator::visitCallInitElementArray(LCallInitElementArray* lir)
9884 {
9885 pushArg(ToValue(lir, LCallInitElementArray::Value));
9886 pushArg(Imm32(lir->mir()->index()));
9887 pushArg(ToRegister(lir->getOperand(0)));
9888 pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
9889 callVM(InitElementArrayInfo, lir);
9890 }
9891
9892 void
visitLoadFixedSlotV(LLoadFixedSlotV * ins)9893 CodeGenerator::visitLoadFixedSlotV(LLoadFixedSlotV* ins)
9894 {
9895 const Register obj = ToRegister(ins->getOperand(0));
9896 size_t slot = ins->mir()->slot();
9897 ValueOperand result = GetValueOutput(ins);
9898
9899 masm.loadValue(Address(obj, NativeObject::getFixedSlotOffset(slot)), result);
9900 }
9901
9902 void
visitLoadFixedSlotT(LLoadFixedSlotT * ins)9903 CodeGenerator::visitLoadFixedSlotT(LLoadFixedSlotT* ins)
9904 {
9905 const Register obj = ToRegister(ins->getOperand(0));
9906 size_t slot = ins->mir()->slot();
9907 AnyRegister result = ToAnyRegister(ins->getDef(0));
9908 MIRType type = ins->mir()->type();
9909
9910 masm.loadUnboxedValue(Address(obj, NativeObject::getFixedSlotOffset(slot)), type, result);
9911 }
9912
9913 void
visitLoadFixedSlotAndUnbox(LLoadFixedSlotAndUnbox * ins)9914 CodeGenerator::visitLoadFixedSlotAndUnbox(LLoadFixedSlotAndUnbox* ins)
9915 {
9916 const MLoadFixedSlotAndUnbox* mir = ins->mir();
9917 MIRType type = mir->type();
9918 const Register input = ToRegister(ins->getOperand(0));
9919 AnyRegister result = ToAnyRegister(ins->output());
9920 size_t slot = mir->slot();
9921
9922 Address address(input, NativeObject::getFixedSlotOffset(slot));
9923 Label bail;
9924 if (type == MIRType::Double) {
9925 MOZ_ASSERT(result.isFloat());
9926 masm.ensureDouble(address, result.fpu(), &bail);
9927 if (mir->fallible())
9928 bailoutFrom(&bail, ins->snapshot());
9929 return;
9930 }
9931 if (mir->fallible()) {
9932 switch (type) {
9933 case MIRType::Int32:
9934 masm.branchTestInt32(Assembler::NotEqual, address, &bail);
9935 break;
9936 case MIRType::Boolean:
9937 masm.branchTestBoolean(Assembler::NotEqual, address, &bail);
9938 break;
9939 default:
9940 MOZ_CRASH("Given MIRType cannot be unboxed.");
9941 }
9942 bailoutFrom(&bail, ins->snapshot());
9943 }
9944 masm.loadUnboxedValue(address, type, result);
9945 }
9946
9947 void
visitStoreFixedSlotV(LStoreFixedSlotV * ins)9948 CodeGenerator::visitStoreFixedSlotV(LStoreFixedSlotV* ins)
9949 {
9950 const Register obj = ToRegister(ins->getOperand(0));
9951 size_t slot = ins->mir()->slot();
9952
9953 const ValueOperand value = ToValue(ins, LStoreFixedSlotV::Value);
9954
9955 Address address(obj, NativeObject::getFixedSlotOffset(slot));
9956 if (ins->mir()->needsBarrier())
9957 emitPreBarrier(address);
9958
9959 masm.storeValue(value, address);
9960 }
9961
9962 void
visitStoreFixedSlotT(LStoreFixedSlotT * ins)9963 CodeGenerator::visitStoreFixedSlotT(LStoreFixedSlotT* ins)
9964 {
9965 const Register obj = ToRegister(ins->getOperand(0));
9966 size_t slot = ins->mir()->slot();
9967
9968 const LAllocation* value = ins->value();
9969 MIRType valueType = ins->mir()->value()->type();
9970
9971 Address address(obj, NativeObject::getFixedSlotOffset(slot));
9972 if (ins->mir()->needsBarrier())
9973 emitPreBarrier(address);
9974
9975 if (valueType == MIRType::ObjectOrNull) {
9976 Register nvalue = ToRegister(value);
9977 masm.storeObjectOrNull(nvalue, address);
9978 } else {
9979 ConstantOrRegister nvalue = value->isConstant()
9980 ? ConstantOrRegister(value->toConstant()->toJSValue())
9981 : TypedOrValueRegister(valueType, ToAnyRegister(value));
9982 masm.storeConstantOrRegister(nvalue, address);
9983 }
9984 }
9985
9986 void
visitGetNameCache(LGetNameCache * ins)9987 CodeGenerator::visitGetNameCache(LGetNameCache* ins)
9988 {
9989 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
9990 Register envChain = ToRegister(ins->envObj());
9991 TypedOrValueRegister output(GetValueOutput(ins));
9992 bool isTypeOf = ins->mir()->accessKind() != MGetNameCache::NAME;
9993
9994 NameIC cache(liveRegs, isTypeOf, envChain, ins->mir()->name(), output);
9995 cache.setProfilerLeavePC(ins->mir()->profilerLeavePc());
9996 addCache(ins, allocateCache(cache));
9997 }
9998
9999 typedef bool (*NameICFn)(JSContext*, HandleScript, size_t, HandleObject, MutableHandleValue);
10000 const VMFunction NameIC::UpdateInfo = FunctionInfo<NameICFn>(NameIC::update, "NameIC::update");
10001
10002 void
visitNameIC(OutOfLineUpdateCache * ool,DataPtr<NameIC> & ic)10003 CodeGenerator::visitNameIC(OutOfLineUpdateCache* ool, DataPtr<NameIC>& ic)
10004 {
10005 LInstruction* lir = ool->lir();
10006 saveLive(lir);
10007
10008 pushArg(ic->environmentChainReg());
10009 pushArg(Imm32(ool->getCacheIndex()));
10010 pushArg(ImmGCPtr(gen->info().script()));
10011 callVM(NameIC::UpdateInfo, lir);
10012 StoreValueTo(ic->outputReg()).generate(this);
10013 restoreLiveIgnore(lir, StoreValueTo(ic->outputReg()).clobbered());
10014
10015 masm.jump(ool->rejoin());
10016 }
10017
10018 void
addGetPropertyCache(LInstruction * ins,LiveRegisterSet liveRegs,Register objReg,const ConstantOrRegister & id,TypedOrValueRegister output,bool monitoredResult,bool allowDoubleResult,jsbytecode * profilerLeavePc)10019 CodeGenerator::addGetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs, Register objReg,
10020 const ConstantOrRegister& id, TypedOrValueRegister output,
10021 bool monitoredResult, bool allowDoubleResult,
10022 jsbytecode* profilerLeavePc)
10023 {
10024 GetPropertyIC cache(liveRegs, objReg, id, output, monitoredResult, allowDoubleResult);
10025 cache.setProfilerLeavePC(profilerLeavePc);
10026 addCache(ins, allocateCache(cache));
10027 }
10028
10029 void
addSetPropertyCache(LInstruction * ins,LiveRegisterSet liveRegs,Register objReg,Register temp,Register tempUnbox,FloatRegister tempDouble,FloatRegister tempF32,const ConstantOrRegister & id,const ConstantOrRegister & value,bool strict,bool needsTypeBarrier,bool guardHoles,jsbytecode * profilerLeavePc)10030 CodeGenerator::addSetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs, Register objReg,
10031 Register temp, Register tempUnbox, FloatRegister tempDouble,
10032 FloatRegister tempF32, const ConstantOrRegister& id,
10033 const ConstantOrRegister& value,
10034 bool strict, bool needsTypeBarrier, bool guardHoles,
10035 jsbytecode* profilerLeavePc)
10036 {
10037 SetPropertyIC cache(liveRegs, objReg, temp, tempUnbox, tempDouble, tempF32, id, value, strict,
10038 needsTypeBarrier, guardHoles);
10039 cache.setProfilerLeavePC(profilerLeavePc);
10040 addCache(ins, allocateCache(cache));
10041 }
10042
10043 ConstantOrRegister
toConstantOrRegister(LInstruction * lir,size_t n,MIRType type)10044 CodeGenerator::toConstantOrRegister(LInstruction* lir, size_t n, MIRType type)
10045 {
10046 if (type == MIRType::Value)
10047 return TypedOrValueRegister(ToValue(lir, n));
10048
10049 const LAllocation* value = lir->getOperand(n);
10050 if (value->isConstant())
10051 return ConstantOrRegister(value->toConstant()->toJSValue());
10052
10053 return TypedOrValueRegister(type, ToAnyRegister(value));
10054 }
10055
10056 void
visitGetPropertyCacheV(LGetPropertyCacheV * ins)10057 CodeGenerator::visitGetPropertyCacheV(LGetPropertyCacheV* ins)
10058 {
10059 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
10060 Register objReg = ToRegister(ins->getOperand(0));
10061 ConstantOrRegister id = toConstantOrRegister(ins, LGetPropertyCacheV::Id, ins->mir()->idval()->type());
10062 bool monitoredResult = ins->mir()->monitoredResult();
10063 TypedOrValueRegister output = TypedOrValueRegister(GetValueOutput(ins));
10064
10065 addGetPropertyCache(ins, liveRegs, objReg, id, output, monitoredResult,
10066 ins->mir()->allowDoubleResult(), ins->mir()->profilerLeavePc());
10067 }
10068
10069 void
visitGetPropertyCacheT(LGetPropertyCacheT * ins)10070 CodeGenerator::visitGetPropertyCacheT(LGetPropertyCacheT* ins)
10071 {
10072 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
10073 Register objReg = ToRegister(ins->getOperand(0));
10074 ConstantOrRegister id = toConstantOrRegister(ins, LGetPropertyCacheT::Id, ins->mir()->idval()->type());
10075 bool monitoredResult = ins->mir()->monitoredResult();
10076 TypedOrValueRegister output(ins->mir()->type(), ToAnyRegister(ins->getDef(0)));
10077
10078 addGetPropertyCache(ins, liveRegs, objReg, id, output, monitoredResult,
10079 ins->mir()->allowDoubleResult(), ins->mir()->profilerLeavePc());
10080 }
10081
10082 typedef bool (*GetPropertyICFn)(JSContext*, HandleScript, size_t, HandleObject, HandleValue,
10083 MutableHandleValue);
10084 const VMFunction GetPropertyIC::UpdateInfo =
10085 FunctionInfo<GetPropertyICFn>(GetPropertyIC::update, "GetPropertyIC::update");
10086
10087 void
visitGetPropertyIC(OutOfLineUpdateCache * ool,DataPtr<GetPropertyIC> & ic)10088 CodeGenerator::visitGetPropertyIC(OutOfLineUpdateCache* ool, DataPtr<GetPropertyIC>& ic)
10089 {
10090 LInstruction* lir = ool->lir();
10091
10092 if (ic->idempotent()) {
10093 size_t numLocs;
10094 CacheLocationList& cacheLocs = lir->mirRaw()->toGetPropertyCache()->location();
10095 size_t locationBase;
10096 if (!addCacheLocations(cacheLocs, &numLocs, &locationBase))
10097 return;
10098 ic->setLocationInfo(locationBase, numLocs);
10099 }
10100
10101 saveLive(lir);
10102
10103 pushArg(ic->id());
10104 pushArg(ic->object());
10105 pushArg(Imm32(ool->getCacheIndex()));
10106 pushArg(ImmGCPtr(gen->info().script()));
10107 callVM(GetPropertyIC::UpdateInfo, lir);
10108 StoreValueTo(ic->output()).generate(this);
10109 restoreLiveIgnore(lir, StoreValueTo(ic->output()).clobbered());
10110
10111 masm.jump(ool->rejoin());
10112 }
10113
10114 void
visitBindNameCache(LBindNameCache * ins)10115 CodeGenerator::visitBindNameCache(LBindNameCache* ins)
10116 {
10117 Register envChain = ToRegister(ins->environmentChain());
10118 Register output = ToRegister(ins->output());
10119 BindNameIC cache(envChain, ins->mir()->name(), output);
10120 cache.setProfilerLeavePC(ins->mir()->profilerLeavePc());
10121
10122 addCache(ins, allocateCache(cache));
10123 }
10124
10125 typedef JSObject* (*BindNameICFn)(JSContext*, HandleScript, size_t, HandleObject);
10126 const VMFunction BindNameIC::UpdateInfo =
10127 FunctionInfo<BindNameICFn>(BindNameIC::update, "BindNameIC::update");
10128
10129 void
visitBindNameIC(OutOfLineUpdateCache * ool,DataPtr<BindNameIC> & ic)10130 CodeGenerator::visitBindNameIC(OutOfLineUpdateCache* ool, DataPtr<BindNameIC>& ic)
10131 {
10132 LInstruction* lir = ool->lir();
10133 saveLive(lir);
10134
10135 pushArg(ic->environmentChainReg());
10136 pushArg(Imm32(ool->getCacheIndex()));
10137 pushArg(ImmGCPtr(gen->info().script()));
10138 callVM(BindNameIC::UpdateInfo, lir);
10139 StoreRegisterTo(ic->outputReg()).generate(this);
10140 restoreLiveIgnore(lir, StoreRegisterTo(ic->outputReg()).clobbered());
10141
10142 masm.jump(ool->rejoin());
10143 }
10144
10145 typedef bool (*SetPropertyFn)(JSContext*, HandleObject,
10146 HandlePropertyName, const HandleValue, bool, jsbytecode*);
10147 static const VMFunction SetPropertyInfo = FunctionInfo<SetPropertyFn>(SetProperty, "SetProperty");
10148
10149 void
visitCallSetProperty(LCallSetProperty * ins)10150 CodeGenerator::visitCallSetProperty(LCallSetProperty* ins)
10151 {
10152 ConstantOrRegister value = TypedOrValueRegister(ToValue(ins, LCallSetProperty::Value));
10153
10154 const Register objReg = ToRegister(ins->getOperand(0));
10155
10156 pushArg(ImmPtr(ins->mir()->resumePoint()->pc()));
10157 pushArg(Imm32(ins->mir()->strict()));
10158
10159 pushArg(value);
10160 pushArg(ImmGCPtr(ins->mir()->name()));
10161 pushArg(objReg);
10162
10163 callVM(SetPropertyInfo, ins);
10164 }
10165
10166 typedef bool (*DeletePropertyFn)(JSContext*, HandleValue, HandlePropertyName, bool*);
10167 static const VMFunction DeletePropertyStrictInfo =
10168 FunctionInfo<DeletePropertyFn>(DeletePropertyJit<true>, "DeletePropertyStrictJit");
10169 static const VMFunction DeletePropertyNonStrictInfo =
10170 FunctionInfo<DeletePropertyFn>(DeletePropertyJit<false>, "DeletePropertyNonStrictJit");
10171
10172 void
visitCallDeleteProperty(LCallDeleteProperty * lir)10173 CodeGenerator::visitCallDeleteProperty(LCallDeleteProperty* lir)
10174 {
10175 pushArg(ImmGCPtr(lir->mir()->name()));
10176 pushArg(ToValue(lir, LCallDeleteProperty::Value));
10177
10178 if (lir->mir()->strict())
10179 callVM(DeletePropertyStrictInfo, lir);
10180 else
10181 callVM(DeletePropertyNonStrictInfo, lir);
10182 }
10183
10184 typedef bool (*DeleteElementFn)(JSContext*, HandleValue, HandleValue, bool*);
10185 static const VMFunction DeleteElementStrictInfo =
10186 FunctionInfo<DeleteElementFn>(DeleteElementJit<true>, "DeleteElementStrictJit");
10187 static const VMFunction DeleteElementNonStrictInfo =
10188 FunctionInfo<DeleteElementFn>(DeleteElementJit<false>, "DeleteElementNonStrictJit");
10189
10190 void
visitCallDeleteElement(LCallDeleteElement * lir)10191 CodeGenerator::visitCallDeleteElement(LCallDeleteElement* lir)
10192 {
10193 pushArg(ToValue(lir, LCallDeleteElement::Index));
10194 pushArg(ToValue(lir, LCallDeleteElement::Value));
10195
10196 if (lir->mir()->strict())
10197 callVM(DeleteElementStrictInfo, lir);
10198 else
10199 callVM(DeleteElementNonStrictInfo, lir);
10200 }
10201
10202 void
visitSetPropertyCache(LSetPropertyCache * ins)10203 CodeGenerator::visitSetPropertyCache(LSetPropertyCache* ins)
10204 {
10205 LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
10206 Register objReg = ToRegister(ins->getOperand(0));
10207 Register temp = ToRegister(ins->temp());
10208 Register tempUnbox = ToTempUnboxRegister(ins->tempToUnboxIndex());
10209 FloatRegister tempDouble = ToTempFloatRegisterOrInvalid(ins->tempDouble());
10210 FloatRegister tempF32 = ToTempFloatRegisterOrInvalid(ins->tempFloat32());
10211
10212 ConstantOrRegister id =
10213 toConstantOrRegister(ins, LSetPropertyCache::Id, ins->mir()->idval()->type());
10214 ConstantOrRegister value =
10215 toConstantOrRegister(ins, LSetPropertyCache::Value, ins->mir()->value()->type());
10216
10217 addSetPropertyCache(ins, liveRegs, objReg, temp, tempUnbox, tempDouble, tempF32,
10218 id, value, ins->mir()->strict(), ins->mir()->needsTypeBarrier(),
10219 ins->mir()->guardHoles(), ins->mir()->profilerLeavePc());
10220 }
10221
10222 typedef bool (*SetPropertyICFn)(JSContext*, HandleScript, size_t, HandleObject, HandleValue,
10223 HandleValue);
10224 const VMFunction SetPropertyIC::UpdateInfo =
10225 FunctionInfo<SetPropertyICFn>(SetPropertyIC::update, "SetPropertyIC::update");
10226
10227 void
visitSetPropertyIC(OutOfLineUpdateCache * ool,DataPtr<SetPropertyIC> & ic)10228 CodeGenerator::visitSetPropertyIC(OutOfLineUpdateCache* ool, DataPtr<SetPropertyIC>& ic)
10229 {
10230 LInstruction* lir = ool->lir();
10231 saveLive(lir);
10232
10233 pushArg(ic->value());
10234 pushArg(ic->id());
10235 pushArg(ic->object());
10236 pushArg(Imm32(ool->getCacheIndex()));
10237 pushArg(ImmGCPtr(gen->info().script()));
10238 callVM(SetPropertyIC::UpdateInfo, lir);
10239 restoreLive(lir);
10240
10241 masm.jump(ool->rejoin());
10242 }
10243
10244 typedef bool (*ThrowFn)(JSContext*, HandleValue);
10245 static const VMFunction ThrowInfoCodeGen = FunctionInfo<ThrowFn>(js::Throw, "Throw");
10246
10247 void
visitThrow(LThrow * lir)10248 CodeGenerator::visitThrow(LThrow* lir)
10249 {
10250 pushArg(ToValue(lir, LThrow::Value));
10251 callVM(ThrowInfoCodeGen, lir);
10252 }
10253
10254 typedef bool (*BitNotFn)(JSContext*, HandleValue, int* p);
10255 static const VMFunction BitNotInfo = FunctionInfo<BitNotFn>(BitNot, "BitNot");
10256
10257 void
visitBitNotV(LBitNotV * lir)10258 CodeGenerator::visitBitNotV(LBitNotV* lir)
10259 {
10260 pushArg(ToValue(lir, LBitNotV::Input));
10261 callVM(BitNotInfo, lir);
10262 }
10263
10264 typedef bool (*BitopFn)(JSContext*, HandleValue, HandleValue, int* p);
10265 static const VMFunction BitAndInfo = FunctionInfo<BitopFn>(BitAnd, "BitAnd");
10266 static const VMFunction BitOrInfo = FunctionInfo<BitopFn>(BitOr, "BitOr");
10267 static const VMFunction BitXorInfo = FunctionInfo<BitopFn>(BitXor, "BitXor");
10268 static const VMFunction BitLhsInfo = FunctionInfo<BitopFn>(BitLsh, "BitLsh");
10269 static const VMFunction BitRhsInfo = FunctionInfo<BitopFn>(BitRsh, "BitRsh");
10270
10271 void
visitBitOpV(LBitOpV * lir)10272 CodeGenerator::visitBitOpV(LBitOpV* lir)
10273 {
10274 pushArg(ToValue(lir, LBitOpV::RhsInput));
10275 pushArg(ToValue(lir, LBitOpV::LhsInput));
10276
10277 switch (lir->jsop()) {
10278 case JSOP_BITAND:
10279 callVM(BitAndInfo, lir);
10280 break;
10281 case JSOP_BITOR:
10282 callVM(BitOrInfo, lir);
10283 break;
10284 case JSOP_BITXOR:
10285 callVM(BitXorInfo, lir);
10286 break;
10287 case JSOP_LSH:
10288 callVM(BitLhsInfo, lir);
10289 break;
10290 case JSOP_RSH:
10291 callVM(BitRhsInfo, lir);
10292 break;
10293 default:
10294 MOZ_CRASH("unexpected bitop");
10295 }
10296 }
10297
10298 class OutOfLineTypeOfV : public OutOfLineCodeBase<CodeGenerator>
10299 {
10300 LTypeOfV* ins_;
10301
10302 public:
OutOfLineTypeOfV(LTypeOfV * ins)10303 explicit OutOfLineTypeOfV(LTypeOfV* ins)
10304 : ins_(ins)
10305 { }
10306
accept(CodeGenerator * codegen)10307 void accept(CodeGenerator* codegen) {
10308 codegen->visitOutOfLineTypeOfV(this);
10309 }
ins() const10310 LTypeOfV* ins() const {
10311 return ins_;
10312 }
10313 };
10314
10315 void
visitTypeOfV(LTypeOfV * lir)10316 CodeGenerator::visitTypeOfV(LTypeOfV* lir)
10317 {
10318 const ValueOperand value = ToValue(lir, LTypeOfV::Input);
10319 Register output = ToRegister(lir->output());
10320 Register tag = masm.splitTagForTest(value);
10321
10322 const JSAtomState& names = GetJitContext()->runtime->names();
10323 Label done;
10324
10325 MDefinition* input = lir->mir()->input();
10326
10327 bool testObject = input->mightBeType(MIRType::Object);
10328 bool testNumber = input->mightBeType(MIRType::Int32) || input->mightBeType(MIRType::Double);
10329 bool testBoolean = input->mightBeType(MIRType::Boolean);
10330 bool testUndefined = input->mightBeType(MIRType::Undefined);
10331 bool testNull = input->mightBeType(MIRType::Null);
10332 bool testString = input->mightBeType(MIRType::String);
10333 bool testSymbol = input->mightBeType(MIRType::Symbol);
10334
10335 unsigned numTests = unsigned(testObject) + unsigned(testNumber) + unsigned(testBoolean) +
10336 unsigned(testUndefined) + unsigned(testNull) + unsigned(testString) + unsigned(testSymbol);
10337
10338 MOZ_ASSERT_IF(!input->emptyResultTypeSet(), numTests > 0);
10339
10340 OutOfLineTypeOfV* ool = nullptr;
10341 if (testObject) {
10342 if (lir->mir()->inputMaybeCallableOrEmulatesUndefined()) {
10343 // The input may be a callable object (result is "function") or may
10344 // emulate undefined (result is "undefined"). Use an OOL path.
10345 ool = new(alloc()) OutOfLineTypeOfV(lir);
10346 addOutOfLineCode(ool, lir->mir());
10347
10348 if (numTests > 1)
10349 masm.branchTestObject(Assembler::Equal, tag, ool->entry());
10350 else
10351 masm.jump(ool->entry());
10352 } else {
10353 // Input is not callable and does not emulate undefined, so if
10354 // it's an object the result is always "object".
10355 Label notObject;
10356 if (numTests > 1)
10357 masm.branchTestObject(Assembler::NotEqual, tag, ¬Object);
10358 masm.movePtr(ImmGCPtr(names.object), output);
10359 if (numTests > 1)
10360 masm.jump(&done);
10361 masm.bind(¬Object);
10362 }
10363 numTests--;
10364 }
10365
10366 if (testNumber) {
10367 Label notNumber;
10368 if (numTests > 1)
10369 masm.branchTestNumber(Assembler::NotEqual, tag, ¬Number);
10370 masm.movePtr(ImmGCPtr(names.number), output);
10371 if (numTests > 1)
10372 masm.jump(&done);
10373 masm.bind(¬Number);
10374 numTests--;
10375 }
10376
10377 if (testUndefined) {
10378 Label notUndefined;
10379 if (numTests > 1)
10380 masm.branchTestUndefined(Assembler::NotEqual, tag, ¬Undefined);
10381 masm.movePtr(ImmGCPtr(names.undefined), output);
10382 if (numTests > 1)
10383 masm.jump(&done);
10384 masm.bind(¬Undefined);
10385 numTests--;
10386 }
10387
10388 if (testNull) {
10389 Label notNull;
10390 if (numTests > 1)
10391 masm.branchTestNull(Assembler::NotEqual, tag, ¬Null);
10392 masm.movePtr(ImmGCPtr(names.object), output);
10393 if (numTests > 1)
10394 masm.jump(&done);
10395 masm.bind(¬Null);
10396 numTests--;
10397 }
10398
10399 if (testBoolean) {
10400 Label notBoolean;
10401 if (numTests > 1)
10402 masm.branchTestBoolean(Assembler::NotEqual, tag, ¬Boolean);
10403 masm.movePtr(ImmGCPtr(names.boolean), output);
10404 if (numTests > 1)
10405 masm.jump(&done);
10406 masm.bind(¬Boolean);
10407 numTests--;
10408 }
10409
10410 if (testString) {
10411 Label notString;
10412 if (numTests > 1)
10413 masm.branchTestString(Assembler::NotEqual, tag, ¬String);
10414 masm.movePtr(ImmGCPtr(names.string), output);
10415 if (numTests > 1)
10416 masm.jump(&done);
10417 masm.bind(¬String);
10418 numTests--;
10419 }
10420
10421 if (testSymbol) {
10422 Label notSymbol;
10423 if (numTests > 1)
10424 masm.branchTestSymbol(Assembler::NotEqual, tag, ¬Symbol);
10425 masm.movePtr(ImmGCPtr(names.symbol), output);
10426 if (numTests > 1)
10427 masm.jump(&done);
10428 masm.bind(¬Symbol);
10429 numTests--;
10430 }
10431
10432 MOZ_ASSERT(numTests == 0);
10433
10434 masm.bind(&done);
10435 if (ool)
10436 masm.bind(ool->rejoin());
10437 }
10438
10439 void
visitOutOfLineTypeOfV(OutOfLineTypeOfV * ool)10440 CodeGenerator::visitOutOfLineTypeOfV(OutOfLineTypeOfV* ool)
10441 {
10442 LTypeOfV* ins = ool->ins();
10443
10444 ValueOperand input = ToValue(ins, LTypeOfV::Input);
10445 Register temp = ToTempUnboxRegister(ins->tempToUnbox());
10446 Register output = ToRegister(ins->output());
10447
10448 Register obj = masm.extractObject(input, temp);
10449
10450 saveVolatile(output);
10451 masm.setupUnalignedABICall(output);
10452 masm.passABIArg(obj);
10453 masm.movePtr(ImmPtr(GetJitContext()->runtime), output);
10454 masm.passABIArg(output);
10455 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::TypeOfObjectOperation));
10456 masm.storeCallPointerResult(output);
10457 restoreVolatile(output);
10458
10459 masm.jump(ool->rejoin());
10460 }
10461
10462 typedef JSObject* (*ToAsyncFn)(JSContext*, HandleFunction);
10463 static const VMFunction ToAsyncInfo = FunctionInfo<ToAsyncFn>(js::WrapAsyncFunction, "ToAsync");
10464
10465 void
visitToAsync(LToAsync * lir)10466 CodeGenerator::visitToAsync(LToAsync* lir)
10467 {
10468 pushArg(ToRegister(lir->unwrapped()));
10469 callVM(ToAsyncInfo, lir);
10470 }
10471
10472 typedef bool (*ToIdFn)(JSContext*, HandleScript, jsbytecode*, HandleValue,
10473 MutableHandleValue);
10474 static const VMFunction ToIdInfo = FunctionInfo<ToIdFn>(ToIdOperation, "ToIdOperation");
10475
10476 void
visitToIdV(LToIdV * lir)10477 CodeGenerator::visitToIdV(LToIdV* lir)
10478 {
10479 Label notInt32;
10480 FloatRegister temp = ToFloatRegister(lir->tempFloat());
10481 const ValueOperand out = ToOutValue(lir);
10482 ValueOperand input = ToValue(lir, LToIdV::Input);
10483
10484 OutOfLineCode* ool = oolCallVM(ToIdInfo, lir,
10485 ArgList(ImmGCPtr(current->mir()->info().script()),
10486 ImmPtr(lir->mir()->resumePoint()->pc()),
10487 ToValue(lir, LToIdV::Input)),
10488 StoreValueTo(out));
10489
10490 Register tag = masm.splitTagForTest(input);
10491
10492 masm.branchTestInt32(Assembler::NotEqual, tag, ¬Int32);
10493 masm.moveValue(input, out);
10494 masm.jump(ool->rejoin());
10495
10496 masm.bind(¬Int32);
10497 masm.branchTestDouble(Assembler::NotEqual, tag, ool->entry());
10498 masm.unboxDouble(input, temp);
10499 masm.convertDoubleToInt32(temp, out.scratchReg(), ool->entry(), true);
10500 masm.tagValue(JSVAL_TYPE_INT32, out.scratchReg(), out);
10501
10502 masm.bind(ool->rejoin());
10503 }
10504
10505 template<typename T>
10506 void
emitLoadElementT(LLoadElementT * lir,const T & source)10507 CodeGenerator::emitLoadElementT(LLoadElementT* lir, const T& source)
10508 {
10509 if (LIRGenerator::allowTypedElementHoleCheck()) {
10510 if (lir->mir()->needsHoleCheck()) {
10511 Label bail;
10512 masm.branchTestMagic(Assembler::Equal, source, &bail);
10513 bailoutFrom(&bail, lir->snapshot());
10514 }
10515 } else {
10516 MOZ_ASSERT(!lir->mir()->needsHoleCheck());
10517 }
10518
10519 AnyRegister output = ToAnyRegister(lir->output());
10520 if (lir->mir()->loadDoubles())
10521 masm.loadDouble(source, output.fpu());
10522 else
10523 masm.loadUnboxedValue(source, lir->mir()->type(), output);
10524 }
10525
10526 void
visitLoadElementT(LLoadElementT * lir)10527 CodeGenerator::visitLoadElementT(LLoadElementT* lir)
10528 {
10529 Register elements = ToRegister(lir->elements());
10530 const LAllocation* index = lir->index();
10531 if (index->isConstant()) {
10532 int32_t offset = ToInt32(index) * sizeof(js::Value) + lir->mir()->offsetAdjustment();
10533 emitLoadElementT(lir, Address(elements, offset));
10534 } else {
10535 emitLoadElementT(lir, BaseIndex(elements, ToRegister(index), TimesEight,
10536 lir->mir()->offsetAdjustment()));
10537 }
10538 }
10539
10540 void
visitLoadElementV(LLoadElementV * load)10541 CodeGenerator::visitLoadElementV(LLoadElementV* load)
10542 {
10543 Register elements = ToRegister(load->elements());
10544 const ValueOperand out = ToOutValue(load);
10545
10546 if (load->index()->isConstant()) {
10547 NativeObject::elementsSizeMustNotOverflow();
10548 int32_t offset = ToInt32(load->index()) * sizeof(Value) + load->mir()->offsetAdjustment();
10549 masm.loadValue(Address(elements, offset), out);
10550 } else {
10551 masm.loadValue(BaseObjectElementIndex(elements, ToRegister(load->index()),
10552 load->mir()->offsetAdjustment()), out);
10553 }
10554
10555 if (load->mir()->needsHoleCheck()) {
10556 Label testMagic;
10557 masm.branchTestMagic(Assembler::Equal, out, &testMagic);
10558 bailoutFrom(&testMagic, load->snapshot());
10559 }
10560 }
10561
10562 void
visitLoadElementHole(LLoadElementHole * lir)10563 CodeGenerator::visitLoadElementHole(LLoadElementHole* lir)
10564 {
10565 Register elements = ToRegister(lir->elements());
10566 Register initLength = ToRegister(lir->initLength());
10567 const ValueOperand out = ToOutValue(lir);
10568
10569 const MLoadElementHole* mir = lir->mir();
10570
10571 // If the index is out of bounds, load |undefined|. Otherwise, load the
10572 // value.
10573 Label undefined, done;
10574 if (lir->index()->isConstant())
10575 masm.branch32(Assembler::BelowOrEqual, initLength, Imm32(ToInt32(lir->index())), &undefined);
10576 else
10577 masm.branch32(Assembler::BelowOrEqual, initLength, ToRegister(lir->index()), &undefined);
10578
10579 if (mir->unboxedType() != JSVAL_TYPE_MAGIC) {
10580 size_t width = UnboxedTypeSize(mir->unboxedType());
10581 if (lir->index()->isConstant()) {
10582 Address addr(elements, ToInt32(lir->index()) * width);
10583 masm.loadUnboxedProperty(addr, mir->unboxedType(), out);
10584 } else {
10585 BaseIndex addr(elements, ToRegister(lir->index()), ScaleFromElemWidth(width));
10586 masm.loadUnboxedProperty(addr, mir->unboxedType(), out);
10587 }
10588 } else {
10589 if (lir->index()->isConstant()) {
10590 NativeObject::elementsSizeMustNotOverflow();
10591 masm.loadValue(Address(elements, ToInt32(lir->index()) * sizeof(Value)), out);
10592 } else {
10593 masm.loadValue(BaseObjectElementIndex(elements, ToRegister(lir->index())), out);
10594 }
10595 }
10596
10597 // If a hole check is needed, and the value wasn't a hole, we're done.
10598 // Otherwise, we'll load undefined.
10599 if (lir->mir()->needsHoleCheck())
10600 masm.branchTestMagic(Assembler::NotEqual, out, &done);
10601 else
10602 masm.jump(&done);
10603
10604 masm.bind(&undefined);
10605
10606 if (mir->needsNegativeIntCheck()) {
10607 if (lir->index()->isConstant()) {
10608 if (ToInt32(lir->index()) < 0)
10609 bailout(lir->snapshot());
10610 } else {
10611 Label negative;
10612 masm.branch32(Assembler::LessThan, ToRegister(lir->index()), Imm32(0), &negative);
10613 bailoutFrom(&negative, lir->snapshot());
10614 }
10615 }
10616
10617 masm.moveValue(UndefinedValue(), out);
10618 masm.bind(&done);
10619 }
10620
10621 void
visitLoadUnboxedPointerV(LLoadUnboxedPointerV * lir)10622 CodeGenerator::visitLoadUnboxedPointerV(LLoadUnboxedPointerV* lir)
10623 {
10624 Register elements = ToRegister(lir->elements());
10625 const ValueOperand out = ToOutValue(lir);
10626
10627 if (lir->index()->isConstant()) {
10628 int32_t offset = ToInt32(lir->index()) * sizeof(uintptr_t) + lir->mir()->offsetAdjustment();
10629 masm.loadPtr(Address(elements, offset), out.scratchReg());
10630 } else {
10631 masm.loadPtr(BaseIndex(elements, ToRegister(lir->index()), ScalePointer,
10632 lir->mir()->offsetAdjustment()), out.scratchReg());
10633 }
10634
10635 Label notNull, done;
10636 masm.branchPtr(Assembler::NotEqual, out.scratchReg(), ImmWord(0), ¬Null);
10637
10638 masm.moveValue(NullValue(), out);
10639 masm.jump(&done);
10640
10641 masm.bind(¬Null);
10642 masm.tagValue(JSVAL_TYPE_OBJECT, out.scratchReg(), out);
10643
10644 masm.bind(&done);
10645 }
10646
10647 void
visitLoadUnboxedPointerT(LLoadUnboxedPointerT * lir)10648 CodeGenerator::visitLoadUnboxedPointerT(LLoadUnboxedPointerT* lir)
10649 {
10650 Register elements = ToRegister(lir->elements());
10651 const LAllocation* index = lir->index();
10652 Register out = ToRegister(lir->output());
10653
10654 bool bailOnNull;
10655 int32_t offsetAdjustment;
10656 if (lir->mir()->isLoadUnboxedObjectOrNull()) {
10657 bailOnNull = lir->mir()->toLoadUnboxedObjectOrNull()->nullBehavior() ==
10658 MLoadUnboxedObjectOrNull::BailOnNull;
10659 offsetAdjustment = lir->mir()->toLoadUnboxedObjectOrNull()->offsetAdjustment();
10660 } else if (lir->mir()->isLoadUnboxedString()) {
10661 bailOnNull = false;
10662 offsetAdjustment = lir->mir()->toLoadUnboxedString()->offsetAdjustment();
10663 } else {
10664 MOZ_CRASH();
10665 }
10666
10667 if (index->isConstant()) {
10668 Address source(elements, ToInt32(index) * sizeof(uintptr_t) + offsetAdjustment);
10669 masm.loadPtr(source, out);
10670 } else {
10671 BaseIndex source(elements, ToRegister(index), ScalePointer, offsetAdjustment);
10672 masm.loadPtr(source, out);
10673 }
10674
10675 if (bailOnNull) {
10676 Label bail;
10677 masm.branchTestPtr(Assembler::Zero, out, out, &bail);
10678 bailoutFrom(&bail, lir->snapshot());
10679 }
10680 }
10681
10682 void
visitUnboxObjectOrNull(LUnboxObjectOrNull * lir)10683 CodeGenerator::visitUnboxObjectOrNull(LUnboxObjectOrNull* lir)
10684 {
10685 Register obj = ToRegister(lir->input());
10686
10687 if (lir->mir()->fallible()) {
10688 Label bail;
10689 masm.branchTestPtr(Assembler::Zero, obj, obj, &bail);
10690 bailoutFrom(&bail, lir->snapshot());
10691 }
10692 }
10693
10694 void
visitLoadUnboxedScalar(LLoadUnboxedScalar * lir)10695 CodeGenerator::visitLoadUnboxedScalar(LLoadUnboxedScalar* lir)
10696 {
10697 Register elements = ToRegister(lir->elements());
10698 Register temp = lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
10699 AnyRegister out = ToAnyRegister(lir->output());
10700
10701 const MLoadUnboxedScalar* mir = lir->mir();
10702
10703 Scalar::Type readType = mir->readType();
10704 unsigned numElems = mir->numElems();
10705
10706 int width = Scalar::byteSize(mir->storageType());
10707 bool canonicalizeDouble = mir->canonicalizeDoubles();
10708
10709 Label fail;
10710 if (lir->index()->isConstant()) {
10711 Address source(elements, ToInt32(lir->index()) * width + mir->offsetAdjustment());
10712 masm.loadFromTypedArray(readType, source, out, temp, &fail, canonicalizeDouble, numElems);
10713 } else {
10714 BaseIndex source(elements, ToRegister(lir->index()), ScaleFromElemWidth(width),
10715 mir->offsetAdjustment());
10716 masm.loadFromTypedArray(readType, source, out, temp, &fail, canonicalizeDouble, numElems);
10717 }
10718
10719 if (fail.used())
10720 bailoutFrom(&fail, lir->snapshot());
10721 }
10722
10723 void
visitLoadTypedArrayElementHole(LLoadTypedArrayElementHole * lir)10724 CodeGenerator::visitLoadTypedArrayElementHole(LLoadTypedArrayElementHole* lir)
10725 {
10726 Register object = ToRegister(lir->object());
10727 const ValueOperand out = ToOutValue(lir);
10728
10729 // Load the length.
10730 Register scratch = out.scratchReg();
10731 RegisterOrInt32Constant key = ToRegisterOrInt32Constant(lir->index());
10732 masm.unboxInt32(Address(object, TypedArrayObject::lengthOffset()), scratch);
10733
10734 // Load undefined unless length > key.
10735 Label inbounds, done;
10736 masm.branch32(Assembler::Above, scratch, key, &inbounds);
10737 masm.moveValue(UndefinedValue(), out);
10738 masm.jump(&done);
10739
10740 // Load the elements vector.
10741 masm.bind(&inbounds);
10742 masm.loadPtr(Address(object, TypedArrayObject::dataOffset()), scratch);
10743
10744 Scalar::Type arrayType = lir->mir()->arrayType();
10745 int width = Scalar::byteSize(arrayType);
10746
10747 Label fail;
10748 if (key.isConstant()) {
10749 Address source(scratch, key.constant() * width);
10750 masm.loadFromTypedArray(arrayType, source, out, lir->mir()->allowDouble(),
10751 out.scratchReg(), &fail);
10752 } else {
10753 BaseIndex source(scratch, key.reg(), ScaleFromElemWidth(width));
10754 masm.loadFromTypedArray(arrayType, source, out, lir->mir()->allowDouble(),
10755 out.scratchReg(), &fail);
10756 }
10757
10758 if (fail.used())
10759 bailoutFrom(&fail, lir->snapshot());
10760
10761 masm.bind(&done);
10762 }
10763
10764 template <typename T>
10765 static inline void
StoreToTypedArray(MacroAssembler & masm,Scalar::Type writeType,const LAllocation * value,const T & dest,unsigned numElems=0)10766 StoreToTypedArray(MacroAssembler& masm, Scalar::Type writeType, const LAllocation* value,
10767 const T& dest, unsigned numElems = 0)
10768 {
10769 if (Scalar::isSimdType(writeType) ||
10770 writeType == Scalar::Float32 ||
10771 writeType == Scalar::Float64)
10772 {
10773 masm.storeToTypedFloatArray(writeType, ToFloatRegister(value), dest, numElems);
10774 } else {
10775 if (value->isConstant())
10776 masm.storeToTypedIntArray(writeType, Imm32(ToInt32(value)), dest);
10777 else
10778 masm.storeToTypedIntArray(writeType, ToRegister(value), dest);
10779 }
10780 }
10781
10782 void
visitStoreUnboxedScalar(LStoreUnboxedScalar * lir)10783 CodeGenerator::visitStoreUnboxedScalar(LStoreUnboxedScalar* lir)
10784 {
10785 Register elements = ToRegister(lir->elements());
10786 const LAllocation* value = lir->value();
10787
10788 const MStoreUnboxedScalar* mir = lir->mir();
10789
10790 Scalar::Type writeType = mir->writeType();
10791 unsigned numElems = mir->numElems();
10792
10793 int width = Scalar::byteSize(mir->storageType());
10794
10795 if (lir->index()->isConstant()) {
10796 Address dest(elements, ToInt32(lir->index()) * width + mir->offsetAdjustment());
10797 StoreToTypedArray(masm, writeType, value, dest, numElems);
10798 } else {
10799 BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width),
10800 mir->offsetAdjustment());
10801 StoreToTypedArray(masm, writeType, value, dest, numElems);
10802 }
10803 }
10804
10805 void
visitStoreTypedArrayElementHole(LStoreTypedArrayElementHole * lir)10806 CodeGenerator::visitStoreTypedArrayElementHole(LStoreTypedArrayElementHole* lir)
10807 {
10808 Register elements = ToRegister(lir->elements());
10809 const LAllocation* value = lir->value();
10810
10811 Scalar::Type arrayType = lir->mir()->arrayType();
10812 int width = Scalar::byteSize(arrayType);
10813
10814 const LAllocation* index = lir->index();
10815 const LAllocation* length = lir->length();
10816
10817 bool guardLength = true;
10818 if (index->isConstant() && length->isConstant()) {
10819 uint32_t idx = ToInt32(index);
10820 uint32_t len = ToInt32(length);
10821 if (idx >= len)
10822 return;
10823 guardLength = false;
10824 }
10825 Label skip;
10826 if (index->isConstant()) {
10827 uint32_t idx = ToInt32(index);
10828 if (guardLength) {
10829 if (length->isRegister())
10830 masm.branch32(Assembler::BelowOrEqual, ToRegister(length), Imm32(idx), &skip);
10831 else
10832 masm.branch32(Assembler::BelowOrEqual, ToAddress(length), Imm32(idx), &skip);
10833 }
10834 Address dest(elements, idx * width);
10835 StoreToTypedArray(masm, arrayType, value, dest);
10836 } else {
10837 Register idxReg = ToRegister(index);
10838 MOZ_ASSERT(guardLength);
10839 if (length->isConstant())
10840 masm.branch32(Assembler::AboveOrEqual, idxReg, Imm32(ToInt32(length)), &skip);
10841 else if (length->isRegister())
10842 masm.branch32(Assembler::BelowOrEqual, ToRegister(length), idxReg, &skip);
10843 else
10844 masm.branch32(Assembler::BelowOrEqual, ToAddress(length), idxReg, &skip);
10845 BaseIndex dest(elements, ToRegister(index), ScaleFromElemWidth(width));
10846 StoreToTypedArray(masm, arrayType, value, dest);
10847 }
10848 if (guardLength)
10849 masm.bind(&skip);
10850 }
10851
10852 void
visitAtomicIsLockFree(LAtomicIsLockFree * lir)10853 CodeGenerator::visitAtomicIsLockFree(LAtomicIsLockFree* lir)
10854 {
10855 Register value = ToRegister(lir->value());
10856 Register output = ToRegister(lir->output());
10857
10858 // Keep this in sync with isLockfree() in jit/AtomicOperations.h.
10859 MOZ_ASSERT(AtomicOperations::isLockfree(1)); // Implementation artifact
10860 MOZ_ASSERT(AtomicOperations::isLockfree(2)); // Implementation artifact
10861 MOZ_ASSERT(AtomicOperations::isLockfree(4)); // Spec requirement
10862 MOZ_ASSERT(!AtomicOperations::isLockfree(8)); // Implementation invariant, for now
10863
10864 Label Ldone, Lfailed;
10865 masm.move32(Imm32(1), output);
10866 masm.branch32(Assembler::Equal, value, Imm32(4), &Ldone);
10867 masm.branch32(Assembler::Equal, value, Imm32(2), &Ldone);
10868 masm.branch32(Assembler::Equal, value, Imm32(1), &Ldone);
10869 masm.move32(Imm32(0), output);
10870 masm.bind(&Ldone);
10871 }
10872
10873 void
visitGuardSharedTypedArray(LGuardSharedTypedArray * guard)10874 CodeGenerator::visitGuardSharedTypedArray(LGuardSharedTypedArray* guard)
10875 {
10876 Register obj = ToRegister(guard->input());
10877 Register tmp = ToRegister(guard->tempInt());
10878
10879 // The shared-memory flag is a bit in the ObjectElements header
10880 // that is set if the TypedArray is mapping a SharedArrayBuffer.
10881 // The flag is set at construction and does not change subsequently.
10882 masm.loadPtr(Address(obj, TypedArrayObject::offsetOfElements()), tmp);
10883 masm.load32(Address(tmp, ObjectElements::offsetOfFlags()), tmp);
10884 bailoutTest32(Assembler::Zero, tmp, Imm32(ObjectElements::SHARED_MEMORY), guard->snapshot());
10885 }
10886
10887 void
visitClampIToUint8(LClampIToUint8 * lir)10888 CodeGenerator::visitClampIToUint8(LClampIToUint8* lir)
10889 {
10890 Register output = ToRegister(lir->output());
10891 MOZ_ASSERT(output == ToRegister(lir->input()));
10892 masm.clampIntToUint8(output);
10893 }
10894
10895 void
visitClampDToUint8(LClampDToUint8 * lir)10896 CodeGenerator::visitClampDToUint8(LClampDToUint8* lir)
10897 {
10898 FloatRegister input = ToFloatRegister(lir->input());
10899 Register output = ToRegister(lir->output());
10900 masm.clampDoubleToUint8(input, output);
10901 }
10902
10903 void
visitClampVToUint8(LClampVToUint8 * lir)10904 CodeGenerator::visitClampVToUint8(LClampVToUint8* lir)
10905 {
10906 ValueOperand operand = ToValue(lir, LClampVToUint8::Input);
10907 FloatRegister tempFloat = ToFloatRegister(lir->tempFloat());
10908 Register output = ToRegister(lir->output());
10909 MDefinition* input = lir->mir()->input();
10910
10911 Label* stringEntry;
10912 Label* stringRejoin;
10913 if (input->mightBeType(MIRType::String)) {
10914 OutOfLineCode* oolString = oolCallVM(StringToNumberInfo, lir, ArgList(output),
10915 StoreFloatRegisterTo(tempFloat));
10916 stringEntry = oolString->entry();
10917 stringRejoin = oolString->rejoin();
10918 } else {
10919 stringEntry = nullptr;
10920 stringRejoin = nullptr;
10921 }
10922
10923 Label fails;
10924 masm.clampValueToUint8(operand, input,
10925 stringEntry, stringRejoin,
10926 output, tempFloat, output, &fails);
10927
10928 bailoutFrom(&fails, lir->snapshot());
10929 }
10930
10931 typedef bool (*OperatorInFn)(JSContext*, HandleValue, HandleObject, bool*);
10932 static const VMFunction OperatorInInfo = FunctionInfo<OperatorInFn>(OperatorIn, "OperatorIn");
10933
10934 void
visitIn(LIn * ins)10935 CodeGenerator::visitIn(LIn* ins)
10936 {
10937 pushArg(ToRegister(ins->rhs()));
10938 pushArg(ToValue(ins, LIn::LHS));
10939
10940 callVM(OperatorInInfo, ins);
10941 }
10942
10943 typedef bool (*OperatorInIFn)(JSContext*, uint32_t, HandleObject, bool*);
10944 static const VMFunction OperatorInIInfo = FunctionInfo<OperatorInIFn>(OperatorInI, "OperatorInI");
10945
10946 void
visitInArray(LInArray * lir)10947 CodeGenerator::visitInArray(LInArray* lir)
10948 {
10949 const MInArray* mir = lir->mir();
10950 Register elements = ToRegister(lir->elements());
10951 Register initLength = ToRegister(lir->initLength());
10952 Register output = ToRegister(lir->output());
10953
10954 // When the array is not packed we need to do a hole check in addition to the bounds check.
10955 Label falseBranch, done, trueBranch;
10956
10957 OutOfLineCode* ool = nullptr;
10958 Label* failedInitLength = &falseBranch;
10959
10960 if (lir->index()->isConstant()) {
10961 int32_t index = ToInt32(lir->index());
10962
10963 MOZ_ASSERT_IF(index < 0, mir->needsNegativeIntCheck());
10964 if (mir->needsNegativeIntCheck()) {
10965 ool = oolCallVM(OperatorInIInfo, lir,
10966 ArgList(Imm32(index), ToRegister(lir->object())),
10967 StoreRegisterTo(output));
10968 failedInitLength = ool->entry();
10969 }
10970
10971 masm.branch32(Assembler::BelowOrEqual, initLength, Imm32(index), failedInitLength);
10972 if (mir->needsHoleCheck() && mir->unboxedType() == JSVAL_TYPE_MAGIC) {
10973 NativeObject::elementsSizeMustNotOverflow();
10974 Address address = Address(elements, index * sizeof(Value));
10975 masm.branchTestMagic(Assembler::Equal, address, &falseBranch);
10976 }
10977 } else {
10978 Label negativeIntCheck;
10979 Register index = ToRegister(lir->index());
10980
10981 if (mir->needsNegativeIntCheck())
10982 failedInitLength = &negativeIntCheck;
10983
10984 masm.branch32(Assembler::BelowOrEqual, initLength, index, failedInitLength);
10985 if (mir->needsHoleCheck() && mir->unboxedType() == JSVAL_TYPE_MAGIC) {
10986 BaseIndex address = BaseIndex(elements, ToRegister(lir->index()), TimesEight);
10987 masm.branchTestMagic(Assembler::Equal, address, &falseBranch);
10988 }
10989 masm.jump(&trueBranch);
10990
10991 if (mir->needsNegativeIntCheck()) {
10992 masm.bind(&negativeIntCheck);
10993 ool = oolCallVM(OperatorInIInfo, lir,
10994 ArgList(index, ToRegister(lir->object())),
10995 StoreRegisterTo(output));
10996
10997 masm.branch32(Assembler::LessThan, index, Imm32(0), ool->entry());
10998 masm.jump(&falseBranch);
10999 }
11000 }
11001
11002 masm.bind(&trueBranch);
11003 masm.move32(Imm32(1), output);
11004 masm.jump(&done);
11005
11006 masm.bind(&falseBranch);
11007 masm.move32(Imm32(0), output);
11008 masm.bind(&done);
11009
11010 if (ool)
11011 masm.bind(ool->rejoin());
11012 }
11013
11014 void
visitInstanceOfO(LInstanceOfO * ins)11015 CodeGenerator::visitInstanceOfO(LInstanceOfO* ins)
11016 {
11017 emitInstanceOf(ins, ins->mir()->prototypeObject());
11018 }
11019
11020 void
visitInstanceOfV(LInstanceOfV * ins)11021 CodeGenerator::visitInstanceOfV(LInstanceOfV* ins)
11022 {
11023 emitInstanceOf(ins, ins->mir()->prototypeObject());
11024 }
11025
11026 // Wrap IsDelegateOfObject, which takes a JSObject*, not a HandleObject
11027 static bool
IsDelegateObject(JSContext * cx,HandleObject protoObj,HandleObject obj,bool * res)11028 IsDelegateObject(JSContext* cx, HandleObject protoObj, HandleObject obj, bool* res)
11029 {
11030 return IsDelegateOfObject(cx, protoObj, obj, res);
11031 }
11032
11033 typedef bool (*IsDelegateObjectFn)(JSContext*, HandleObject, HandleObject, bool*);
11034 static const VMFunction IsDelegateObjectInfo =
11035 FunctionInfo<IsDelegateObjectFn>(IsDelegateObject, "IsDelegateObject");
11036
11037 void
emitInstanceOf(LInstruction * ins,JSObject * prototypeObject)11038 CodeGenerator::emitInstanceOf(LInstruction* ins, JSObject* prototypeObject)
11039 {
11040 // This path implements fun_hasInstance when the function's prototype is
11041 // known to be prototypeObject.
11042
11043 Label done;
11044 Register output = ToRegister(ins->getDef(0));
11045
11046 // If the lhs is a primitive, the result is false.
11047 Register objReg;
11048 if (ins->isInstanceOfV()) {
11049 Label isObject;
11050 ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LHS);
11051 masm.branchTestObject(Assembler::Equal, lhsValue, &isObject);
11052 masm.mov(ImmWord(0), output);
11053 masm.jump(&done);
11054 masm.bind(&isObject);
11055 objReg = masm.extractObject(lhsValue, output);
11056 } else {
11057 objReg = ToRegister(ins->toInstanceOfO()->lhs());
11058 }
11059
11060 // Crawl the lhs's prototype chain in a loop to search for prototypeObject.
11061 // This follows the main loop of js::IsDelegate, though additionally breaks
11062 // out of the loop on Proxy::LazyProto.
11063
11064 // Load the lhs's prototype.
11065 masm.loadObjProto(objReg, output);
11066
11067 Label testLazy;
11068 {
11069 Label loopPrototypeChain;
11070 masm.bind(&loopPrototypeChain);
11071
11072 // Test for the target prototype object.
11073 Label notPrototypeObject;
11074 masm.branchPtr(Assembler::NotEqual, output, ImmGCPtr(prototypeObject), ¬PrototypeObject);
11075 masm.mov(ImmWord(1), output);
11076 masm.jump(&done);
11077 masm.bind(¬PrototypeObject);
11078
11079 MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
11080
11081 // Test for nullptr or Proxy::LazyProto
11082 masm.branchPtr(Assembler::BelowOrEqual, output, ImmWord(1), &testLazy);
11083
11084 // Load the current object's prototype.
11085 masm.loadObjProto(output, output);
11086
11087 masm.jump(&loopPrototypeChain);
11088 }
11089
11090 // Make a VM call if an object with a lazy proto was found on the prototype
11091 // chain. This currently occurs only for cross compartment wrappers, which
11092 // we do not expect to be compared with non-wrapper functions from this
11093 // compartment. Otherwise, we stopped on a nullptr prototype and the output
11094 // register is already correct.
11095
11096 OutOfLineCode* ool = oolCallVM(IsDelegateObjectInfo, ins,
11097 ArgList(ImmGCPtr(prototypeObject), objReg),
11098 StoreRegisterTo(output));
11099
11100 // Regenerate the original lhs object for the VM call.
11101 Label regenerate, *lazyEntry;
11102 if (objReg != output) {
11103 lazyEntry = ool->entry();
11104 } else {
11105 masm.bind(®enerate);
11106 lazyEntry = ®enerate;
11107 if (ins->isInstanceOfV()) {
11108 ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LHS);
11109 objReg = masm.extractObject(lhsValue, output);
11110 } else {
11111 objReg = ToRegister(ins->toInstanceOfO()->lhs());
11112 }
11113 MOZ_ASSERT(objReg == output);
11114 masm.jump(ool->entry());
11115 }
11116
11117 masm.bind(&testLazy);
11118 masm.branchPtr(Assembler::Equal, output, ImmWord(1), lazyEntry);
11119
11120 masm.bind(&done);
11121 masm.bind(ool->rejoin());
11122 }
11123
11124 typedef bool (*HasInstanceFn)(JSContext*, HandleObject, HandleValue, bool*);
11125 static const VMFunction HasInstanceInfo = FunctionInfo<HasInstanceFn>(js::HasInstance, "HasInstance");
11126
11127 void
visitCallInstanceOf(LCallInstanceOf * ins)11128 CodeGenerator::visitCallInstanceOf(LCallInstanceOf* ins)
11129 {
11130 ValueOperand lhs = ToValue(ins, LCallInstanceOf::LHS);
11131 Register rhs = ToRegister(ins->rhs());
11132 MOZ_ASSERT(ToRegister(ins->output()) == ReturnReg);
11133
11134 pushArg(lhs);
11135 pushArg(rhs);
11136 callVM(HasInstanceInfo, ins);
11137 }
11138
11139 void
visitGetDOMProperty(LGetDOMProperty * ins)11140 CodeGenerator::visitGetDOMProperty(LGetDOMProperty* ins)
11141 {
11142 const Register JSContextReg = ToRegister(ins->getJSContextReg());
11143 const Register ObjectReg = ToRegister(ins->getObjectReg());
11144 const Register PrivateReg = ToRegister(ins->getPrivReg());
11145 const Register ValueReg = ToRegister(ins->getValueReg());
11146
11147 Label haveValue;
11148 if (ins->mir()->valueMayBeInSlot()) {
11149 size_t slot = ins->mir()->domMemberSlotIndex();
11150 // It's a bit annoying to redo these slot calculations, which duplcate
11151 // LSlots and a few other things like that, but I'm not sure there's a
11152 // way to reuse those here.
11153 if (slot < NativeObject::MAX_FIXED_SLOTS) {
11154 masm.loadValue(Address(ObjectReg, NativeObject::getFixedSlotOffset(slot)),
11155 JSReturnOperand);
11156 } else {
11157 // It's a dynamic slot.
11158 slot -= NativeObject::MAX_FIXED_SLOTS;
11159 // Use PrivateReg as a scratch register for the slots pointer.
11160 masm.loadPtr(Address(ObjectReg, NativeObject::offsetOfSlots()),
11161 PrivateReg);
11162 masm.loadValue(Address(PrivateReg, slot*sizeof(js::Value)),
11163 JSReturnOperand);
11164 }
11165 masm.branchTestUndefined(Assembler::NotEqual, JSReturnOperand, &haveValue);
11166 }
11167
11168 DebugOnly<uint32_t> initialStack = masm.framePushed();
11169
11170 masm.checkStackAlignment();
11171
11172 // Make space for the outparam. Pre-initialize it to UndefinedValue so we
11173 // can trace it at GC time.
11174 masm.Push(UndefinedValue());
11175 // We pass the pointer to our out param as an instance of
11176 // JSJitGetterCallArgs, since on the binary level it's the same thing.
11177 JS_STATIC_ASSERT(sizeof(JSJitGetterCallArgs) == sizeof(Value*));
11178 masm.moveStackPtrTo(ValueReg);
11179
11180 masm.Push(ObjectReg);
11181
11182 LoadDOMPrivate(masm, ObjectReg, PrivateReg);
11183
11184 // Rooting will happen at GC time.
11185 masm.moveStackPtrTo(ObjectReg);
11186
11187 uint32_t safepointOffset = masm.buildFakeExitFrame(JSContextReg);
11188 masm.enterFakeExitFrame(IonDOMExitFrameLayoutGetterToken);
11189
11190 markSafepointAt(safepointOffset, ins);
11191
11192 masm.setupUnalignedABICall(JSContextReg);
11193
11194 masm.loadJSContext(JSContextReg);
11195
11196 masm.passABIArg(JSContextReg);
11197 masm.passABIArg(ObjectReg);
11198 masm.passABIArg(PrivateReg);
11199 masm.passABIArg(ValueReg);
11200 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ins->mir()->fun()));
11201
11202 if (ins->mir()->isInfallible()) {
11203 masm.loadValue(Address(masm.getStackPointer(), IonDOMExitFrameLayout::offsetOfResult()),
11204 JSReturnOperand);
11205 } else {
11206 masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
11207
11208 masm.loadValue(Address(masm.getStackPointer(), IonDOMExitFrameLayout::offsetOfResult()),
11209 JSReturnOperand);
11210 }
11211 masm.adjustStack(IonDOMExitFrameLayout::Size());
11212
11213 masm.bind(&haveValue);
11214
11215 MOZ_ASSERT(masm.framePushed() == initialStack);
11216 }
11217
11218 void
visitGetDOMMemberV(LGetDOMMemberV * ins)11219 CodeGenerator::visitGetDOMMemberV(LGetDOMMemberV* ins)
11220 {
11221 // It's simpler to duplicate visitLoadFixedSlotV here than it is to try to
11222 // use an LLoadFixedSlotV or some subclass of it for this case: that would
11223 // require us to have MGetDOMMember inherit from MLoadFixedSlot, and then
11224 // we'd have to duplicate a bunch of stuff we now get for free from
11225 // MGetDOMProperty.
11226 Register object = ToRegister(ins->object());
11227 size_t slot = ins->mir()->domMemberSlotIndex();
11228 ValueOperand result = GetValueOutput(ins);
11229
11230 masm.loadValue(Address(object, NativeObject::getFixedSlotOffset(slot)), result);
11231 }
11232
11233 void
visitGetDOMMemberT(LGetDOMMemberT * ins)11234 CodeGenerator::visitGetDOMMemberT(LGetDOMMemberT* ins)
11235 {
11236 // It's simpler to duplicate visitLoadFixedSlotT here than it is to try to
11237 // use an LLoadFixedSlotT or some subclass of it for this case: that would
11238 // require us to have MGetDOMMember inherit from MLoadFixedSlot, and then
11239 // we'd have to duplicate a bunch of stuff we now get for free from
11240 // MGetDOMProperty.
11241 Register object = ToRegister(ins->object());
11242 size_t slot = ins->mir()->domMemberSlotIndex();
11243 AnyRegister result = ToAnyRegister(ins->getDef(0));
11244 MIRType type = ins->mir()->type();
11245
11246 masm.loadUnboxedValue(Address(object, NativeObject::getFixedSlotOffset(slot)), type, result);
11247 }
11248
11249 void
visitSetDOMProperty(LSetDOMProperty * ins)11250 CodeGenerator::visitSetDOMProperty(LSetDOMProperty* ins)
11251 {
11252 const Register JSContextReg = ToRegister(ins->getJSContextReg());
11253 const Register ObjectReg = ToRegister(ins->getObjectReg());
11254 const Register PrivateReg = ToRegister(ins->getPrivReg());
11255 const Register ValueReg = ToRegister(ins->getValueReg());
11256
11257 DebugOnly<uint32_t> initialStack = masm.framePushed();
11258
11259 masm.checkStackAlignment();
11260
11261 // Push the argument. Rooting will happen at GC time.
11262 ValueOperand argVal = ToValue(ins, LSetDOMProperty::Value);
11263 masm.Push(argVal);
11264 // We pass the pointer to our out param as an instance of
11265 // JSJitGetterCallArgs, since on the binary level it's the same thing.
11266 JS_STATIC_ASSERT(sizeof(JSJitSetterCallArgs) == sizeof(Value*));
11267 masm.moveStackPtrTo(ValueReg);
11268
11269 masm.Push(ObjectReg);
11270
11271 LoadDOMPrivate(masm, ObjectReg, PrivateReg);
11272
11273 // Rooting will happen at GC time.
11274 masm.moveStackPtrTo(ObjectReg);
11275
11276 uint32_t safepointOffset = masm.buildFakeExitFrame(JSContextReg);
11277 masm.enterFakeExitFrame(IonDOMExitFrameLayoutSetterToken);
11278
11279 markSafepointAt(safepointOffset, ins);
11280
11281 masm.setupUnalignedABICall(JSContextReg);
11282
11283 masm.loadJSContext(JSContextReg);
11284
11285 masm.passABIArg(JSContextReg);
11286 masm.passABIArg(ObjectReg);
11287 masm.passABIArg(PrivateReg);
11288 masm.passABIArg(ValueReg);
11289 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ins->mir()->fun()));
11290
11291 masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
11292
11293 masm.adjustStack(IonDOMExitFrameLayout::Size());
11294
11295 MOZ_ASSERT(masm.framePushed() == initialStack);
11296 }
11297
11298 class OutOfLineIsCallable : public OutOfLineCodeBase<CodeGenerator>
11299 {
11300 LIsCallable* ins_;
11301
11302 public:
OutOfLineIsCallable(LIsCallable * ins)11303 explicit OutOfLineIsCallable(LIsCallable* ins)
11304 : ins_(ins)
11305 { }
11306
accept(CodeGenerator * codegen)11307 void accept(CodeGenerator* codegen) {
11308 codegen->visitOutOfLineIsCallable(this);
11309 }
ins() const11310 LIsCallable* ins() const {
11311 return ins_;
11312 }
11313 };
11314
11315 void
visitIsCallable(LIsCallable * ins)11316 CodeGenerator::visitIsCallable(LIsCallable* ins)
11317 {
11318 Register object = ToRegister(ins->object());
11319 Register output = ToRegister(ins->output());
11320
11321 OutOfLineIsCallable* ool = new(alloc()) OutOfLineIsCallable(ins);
11322 addOutOfLineCode(ool, ins->mir());
11323
11324 Label notFunction, hasCOps, done;
11325 masm.loadObjClass(object, output);
11326
11327 // Just skim proxies off. Their notion of isCallable() is more complicated.
11328 masm.branchTestClassIsProxy(true, output, ool->entry());
11329
11330 // An object is callable iff:
11331 // is<JSFunction>() || (getClass()->cOps && getClass()->cOps->call).
11332 masm.branchPtr(Assembler::NotEqual, output, ImmPtr(&JSFunction::class_), ¬Function);
11333 masm.move32(Imm32(1), output);
11334 masm.jump(&done);
11335
11336 masm.bind(¬Function);
11337 masm.branchPtr(Assembler::NonZero, Address(output, offsetof(js::Class, cOps)),
11338 ImmPtr(nullptr), &hasCOps);
11339 masm.move32(Imm32(0), output);
11340 masm.jump(&done);
11341
11342 masm.bind(&hasCOps);
11343 masm.loadPtr(Address(output, offsetof(js::Class, cOps)), output);
11344 masm.cmpPtrSet(Assembler::NonZero, Address(output, offsetof(js::ClassOps, call)),
11345 ImmPtr(nullptr), output);
11346
11347 masm.bind(&done);
11348 masm.bind(ool->rejoin());
11349 }
11350
11351 void
visitOutOfLineIsCallable(OutOfLineIsCallable * ool)11352 CodeGenerator::visitOutOfLineIsCallable(OutOfLineIsCallable* ool)
11353 {
11354 LIsCallable* ins = ool->ins();
11355 Register object = ToRegister(ins->object());
11356 Register output = ToRegister(ins->output());
11357
11358 saveVolatile(output);
11359 masm.setupUnalignedABICall(output);
11360 masm.passABIArg(object);
11361 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ObjectIsCallable));
11362 masm.storeCallBoolResult(output);
11363 restoreVolatile(output);
11364 masm.jump(ool->rejoin());
11365 }
11366
11367 class OutOfLineIsConstructor : public OutOfLineCodeBase<CodeGenerator>
11368 {
11369 LIsConstructor* ins_;
11370
11371 public:
OutOfLineIsConstructor(LIsConstructor * ins)11372 explicit OutOfLineIsConstructor(LIsConstructor* ins)
11373 : ins_(ins)
11374 { }
11375
accept(CodeGenerator * codegen)11376 void accept(CodeGenerator* codegen) {
11377 codegen->visitOutOfLineIsConstructor(this);
11378 }
ins() const11379 LIsConstructor* ins() const {
11380 return ins_;
11381 }
11382 };
11383
11384 void
visitIsConstructor(LIsConstructor * ins)11385 CodeGenerator::visitIsConstructor(LIsConstructor* ins)
11386 {
11387 Register object = ToRegister(ins->object());
11388 Register output = ToRegister(ins->output());
11389
11390 OutOfLineIsConstructor* ool = new(alloc()) OutOfLineIsConstructor(ins);
11391 addOutOfLineCode(ool, ins->mir());
11392
11393 Label notFunction, notConstructor, hasCOps, done;
11394 masm.loadObjClass(object, output);
11395
11396 // Just skim proxies off. Their notion of isConstructor() is more complicated.
11397 masm.branchTestClassIsProxy(true, output, ool->entry());
11398
11399 // An object is constructor iff
11400 // ((is<JSFunction>() && as<JSFunction>().isConstructor) ||
11401 // (getClass()->cOps && getClass()->cOps->construct)).
11402 masm.branchPtr(Assembler::NotEqual, output, ImmPtr(&JSFunction::class_), ¬Function);
11403 masm.load16ZeroExtend(Address(object, JSFunction::offsetOfFlags()), output);
11404 masm.and32(Imm32(JSFunction::CONSTRUCTOR), output);
11405 masm.branchTest32(Assembler::Zero, output, output, ¬Constructor);
11406 masm.move32(Imm32(1), output);
11407 masm.jump(&done);
11408 masm.bind(¬Constructor);
11409 masm.move32(Imm32(0), output);
11410 masm.jump(&done);
11411
11412 masm.bind(¬Function);
11413 masm.branchPtr(Assembler::NonZero, Address(output, offsetof(js::Class, cOps)),
11414 ImmPtr(nullptr), &hasCOps);
11415 masm.move32(Imm32(0), output);
11416 masm.jump(&done);
11417
11418 masm.bind(&hasCOps);
11419 masm.loadPtr(Address(output, offsetof(js::Class, cOps)), output);
11420 masm.cmpPtrSet(Assembler::NonZero, Address(output, offsetof(js::ClassOps, construct)),
11421 ImmPtr(nullptr), output);
11422
11423 masm.bind(&done);
11424 masm.bind(ool->rejoin());
11425 }
11426
11427 void
visitOutOfLineIsConstructor(OutOfLineIsConstructor * ool)11428 CodeGenerator::visitOutOfLineIsConstructor(OutOfLineIsConstructor* ool)
11429 {
11430 LIsConstructor* ins = ool->ins();
11431 Register object = ToRegister(ins->object());
11432 Register output = ToRegister(ins->output());
11433
11434 saveVolatile(output);
11435 masm.setupUnalignedABICall(output);
11436 masm.passABIArg(object);
11437 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ObjectIsConstructor));
11438 masm.storeCallBoolResult(output);
11439 restoreVolatile(output);
11440 masm.jump(ool->rejoin());
11441 }
11442
11443 void
visitIsObject(LIsObject * ins)11444 CodeGenerator::visitIsObject(LIsObject* ins)
11445 {
11446 Register output = ToRegister(ins->output());
11447 ValueOperand value = ToValue(ins, LIsObject::Input);
11448 masm.testObjectSet(Assembler::Equal, value, output);
11449 }
11450
11451 void
visitIsObjectAndBranch(LIsObjectAndBranch * ins)11452 CodeGenerator::visitIsObjectAndBranch(LIsObjectAndBranch* ins)
11453 {
11454 ValueOperand value = ToValue(ins, LIsObjectAndBranch::Input);
11455 testObjectEmitBranch(Assembler::Equal, value, ins->ifTrue(), ins->ifFalse());
11456 }
11457
11458 void
loadOutermostJSScript(Register reg)11459 CodeGenerator::loadOutermostJSScript(Register reg)
11460 {
11461 // The "outermost" JSScript means the script that we are compiling
11462 // basically; this is not always the script associated with the
11463 // current basic block, which might be an inlined script.
11464
11465 MIRGraph& graph = current->mir()->graph();
11466 MBasicBlock* entryBlock = graph.entryBlock();
11467 masm.movePtr(ImmGCPtr(entryBlock->info().script()), reg);
11468 }
11469
11470 void
loadJSScriptForBlock(MBasicBlock * block,Register reg)11471 CodeGenerator::loadJSScriptForBlock(MBasicBlock* block, Register reg)
11472 {
11473 // The current JSScript means the script for the current
11474 // basic block. This may be an inlined script.
11475
11476 JSScript* script = block->info().script();
11477 masm.movePtr(ImmGCPtr(script), reg);
11478 }
11479
11480 void
visitHasClass(LHasClass * ins)11481 CodeGenerator::visitHasClass(LHasClass* ins)
11482 {
11483 Register lhs = ToRegister(ins->lhs());
11484 Register output = ToRegister(ins->output());
11485
11486 masm.loadObjClass(lhs, output);
11487 masm.cmpPtrSet(Assembler::Equal, output, ImmPtr(ins->mir()->getClass()), output);
11488 }
11489
11490 void
visitWasmParameter(LWasmParameter * lir)11491 CodeGenerator::visitWasmParameter(LWasmParameter* lir)
11492 {
11493 }
11494
11495 void
visitWasmParameterI64(LWasmParameterI64 * lir)11496 CodeGenerator::visitWasmParameterI64(LWasmParameterI64* lir)
11497 {
11498 }
11499
11500 void
visitWasmReturn(LWasmReturn * lir)11501 CodeGenerator::visitWasmReturn(LWasmReturn* lir)
11502 {
11503 // Don't emit a jump to the return label if this is the last block.
11504 if (current->mir() != *gen->graph().poBegin())
11505 masm.jump(&returnLabel_);
11506 }
11507
11508 void
visitWasmReturnI64(LWasmReturnI64 * lir)11509 CodeGenerator::visitWasmReturnI64(LWasmReturnI64* lir)
11510 {
11511 // Don't emit a jump to the return label if this is the last block.
11512 if (current->mir() != *gen->graph().poBegin())
11513 masm.jump(&returnLabel_);
11514 }
11515
11516 void
visitWasmReturnVoid(LWasmReturnVoid * lir)11517 CodeGenerator::visitWasmReturnVoid(LWasmReturnVoid* lir)
11518 {
11519 // Don't emit a jump to the return label if this is the last block.
11520 if (current->mir() != *gen->graph().poBegin())
11521 masm.jump(&returnLabel_);
11522 }
11523
11524 void
emitAssertRangeI(const Range * r,Register input)11525 CodeGenerator::emitAssertRangeI(const Range* r, Register input)
11526 {
11527 // Check the lower bound.
11528 if (r->hasInt32LowerBound() && r->lower() > INT32_MIN) {
11529 Label success;
11530 masm.branch32(Assembler::GreaterThanOrEqual, input, Imm32(r->lower()), &success);
11531 masm.assumeUnreachable("Integer input should be equal or higher than Lowerbound.");
11532 masm.bind(&success);
11533 }
11534
11535 // Check the upper bound.
11536 if (r->hasInt32UpperBound() && r->upper() < INT32_MAX) {
11537 Label success;
11538 masm.branch32(Assembler::LessThanOrEqual, input, Imm32(r->upper()), &success);
11539 masm.assumeUnreachable("Integer input should be lower or equal than Upperbound.");
11540 masm.bind(&success);
11541 }
11542
11543 // For r->canHaveFractionalPart(), r->canBeNegativeZero(), and
11544 // r->exponent(), there's nothing to check, because if we ended up in the
11545 // integer range checking code, the value is already in an integer register
11546 // in the integer range.
11547 }
11548
11549 void
emitAssertRangeD(const Range * r,FloatRegister input,FloatRegister temp)11550 CodeGenerator::emitAssertRangeD(const Range* r, FloatRegister input, FloatRegister temp)
11551 {
11552 // Check the lower bound.
11553 if (r->hasInt32LowerBound()) {
11554 Label success;
11555 masm.loadConstantDouble(r->lower(), temp);
11556 if (r->canBeNaN())
11557 masm.branchDouble(Assembler::DoubleUnordered, input, input, &success);
11558 masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, input, temp, &success);
11559 masm.assumeUnreachable("Double input should be equal or higher than Lowerbound.");
11560 masm.bind(&success);
11561 }
11562 // Check the upper bound.
11563 if (r->hasInt32UpperBound()) {
11564 Label success;
11565 masm.loadConstantDouble(r->upper(), temp);
11566 if (r->canBeNaN())
11567 masm.branchDouble(Assembler::DoubleUnordered, input, input, &success);
11568 masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, temp, &success);
11569 masm.assumeUnreachable("Double input should be lower or equal than Upperbound.");
11570 masm.bind(&success);
11571 }
11572
11573 // This code does not yet check r->canHaveFractionalPart(). This would require new
11574 // assembler interfaces to make rounding instructions available.
11575
11576 if (!r->canBeNegativeZero()) {
11577 Label success;
11578
11579 // First, test for being equal to 0.0, which also includes -0.0.
11580 masm.loadConstantDouble(0.0, temp);
11581 masm.branchDouble(Assembler::DoubleNotEqualOrUnordered, input, temp, &success);
11582
11583 // The easiest way to distinguish -0.0 from 0.0 is that 1.0/-0.0 is
11584 // -Infinity instead of Infinity.
11585 masm.loadConstantDouble(1.0, temp);
11586 masm.divDouble(input, temp);
11587 masm.branchDouble(Assembler::DoubleGreaterThan, temp, input, &success);
11588
11589 masm.assumeUnreachable("Input shouldn't be negative zero.");
11590
11591 masm.bind(&success);
11592 }
11593
11594 if (!r->hasInt32Bounds() && !r->canBeInfiniteOrNaN() &&
11595 r->exponent() < FloatingPoint<double>::kExponentBias)
11596 {
11597 // Check the bounds implied by the maximum exponent.
11598 Label exponentLoOk;
11599 masm.loadConstantDouble(pow(2.0, r->exponent() + 1), temp);
11600 masm.branchDouble(Assembler::DoubleUnordered, input, input, &exponentLoOk);
11601 masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, temp, &exponentLoOk);
11602 masm.assumeUnreachable("Check for exponent failed.");
11603 masm.bind(&exponentLoOk);
11604
11605 Label exponentHiOk;
11606 masm.loadConstantDouble(-pow(2.0, r->exponent() + 1), temp);
11607 masm.branchDouble(Assembler::DoubleUnordered, input, input, &exponentHiOk);
11608 masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, input, temp, &exponentHiOk);
11609 masm.assumeUnreachable("Check for exponent failed.");
11610 masm.bind(&exponentHiOk);
11611 } else if (!r->hasInt32Bounds() && !r->canBeNaN()) {
11612 // If we think the value can't be NaN, check that it isn't.
11613 Label notnan;
11614 masm.branchDouble(Assembler::DoubleOrdered, input, input, ¬nan);
11615 masm.assumeUnreachable("Input shouldn't be NaN.");
11616 masm.bind(¬nan);
11617
11618 // If we think the value also can't be an infinity, check that it isn't.
11619 if (!r->canBeInfiniteOrNaN()) {
11620 Label notposinf;
11621 masm.loadConstantDouble(PositiveInfinity<double>(), temp);
11622 masm.branchDouble(Assembler::DoubleLessThan, input, temp, ¬posinf);
11623 masm.assumeUnreachable("Input shouldn't be +Inf.");
11624 masm.bind(¬posinf);
11625
11626 Label notneginf;
11627 masm.loadConstantDouble(NegativeInfinity<double>(), temp);
11628 masm.branchDouble(Assembler::DoubleGreaterThan, input, temp, ¬neginf);
11629 masm.assumeUnreachable("Input shouldn't be -Inf.");
11630 masm.bind(¬neginf);
11631 }
11632 }
11633 }
11634
11635 void
visitAssertResultV(LAssertResultV * ins)11636 CodeGenerator::visitAssertResultV(LAssertResultV* ins)
11637 {
11638 const ValueOperand value = ToValue(ins, LAssertResultV::Input);
11639 emitAssertResultV(value, ins->mirRaw()->resultTypeSet());
11640 }
11641
11642 void
visitAssertResultT(LAssertResultT * ins)11643 CodeGenerator::visitAssertResultT(LAssertResultT* ins)
11644 {
11645 Register input = ToRegister(ins->input());
11646 MDefinition* mir = ins->mirRaw();
11647
11648 emitAssertObjectOrStringResult(input, mir->type(), mir->resultTypeSet());
11649 }
11650
11651 void
visitAssertRangeI(LAssertRangeI * ins)11652 CodeGenerator::visitAssertRangeI(LAssertRangeI* ins)
11653 {
11654 Register input = ToRegister(ins->input());
11655 const Range* r = ins->range();
11656
11657 emitAssertRangeI(r, input);
11658 }
11659
11660 void
visitAssertRangeD(LAssertRangeD * ins)11661 CodeGenerator::visitAssertRangeD(LAssertRangeD* ins)
11662 {
11663 FloatRegister input = ToFloatRegister(ins->input());
11664 FloatRegister temp = ToFloatRegister(ins->temp());
11665 const Range* r = ins->range();
11666
11667 emitAssertRangeD(r, input, temp);
11668 }
11669
11670 void
visitAssertRangeF(LAssertRangeF * ins)11671 CodeGenerator::visitAssertRangeF(LAssertRangeF* ins)
11672 {
11673 FloatRegister input = ToFloatRegister(ins->input());
11674 FloatRegister temp = ToFloatRegister(ins->temp());
11675 FloatRegister temp2 = ToFloatRegister(ins->temp2());
11676
11677 const Range* r = ins->range();
11678
11679 masm.convertFloat32ToDouble(input, temp);
11680 emitAssertRangeD(r, temp, temp2);
11681 }
11682
11683 void
visitAssertRangeV(LAssertRangeV * ins)11684 CodeGenerator::visitAssertRangeV(LAssertRangeV* ins)
11685 {
11686 const Range* r = ins->range();
11687 const ValueOperand value = ToValue(ins, LAssertRangeV::Input);
11688 Register tag = masm.splitTagForTest(value);
11689 Label done;
11690
11691 {
11692 Label isNotInt32;
11693 masm.branchTestInt32(Assembler::NotEqual, tag, &isNotInt32);
11694 Register unboxInt32 = ToTempUnboxRegister(ins->temp());
11695 Register input = masm.extractInt32(value, unboxInt32);
11696 emitAssertRangeI(r, input);
11697 masm.jump(&done);
11698 masm.bind(&isNotInt32);
11699 }
11700
11701 {
11702 Label isNotDouble;
11703 masm.branchTestDouble(Assembler::NotEqual, tag, &isNotDouble);
11704 FloatRegister input = ToFloatRegister(ins->floatTemp1());
11705 FloatRegister temp = ToFloatRegister(ins->floatTemp2());
11706 masm.unboxDouble(value, input);
11707 emitAssertRangeD(r, input, temp);
11708 masm.jump(&done);
11709 masm.bind(&isNotDouble);
11710 }
11711
11712 masm.assumeUnreachable("Incorrect range for Value.");
11713 masm.bind(&done);
11714 }
11715
11716 void
visitInterruptCheck(LInterruptCheck * lir)11717 CodeGenerator::visitInterruptCheck(LInterruptCheck* lir)
11718 {
11719 if (lir->implicit()) {
11720 OutOfLineInterruptCheckImplicit* ool = new(alloc()) OutOfLineInterruptCheckImplicit(current, lir);
11721 addOutOfLineCode(ool, lir->mir());
11722
11723 lir->setOolEntry(ool->entry());
11724 masm.bind(ool->rejoin());
11725 return;
11726 }
11727
11728 OutOfLineCode* ool = oolCallVM(InterruptCheckInfo, lir, ArgList(), StoreNothing());
11729
11730 AbsoluteAddress interruptAddr(GetJitContext()->runtime->addressOfInterruptUint32());
11731 masm.branch32(Assembler::NotEqual, interruptAddr, Imm32(0), ool->entry());
11732 masm.bind(ool->rejoin());
11733 }
11734
11735 void
visitWasmTrap(LWasmTrap * lir)11736 CodeGenerator::visitWasmTrap(LWasmTrap* lir)
11737 {
11738 MOZ_ASSERT(gen->compilingWasm());
11739 const MWasmTrap* mir = lir->mir();
11740
11741 masm.jump(trap(mir, mir->trap()));
11742 }
11743
11744 void
visitWasmBoundsCheck(LWasmBoundsCheck * ins)11745 CodeGenerator::visitWasmBoundsCheck(LWasmBoundsCheck* ins)
11746 {
11747 const MWasmBoundsCheck* mir = ins->mir();
11748 Register ptr = ToRegister(ins->ptr());
11749 masm.wasmBoundsCheck(Assembler::AboveOrEqual, ptr, trap(mir, wasm::Trap::OutOfBounds));
11750 }
11751
11752 typedef bool (*RecompileFn)(JSContext*);
11753 static const VMFunction RecompileFnInfo = FunctionInfo<RecompileFn>(Recompile, "Recompile");
11754
11755 typedef bool (*ForcedRecompileFn)(JSContext*);
11756 static const VMFunction ForcedRecompileFnInfo =
11757 FunctionInfo<ForcedRecompileFn>(ForcedRecompile, "ForcedRecompile");
11758
11759 void
visitRecompileCheck(LRecompileCheck * ins)11760 CodeGenerator::visitRecompileCheck(LRecompileCheck* ins)
11761 {
11762 Label done;
11763 Register tmp = ToRegister(ins->scratch());
11764 OutOfLineCode* ool;
11765 if (ins->mir()->forceRecompilation())
11766 ool = oolCallVM(ForcedRecompileFnInfo, ins, ArgList(), StoreRegisterTo(tmp));
11767 else
11768 ool = oolCallVM(RecompileFnInfo, ins, ArgList(), StoreRegisterTo(tmp));
11769
11770 // Check if warm-up counter is high enough.
11771 AbsoluteAddress warmUpCount = AbsoluteAddress(ins->mir()->script()->addressOfWarmUpCounter());
11772 if (ins->mir()->increaseWarmUpCounter()) {
11773 masm.load32(warmUpCount, tmp);
11774 masm.add32(Imm32(1), tmp);
11775 masm.store32(tmp, warmUpCount);
11776 masm.branch32(Assembler::BelowOrEqual, tmp, Imm32(ins->mir()->recompileThreshold()), &done);
11777 } else {
11778 masm.branch32(Assembler::BelowOrEqual, warmUpCount, Imm32(ins->mir()->recompileThreshold()),
11779 &done);
11780 }
11781
11782 // Check if not yet recompiling.
11783 CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), tmp);
11784 masm.propagateOOM(ionScriptLabels_.append(label));
11785 masm.branch32(Assembler::Equal,
11786 Address(tmp, IonScript::offsetOfRecompiling()),
11787 Imm32(0),
11788 ool->entry());
11789 masm.bind(ool->rejoin());
11790 masm.bind(&done);
11791 }
11792
11793 void
visitLexicalCheck(LLexicalCheck * ins)11794 CodeGenerator::visitLexicalCheck(LLexicalCheck* ins)
11795 {
11796 ValueOperand inputValue = ToValue(ins, LLexicalCheck::Input);
11797 Label bail;
11798 masm.branchTestMagicValue(Assembler::Equal, inputValue, JS_UNINITIALIZED_LEXICAL, &bail);
11799 bailoutFrom(&bail, ins->snapshot());
11800 }
11801
11802 typedef bool (*ThrowRuntimeLexicalErrorFn)(JSContext*, unsigned);
11803 static const VMFunction ThrowRuntimeLexicalErrorInfo =
11804 FunctionInfo<ThrowRuntimeLexicalErrorFn>(ThrowRuntimeLexicalError, "ThrowRuntimeLexicalError");
11805
11806 void
visitThrowRuntimeLexicalError(LThrowRuntimeLexicalError * ins)11807 CodeGenerator::visitThrowRuntimeLexicalError(LThrowRuntimeLexicalError* ins)
11808 {
11809 pushArg(Imm32(ins->mir()->errorNumber()));
11810 callVM(ThrowRuntimeLexicalErrorInfo, ins);
11811 }
11812
11813 typedef bool (*GlobalNameConflictsCheckFromIonFn)(JSContext*, HandleScript);
11814 static const VMFunction GlobalNameConflictsCheckFromIonInfo =
11815 FunctionInfo<GlobalNameConflictsCheckFromIonFn>(GlobalNameConflictsCheckFromIon,
11816 "GlobalNameConflictsCheckFromIon");
11817
11818 void
visitGlobalNameConflictsCheck(LGlobalNameConflictsCheck * ins)11819 CodeGenerator::visitGlobalNameConflictsCheck(LGlobalNameConflictsCheck* ins)
11820 {
11821 pushArg(ImmGCPtr(ins->mirRaw()->block()->info().script()));
11822 callVM(GlobalNameConflictsCheckFromIonInfo, ins);
11823 }
11824
11825 void
visitDebugger(LDebugger * ins)11826 CodeGenerator::visitDebugger(LDebugger* ins)
11827 {
11828 Register cx = ToRegister(ins->getTemp(0));
11829 Register temp = ToRegister(ins->getTemp(1));
11830
11831 masm.loadJSContext(cx);
11832 masm.setupUnalignedABICall(temp);
11833 masm.passABIArg(cx);
11834 masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, GlobalHasLiveOnDebuggerStatement));
11835
11836 Label bail;
11837 masm.branchIfTrueBool(ReturnReg, &bail);
11838 bailoutFrom(&bail, ins->snapshot());
11839 }
11840
11841 void
visitNewTarget(LNewTarget * ins)11842 CodeGenerator::visitNewTarget(LNewTarget *ins)
11843 {
11844 ValueOperand output = GetValueOutput(ins);
11845
11846 // if (isConstructing) output = argv[Max(numActualArgs, numFormalArgs)]
11847 Label notConstructing, done;
11848 Address calleeToken(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfCalleeToken());
11849 masm.branchTestPtr(Assembler::Zero, calleeToken,
11850 Imm32(CalleeToken_FunctionConstructing), ¬Constructing);
11851
11852 Register argvLen = output.scratchReg();
11853
11854 Address actualArgsPtr(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfNumActualArgs());
11855 masm.loadPtr(actualArgsPtr, argvLen);
11856
11857 Label useNFormals;
11858
11859 size_t numFormalArgs = ins->mirRaw()->block()->info().funMaybeLazy()->nargs();
11860 masm.branchPtr(Assembler::Below, argvLen, Imm32(numFormalArgs),
11861 &useNFormals);
11862
11863 size_t argsOffset = frameSize() + JitFrameLayout::offsetOfActualArgs();
11864 {
11865 BaseValueIndex newTarget(masm.getStackPointer(), argvLen, argsOffset);
11866 masm.loadValue(newTarget, output);
11867 masm.jump(&done);
11868 }
11869
11870 masm.bind(&useNFormals);
11871
11872 {
11873 Address newTarget(masm.getStackPointer(), argsOffset + (numFormalArgs * sizeof(Value)));
11874 masm.loadValue(newTarget, output);
11875 masm.jump(&done);
11876 }
11877
11878 // else output = undefined
11879 masm.bind(¬Constructing);
11880 masm.moveValue(UndefinedValue(), output);
11881 masm.bind(&done);
11882 }
11883
11884 void
visitCheckReturn(LCheckReturn * ins)11885 CodeGenerator::visitCheckReturn(LCheckReturn* ins)
11886 {
11887 ValueOperand returnValue = ToValue(ins, LCheckReturn::ReturnValue);
11888 ValueOperand thisValue = ToValue(ins, LCheckReturn::ThisValue);
11889 Label bail, noChecks;
11890 masm.branchTestObject(Assembler::Equal, returnValue, &noChecks);
11891 masm.branchTestUndefined(Assembler::NotEqual, returnValue, &bail);
11892 masm.branchTestMagicValue(Assembler::Equal, thisValue, JS_UNINITIALIZED_LEXICAL, &bail);
11893 bailoutFrom(&bail, ins->snapshot());
11894 masm.bind(&noChecks);
11895 }
11896
11897 typedef bool (*ThrowCheckIsObjectFn)(JSContext*, CheckIsObjectKind);
11898 static const VMFunction ThrowCheckIsObjectInfo =
11899 FunctionInfo<ThrowCheckIsObjectFn>(ThrowCheckIsObject, "ThrowCheckIsObject");
11900
11901 void
visitCheckIsObj(LCheckIsObj * ins)11902 CodeGenerator::visitCheckIsObj(LCheckIsObj* ins)
11903 {
11904 ValueOperand checkValue = ToValue(ins, LCheckIsObj::CheckValue);
11905
11906 OutOfLineCode* ool = oolCallVM(ThrowCheckIsObjectInfo, ins,
11907 ArgList(Imm32(ins->mir()->checkKind())),
11908 StoreNothing());
11909 masm.branchTestObject(Assembler::NotEqual, checkValue, ool->entry());
11910 masm.bind(ool->rejoin());
11911 }
11912
11913 typedef bool (*ThrowObjCoercibleFn)(JSContext*, HandleValue);
11914 static const VMFunction ThrowObjectCoercibleInfo =
11915 FunctionInfo<ThrowObjCoercibleFn>(ThrowObjectCoercible, "ThrowObjectCoercible");
11916
11917 void
visitCheckObjCoercible(LCheckObjCoercible * ins)11918 CodeGenerator::visitCheckObjCoercible(LCheckObjCoercible* ins)
11919 {
11920 ValueOperand checkValue = ToValue(ins, LCheckObjCoercible::CheckValue);
11921 Label fail, done;
11922 masm.branchTestNull(Assembler::Equal, checkValue, &fail);
11923 masm.branchTestUndefined(Assembler::NotEqual, checkValue, &done);
11924 masm.bind(&fail);
11925 pushArg(checkValue);
11926 callVM(ThrowObjectCoercibleInfo, ins);
11927 masm.bind(&done);
11928 }
11929
11930 typedef bool (*CheckSelfHostedFn)(JSContext*, HandleValue);
11931 static const VMFunction CheckSelfHostedInfo =
11932 FunctionInfo<CheckSelfHostedFn>(js::Debug_CheckSelfHosted, "Debug_CheckSelfHosted");
11933
11934 void
visitDebugCheckSelfHosted(LDebugCheckSelfHosted * ins)11935 CodeGenerator::visitDebugCheckSelfHosted(LDebugCheckSelfHosted* ins)
11936 {
11937 ValueOperand checkValue = ToValue(ins, LDebugCheckSelfHosted::CheckValue);
11938 pushArg(checkValue);
11939 callVM(CheckSelfHostedInfo, ins);
11940 }
11941
11942 void
visitRandom(LRandom * ins)11943 CodeGenerator::visitRandom(LRandom* ins)
11944 {
11945 using mozilla::non_crypto::XorShift128PlusRNG;
11946
11947 FloatRegister output = ToFloatRegister(ins->output());
11948 Register tempReg = ToRegister(ins->temp0());
11949
11950 #ifdef JS_PUNBOX64
11951 Register64 s0Reg(ToRegister(ins->temp1()));
11952 Register64 s1Reg(ToRegister(ins->temp2()));
11953 #else
11954 Register64 s0Reg(ToRegister(ins->temp1()), ToRegister(ins->temp2()));
11955 Register64 s1Reg(ToRegister(ins->temp3()), ToRegister(ins->temp4()));
11956 #endif
11957
11958 const void* rng = gen->compartment->addressOfRandomNumberGenerator();
11959 masm.movePtr(ImmPtr(rng), tempReg);
11960
11961 static_assert(sizeof(XorShift128PlusRNG) == 2 * sizeof(uint64_t),
11962 "Code below assumes XorShift128PlusRNG contains two uint64_t values");
11963
11964 Address state0Addr(tempReg, XorShift128PlusRNG::offsetOfState0());
11965 Address state1Addr(tempReg, XorShift128PlusRNG::offsetOfState1());
11966
11967 // uint64_t s1 = mState[0];
11968 masm.load64(state0Addr, s1Reg);
11969
11970 // s1 ^= s1 << 23;
11971 masm.move64(s1Reg, s0Reg);
11972 masm.lshift64(Imm32(23), s1Reg);
11973 masm.xor64(s0Reg, s1Reg);
11974
11975 // s1 ^= s1 >> 17
11976 masm.move64(s1Reg, s0Reg);
11977 masm.rshift64(Imm32(17), s1Reg);
11978 masm.xor64(s0Reg, s1Reg);
11979
11980 // const uint64_t s0 = mState[1];
11981 masm.load64(state1Addr, s0Reg);
11982
11983 // mState[0] = s0;
11984 masm.store64(s0Reg, state0Addr);
11985
11986 // s1 ^= s0
11987 masm.xor64(s0Reg, s1Reg);
11988
11989 // s1 ^= s0 >> 26
11990 masm.rshift64(Imm32(26), s0Reg);
11991 masm.xor64(s0Reg, s1Reg);
11992
11993 // mState[1] = s1
11994 masm.store64(s1Reg, state1Addr);
11995
11996 // s1 += mState[0]
11997 masm.load64(state0Addr, s0Reg);
11998 masm.add64(s0Reg, s1Reg);
11999
12000 // See comment in XorShift128PlusRNG::nextDouble().
12001 static const int MantissaBits = FloatingPoint<double>::kExponentShift + 1;
12002 static const double ScaleInv = double(1) / (1ULL << MantissaBits);
12003
12004 masm.and64(Imm64((1ULL << MantissaBits) - 1), s1Reg);
12005
12006 if (masm.convertUInt64ToDoubleNeedsTemp())
12007 masm.convertUInt64ToDouble(s1Reg, output, tempReg);
12008 else
12009 masm.convertUInt64ToDouble(s1Reg, output, Register::Invalid());
12010
12011 // output *= ScaleInv
12012 masm.mulDoublePtr(ImmPtr(&ScaleInv), tempReg, output);
12013 }
12014
12015 void
visitSignExtend(LSignExtend * ins)12016 CodeGenerator::visitSignExtend(LSignExtend* ins)
12017 {
12018 Register input = ToRegister(ins->input());
12019 Register output = ToRegister(ins->output());
12020
12021 switch (ins->mode()) {
12022 case MSignExtend::Byte:
12023 masm.move8SignExtend(input, output);
12024 break;
12025 case MSignExtend::Half:
12026 masm.move16SignExtend(input, output);
12027 break;
12028 }
12029 }
12030
12031 void
visitRotate(LRotate * ins)12032 CodeGenerator::visitRotate(LRotate* ins)
12033 {
12034 MRotate* mir = ins->mir();
12035 Register input = ToRegister(ins->input());
12036 Register dest = ToRegister(ins->output());
12037
12038 const LAllocation* count = ins->count();
12039 if (count->isConstant()) {
12040 int32_t c = ToInt32(count) & 0x1F;
12041 if (mir->isLeftRotate())
12042 masm.rotateLeft(Imm32(c), input, dest);
12043 else
12044 masm.rotateRight(Imm32(c), input, dest);
12045 } else {
12046 Register creg = ToRegister(count);
12047 if (mir->isLeftRotate())
12048 masm.rotateLeft(creg, input, dest);
12049 else
12050 masm.rotateRight(creg, input, dest);
12051 }
12052 }
12053
12054 class OutOfLineNaNToZero : public OutOfLineCodeBase<CodeGenerator>
12055 {
12056 LNaNToZero* lir_;
12057
12058 public:
OutOfLineNaNToZero(LNaNToZero * lir)12059 explicit OutOfLineNaNToZero(LNaNToZero* lir)
12060 : lir_(lir)
12061 {}
12062
accept(CodeGenerator * codegen)12063 void accept(CodeGenerator* codegen) {
12064 codegen->visitOutOfLineNaNToZero(this);
12065 }
lir() const12066 LNaNToZero* lir() const {
12067 return lir_;
12068 }
12069 };
12070
12071 void
visitOutOfLineNaNToZero(OutOfLineNaNToZero * ool)12072 CodeGenerator::visitOutOfLineNaNToZero(OutOfLineNaNToZero* ool)
12073 {
12074 FloatRegister output = ToFloatRegister(ool->lir()->output());
12075 masm.loadConstantDouble(0.0, output);
12076 masm.jump(ool->rejoin());
12077 }
12078
12079 void
visitNaNToZero(LNaNToZero * lir)12080 CodeGenerator::visitNaNToZero(LNaNToZero* lir)
12081 {
12082 FloatRegister input = ToFloatRegister(lir->input());
12083
12084 OutOfLineNaNToZero* ool = new(alloc()) OutOfLineNaNToZero(lir);
12085 addOutOfLineCode(ool, lir->mir());
12086
12087 if (lir->mir()->operandIsNeverNegativeZero()){
12088 masm.branchDouble(Assembler::DoubleUnordered, input, input, ool->entry());
12089 } else {
12090 FloatRegister scratch = ToFloatRegister(lir->tempDouble());
12091 masm.loadConstantDouble(0.0, scratch);
12092 masm.branchDouble(Assembler::DoubleEqualOrUnordered, input, scratch, ool->entry());
12093 }
12094 masm.bind(ool->rejoin());
12095 }
12096
12097 } // namespace jit
12098 } // namespace js
12099