1 // Copyright 2015 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/codegen/assembler-inl.h"
6 #include "src/codegen/callable.h"
7 #include "src/codegen/macro-assembler.h"
8 #include "src/codegen/optimized-compilation-info.h"
9 #include "src/compiler/backend/code-generator-impl.h"
10 #include "src/compiler/backend/code-generator.h"
11 #include "src/compiler/backend/gap-resolver.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/osr.h"
14 #include "src/heap/memory-chunk.h"
15
16 #if V8_ENABLE_WEBASSEMBLY
17 #include "src/wasm/wasm-code-manager.h"
18 #include "src/wasm/wasm-objects.h"
19 #endif // V8_ENABLE_WEBASSEMBLY
20
21 namespace v8 {
22 namespace internal {
23 namespace compiler {
24
25 #define __ tasm()->
26
27 #define kScratchReg ip
28
29 // Adds S390-specific methods to convert InstructionOperands.
30 class S390OperandConverter final : public InstructionOperandConverter {
31 public:
S390OperandConverter(CodeGenerator * gen,Instruction * instr)32 S390OperandConverter(CodeGenerator* gen, Instruction* instr)
33 : InstructionOperandConverter(gen, instr) {}
34
OutputCount()35 size_t OutputCount() { return instr_->OutputCount(); }
36
Is64BitOperand(int index)37 bool Is64BitOperand(int index) {
38 return LocationOperand::cast(instr_->InputAt(index))->representation() ==
39 MachineRepresentation::kWord64;
40 }
41
Is32BitOperand(int index)42 bool Is32BitOperand(int index) {
43 return LocationOperand::cast(instr_->InputAt(index))->representation() ==
44 MachineRepresentation::kWord32;
45 }
46
CompareLogical() const47 bool CompareLogical() const {
48 switch (instr_->flags_condition()) {
49 case kUnsignedLessThan:
50 case kUnsignedGreaterThanOrEqual:
51 case kUnsignedLessThanOrEqual:
52 case kUnsignedGreaterThan:
53 return true;
54 default:
55 return false;
56 }
57 UNREACHABLE();
58 }
59
InputImmediate(size_t index)60 Operand InputImmediate(size_t index) {
61 Constant constant = ToConstant(instr_->InputAt(index));
62 switch (constant.type()) {
63 case Constant::kInt32:
64 return Operand(constant.ToInt32());
65 case Constant::kFloat32:
66 return Operand::EmbeddedNumber(constant.ToFloat32());
67 case Constant::kFloat64:
68 return Operand::EmbeddedNumber(constant.ToFloat64().value());
69 case Constant::kInt64:
70 #if V8_TARGET_ARCH_S390X
71 return Operand(constant.ToInt64());
72 #endif
73 case Constant::kExternalReference:
74 return Operand(constant.ToExternalReference());
75 case Constant::kDelayedStringConstant:
76 return Operand::EmbeddedStringConstant(
77 constant.ToDelayedStringConstant());
78 case Constant::kCompressedHeapObject:
79 case Constant::kHeapObject:
80 case Constant::kRpoNumber:
81 break;
82 }
83 UNREACHABLE();
84 }
85
MemoryOperand(AddressingMode * mode,size_t * first_index)86 MemOperand MemoryOperand(AddressingMode* mode, size_t* first_index) {
87 const size_t index = *first_index;
88 if (mode) *mode = AddressingModeField::decode(instr_->opcode());
89 switch (AddressingModeField::decode(instr_->opcode())) {
90 case kMode_None:
91 break;
92 case kMode_MR:
93 *first_index += 1;
94 return MemOperand(InputRegister(index + 0), 0);
95 case kMode_MRI:
96 *first_index += 2;
97 return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
98 case kMode_MRR:
99 *first_index += 2;
100 return MemOperand(InputRegister(index + 0), InputRegister(index + 1));
101 case kMode_MRRI:
102 *first_index += 3;
103 return MemOperand(InputRegister(index + 0), InputRegister(index + 1),
104 InputInt32(index + 2));
105 }
106 UNREACHABLE();
107 }
108
MemoryOperand(AddressingMode * mode=nullptr,size_t first_index=0)109 MemOperand MemoryOperand(AddressingMode* mode = nullptr,
110 size_t first_index = 0) {
111 return MemoryOperand(mode, &first_index);
112 }
113
ToMemOperand(InstructionOperand * op) const114 MemOperand ToMemOperand(InstructionOperand* op) const {
115 DCHECK_NOT_NULL(op);
116 DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
117 return SlotToMemOperand(AllocatedOperand::cast(op)->index());
118 }
119
SlotToMemOperand(int slot) const120 MemOperand SlotToMemOperand(int slot) const {
121 FrameOffset offset = frame_access_state()->GetFrameOffset(slot);
122 return MemOperand(offset.from_stack_pointer() ? sp : fp, offset.offset());
123 }
124
InputStackSlot(size_t index)125 MemOperand InputStackSlot(size_t index) {
126 InstructionOperand* op = instr_->InputAt(index);
127 return SlotToMemOperand(AllocatedOperand::cast(op)->index());
128 }
129
InputStackSlot32(size_t index)130 MemOperand InputStackSlot32(size_t index) {
131 #if V8_TARGET_ARCH_S390X && !V8_TARGET_LITTLE_ENDIAN
132 // We want to read the 32-bits directly from memory
133 MemOperand mem = InputStackSlot(index);
134 return MemOperand(mem.rx(), mem.rb(), mem.offset() + 4);
135 #else
136 return InputStackSlot(index);
137 #endif
138 }
139 };
140
HasRegisterOutput(Instruction * instr,int index=0)141 static inline bool HasRegisterOutput(Instruction* instr, int index = 0) {
142 return instr->OutputCount() > 0 && instr->OutputAt(index)->IsRegister();
143 }
144
HasFPRegisterInput(Instruction * instr,int index)145 static inline bool HasFPRegisterInput(Instruction* instr, int index) {
146 return instr->InputAt(index)->IsFPRegister();
147 }
148
HasRegisterInput(Instruction * instr,int index)149 static inline bool HasRegisterInput(Instruction* instr, int index) {
150 return instr->InputAt(index)->IsRegister() ||
151 HasFPRegisterInput(instr, index);
152 }
153
HasImmediateInput(Instruction * instr,size_t index)154 static inline bool HasImmediateInput(Instruction* instr, size_t index) {
155 return instr->InputAt(index)->IsImmediate();
156 }
157
HasFPStackSlotInput(Instruction * instr,size_t index)158 static inline bool HasFPStackSlotInput(Instruction* instr, size_t index) {
159 return instr->InputAt(index)->IsFPStackSlot();
160 }
161
HasStackSlotInput(Instruction * instr,size_t index)162 static inline bool HasStackSlotInput(Instruction* instr, size_t index) {
163 return instr->InputAt(index)->IsStackSlot() ||
164 HasFPStackSlotInput(instr, index);
165 }
166
167 namespace {
168
169 class OutOfLineRecordWrite final : public OutOfLineCode {
170 public:
OutOfLineRecordWrite(CodeGenerator * gen,Register object,Register offset,Register value,Register scratch0,Register scratch1,RecordWriteMode mode,StubCallMode stub_mode,UnwindingInfoWriter * unwinding_info_writer)171 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Register offset,
172 Register value, Register scratch0, Register scratch1,
173 RecordWriteMode mode, StubCallMode stub_mode,
174 UnwindingInfoWriter* unwinding_info_writer)
175 : OutOfLineCode(gen),
176 object_(object),
177 offset_(offset),
178 offset_immediate_(0),
179 value_(value),
180 scratch0_(scratch0),
181 scratch1_(scratch1),
182 mode_(mode),
183 #if V8_ENABLE_WEBASSEMBLY
184 stub_mode_(stub_mode),
185 #endif // V8_ENABLE_WEBASSEMBLY
186 must_save_lr_(!gen->frame_access_state()->has_frame()),
187 unwinding_info_writer_(unwinding_info_writer),
188 zone_(gen->zone()) {
189 DCHECK(!AreAliased(object, offset, scratch0, scratch1));
190 DCHECK(!AreAliased(value, offset, scratch0, scratch1));
191 }
192
OutOfLineRecordWrite(CodeGenerator * gen,Register object,int32_t offset,Register value,Register scratch0,Register scratch1,RecordWriteMode mode,StubCallMode stub_mode,UnwindingInfoWriter * unwinding_info_writer)193 OutOfLineRecordWrite(CodeGenerator* gen, Register object, int32_t offset,
194 Register value, Register scratch0, Register scratch1,
195 RecordWriteMode mode, StubCallMode stub_mode,
196 UnwindingInfoWriter* unwinding_info_writer)
197 : OutOfLineCode(gen),
198 object_(object),
199 offset_(no_reg),
200 offset_immediate_(offset),
201 value_(value),
202 scratch0_(scratch0),
203 scratch1_(scratch1),
204 mode_(mode),
205 #if V8_ENABLE_WEBASSEMBLY
206 stub_mode_(stub_mode),
207 #endif // V8_ENABLE_WEBASSEMBLY
208 must_save_lr_(!gen->frame_access_state()->has_frame()),
209 unwinding_info_writer_(unwinding_info_writer),
210 zone_(gen->zone()) {
211 }
212
Generate()213 void Generate() final {
214 if (COMPRESS_POINTERS_BOOL) {
215 __ DecompressTaggedPointer(value_, value_);
216 }
217 __ CheckPageFlag(value_, scratch0_,
218 MemoryChunk::kPointersToHereAreInterestingMask, eq,
219 exit());
220 if (offset_ == no_reg) {
221 __ AddS64(scratch1_, object_, Operand(offset_immediate_));
222 } else {
223 DCHECK_EQ(0, offset_immediate_);
224 __ AddS64(scratch1_, object_, offset_);
225 }
226 RememberedSetAction const remembered_set_action =
227 mode_ > RecordWriteMode::kValueIsMap ? RememberedSetAction::kEmit
228 : RememberedSetAction::kOmit;
229 SaveFPRegsMode const save_fp_mode = frame()->DidAllocateDoubleRegisters()
230 ? SaveFPRegsMode::kSave
231 : SaveFPRegsMode::kIgnore;
232 if (must_save_lr_) {
233 // We need to save and restore r14 if the frame was elided.
234 __ Push(r14);
235 unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset());
236 }
237 if (mode_ == RecordWriteMode::kValueIsEphemeronKey) {
238 __ CallEphemeronKeyBarrier(object_, scratch1_, save_fp_mode);
239 #if V8_ENABLE_WEBASSEMBLY
240 } else if (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) {
241 __ CallRecordWriteStubSaveRegisters(object_, scratch1_,
242 remembered_set_action, save_fp_mode,
243 StubCallMode::kCallWasmRuntimeStub);
244 #endif // V8_ENABLE_WEBASSEMBLY
245 } else {
246 __ CallRecordWriteStubSaveRegisters(object_, scratch1_,
247 remembered_set_action, save_fp_mode);
248 }
249 if (must_save_lr_) {
250 // We need to save and restore r14 if the frame was elided.
251 __ Pop(r14);
252 unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset());
253 }
254 }
255
256 private:
257 Register const object_;
258 Register const offset_;
259 int32_t const offset_immediate_; // Valid if offset_ == no_reg.
260 Register const value_;
261 Register const scratch0_;
262 Register const scratch1_;
263 RecordWriteMode const mode_;
264 #if V8_ENABLE_WEBASSEMBLY
265 StubCallMode stub_mode_;
266 #endif // V8_ENABLE_WEBASSEMBLY
267 bool must_save_lr_;
268 UnwindingInfoWriter* const unwinding_info_writer_;
269 Zone* zone_;
270 };
271
FlagsConditionToCondition(FlagsCondition condition,ArchOpcode op)272 Condition FlagsConditionToCondition(FlagsCondition condition, ArchOpcode op) {
273 switch (condition) {
274 case kEqual:
275 return eq;
276 case kNotEqual:
277 return ne;
278 case kUnsignedLessThan:
279 // unsigned number never less than 0
280 if (op == kS390_LoadAndTestWord32 || op == kS390_LoadAndTestWord64)
281 return CC_NOP;
282 V8_FALLTHROUGH;
283 case kSignedLessThan:
284 return lt;
285 case kUnsignedGreaterThanOrEqual:
286 // unsigned number always greater than or equal 0
287 if (op == kS390_LoadAndTestWord32 || op == kS390_LoadAndTestWord64)
288 return CC_ALWAYS;
289 V8_FALLTHROUGH;
290 case kSignedGreaterThanOrEqual:
291 return ge;
292 case kUnsignedLessThanOrEqual:
293 // unsigned number never less than 0
294 if (op == kS390_LoadAndTestWord32 || op == kS390_LoadAndTestWord64)
295 return CC_EQ;
296 V8_FALLTHROUGH;
297 case kSignedLessThanOrEqual:
298 return le;
299 case kUnsignedGreaterThan:
300 // unsigned number always greater than or equal 0
301 if (op == kS390_LoadAndTestWord32 || op == kS390_LoadAndTestWord64)
302 return ne;
303 V8_FALLTHROUGH;
304 case kSignedGreaterThan:
305 return gt;
306 case kOverflow:
307 // Overflow checked for AddS64/SubS64 only.
308 switch (op) {
309 case kS390_Add32:
310 case kS390_Add64:
311 case kS390_Sub32:
312 case kS390_Sub64:
313 case kS390_Abs64:
314 case kS390_Abs32:
315 case kS390_Mul32:
316 return overflow;
317 default:
318 break;
319 }
320 break;
321 case kNotOverflow:
322 switch (op) {
323 case kS390_Add32:
324 case kS390_Add64:
325 case kS390_Sub32:
326 case kS390_Sub64:
327 case kS390_Abs64:
328 case kS390_Abs32:
329 case kS390_Mul32:
330 return nooverflow;
331 default:
332 break;
333 }
334 break;
335 default:
336 break;
337 }
338 UNREACHABLE();
339 }
340
341 #define GET_MEMOPERAND32(ret, fi) \
342 ([&](int& ret) { \
343 AddressingMode mode = AddressingModeField::decode(instr->opcode()); \
344 MemOperand mem(r0); \
345 if (mode != kMode_None) { \
346 size_t first_index = (fi); \
347 mem = i.MemoryOperand(&mode, &first_index); \
348 ret = first_index; \
349 } else { \
350 mem = i.InputStackSlot32(fi); \
351 } \
352 return mem; \
353 })(ret)
354
355 #define GET_MEMOPERAND(ret, fi) \
356 ([&](int& ret) { \
357 AddressingMode mode = AddressingModeField::decode(instr->opcode()); \
358 MemOperand mem(r0); \
359 if (mode != kMode_None) { \
360 size_t first_index = (fi); \
361 mem = i.MemoryOperand(&mode, &first_index); \
362 ret = first_index; \
363 } else { \
364 mem = i.InputStackSlot(fi); \
365 } \
366 return mem; \
367 })(ret)
368
369 #define RRInstr(instr) \
370 [&]() { \
371 DCHECK(i.OutputRegister() == i.InputRegister(0)); \
372 __ instr(i.OutputRegister(), i.InputRegister(1)); \
373 return 2; \
374 }
375 #define RIInstr(instr) \
376 [&]() { \
377 DCHECK(i.OutputRegister() == i.InputRegister(0)); \
378 __ instr(i.OutputRegister(), i.InputImmediate(1)); \
379 return 2; \
380 }
381 #define RMInstr(instr, GETMEM) \
382 [&]() { \
383 DCHECK(i.OutputRegister() == i.InputRegister(0)); \
384 int ret = 2; \
385 __ instr(i.OutputRegister(), GETMEM(ret, 1)); \
386 return ret; \
387 }
388 #define RM32Instr(instr) RMInstr(instr, GET_MEMOPERAND32)
389 #define RM64Instr(instr) RMInstr(instr, GET_MEMOPERAND)
390
391 #define RRRInstr(instr) \
392 [&]() { \
393 __ instr(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); \
394 return 2; \
395 }
396 #define RRIInstr(instr) \
397 [&]() { \
398 __ instr(i.OutputRegister(), i.InputRegister(0), i.InputImmediate(1)); \
399 return 2; \
400 }
401 #define RRMInstr(instr, GETMEM) \
402 [&]() { \
403 int ret = 2; \
404 __ instr(i.OutputRegister(), i.InputRegister(0), GETMEM(ret, 1)); \
405 return ret; \
406 }
407 #define RRM32Instr(instr) RRMInstr(instr, GET_MEMOPERAND32)
408 #define RRM64Instr(instr) RRMInstr(instr, GET_MEMOPERAND)
409
410 #define DDInstr(instr) \
411 [&]() { \
412 DCHECK(i.OutputDoubleRegister() == i.InputDoubleRegister(0)); \
413 __ instr(i.OutputDoubleRegister(), i.InputDoubleRegister(1)); \
414 return 2; \
415 }
416
417 #define DMInstr(instr) \
418 [&]() { \
419 DCHECK(i.OutputDoubleRegister() == i.InputDoubleRegister(0)); \
420 int ret = 2; \
421 __ instr(i.OutputDoubleRegister(), GET_MEMOPERAND(ret, 1)); \
422 return ret; \
423 }
424
425 #define DMTInstr(instr) \
426 [&]() { \
427 DCHECK(i.OutputDoubleRegister() == i.InputDoubleRegister(0)); \
428 int ret = 2; \
429 __ instr(i.OutputDoubleRegister(), GET_MEMOPERAND(ret, 1), \
430 kScratchDoubleReg); \
431 return ret; \
432 }
433
434 #define R_MInstr(instr) \
435 [&]() { \
436 int ret = 2; \
437 __ instr(i.OutputRegister(), GET_MEMOPERAND(ret, 0)); \
438 return ret; \
439 }
440
441 #define R_DInstr(instr) \
442 [&]() { \
443 __ instr(i.OutputRegister(), i.InputDoubleRegister(0)); \
444 return 2; \
445 }
446
447 #define D_DInstr(instr) \
448 [&]() { \
449 __ instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \
450 return 2; \
451 }
452
453 #define D_MInstr(instr) \
454 [&]() { \
455 int ret = 2; \
456 __ instr(i.OutputDoubleRegister(), GET_MEMOPERAND(ret, 0)); \
457 return ret; \
458 }
459
460 #define D_MTInstr(instr) \
461 [&]() { \
462 int ret = 2; \
463 __ instr(i.OutputDoubleRegister(), GET_MEMOPERAND(ret, 0), \
464 kScratchDoubleReg); \
465 return ret; \
466 }
467
nullInstr()468 static int nullInstr() { UNREACHABLE(); }
469
470 template <int numOfOperand, class RType, class MType, class IType>
AssembleOp(Instruction * instr,RType r,MType m,IType i)471 static inline int AssembleOp(Instruction* instr, RType r, MType m, IType i) {
472 AddressingMode mode = AddressingModeField::decode(instr->opcode());
473 if (mode != kMode_None || HasStackSlotInput(instr, numOfOperand - 1)) {
474 return m();
475 } else if (HasRegisterInput(instr, numOfOperand - 1)) {
476 return r();
477 } else if (HasImmediateInput(instr, numOfOperand - 1)) {
478 return i();
479 } else {
480 UNREACHABLE();
481 }
482 }
483
484 template <class _RR, class _RM, class _RI>
AssembleBinOp(Instruction * instr,_RR _rr,_RM _rm,_RI _ri)485 static inline int AssembleBinOp(Instruction* instr, _RR _rr, _RM _rm, _RI _ri) {
486 return AssembleOp<2>(instr, _rr, _rm, _ri);
487 }
488
489 template <class _R, class _M, class _I>
AssembleUnaryOp(Instruction * instr,_R _r,_M _m,_I _i)490 static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
491 return AssembleOp<1>(instr, _r, _m, _i);
492 }
493
494 #define ASSEMBLE_BIN_OP(_rr, _rm, _ri) AssembleBinOp(instr, _rr, _rm, _ri)
495 #define ASSEMBLE_UNARY_OP(_r, _m, _i) AssembleUnaryOp(instr, _r, _m, _i)
496
497 #ifdef V8_TARGET_ARCH_S390X
498 #define CHECK_AND_ZERO_EXT_OUTPUT(num) \
499 ([&](int index) { \
500 DCHECK(HasImmediateInput(instr, (index))); \
501 int doZeroExt = i.InputInt32(index); \
502 if (doZeroExt) __ LoadU32(i.OutputRegister(), i.OutputRegister()); \
503 })(num)
504
505 #define ASSEMBLE_BIN32_OP(_rr, _rm, _ri) \
506 { CHECK_AND_ZERO_EXT_OUTPUT(AssembleBinOp(instr, _rr, _rm, _ri)); }
507 #else
508 #define ASSEMBLE_BIN32_OP ASSEMBLE_BIN_OP
509 #define CHECK_AND_ZERO_EXT_OUTPUT(num)
510 #endif
511
512 } // namespace
513
514 #define ASSEMBLE_FLOAT_UNOP(asm_instr) \
515 do { \
516 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \
517 } while (0)
518
519 #define ASSEMBLE_FLOAT_BINOP(asm_instr) \
520 do { \
521 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
522 i.InputDoubleRegister(1)); \
523 } while (0)
524
525 #define ASSEMBLE_COMPARE(cmp_instr, cmpl_instr) \
526 do { \
527 AddressingMode mode = AddressingModeField::decode(instr->opcode()); \
528 if (mode != kMode_None) { \
529 size_t first_index = 1; \
530 MemOperand operand = i.MemoryOperand(&mode, &first_index); \
531 if (i.CompareLogical()) { \
532 __ cmpl_instr(i.InputRegister(0), operand); \
533 } else { \
534 __ cmp_instr(i.InputRegister(0), operand); \
535 } \
536 } else if (HasRegisterInput(instr, 1)) { \
537 if (i.CompareLogical()) { \
538 __ cmpl_instr(i.InputRegister(0), i.InputRegister(1)); \
539 } else { \
540 __ cmp_instr(i.InputRegister(0), i.InputRegister(1)); \
541 } \
542 } else if (HasImmediateInput(instr, 1)) { \
543 if (i.CompareLogical()) { \
544 __ cmpl_instr(i.InputRegister(0), i.InputImmediate(1)); \
545 } else { \
546 __ cmp_instr(i.InputRegister(0), i.InputImmediate(1)); \
547 } \
548 } else { \
549 DCHECK(HasStackSlotInput(instr, 1)); \
550 if (i.CompareLogical()) { \
551 __ cmpl_instr(i.InputRegister(0), i.InputStackSlot(1)); \
552 } else { \
553 __ cmp_instr(i.InputRegister(0), i.InputStackSlot(1)); \
554 } \
555 } \
556 } while (0)
557
558 #define ASSEMBLE_COMPARE32(cmp_instr, cmpl_instr) \
559 do { \
560 AddressingMode mode = AddressingModeField::decode(instr->opcode()); \
561 if (mode != kMode_None) { \
562 size_t first_index = 1; \
563 MemOperand operand = i.MemoryOperand(&mode, &first_index); \
564 if (i.CompareLogical()) { \
565 __ cmpl_instr(i.InputRegister(0), operand); \
566 } else { \
567 __ cmp_instr(i.InputRegister(0), operand); \
568 } \
569 } else if (HasRegisterInput(instr, 1)) { \
570 if (i.CompareLogical()) { \
571 __ cmpl_instr(i.InputRegister(0), i.InputRegister(1)); \
572 } else { \
573 __ cmp_instr(i.InputRegister(0), i.InputRegister(1)); \
574 } \
575 } else if (HasImmediateInput(instr, 1)) { \
576 if (i.CompareLogical()) { \
577 __ cmpl_instr(i.InputRegister(0), i.InputImmediate(1)); \
578 } else { \
579 __ cmp_instr(i.InputRegister(0), i.InputImmediate(1)); \
580 } \
581 } else { \
582 DCHECK(HasStackSlotInput(instr, 1)); \
583 if (i.CompareLogical()) { \
584 __ cmpl_instr(i.InputRegister(0), i.InputStackSlot32(1)); \
585 } else { \
586 __ cmp_instr(i.InputRegister(0), i.InputStackSlot32(1)); \
587 } \
588 } \
589 } while (0)
590
591 #define ASSEMBLE_FLOAT_COMPARE(cmp_rr_instr, cmp_rm_instr, load_instr) \
592 do { \
593 AddressingMode mode = AddressingModeField::decode(instr->opcode()); \
594 if (mode != kMode_None) { \
595 size_t first_index = 1; \
596 MemOperand operand = i.MemoryOperand(&mode, &first_index); \
597 __ cmp_rm_instr(i.InputDoubleRegister(0), operand); \
598 } else if (HasFPRegisterInput(instr, 1)) { \
599 __ cmp_rr_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
600 } else { \
601 USE(HasFPStackSlotInput); \
602 DCHECK(HasFPStackSlotInput(instr, 1)); \
603 MemOperand operand = i.InputStackSlot(1); \
604 if (operand.offset() >= 0) { \
605 __ cmp_rm_instr(i.InputDoubleRegister(0), operand); \
606 } else { \
607 __ load_instr(kScratchDoubleReg, operand); \
608 __ cmp_rr_instr(i.InputDoubleRegister(0), kScratchDoubleReg); \
609 } \
610 } \
611 } while (0)
612
613 // Divide instruction dr will implicity use register pair
614 // r0 & r1 below.
615 // R0:R1 = R1 / divisor - R0 remainder
616 // Copy remainder to output reg
617 #define ASSEMBLE_MODULO(div_instr, shift_instr) \
618 do { \
619 __ mov(r0, i.InputRegister(0)); \
620 __ shift_instr(r0, Operand(32)); \
621 __ div_instr(r0, i.InputRegister(1)); \
622 __ LoadU32(i.OutputRegister(), r0); \
623 } while (0)
624
625 #define ASSEMBLE_FLOAT_MODULO() \
626 do { \
627 FrameScope scope(tasm(), StackFrame::MANUAL); \
628 __ PrepareCallCFunction(0, 2, kScratchReg); \
629 __ MovToFloatParameters(i.InputDoubleRegister(0), \
630 i.InputDoubleRegister(1)); \
631 __ CallCFunction(ExternalReference::mod_two_doubles_operation(), 0, 2); \
632 __ MovFromFloatResult(i.OutputDoubleRegister()); \
633 } while (0)
634
635 #define ASSEMBLE_IEEE754_UNOP(name) \
636 do { \
637 /* TODO(bmeurer): We should really get rid of this special instruction, */ \
638 /* and generate a CallAddress instruction instead. */ \
639 FrameScope scope(tasm(), StackFrame::MANUAL); \
640 __ PrepareCallCFunction(0, 1, kScratchReg); \
641 __ MovToFloatParameter(i.InputDoubleRegister(0)); \
642 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 1); \
643 /* Move the result in the double result register. */ \
644 __ MovFromFloatResult(i.OutputDoubleRegister()); \
645 } while (0)
646
647 #define ASSEMBLE_IEEE754_BINOP(name) \
648 do { \
649 /* TODO(bmeurer): We should really get rid of this special instruction, */ \
650 /* and generate a CallAddress instruction instead. */ \
651 FrameScope scope(tasm(), StackFrame::MANUAL); \
652 __ PrepareCallCFunction(0, 2, kScratchReg); \
653 __ MovToFloatParameters(i.InputDoubleRegister(0), \
654 i.InputDoubleRegister(1)); \
655 __ CallCFunction(ExternalReference::ieee754_##name##_function(), 0, 2); \
656 /* Move the result in the double result register. */ \
657 __ MovFromFloatResult(i.OutputDoubleRegister()); \
658 } while (0)
659
660 //
661 // Only MRI mode for these instructions available
662 #define ASSEMBLE_LOAD_FLOAT(asm_instr) \
663 do { \
664 DoubleRegister result = i.OutputDoubleRegister(); \
665 AddressingMode mode = kMode_None; \
666 MemOperand operand = i.MemoryOperand(&mode); \
667 __ asm_instr(result, operand); \
668 } while (0)
669
670 #define ASSEMBLE_LOAD_INTEGER(asm_instr) \
671 do { \
672 Register result = i.OutputRegister(); \
673 AddressingMode mode = kMode_None; \
674 MemOperand operand = i.MemoryOperand(&mode); \
675 __ asm_instr(result, operand); \
676 } while (0)
677
678 #define ASSEMBLE_LOADANDTEST64(asm_instr_rr, asm_instr_rm) \
679 { \
680 AddressingMode mode = AddressingModeField::decode(instr->opcode()); \
681 Register dst = HasRegisterOutput(instr) ? i.OutputRegister() : r0; \
682 if (mode != kMode_None) { \
683 size_t first_index = 0; \
684 MemOperand operand = i.MemoryOperand(&mode, &first_index); \
685 __ asm_instr_rm(dst, operand); \
686 } else if (HasRegisterInput(instr, 0)) { \
687 __ asm_instr_rr(dst, i.InputRegister(0)); \
688 } else { \
689 DCHECK(HasStackSlotInput(instr, 0)); \
690 __ asm_instr_rm(dst, i.InputStackSlot(0)); \
691 } \
692 }
693
694 #define ASSEMBLE_LOADANDTEST32(asm_instr_rr, asm_instr_rm) \
695 { \
696 AddressingMode mode = AddressingModeField::decode(instr->opcode()); \
697 Register dst = HasRegisterOutput(instr) ? i.OutputRegister() : r0; \
698 if (mode != kMode_None) { \
699 size_t first_index = 0; \
700 MemOperand operand = i.MemoryOperand(&mode, &first_index); \
701 __ asm_instr_rm(dst, operand); \
702 } else if (HasRegisterInput(instr, 0)) { \
703 __ asm_instr_rr(dst, i.InputRegister(0)); \
704 } else { \
705 DCHECK(HasStackSlotInput(instr, 0)); \
706 __ asm_instr_rm(dst, i.InputStackSlot32(0)); \
707 } \
708 }
709
710 #define ASSEMBLE_STORE_FLOAT32() \
711 do { \
712 size_t index = 0; \
713 AddressingMode mode = kMode_None; \
714 MemOperand operand = i.MemoryOperand(&mode, &index); \
715 DoubleRegister value = i.InputDoubleRegister(index); \
716 __ StoreF32(value, operand); \
717 } while (0)
718
719 #define ASSEMBLE_STORE_DOUBLE() \
720 do { \
721 size_t index = 0; \
722 AddressingMode mode = kMode_None; \
723 MemOperand operand = i.MemoryOperand(&mode, &index); \
724 DoubleRegister value = i.InputDoubleRegister(index); \
725 __ StoreF64(value, operand); \
726 } while (0)
727
728 #define ASSEMBLE_STORE_INTEGER(asm_instr) \
729 do { \
730 size_t index = 0; \
731 AddressingMode mode = kMode_None; \
732 MemOperand operand = i.MemoryOperand(&mode, &index); \
733 Register value = i.InputRegister(index); \
734 __ asm_instr(value, operand); \
735 } while (0)
736
737 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_BYTE(load_and_ext) \
738 do { \
739 Register old_value = i.InputRegister(0); \
740 Register new_value = i.InputRegister(1); \
741 Register output = i.OutputRegister(); \
742 Register addr = kScratchReg; \
743 Register temp0 = r0; \
744 Register temp1 = r1; \
745 size_t index = 2; \
746 AddressingMode mode = kMode_None; \
747 MemOperand op = i.MemoryOperand(&mode, &index); \
748 __ lay(addr, op); \
749 __ AtomicCmpExchangeU8(addr, output, old_value, new_value, temp0, temp1); \
750 __ load_and_ext(output, output); \
751 } while (false)
752
753 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_HALFWORD(load_and_ext) \
754 do { \
755 Register old_value = i.InputRegister(0); \
756 Register new_value = i.InputRegister(1); \
757 Register output = i.OutputRegister(); \
758 Register addr = kScratchReg; \
759 Register temp0 = r0; \
760 Register temp1 = r1; \
761 size_t index = 2; \
762 AddressingMode mode = kMode_None; \
763 MemOperand op = i.MemoryOperand(&mode, &index); \
764 __ lay(addr, op); \
765 __ AtomicCmpExchangeU16(addr, output, old_value, new_value, temp0, temp1); \
766 __ load_and_ext(output, output); \
767 } while (false)
768
769 #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_WORD() \
770 do { \
771 Register new_val = i.InputRegister(1); \
772 Register output = i.OutputRegister(); \
773 Register addr = kScratchReg; \
774 size_t index = 2; \
775 AddressingMode mode = kMode_None; \
776 MemOperand op = i.MemoryOperand(&mode, &index); \
777 __ lay(addr, op); \
778 __ CmpAndSwap(output, new_val, MemOperand(addr)); \
779 __ LoadU32(output, output); \
780 } while (false)
781
782 #define ASSEMBLE_ATOMIC_BINOP_WORD(load_and_op) \
783 do { \
784 Register value = i.InputRegister(2); \
785 Register result = i.OutputRegister(0); \
786 Register addr = r1; \
787 AddressingMode mode = kMode_None; \
788 MemOperand op = i.MemoryOperand(&mode); \
789 __ lay(addr, op); \
790 __ load_and_op(result, value, MemOperand(addr)); \
791 __ LoadU32(result, result); \
792 } while (false)
793
794 #define ASSEMBLE_ATOMIC_BINOP_WORD64(load_and_op) \
795 do { \
796 Register value = i.InputRegister(2); \
797 Register result = i.OutputRegister(0); \
798 Register addr = r1; \
799 AddressingMode mode = kMode_None; \
800 MemOperand op = i.MemoryOperand(&mode); \
801 __ lay(addr, op); \
802 __ load_and_op(result, value, MemOperand(addr)); \
803 } while (false)
804
805 #define ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end) \
806 do { \
807 Label do_cs; \
808 __ LoadU32(prev, MemOperand(addr, offset)); \
809 __ bind(&do_cs); \
810 __ RotateInsertSelectBits(temp, value, Operand(start), Operand(end), \
811 Operand(static_cast<intptr_t>(shift_amount)), \
812 true); \
813 __ bin_inst(new_val, prev, temp); \
814 __ lr(temp, prev); \
815 __ RotateInsertSelectBits(temp, new_val, Operand(start), Operand(end), \
816 Operand::Zero(), false); \
817 __ CmpAndSwap(prev, temp, MemOperand(addr, offset)); \
818 __ bne(&do_cs, Label::kNear); \
819 } while (false)
820
821 #ifdef V8_TARGET_BIG_ENDIAN
822 #define ATOMIC_BIN_OP_HALFWORD(bin_inst, index, extract_result) \
823 { \
824 constexpr int offset = -(2 * index); \
825 constexpr int shift_amount = 16 - (index * 16); \
826 constexpr int start = 48 - shift_amount; \
827 constexpr int end = start + 15; \
828 ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \
829 extract_result(); \
830 }
831 #define ATOMIC_BIN_OP_BYTE(bin_inst, index, extract_result) \
832 { \
833 constexpr int offset = -(index); \
834 constexpr int shift_amount = 24 - (index * 8); \
835 constexpr int start = 56 - shift_amount; \
836 constexpr int end = start + 7; \
837 ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \
838 extract_result(); \
839 }
840 #else
841 #define ATOMIC_BIN_OP_HALFWORD(bin_inst, index, extract_result) \
842 { \
843 constexpr int offset = -(2 * index); \
844 constexpr int shift_amount = index * 16; \
845 constexpr int start = 48 - shift_amount; \
846 constexpr int end = start + 15; \
847 ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \
848 extract_result(); \
849 }
850 #define ATOMIC_BIN_OP_BYTE(bin_inst, index, extract_result) \
851 { \
852 constexpr int offset = -(index); \
853 constexpr int shift_amount = index * 8; \
854 constexpr int start = 56 - shift_amount; \
855 constexpr int end = start + 7; \
856 ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \
857 extract_result(); \
858 }
859 #endif // V8_TARGET_BIG_ENDIAN
860
861 #define ASSEMBLE_ATOMIC_BINOP_HALFWORD(bin_inst, extract_result) \
862 do { \
863 Register value = i.InputRegister(2); \
864 Register result = i.OutputRegister(0); \
865 Register prev = i.TempRegister(0); \
866 Register new_val = r0; \
867 Register addr = r1; \
868 Register temp = kScratchReg; \
869 AddressingMode mode = kMode_None; \
870 MemOperand op = i.MemoryOperand(&mode); \
871 Label two, done; \
872 __ lay(addr, op); \
873 __ tmll(addr, Operand(3)); \
874 __ b(Condition(2), &two); \
875 /* word boundary */ \
876 ATOMIC_BIN_OP_HALFWORD(bin_inst, 0, extract_result); \
877 __ b(&done); \
878 __ bind(&two); \
879 /* halfword boundary */ \
880 ATOMIC_BIN_OP_HALFWORD(bin_inst, 1, extract_result); \
881 __ bind(&done); \
882 } while (false)
883
884 #define ASSEMBLE_ATOMIC_BINOP_BYTE(bin_inst, extract_result) \
885 do { \
886 Register value = i.InputRegister(2); \
887 Register result = i.OutputRegister(0); \
888 Register addr = i.TempRegister(0); \
889 Register prev = r0; \
890 Register new_val = r1; \
891 Register temp = kScratchReg; \
892 AddressingMode mode = kMode_None; \
893 MemOperand op = i.MemoryOperand(&mode); \
894 Label done, one, two, three; \
895 __ lay(addr, op); \
896 __ tmll(addr, Operand(3)); \
897 __ b(Condition(1), &three); \
898 __ b(Condition(2), &two); \
899 __ b(Condition(4), &one); \
900 /* ending with 0b00 (word boundary) */ \
901 ATOMIC_BIN_OP_BYTE(bin_inst, 0, extract_result); \
902 __ b(&done); \
903 /* ending with 0b01 */ \
904 __ bind(&one); \
905 ATOMIC_BIN_OP_BYTE(bin_inst, 1, extract_result); \
906 __ b(&done); \
907 /* ending with 0b10 (hw boundary) */ \
908 __ bind(&two); \
909 ATOMIC_BIN_OP_BYTE(bin_inst, 2, extract_result); \
910 __ b(&done); \
911 /* ending with 0b11 */ \
912 __ bind(&three); \
913 ATOMIC_BIN_OP_BYTE(bin_inst, 3, extract_result); \
914 __ bind(&done); \
915 } while (false)
916
917 #define ASSEMBLE_ATOMIC64_COMP_EXCHANGE_WORD64() \
918 do { \
919 Register new_val = i.InputRegister(1); \
920 Register output = i.OutputRegister(); \
921 Register addr = kScratchReg; \
922 size_t index = 2; \
923 AddressingMode mode = kMode_None; \
924 MemOperand op = i.MemoryOperand(&mode, &index); \
925 __ lay(addr, op); \
926 __ CmpAndSwap64(output, new_val, MemOperand(addr)); \
927 } while (false)
928
AssembleDeconstructFrame()929 void CodeGenerator::AssembleDeconstructFrame() {
930 __ LeaveFrame(StackFrame::MANUAL);
931 unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
932 }
933
AssemblePrepareTailCall()934 void CodeGenerator::AssemblePrepareTailCall() {
935 if (frame_access_state()->has_frame()) {
936 __ RestoreFrameStateForTailCall();
937 }
938 frame_access_state()->SetFrameAccessToSP();
939 }
940
941 namespace {
942
FlushPendingPushRegisters(TurboAssembler * tasm,FrameAccessState * frame_access_state,ZoneVector<Register> * pending_pushes)943 void FlushPendingPushRegisters(TurboAssembler* tasm,
944 FrameAccessState* frame_access_state,
945 ZoneVector<Register>* pending_pushes) {
946 switch (pending_pushes->size()) {
947 case 0:
948 break;
949 case 1:
950 tasm->Push((*pending_pushes)[0]);
951 break;
952 case 2:
953 tasm->Push((*pending_pushes)[0], (*pending_pushes)[1]);
954 break;
955 case 3:
956 tasm->Push((*pending_pushes)[0], (*pending_pushes)[1],
957 (*pending_pushes)[2]);
958 break;
959 default:
960 UNREACHABLE();
961 }
962 frame_access_state->IncreaseSPDelta(pending_pushes->size());
963 pending_pushes->clear();
964 }
965
AdjustStackPointerForTailCall(TurboAssembler * tasm,FrameAccessState * state,int new_slot_above_sp,ZoneVector<Register> * pending_pushes=nullptr,bool allow_shrinkage=true)966 void AdjustStackPointerForTailCall(
967 TurboAssembler* tasm, FrameAccessState* state, int new_slot_above_sp,
968 ZoneVector<Register>* pending_pushes = nullptr,
969 bool allow_shrinkage = true) {
970 int current_sp_offset = state->GetSPToFPSlotCount() +
971 StandardFrameConstants::kFixedSlotCountAboveFp;
972 int stack_slot_delta = new_slot_above_sp - current_sp_offset;
973 if (stack_slot_delta > 0) {
974 if (pending_pushes != nullptr) {
975 FlushPendingPushRegisters(tasm, state, pending_pushes);
976 }
977 tasm->AddS64(sp, sp, Operand(-stack_slot_delta * kSystemPointerSize));
978 state->IncreaseSPDelta(stack_slot_delta);
979 } else if (allow_shrinkage && stack_slot_delta < 0) {
980 if (pending_pushes != nullptr) {
981 FlushPendingPushRegisters(tasm, state, pending_pushes);
982 }
983 tasm->AddS64(sp, sp, Operand(-stack_slot_delta * kSystemPointerSize));
984 state->IncreaseSPDelta(stack_slot_delta);
985 }
986 }
987
988 } // namespace
989
AssembleTailCallBeforeGap(Instruction * instr,int first_unused_slot_offset)990 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
991 int first_unused_slot_offset) {
992 ZoneVector<MoveOperands*> pushes(zone());
993 GetPushCompatibleMoves(instr, kRegisterPush, &pushes);
994
995 if (!pushes.empty() &&
996 (LocationOperand::cast(pushes.back()->destination()).index() + 1 ==
997 first_unused_slot_offset)) {
998 S390OperandConverter g(this, instr);
999 ZoneVector<Register> pending_pushes(zone());
1000 for (auto move : pushes) {
1001 LocationOperand destination_location(
1002 LocationOperand::cast(move->destination()));
1003 InstructionOperand source(move->source());
1004 AdjustStackPointerForTailCall(
1005 tasm(), frame_access_state(),
1006 destination_location.index() - pending_pushes.size(),
1007 &pending_pushes);
1008 // Pushes of non-register data types are not supported.
1009 DCHECK(source.IsRegister());
1010 LocationOperand source_location(LocationOperand::cast(source));
1011 pending_pushes.push_back(source_location.GetRegister());
1012 // TODO(arm): We can push more than 3 registers at once. Add support in
1013 // the macro-assembler for pushing a list of registers.
1014 if (pending_pushes.size() == 3) {
1015 FlushPendingPushRegisters(tasm(), frame_access_state(),
1016 &pending_pushes);
1017 }
1018 move->Eliminate();
1019 }
1020 FlushPendingPushRegisters(tasm(), frame_access_state(), &pending_pushes);
1021 }
1022 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
1023 first_unused_slot_offset, nullptr, false);
1024 }
1025
AssembleTailCallAfterGap(Instruction * instr,int first_unused_slot_offset)1026 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
1027 int first_unused_slot_offset) {
1028 AdjustStackPointerForTailCall(tasm(), frame_access_state(),
1029 first_unused_slot_offset);
1030 }
1031
1032 // Check that {kJavaScriptCallCodeStartRegister} is correct.
AssembleCodeStartRegisterCheck()1033 void CodeGenerator::AssembleCodeStartRegisterCheck() {
1034 Register scratch = r1;
1035 __ ComputeCodeStartAddress(scratch);
1036 __ CmpS64(scratch, kJavaScriptCallCodeStartRegister);
1037 __ Assert(eq, AbortReason::kWrongFunctionCodeStart);
1038 }
1039
1040 // Check if the code object is marked for deoptimization. If it is, then it
1041 // jumps to the CompileLazyDeoptimizedCode builtin. In order to do this we need
1042 // to:
1043 // 1. read from memory the word that contains that bit, which can be found in
1044 // the flags in the referenced {CodeDataContainer} object;
1045 // 2. test kMarkedForDeoptimizationBit in those flags; and
1046 // 3. if it is not zero then it jumps to the builtin.
BailoutIfDeoptimized()1047 void CodeGenerator::BailoutIfDeoptimized() {
1048 if (FLAG_debug_code) {
1049 // Check that {kJavaScriptCallCodeStartRegister} is correct.
1050 __ ComputeCodeStartAddress(ip);
1051 __ CmpS64(ip, kJavaScriptCallCodeStartRegister);
1052 __ Assert(eq, AbortReason::kWrongFunctionCodeStart);
1053 }
1054
1055 int offset = Code::kCodeDataContainerOffset - Code::kHeaderSize;
1056 __ LoadTaggedPointerField(
1057 ip, MemOperand(kJavaScriptCallCodeStartRegister, offset), r0);
1058 __ LoadS32(ip,
1059 FieldMemOperand(ip, CodeDataContainer::kKindSpecificFlagsOffset));
1060 __ TestBit(ip, Code::kMarkedForDeoptimizationBit);
1061 __ Jump(BUILTIN_CODE(isolate(), CompileLazyDeoptimizedCode),
1062 RelocInfo::CODE_TARGET, ne);
1063 }
1064
1065 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction(Instruction * instr)1066 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
1067 Instruction* instr) {
1068 S390OperandConverter i(this, instr);
1069 ArchOpcode opcode = ArchOpcodeField::decode(instr->opcode());
1070
1071 switch (opcode) {
1072 case kArchComment:
1073 #ifdef V8_TARGET_ARCH_S390X
1074 __ RecordComment(reinterpret_cast<const char*>(i.InputInt64(0)));
1075 #else
1076 __ RecordComment(reinterpret_cast<const char*>(i.InputInt32(0)));
1077 #endif
1078 break;
1079 case kArchCallCodeObject: {
1080 if (HasRegisterInput(instr, 0)) {
1081 Register reg = i.InputRegister(0);
1082 DCHECK_IMPLIES(
1083 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
1084 reg == kJavaScriptCallCodeStartRegister);
1085 __ CallCodeObject(reg);
1086 } else {
1087 __ Call(i.InputCode(0), RelocInfo::CODE_TARGET);
1088 }
1089 RecordCallPosition(instr);
1090 frame_access_state()->ClearSPDelta();
1091 break;
1092 }
1093 case kArchCallBuiltinPointer: {
1094 DCHECK(!instr->InputAt(0)->IsImmediate());
1095 Register builtin_index = i.InputRegister(0);
1096 __ CallBuiltinByIndex(builtin_index);
1097 RecordCallPosition(instr);
1098 frame_access_state()->ClearSPDelta();
1099 break;
1100 }
1101 #if V8_ENABLE_WEBASSEMBLY
1102 case kArchCallWasmFunction: {
1103 // We must not share code targets for calls to builtins for wasm code, as
1104 // they might need to be patched individually.
1105 if (instr->InputAt(0)->IsImmediate()) {
1106 Constant constant = i.ToConstant(instr->InputAt(0));
1107 Address wasm_code = static_cast<Address>(constant.ToInt64());
1108 __ Call(wasm_code, constant.rmode());
1109 } else {
1110 __ Call(i.InputRegister(0));
1111 }
1112 RecordCallPosition(instr);
1113 frame_access_state()->ClearSPDelta();
1114 break;
1115 }
1116 case kArchTailCallWasm: {
1117 // We must not share code targets for calls to builtins for wasm code, as
1118 // they might need to be patched individually.
1119 if (instr->InputAt(0)->IsImmediate()) {
1120 Constant constant = i.ToConstant(instr->InputAt(0));
1121 Address wasm_code = static_cast<Address>(constant.ToInt64());
1122 __ Jump(wasm_code, constant.rmode());
1123 } else {
1124 __ Jump(i.InputRegister(0));
1125 }
1126 frame_access_state()->ClearSPDelta();
1127 frame_access_state()->SetFrameAccessToDefault();
1128 break;
1129 }
1130 #endif // V8_ENABLE_WEBASSEMBLY
1131 case kArchTailCallCodeObject: {
1132 if (HasRegisterInput(instr, 0)) {
1133 Register reg = i.InputRegister(0);
1134 DCHECK_IMPLIES(
1135 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
1136 reg == kJavaScriptCallCodeStartRegister);
1137 __ JumpCodeObject(reg);
1138 } else {
1139 // We cannot use the constant pool to load the target since
1140 // we've already restored the caller's frame.
1141 ConstantPoolUnavailableScope constant_pool_unavailable(tasm());
1142 __ Jump(i.InputCode(0), RelocInfo::CODE_TARGET);
1143 }
1144 frame_access_state()->ClearSPDelta();
1145 frame_access_state()->SetFrameAccessToDefault();
1146 break;
1147 }
1148 case kArchTailCallAddress: {
1149 CHECK(!instr->InputAt(0)->IsImmediate());
1150 Register reg = i.InputRegister(0);
1151 DCHECK_IMPLIES(
1152 instr->HasCallDescriptorFlag(CallDescriptor::kFixedTargetRegister),
1153 reg == kJavaScriptCallCodeStartRegister);
1154 __ Jump(reg);
1155 frame_access_state()->ClearSPDelta();
1156 frame_access_state()->SetFrameAccessToDefault();
1157 break;
1158 }
1159 case kArchCallJSFunction: {
1160 Register func = i.InputRegister(0);
1161 if (FLAG_debug_code) {
1162 // Check the function's context matches the context argument.
1163 __ LoadTaggedPointerField(
1164 kScratchReg, FieldMemOperand(func, JSFunction::kContextOffset));
1165 __ CmpS64(cp, kScratchReg);
1166 __ Assert(eq, AbortReason::kWrongFunctionContext);
1167 }
1168 static_assert(kJavaScriptCallCodeStartRegister == r4, "ABI mismatch");
1169 __ LoadTaggedPointerField(r4,
1170 FieldMemOperand(func, JSFunction::kCodeOffset));
1171 __ CallCodeObject(r4);
1172 RecordCallPosition(instr);
1173 frame_access_state()->ClearSPDelta();
1174 break;
1175 }
1176 case kArchPrepareCallCFunction: {
1177 int const num_gp_parameters = ParamField::decode(instr->opcode());
1178 int const num_fp_parameters = FPParamField::decode(instr->opcode());
1179 __ PrepareCallCFunction(num_gp_parameters + num_fp_parameters,
1180 kScratchReg);
1181 // Frame alignment requires using FP-relative frame addressing.
1182 frame_access_state()->SetFrameAccessToFP();
1183 break;
1184 }
1185 case kArchSaveCallerRegisters: {
1186 fp_mode_ =
1187 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()));
1188 DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
1189 fp_mode_ == SaveFPRegsMode::kSave);
1190 // kReturnRegister0 should have been saved before entering the stub.
1191 int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0);
1192 DCHECK(IsAligned(bytes, kSystemPointerSize));
1193 DCHECK_EQ(0, frame_access_state()->sp_delta());
1194 frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
1195 DCHECK(!caller_registers_saved_);
1196 caller_registers_saved_ = true;
1197 break;
1198 }
1199 case kArchRestoreCallerRegisters: {
1200 DCHECK(fp_mode_ ==
1201 static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())));
1202 DCHECK(fp_mode_ == SaveFPRegsMode::kIgnore ||
1203 fp_mode_ == SaveFPRegsMode::kSave);
1204 // Don't overwrite the returned value.
1205 int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0);
1206 frame_access_state()->IncreaseSPDelta(-(bytes / kSystemPointerSize));
1207 DCHECK_EQ(0, frame_access_state()->sp_delta());
1208 DCHECK(caller_registers_saved_);
1209 caller_registers_saved_ = false;
1210 break;
1211 }
1212 case kArchPrepareTailCall:
1213 AssemblePrepareTailCall();
1214 break;
1215 case kArchCallCFunction: {
1216 int const num_gp_parameters = ParamField::decode(instr->opcode());
1217 int const num_fp_parameters = FPParamField::decode(instr->opcode());
1218 Label return_location;
1219 // Put the return address in a stack slot.
1220 #if V8_ENABLE_WEBASSEMBLY
1221 if (linkage()->GetIncomingDescriptor()->IsWasmCapiFunction()) {
1222 // Put the return address in a stack slot.
1223 __ larl(r0, &return_location);
1224 __ StoreU64(r0,
1225 MemOperand(fp, WasmExitFrameConstants::kCallingPCOffset));
1226 }
1227 #endif // V8_ENABLE_WEBASSEMBLY
1228 if (instr->InputAt(0)->IsImmediate()) {
1229 ExternalReference ref = i.InputExternalReference(0);
1230 __ CallCFunction(ref, num_gp_parameters, num_fp_parameters);
1231 } else {
1232 Register func = i.InputRegister(0);
1233 __ CallCFunction(func, num_gp_parameters, num_fp_parameters);
1234 }
1235 __ bind(&return_location);
1236 #if V8_ENABLE_WEBASSEMBLY
1237 if (linkage()->GetIncomingDescriptor()->IsWasmCapiFunction()) {
1238 RecordSafepoint(instr->reference_map());
1239 }
1240 #endif // V8_ENABLE_WEBASSEMBLY
1241 frame_access_state()->SetFrameAccessToDefault();
1242 // Ideally, we should decrement SP delta to match the change of stack
1243 // pointer in CallCFunction. However, for certain architectures (e.g.
1244 // ARM), there may be more strict alignment requirement, causing old SP
1245 // to be saved on the stack. In those cases, we can not calculate the SP
1246 // delta statically.
1247 frame_access_state()->ClearSPDelta();
1248 if (caller_registers_saved_) {
1249 // Need to re-sync SP delta introduced in kArchSaveCallerRegisters.
1250 // Here, we assume the sequence to be:
1251 // kArchSaveCallerRegisters;
1252 // kArchCallCFunction;
1253 // kArchRestoreCallerRegisters;
1254 int bytes =
1255 __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0);
1256 frame_access_state()->IncreaseSPDelta(bytes / kSystemPointerSize);
1257 }
1258 break;
1259 }
1260 case kArchJmp:
1261 AssembleArchJump(i.InputRpo(0));
1262 break;
1263 case kArchBinarySearchSwitch:
1264 AssembleArchBinarySearchSwitch(instr);
1265 break;
1266 case kArchTableSwitch:
1267 AssembleArchTableSwitch(instr);
1268 break;
1269 case kArchAbortCSADcheck:
1270 DCHECK(i.InputRegister(0) == r3);
1271 {
1272 // We don't actually want to generate a pile of code for this, so just
1273 // claim there is a stack frame, without generating one.
1274 FrameScope scope(tasm(), StackFrame::NO_FRAME_TYPE);
1275 __ Call(isolate()->builtins()->code_handle(Builtin::kAbortCSADcheck),
1276 RelocInfo::CODE_TARGET);
1277 }
1278 __ stop();
1279 break;
1280 case kArchDebugBreak:
1281 __ DebugBreak();
1282 break;
1283 case kArchNop:
1284 case kArchThrowTerminator:
1285 // don't emit code for nops.
1286 break;
1287 case kArchDeoptimize: {
1288 DeoptimizationExit* exit =
1289 BuildTranslation(instr, -1, 0, 0, OutputFrameStateCombine::Ignore());
1290 __ b(exit->label());
1291 break;
1292 }
1293 case kArchRet:
1294 AssembleReturn(instr->InputAt(0));
1295 break;
1296 case kArchFramePointer:
1297 __ mov(i.OutputRegister(), fp);
1298 break;
1299 case kArchParentFramePointer:
1300 if (frame_access_state()->has_frame()) {
1301 __ LoadU64(i.OutputRegister(), MemOperand(fp, 0));
1302 } else {
1303 __ mov(i.OutputRegister(), fp);
1304 }
1305 break;
1306 case kArchStackPointerGreaterThan: {
1307 // Potentially apply an offset to the current stack pointer before the
1308 // comparison to consider the size difference of an optimized frame versus
1309 // the contained unoptimized frames.
1310
1311 Register lhs_register = sp;
1312 uint32_t offset;
1313
1314 if (ShouldApplyOffsetToStackCheck(instr, &offset)) {
1315 lhs_register = i.TempRegister(0);
1316 __ SubS64(lhs_register, sp, Operand(offset));
1317 }
1318
1319 constexpr size_t kValueIndex = 0;
1320 DCHECK(instr->InputAt(kValueIndex)->IsRegister());
1321 __ CmpU64(lhs_register, i.InputRegister(kValueIndex));
1322 break;
1323 }
1324 case kArchStackCheckOffset:
1325 __ LoadSmiLiteral(i.OutputRegister(),
1326 Smi::FromInt(GetStackCheckOffset()));
1327 break;
1328 case kArchTruncateDoubleToI:
1329 __ TruncateDoubleToI(isolate(), zone(), i.OutputRegister(),
1330 i.InputDoubleRegister(0), DetermineStubCallMode());
1331 break;
1332 case kArchStoreWithWriteBarrier: {
1333 RecordWriteMode mode =
1334 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
1335 Register object = i.InputRegister(0);
1336 Register value = i.InputRegister(2);
1337 Register scratch0 = i.TempRegister(0);
1338 Register scratch1 = i.TempRegister(1);
1339 OutOfLineRecordWrite* ool;
1340
1341 AddressingMode addressing_mode =
1342 AddressingModeField::decode(instr->opcode());
1343 if (addressing_mode == kMode_MRI) {
1344 int32_t offset = i.InputInt32(1);
1345 ool = zone()->New<OutOfLineRecordWrite>(
1346 this, object, offset, value, scratch0, scratch1, mode,
1347 DetermineStubCallMode(), &unwinding_info_writer_);
1348 __ StoreTaggedField(value, MemOperand(object, offset), r0);
1349 } else {
1350 DCHECK_EQ(kMode_MRR, addressing_mode);
1351 Register offset(i.InputRegister(1));
1352 ool = zone()->New<OutOfLineRecordWrite>(
1353 this, object, offset, value, scratch0, scratch1, mode,
1354 DetermineStubCallMode(), &unwinding_info_writer_);
1355 __ StoreTaggedField(value, MemOperand(object, offset));
1356 }
1357 if (mode > RecordWriteMode::kValueIsPointer) {
1358 __ JumpIfSmi(value, ool->exit());
1359 }
1360 __ CheckPageFlag(object, scratch0,
1361 MemoryChunk::kPointersFromHereAreInterestingMask, ne,
1362 ool->entry());
1363 __ bind(ool->exit());
1364 break;
1365 }
1366 case kArchStackSlot: {
1367 FrameOffset offset =
1368 frame_access_state()->GetFrameOffset(i.InputInt32(0));
1369 __ AddS64(i.OutputRegister(), offset.from_stack_pointer() ? sp : fp,
1370 Operand(offset.offset()));
1371 break;
1372 }
1373 case kS390_Peek: {
1374 int reverse_slot = i.InputInt32(0);
1375 int offset =
1376 FrameSlotToFPOffset(frame()->GetTotalFrameSlotCount() - reverse_slot);
1377 if (instr->OutputAt(0)->IsFPRegister()) {
1378 LocationOperand* op = LocationOperand::cast(instr->OutputAt(0));
1379 if (op->representation() == MachineRepresentation::kFloat64) {
1380 __ LoadF64(i.OutputDoubleRegister(), MemOperand(fp, offset));
1381 } else if (op->representation() == MachineRepresentation::kFloat32) {
1382 __ LoadF32(i.OutputFloatRegister(), MemOperand(fp, offset));
1383 } else {
1384 DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
1385 __ LoadV128(i.OutputSimd128Register(), MemOperand(fp, offset),
1386 kScratchReg);
1387 }
1388 } else {
1389 __ LoadU64(i.OutputRegister(), MemOperand(fp, offset));
1390 }
1391 break;
1392 }
1393 case kS390_Abs32:
1394 // TODO(john.yan): zero-ext
1395 __ lpr(i.OutputRegister(0), i.InputRegister(0));
1396 break;
1397 case kS390_Abs64:
1398 __ lpgr(i.OutputRegister(0), i.InputRegister(0));
1399 break;
1400 case kS390_And32:
1401 // zero-ext
1402 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1403 ASSEMBLE_BIN32_OP(RRRInstr(nrk), RM32Instr(And), RIInstr(nilf));
1404 } else {
1405 ASSEMBLE_BIN32_OP(RRInstr(nr), RM32Instr(And), RIInstr(nilf));
1406 }
1407 break;
1408 case kS390_And64:
1409 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1410 ASSEMBLE_BIN_OP(RRRInstr(ngrk), RM64Instr(ng), nullInstr);
1411 } else {
1412 ASSEMBLE_BIN_OP(RRInstr(ngr), RM64Instr(ng), nullInstr);
1413 }
1414 break;
1415 case kS390_Or32:
1416 // zero-ext
1417 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1418 ASSEMBLE_BIN32_OP(RRRInstr(ork), RM32Instr(Or), RIInstr(oilf));
1419 } else {
1420 ASSEMBLE_BIN32_OP(RRInstr(or_z), RM32Instr(Or), RIInstr(oilf));
1421 }
1422 break;
1423 case kS390_Or64:
1424 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1425 ASSEMBLE_BIN_OP(RRRInstr(ogrk), RM64Instr(og), nullInstr);
1426 } else {
1427 ASSEMBLE_BIN_OP(RRInstr(ogr), RM64Instr(og), nullInstr);
1428 }
1429 break;
1430 case kS390_Xor32:
1431 // zero-ext
1432 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1433 ASSEMBLE_BIN32_OP(RRRInstr(xrk), RM32Instr(Xor), RIInstr(xilf));
1434 } else {
1435 ASSEMBLE_BIN32_OP(RRInstr(xr), RM32Instr(Xor), RIInstr(xilf));
1436 }
1437 break;
1438 case kS390_Xor64:
1439 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1440 ASSEMBLE_BIN_OP(RRRInstr(xgrk), RM64Instr(xg), nullInstr);
1441 } else {
1442 ASSEMBLE_BIN_OP(RRInstr(xgr), RM64Instr(xg), nullInstr);
1443 }
1444 break;
1445 case kS390_ShiftLeft32:
1446 // zero-ext
1447 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1448 ASSEMBLE_BIN32_OP(RRRInstr(ShiftLeftU32), nullInstr,
1449 RRIInstr(ShiftLeftU32));
1450 } else {
1451 ASSEMBLE_BIN32_OP(RRInstr(sll), nullInstr, RIInstr(sll));
1452 }
1453 break;
1454 case kS390_ShiftLeft64:
1455 ASSEMBLE_BIN_OP(RRRInstr(sllg), nullInstr, RRIInstr(sllg));
1456 break;
1457 case kS390_ShiftRight32:
1458 // zero-ext
1459 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1460 ASSEMBLE_BIN32_OP(RRRInstr(srlk), nullInstr, RRIInstr(srlk));
1461 } else {
1462 ASSEMBLE_BIN32_OP(RRInstr(srl), nullInstr, RIInstr(srl));
1463 }
1464 break;
1465 case kS390_ShiftRight64:
1466 ASSEMBLE_BIN_OP(RRRInstr(srlg), nullInstr, RRIInstr(srlg));
1467 break;
1468 case kS390_ShiftRightArith32:
1469 // zero-ext
1470 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1471 ASSEMBLE_BIN32_OP(RRRInstr(srak), nullInstr, RRIInstr(srak));
1472 } else {
1473 ASSEMBLE_BIN32_OP(RRInstr(sra), nullInstr, RIInstr(sra));
1474 }
1475 break;
1476 case kS390_ShiftRightArith64:
1477 ASSEMBLE_BIN_OP(RRRInstr(srag), nullInstr, RRIInstr(srag));
1478 break;
1479 case kS390_RotRight32: {
1480 // zero-ext
1481 if (HasRegisterInput(instr, 1)) {
1482 __ lcgr(kScratchReg, i.InputRegister(1));
1483 __ rll(i.OutputRegister(), i.InputRegister(0), kScratchReg);
1484 } else {
1485 __ rll(i.OutputRegister(), i.InputRegister(0),
1486 Operand(32 - i.InputInt32(1)));
1487 }
1488 CHECK_AND_ZERO_EXT_OUTPUT(2);
1489 break;
1490 }
1491 case kS390_RotRight64:
1492 if (HasRegisterInput(instr, 1)) {
1493 __ lcgr(kScratchReg, i.InputRegister(1));
1494 __ rllg(i.OutputRegister(), i.InputRegister(0), kScratchReg);
1495 } else {
1496 DCHECK(HasImmediateInput(instr, 1));
1497 __ rllg(i.OutputRegister(), i.InputRegister(0),
1498 Operand(64 - i.InputInt32(1)));
1499 }
1500 break;
1501 // TODO(john.yan): clean up kS390_RotLeftAnd...
1502 case kS390_RotLeftAndClear64:
1503 if (CpuFeatures::IsSupported(GENERAL_INSTR_EXT)) {
1504 int shiftAmount = i.InputInt32(1);
1505 int endBit = 63 - shiftAmount;
1506 int startBit = 63 - i.InputInt32(2);
1507 __ RotateInsertSelectBits(i.OutputRegister(), i.InputRegister(0),
1508 Operand(startBit), Operand(endBit),
1509 Operand(shiftAmount), true);
1510 } else {
1511 int shiftAmount = i.InputInt32(1);
1512 int clearBit = 63 - i.InputInt32(2);
1513 __ rllg(i.OutputRegister(), i.InputRegister(0), Operand(shiftAmount));
1514 __ sllg(i.OutputRegister(), i.OutputRegister(), Operand(clearBit));
1515 __ srlg(i.OutputRegister(), i.OutputRegister(),
1516 Operand(clearBit + shiftAmount));
1517 __ sllg(i.OutputRegister(), i.OutputRegister(), Operand(shiftAmount));
1518 }
1519 break;
1520 case kS390_RotLeftAndClearLeft64:
1521 if (CpuFeatures::IsSupported(GENERAL_INSTR_EXT)) {
1522 int shiftAmount = i.InputInt32(1);
1523 int endBit = 63;
1524 int startBit = 63 - i.InputInt32(2);
1525 __ RotateInsertSelectBits(i.OutputRegister(), i.InputRegister(0),
1526 Operand(startBit), Operand(endBit),
1527 Operand(shiftAmount), true);
1528 } else {
1529 int shiftAmount = i.InputInt32(1);
1530 int clearBit = 63 - i.InputInt32(2);
1531 __ rllg(i.OutputRegister(), i.InputRegister(0), Operand(shiftAmount));
1532 __ sllg(i.OutputRegister(), i.OutputRegister(), Operand(clearBit));
1533 __ srlg(i.OutputRegister(), i.OutputRegister(), Operand(clearBit));
1534 }
1535 break;
1536 case kS390_RotLeftAndClearRight64:
1537 if (CpuFeatures::IsSupported(GENERAL_INSTR_EXT)) {
1538 int shiftAmount = i.InputInt32(1);
1539 int endBit = 63 - i.InputInt32(2);
1540 int startBit = 0;
1541 __ RotateInsertSelectBits(i.OutputRegister(), i.InputRegister(0),
1542 Operand(startBit), Operand(endBit),
1543 Operand(shiftAmount), true);
1544 } else {
1545 int shiftAmount = i.InputInt32(1);
1546 int clearBit = i.InputInt32(2);
1547 __ rllg(i.OutputRegister(), i.InputRegister(0), Operand(shiftAmount));
1548 __ srlg(i.OutputRegister(), i.OutputRegister(), Operand(clearBit));
1549 __ sllg(i.OutputRegister(), i.OutputRegister(), Operand(clearBit));
1550 }
1551 break;
1552 case kS390_Add32: {
1553 // zero-ext
1554 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1555 ASSEMBLE_BIN32_OP(RRRInstr(ark), RM32Instr(AddS32), RRIInstr(AddS32));
1556 } else {
1557 ASSEMBLE_BIN32_OP(RRInstr(ar), RM32Instr(AddS32), RIInstr(AddS32));
1558 }
1559 break;
1560 }
1561 case kS390_Add64:
1562 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1563 ASSEMBLE_BIN_OP(RRRInstr(agrk), RM64Instr(ag), RRIInstr(AddS64));
1564 } else {
1565 ASSEMBLE_BIN_OP(RRInstr(agr), RM64Instr(ag), RIInstr(agfi));
1566 }
1567 break;
1568 case kS390_AddFloat:
1569 ASSEMBLE_BIN_OP(DDInstr(aebr), DMTInstr(AddFloat32), nullInstr);
1570 break;
1571 case kS390_AddDouble:
1572 ASSEMBLE_BIN_OP(DDInstr(adbr), DMTInstr(AddFloat64), nullInstr);
1573 break;
1574 case kS390_Sub32:
1575 // zero-ext
1576 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1577 ASSEMBLE_BIN32_OP(RRRInstr(srk), RM32Instr(SubS32), RRIInstr(SubS32));
1578 } else {
1579 ASSEMBLE_BIN32_OP(RRInstr(sr), RM32Instr(SubS32), RIInstr(SubS32));
1580 }
1581 break;
1582 case kS390_Sub64:
1583 if (CpuFeatures::IsSupported(DISTINCT_OPS)) {
1584 ASSEMBLE_BIN_OP(RRRInstr(sgrk), RM64Instr(sg), RRIInstr(SubS64));
1585 } else {
1586 ASSEMBLE_BIN_OP(RRInstr(sgr), RM64Instr(sg), RIInstr(SubS64));
1587 }
1588 break;
1589 case kS390_SubFloat:
1590 ASSEMBLE_BIN_OP(DDInstr(sebr), DMTInstr(SubFloat32), nullInstr);
1591 break;
1592 case kS390_SubDouble:
1593 ASSEMBLE_BIN_OP(DDInstr(sdbr), DMTInstr(SubFloat64), nullInstr);
1594 break;
1595 case kS390_Mul32:
1596 // zero-ext
1597 if (CpuFeatures::IsSupported(MISC_INSTR_EXT2)) {
1598 ASSEMBLE_BIN32_OP(RRRInstr(msrkc), RM32Instr(msc), RIInstr(MulS32));
1599 } else {
1600 ASSEMBLE_BIN32_OP(RRInstr(MulS32), RM32Instr(MulS32), RIInstr(MulS32));
1601 }
1602 break;
1603 case kS390_Mul32WithOverflow:
1604 // zero-ext
1605 ASSEMBLE_BIN32_OP(RRRInstr(Mul32WithOverflowIfCCUnequal),
1606 RRM32Instr(Mul32WithOverflowIfCCUnequal),
1607 RRIInstr(Mul32WithOverflowIfCCUnequal));
1608 break;
1609 case kS390_Mul64:
1610 ASSEMBLE_BIN_OP(RRInstr(MulS64), RM64Instr(MulS64), RIInstr(MulS64));
1611 break;
1612 case kS390_MulHigh32:
1613 // zero-ext
1614 ASSEMBLE_BIN_OP(RRRInstr(MulHighS32), RRM32Instr(MulHighS32),
1615 RRIInstr(MulHighS32));
1616 break;
1617 case kS390_MulHighU32:
1618 // zero-ext
1619 ASSEMBLE_BIN_OP(RRRInstr(MulHighU32), RRM32Instr(MulHighU32),
1620 RRIInstr(MulHighU32));
1621 break;
1622 case kS390_MulFloat:
1623 ASSEMBLE_BIN_OP(DDInstr(meebr), DMTInstr(MulFloat32), nullInstr);
1624 break;
1625 case kS390_MulDouble:
1626 ASSEMBLE_BIN_OP(DDInstr(mdbr), DMTInstr(MulFloat64), nullInstr);
1627 break;
1628 case kS390_Div64:
1629 ASSEMBLE_BIN_OP(RRRInstr(DivS64), RRM64Instr(DivS64), nullInstr);
1630 break;
1631 case kS390_Div32: {
1632 // zero-ext
1633 ASSEMBLE_BIN_OP(RRRInstr(DivS32), RRM32Instr(DivS32), nullInstr);
1634 break;
1635 }
1636 case kS390_DivU64:
1637 ASSEMBLE_BIN_OP(RRRInstr(DivU64), RRM64Instr(DivU64), nullInstr);
1638 break;
1639 case kS390_DivU32: {
1640 // zero-ext
1641 ASSEMBLE_BIN_OP(RRRInstr(DivU32), RRM32Instr(DivU32), nullInstr);
1642 break;
1643 }
1644 case kS390_DivFloat:
1645 ASSEMBLE_BIN_OP(DDInstr(debr), DMTInstr(DivFloat32), nullInstr);
1646 break;
1647 case kS390_DivDouble:
1648 ASSEMBLE_BIN_OP(DDInstr(ddbr), DMTInstr(DivFloat64), nullInstr);
1649 break;
1650 case kS390_Mod32:
1651 // zero-ext
1652 ASSEMBLE_BIN_OP(RRRInstr(ModS32), RRM32Instr(ModS32), nullInstr);
1653 break;
1654 case kS390_ModU32:
1655 // zero-ext
1656 ASSEMBLE_BIN_OP(RRRInstr(ModU32), RRM32Instr(ModU32), nullInstr);
1657 break;
1658 case kS390_Mod64:
1659 ASSEMBLE_BIN_OP(RRRInstr(ModS64), RRM64Instr(ModS64), nullInstr);
1660 break;
1661 case kS390_ModU64:
1662 ASSEMBLE_BIN_OP(RRRInstr(ModU64), RRM64Instr(ModU64), nullInstr);
1663 break;
1664 case kS390_AbsFloat:
1665 __ lpebr(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1666 break;
1667 case kS390_SqrtFloat:
1668 ASSEMBLE_UNARY_OP(D_DInstr(sqebr), nullInstr, nullInstr);
1669 break;
1670 case kS390_SqrtDouble:
1671 ASSEMBLE_UNARY_OP(D_DInstr(sqdbr), nullInstr, nullInstr);
1672 break;
1673 case kS390_FloorFloat:
1674 __ FloorF32(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1675 break;
1676 case kS390_CeilFloat:
1677 __ CeilF32(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1678 break;
1679 case kS390_TruncateFloat:
1680 __ TruncF32(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1681 break;
1682 // Double operations
1683 case kS390_ModDouble:
1684 ASSEMBLE_FLOAT_MODULO();
1685 break;
1686 case kIeee754Float64Acos:
1687 ASSEMBLE_IEEE754_UNOP(acos);
1688 break;
1689 case kIeee754Float64Acosh:
1690 ASSEMBLE_IEEE754_UNOP(acosh);
1691 break;
1692 case kIeee754Float64Asin:
1693 ASSEMBLE_IEEE754_UNOP(asin);
1694 break;
1695 case kIeee754Float64Asinh:
1696 ASSEMBLE_IEEE754_UNOP(asinh);
1697 break;
1698 case kIeee754Float64Atanh:
1699 ASSEMBLE_IEEE754_UNOP(atanh);
1700 break;
1701 case kIeee754Float64Atan:
1702 ASSEMBLE_IEEE754_UNOP(atan);
1703 break;
1704 case kIeee754Float64Atan2:
1705 ASSEMBLE_IEEE754_BINOP(atan2);
1706 break;
1707 case kIeee754Float64Tan:
1708 ASSEMBLE_IEEE754_UNOP(tan);
1709 break;
1710 case kIeee754Float64Tanh:
1711 ASSEMBLE_IEEE754_UNOP(tanh);
1712 break;
1713 case kIeee754Float64Cbrt:
1714 ASSEMBLE_IEEE754_UNOP(cbrt);
1715 break;
1716 case kIeee754Float64Sin:
1717 ASSEMBLE_IEEE754_UNOP(sin);
1718 break;
1719 case kIeee754Float64Sinh:
1720 ASSEMBLE_IEEE754_UNOP(sinh);
1721 break;
1722 case kIeee754Float64Cos:
1723 ASSEMBLE_IEEE754_UNOP(cos);
1724 break;
1725 case kIeee754Float64Cosh:
1726 ASSEMBLE_IEEE754_UNOP(cosh);
1727 break;
1728 case kIeee754Float64Exp:
1729 ASSEMBLE_IEEE754_UNOP(exp);
1730 break;
1731 case kIeee754Float64Expm1:
1732 ASSEMBLE_IEEE754_UNOP(expm1);
1733 break;
1734 case kIeee754Float64Log:
1735 ASSEMBLE_IEEE754_UNOP(log);
1736 break;
1737 case kIeee754Float64Log1p:
1738 ASSEMBLE_IEEE754_UNOP(log1p);
1739 break;
1740 case kIeee754Float64Log2:
1741 ASSEMBLE_IEEE754_UNOP(log2);
1742 break;
1743 case kIeee754Float64Log10:
1744 ASSEMBLE_IEEE754_UNOP(log10);
1745 break;
1746 case kIeee754Float64Pow:
1747 ASSEMBLE_IEEE754_BINOP(pow);
1748 break;
1749 case kS390_Neg32:
1750 __ lcr(i.OutputRegister(), i.InputRegister(0));
1751 CHECK_AND_ZERO_EXT_OUTPUT(1);
1752 break;
1753 case kS390_Neg64:
1754 __ lcgr(i.OutputRegister(), i.InputRegister(0));
1755 break;
1756 case kS390_MaxFloat:
1757 __ FloatMax(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1758 i.InputDoubleRegister(1));
1759 break;
1760 case kS390_MaxDouble:
1761 __ DoubleMax(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1762 i.InputDoubleRegister(1));
1763 break;
1764 case kS390_MinFloat:
1765 __ FloatMin(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1766 i.InputDoubleRegister(1));
1767 break;
1768 case kS390_FloatNearestInt:
1769 __ NearestIntF32(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1770 break;
1771 case kS390_MinDouble:
1772 __ DoubleMin(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
1773 i.InputDoubleRegister(1));
1774 break;
1775 case kS390_AbsDouble:
1776 __ lpdbr(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1777 break;
1778 case kS390_FloorDouble:
1779 __ FloorF64(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1780 break;
1781 case kS390_CeilDouble:
1782 __ CeilF64(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1783 break;
1784 case kS390_TruncateDouble:
1785 __ TruncF64(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1786 break;
1787 case kS390_RoundDouble:
1788 __ fidbra(ROUND_TO_NEAREST_AWAY_FROM_0, i.OutputDoubleRegister(),
1789 i.InputDoubleRegister(0));
1790 break;
1791 case kS390_DoubleNearestInt:
1792 __ NearestIntF64(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
1793 break;
1794 case kS390_NegFloat:
1795 ASSEMBLE_UNARY_OP(D_DInstr(lcebr), nullInstr, nullInstr);
1796 break;
1797 case kS390_NegDouble:
1798 ASSEMBLE_UNARY_OP(D_DInstr(lcdbr), nullInstr, nullInstr);
1799 break;
1800 case kS390_Cntlz32: {
1801 __ CountLeadingZerosU32(i.OutputRegister(), i.InputRegister(0), r0);
1802 break;
1803 }
1804 #if V8_TARGET_ARCH_S390X
1805 case kS390_Cntlz64: {
1806 __ CountLeadingZerosU64(i.OutputRegister(), i.InputRegister(0), r0);
1807 break;
1808 }
1809 #endif
1810 case kS390_Popcnt32:
1811 __ Popcnt32(i.OutputRegister(), i.InputRegister(0));
1812 break;
1813 #if V8_TARGET_ARCH_S390X
1814 case kS390_Popcnt64:
1815 __ Popcnt64(i.OutputRegister(), i.InputRegister(0));
1816 break;
1817 #endif
1818 case kS390_Cmp32:
1819 ASSEMBLE_COMPARE32(CmpS32, CmpU32);
1820 break;
1821 #if V8_TARGET_ARCH_S390X
1822 case kS390_Cmp64:
1823 ASSEMBLE_COMPARE(CmpS64, CmpU64);
1824 break;
1825 #endif
1826 case kS390_CmpFloat:
1827 ASSEMBLE_FLOAT_COMPARE(cebr, ceb, ley);
1828 // __ cebr(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1829 break;
1830 case kS390_CmpDouble:
1831 ASSEMBLE_FLOAT_COMPARE(cdbr, cdb, ldy);
1832 // __ cdbr(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1833 break;
1834 case kS390_Tst32:
1835 if (HasRegisterInput(instr, 1)) {
1836 __ And(r0, i.InputRegister(0), i.InputRegister(1));
1837 } else {
1838 // detect tmlh/tmhl/tmhh case
1839 Operand opnd = i.InputImmediate(1);
1840 if (is_uint16(opnd.immediate())) {
1841 __ tmll(i.InputRegister(0), opnd);
1842 } else {
1843 __ lr(r0, i.InputRegister(0));
1844 __ nilf(r0, opnd);
1845 }
1846 }
1847 break;
1848 case kS390_Tst64:
1849 if (HasRegisterInput(instr, 1)) {
1850 __ AndP(r0, i.InputRegister(0), i.InputRegister(1));
1851 } else {
1852 Operand opnd = i.InputImmediate(1);
1853 if (is_uint16(opnd.immediate())) {
1854 __ tmll(i.InputRegister(0), opnd);
1855 } else {
1856 __ AndP(r0, i.InputRegister(0), opnd);
1857 }
1858 }
1859 break;
1860 case kS390_Float64SilenceNaN: {
1861 DoubleRegister value = i.InputDoubleRegister(0);
1862 DoubleRegister result = i.OutputDoubleRegister();
1863 __ CanonicalizeNaN(result, value);
1864 break;
1865 }
1866 case kS390_Push: {
1867 int stack_decrement = i.InputInt32(0);
1868 int slots = stack_decrement / kSystemPointerSize;
1869 LocationOperand* op = LocationOperand::cast(instr->InputAt(1));
1870 MachineRepresentation rep = op->representation();
1871 int pushed_slots = ElementSizeInPointers(rep);
1872 // Slot-sized arguments are never padded but there may be a gap if
1873 // the slot allocator reclaimed other padding slots. Adjust the stack
1874 // here to skip any gap.
1875 __ AllocateStackSpace((slots - pushed_slots) * kSystemPointerSize);
1876 switch (rep) {
1877 case MachineRepresentation::kFloat32:
1878 __ lay(sp, MemOperand(sp, -kSystemPointerSize));
1879 __ StoreF32(i.InputDoubleRegister(1), MemOperand(sp));
1880 break;
1881 case MachineRepresentation::kFloat64:
1882 __ lay(sp, MemOperand(sp, -kDoubleSize));
1883 __ StoreF64(i.InputDoubleRegister(1), MemOperand(sp));
1884 break;
1885 case MachineRepresentation::kSimd128:
1886 __ lay(sp, MemOperand(sp, -kSimd128Size));
1887 __ StoreV128(i.InputDoubleRegister(1), MemOperand(sp), kScratchReg);
1888 break;
1889 default:
1890 __ Push(i.InputRegister(1));
1891 break;
1892 }
1893 frame_access_state()->IncreaseSPDelta(slots);
1894 break;
1895 }
1896 case kS390_PushFrame: {
1897 int num_slots = i.InputInt32(1);
1898 __ lay(sp, MemOperand(sp, -num_slots * kSystemPointerSize));
1899 if (instr->InputAt(0)->IsFPRegister()) {
1900 LocationOperand* op = LocationOperand::cast(instr->InputAt(0));
1901 if (op->representation() == MachineRepresentation::kFloat64) {
1902 __ StoreF64(i.InputDoubleRegister(0), MemOperand(sp));
1903 } else {
1904 DCHECK_EQ(MachineRepresentation::kFloat32, op->representation());
1905 __ StoreF32(i.InputDoubleRegister(0), MemOperand(sp));
1906 }
1907 } else {
1908 __ StoreU64(i.InputRegister(0), MemOperand(sp));
1909 }
1910 break;
1911 }
1912 case kS390_StoreToStackSlot: {
1913 int slot = i.InputInt32(1);
1914 if (instr->InputAt(0)->IsFPRegister()) {
1915 LocationOperand* op = LocationOperand::cast(instr->InputAt(0));
1916 if (op->representation() == MachineRepresentation::kFloat64) {
1917 __ StoreF64(i.InputDoubleRegister(0),
1918 MemOperand(sp, slot * kSystemPointerSize));
1919 } else if (op->representation() == MachineRepresentation::kFloat32) {
1920 __ StoreF32(i.InputDoubleRegister(0),
1921 MemOperand(sp, slot * kSystemPointerSize));
1922 } else {
1923 DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
1924 __ StoreV128(i.InputDoubleRegister(0),
1925 MemOperand(sp, slot * kSystemPointerSize), kScratchReg);
1926 }
1927 } else {
1928 __ StoreU64(i.InputRegister(0),
1929 MemOperand(sp, slot * kSystemPointerSize));
1930 }
1931 break;
1932 }
1933 case kS390_SignExtendWord8ToInt32:
1934 __ lbr(i.OutputRegister(), i.InputRegister(0));
1935 CHECK_AND_ZERO_EXT_OUTPUT(1);
1936 break;
1937 case kS390_SignExtendWord16ToInt32:
1938 __ lhr(i.OutputRegister(), i.InputRegister(0));
1939 CHECK_AND_ZERO_EXT_OUTPUT(1);
1940 break;
1941 case kS390_SignExtendWord8ToInt64:
1942 __ lgbr(i.OutputRegister(), i.InputRegister(0));
1943 break;
1944 case kS390_SignExtendWord16ToInt64:
1945 __ lghr(i.OutputRegister(), i.InputRegister(0));
1946 break;
1947 case kS390_SignExtendWord32ToInt64:
1948 __ lgfr(i.OutputRegister(), i.InputRegister(0));
1949 break;
1950 case kS390_Uint32ToUint64:
1951 // Zero extend
1952 __ llgfr(i.OutputRegister(), i.InputRegister(0));
1953 break;
1954 case kS390_Int64ToInt32:
1955 // sign extend
1956 __ lgfr(i.OutputRegister(), i.InputRegister(0));
1957 break;
1958 // Convert Fixed to Floating Point
1959 case kS390_Int64ToFloat32:
1960 __ ConvertInt64ToFloat(i.OutputDoubleRegister(), i.InputRegister(0));
1961 break;
1962 case kS390_Int64ToDouble:
1963 __ ConvertInt64ToDouble(i.OutputDoubleRegister(), i.InputRegister(0));
1964 break;
1965 case kS390_Uint64ToFloat32:
1966 __ ConvertUnsignedInt64ToFloat(i.OutputDoubleRegister(),
1967 i.InputRegister(0));
1968 break;
1969 case kS390_Uint64ToDouble:
1970 __ ConvertUnsignedInt64ToDouble(i.OutputDoubleRegister(),
1971 i.InputRegister(0));
1972 break;
1973 case kS390_Int32ToFloat32:
1974 __ ConvertIntToFloat(i.OutputDoubleRegister(), i.InputRegister(0));
1975 break;
1976 case kS390_Int32ToDouble:
1977 __ ConvertIntToDouble(i.OutputDoubleRegister(), i.InputRegister(0));
1978 break;
1979 case kS390_Uint32ToFloat32:
1980 __ ConvertUnsignedIntToFloat(i.OutputDoubleRegister(),
1981 i.InputRegister(0));
1982 break;
1983 case kS390_Uint32ToDouble:
1984 __ ConvertUnsignedIntToDouble(i.OutputDoubleRegister(),
1985 i.InputRegister(0));
1986 break;
1987 case kS390_DoubleToInt32: {
1988 Label done;
1989 __ ConvertDoubleToInt32(i.OutputRegister(0), i.InputDoubleRegister(0),
1990 kRoundToNearest);
1991 __ b(Condition(0xE), &done, Label::kNear); // normal case
1992 __ mov(i.OutputRegister(0), Operand::Zero());
1993 __ bind(&done);
1994 break;
1995 }
1996 case kS390_DoubleToUint32: {
1997 Label done;
1998 __ ConvertDoubleToUnsignedInt32(i.OutputRegister(0),
1999 i.InputDoubleRegister(0));
2000 __ b(Condition(0xE), &done, Label::kNear); // normal case
2001 __ mov(i.OutputRegister(0), Operand::Zero());
2002 __ bind(&done);
2003 break;
2004 }
2005 case kS390_DoubleToInt64: {
2006 Label done;
2007 if (i.OutputCount() > 1) {
2008 __ mov(i.OutputRegister(1), Operand(1));
2009 }
2010 __ ConvertDoubleToInt64(i.OutputRegister(0), i.InputDoubleRegister(0));
2011 __ b(Condition(0xE), &done, Label::kNear); // normal case
2012 if (i.OutputCount() > 1) {
2013 __ mov(i.OutputRegister(1), Operand::Zero());
2014 } else {
2015 __ mov(i.OutputRegister(0), Operand::Zero());
2016 }
2017 __ bind(&done);
2018 break;
2019 }
2020 case kS390_DoubleToUint64: {
2021 Label done;
2022 if (i.OutputCount() > 1) {
2023 __ mov(i.OutputRegister(1), Operand(1));
2024 }
2025 __ ConvertDoubleToUnsignedInt64(i.OutputRegister(0),
2026 i.InputDoubleRegister(0));
2027 __ b(Condition(0xE), &done, Label::kNear); // normal case
2028 if (i.OutputCount() > 1) {
2029 __ mov(i.OutputRegister(1), Operand::Zero());
2030 } else {
2031 __ mov(i.OutputRegister(0), Operand::Zero());
2032 }
2033 __ bind(&done);
2034 break;
2035 }
2036 case kS390_Float32ToInt32: {
2037 Label done;
2038 __ ConvertFloat32ToInt32(i.OutputRegister(0), i.InputDoubleRegister(0),
2039 kRoundToZero);
2040 bool set_overflow_to_min_i32 = MiscField::decode(instr->opcode());
2041 if (set_overflow_to_min_i32) {
2042 // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
2043 // because INT32_MIN allows easier out-of-bounds detection.
2044 __ b(Condition(0xE), &done, Label::kNear); // normal case
2045 __ llilh(i.OutputRegister(0), Operand(0x8000));
2046 }
2047 __ bind(&done);
2048 break;
2049 }
2050 case kS390_Float32ToUint32: {
2051 Label done;
2052 __ ConvertFloat32ToUnsignedInt32(i.OutputRegister(0),
2053 i.InputDoubleRegister(0));
2054 bool set_overflow_to_min_u32 = MiscField::decode(instr->opcode());
2055 if (set_overflow_to_min_u32) {
2056 // Avoid UINT32_MAX as an overflow indicator and use 0 instead,
2057 // because 0 allows easier out-of-bounds detection.
2058 __ b(Condition(0xE), &done, Label::kNear); // normal case
2059 __ mov(i.OutputRegister(0), Operand::Zero());
2060 }
2061 __ bind(&done);
2062 break;
2063 }
2064 case kS390_Float32ToUint64: {
2065 Label done;
2066 if (i.OutputCount() > 1) {
2067 __ mov(i.OutputRegister(1), Operand(1));
2068 }
2069 __ ConvertFloat32ToUnsignedInt64(i.OutputRegister(0),
2070 i.InputDoubleRegister(0));
2071 __ b(Condition(0xE), &done, Label::kNear); // normal case
2072 if (i.OutputCount() > 1) {
2073 __ mov(i.OutputRegister(1), Operand::Zero());
2074 } else {
2075 __ mov(i.OutputRegister(0), Operand::Zero());
2076 }
2077 __ bind(&done);
2078 break;
2079 }
2080 case kS390_Float32ToInt64: {
2081 Label done;
2082 if (i.OutputCount() > 1) {
2083 __ mov(i.OutputRegister(1), Operand(1));
2084 }
2085 __ ConvertFloat32ToInt64(i.OutputRegister(0), i.InputDoubleRegister(0));
2086 __ b(Condition(0xE), &done, Label::kNear); // normal case
2087 if (i.OutputCount() > 1) {
2088 __ mov(i.OutputRegister(1), Operand::Zero());
2089 } else {
2090 __ mov(i.OutputRegister(0), Operand::Zero());
2091 }
2092 __ bind(&done);
2093 break;
2094 }
2095 case kS390_DoubleToFloat32:
2096 ASSEMBLE_UNARY_OP(D_DInstr(ledbr), nullInstr, nullInstr);
2097 break;
2098 case kS390_Float32ToDouble:
2099 ASSEMBLE_UNARY_OP(D_DInstr(ldebr), D_MTInstr(LoadF32AsF64), nullInstr);
2100 break;
2101 case kS390_DoubleExtractLowWord32:
2102 __ lgdr(i.OutputRegister(), i.InputDoubleRegister(0));
2103 __ llgfr(i.OutputRegister(), i.OutputRegister());
2104 break;
2105 case kS390_DoubleExtractHighWord32:
2106 __ lgdr(i.OutputRegister(), i.InputDoubleRegister(0));
2107 __ srlg(i.OutputRegister(), i.OutputRegister(), Operand(32));
2108 break;
2109 case kS390_DoubleInsertLowWord32:
2110 __ lgdr(kScratchReg, i.InputDoubleRegister(0));
2111 __ lr(kScratchReg, i.InputRegister(1));
2112 __ ldgr(i.OutputDoubleRegister(), kScratchReg);
2113 break;
2114 case kS390_DoubleInsertHighWord32:
2115 __ sllg(kScratchReg, i.InputRegister(1), Operand(32));
2116 __ lgdr(r0, i.InputDoubleRegister(0));
2117 __ lr(kScratchReg, r0);
2118 __ ldgr(i.OutputDoubleRegister(), kScratchReg);
2119 break;
2120 case kS390_DoubleConstruct:
2121 __ sllg(kScratchReg, i.InputRegister(0), Operand(32));
2122 __ lr(kScratchReg, i.InputRegister(1));
2123
2124 // Bitwise convert from GPR to FPR
2125 __ ldgr(i.OutputDoubleRegister(), kScratchReg);
2126 break;
2127 case kS390_LoadWordS8:
2128 ASSEMBLE_LOAD_INTEGER(LoadS8);
2129 break;
2130 case kS390_BitcastFloat32ToInt32:
2131 ASSEMBLE_UNARY_OP(R_DInstr(MovFloatToInt), R_MInstr(LoadU32), nullInstr);
2132 break;
2133 case kS390_BitcastInt32ToFloat32:
2134 __ MovIntToFloat(i.OutputDoubleRegister(), i.InputRegister(0));
2135 break;
2136 #if V8_TARGET_ARCH_S390X
2137 case kS390_BitcastDoubleToInt64:
2138 __ MovDoubleToInt64(i.OutputRegister(), i.InputDoubleRegister(0));
2139 break;
2140 case kS390_BitcastInt64ToDouble:
2141 __ MovInt64ToDouble(i.OutputDoubleRegister(), i.InputRegister(0));
2142 break;
2143 #endif
2144 case kS390_LoadWordU8:
2145 ASSEMBLE_LOAD_INTEGER(LoadU8);
2146 break;
2147 case kS390_LoadWordU16:
2148 ASSEMBLE_LOAD_INTEGER(LoadU16);
2149 break;
2150 case kS390_LoadWordS16:
2151 ASSEMBLE_LOAD_INTEGER(LoadS16);
2152 break;
2153 case kS390_LoadWordU32:
2154 ASSEMBLE_LOAD_INTEGER(LoadU32);
2155 break;
2156 case kS390_LoadWordS32:
2157 ASSEMBLE_LOAD_INTEGER(LoadS32);
2158 break;
2159 case kS390_LoadReverse16:
2160 ASSEMBLE_LOAD_INTEGER(lrvh);
2161 break;
2162 case kS390_LoadReverse32:
2163 ASSEMBLE_LOAD_INTEGER(lrv);
2164 break;
2165 case kS390_LoadReverse64:
2166 ASSEMBLE_LOAD_INTEGER(lrvg);
2167 break;
2168 case kS390_LoadReverse16RR:
2169 __ lrvr(i.OutputRegister(), i.InputRegister(0));
2170 __ rll(i.OutputRegister(), i.OutputRegister(), Operand(16));
2171 break;
2172 case kS390_LoadReverse32RR:
2173 __ lrvr(i.OutputRegister(), i.InputRegister(0));
2174 break;
2175 case kS390_LoadReverse64RR:
2176 __ lrvgr(i.OutputRegister(), i.InputRegister(0));
2177 break;
2178 case kS390_LoadReverseSimd128RR:
2179 __ vlgv(r0, i.InputSimd128Register(0), MemOperand(r0, 0), Condition(3));
2180 __ vlgv(r1, i.InputSimd128Register(0), MemOperand(r0, 1), Condition(3));
2181 __ lrvgr(r0, r0);
2182 __ lrvgr(r1, r1);
2183 __ vlvg(i.OutputSimd128Register(), r0, MemOperand(r0, 1), Condition(3));
2184 __ vlvg(i.OutputSimd128Register(), r1, MemOperand(r0, 0), Condition(3));
2185 break;
2186 case kS390_LoadReverseSimd128: {
2187 AddressingMode mode = kMode_None;
2188 MemOperand operand = i.MemoryOperand(&mode);
2189 Simd128Register dst = i.OutputSimd128Register();
2190 if (CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_2) &&
2191 is_uint12(operand.offset())) {
2192 __ vlbr(dst, operand, Condition(4));
2193 } else {
2194 __ lrvg(r0, operand);
2195 __ lrvg(r1, MemOperand(operand.rx(), operand.rb(),
2196 operand.offset() + kSystemPointerSize));
2197 __ vlvgp(dst, r1, r0);
2198 }
2199 break;
2200 }
2201 case kS390_LoadWord64:
2202 ASSEMBLE_LOAD_INTEGER(lg);
2203 break;
2204 case kS390_LoadAndTestWord32: {
2205 ASSEMBLE_LOADANDTEST32(ltr, lt_z);
2206 break;
2207 }
2208 case kS390_LoadAndTestWord64: {
2209 ASSEMBLE_LOADANDTEST64(ltgr, ltg);
2210 break;
2211 }
2212 case kS390_LoadFloat32:
2213 ASSEMBLE_LOAD_FLOAT(LoadF32);
2214 break;
2215 case kS390_LoadDouble:
2216 ASSEMBLE_LOAD_FLOAT(LoadF64);
2217 break;
2218 case kS390_LoadSimd128: {
2219 AddressingMode mode = kMode_None;
2220 MemOperand operand = i.MemoryOperand(&mode);
2221 __ vl(i.OutputSimd128Register(), operand, Condition(0));
2222 break;
2223 }
2224 case kS390_StoreWord8:
2225 ASSEMBLE_STORE_INTEGER(StoreU8);
2226 break;
2227 case kS390_StoreWord16:
2228 ASSEMBLE_STORE_INTEGER(StoreU16);
2229 break;
2230 case kS390_StoreWord32:
2231 ASSEMBLE_STORE_INTEGER(StoreU32);
2232 break;
2233 #if V8_TARGET_ARCH_S390X
2234 case kS390_StoreWord64:
2235 ASSEMBLE_STORE_INTEGER(StoreU64);
2236 break;
2237 #endif
2238 case kS390_StoreReverse16:
2239 ASSEMBLE_STORE_INTEGER(strvh);
2240 break;
2241 case kS390_StoreReverse32:
2242 ASSEMBLE_STORE_INTEGER(strv);
2243 break;
2244 case kS390_StoreReverse64:
2245 ASSEMBLE_STORE_INTEGER(strvg);
2246 break;
2247 case kS390_StoreReverseSimd128: {
2248 size_t index = 0;
2249 AddressingMode mode = kMode_None;
2250 MemOperand operand = i.MemoryOperand(&mode, &index);
2251 if (CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_2) &&
2252 is_uint12(operand.offset())) {
2253 __ vstbr(i.InputSimd128Register(index), operand, Condition(4));
2254 } else {
2255 __ vlgv(r0, i.InputSimd128Register(index), MemOperand(r0, 1),
2256 Condition(3));
2257 __ vlgv(r1, i.InputSimd128Register(index), MemOperand(r0, 0),
2258 Condition(3));
2259 __ strvg(r0, operand);
2260 __ strvg(r1, MemOperand(operand.rx(), operand.rb(),
2261 operand.offset() + kSystemPointerSize));
2262 }
2263 break;
2264 }
2265 case kS390_StoreFloat32:
2266 ASSEMBLE_STORE_FLOAT32();
2267 break;
2268 case kS390_StoreDouble:
2269 ASSEMBLE_STORE_DOUBLE();
2270 break;
2271 case kS390_StoreSimd128: {
2272 size_t index = 0;
2273 AddressingMode mode = kMode_None;
2274 MemOperand operand = i.MemoryOperand(&mode, &index);
2275 __ vst(i.InputSimd128Register(index), operand, Condition(0));
2276 break;
2277 }
2278 case kS390_Lay: {
2279 MemOperand mem = i.MemoryOperand();
2280 if (!is_int20(mem.offset())) {
2281 // Add directly to the base register in case the index register (rx) is
2282 // r0.
2283 DCHECK(is_int32(mem.offset()));
2284 __ AddS64(ip, mem.rb(), Operand(mem.offset()));
2285 mem = MemOperand(mem.rx(), ip);
2286 }
2287 __ lay(i.OutputRegister(), mem);
2288 break;
2289 }
2290 case kAtomicExchangeInt8:
2291 case kAtomicExchangeUint8: {
2292 Register base = i.InputRegister(0);
2293 Register index = i.InputRegister(1);
2294 Register value = i.InputRegister(2);
2295 Register output = i.OutputRegister();
2296 __ la(r1, MemOperand(base, index));
2297 __ AtomicExchangeU8(r1, value, output, r0);
2298 if (opcode == kAtomicExchangeInt8) {
2299 __ LoadS8(output, output);
2300 } else {
2301 __ LoadU8(output, output);
2302 }
2303 break;
2304 }
2305 case kAtomicExchangeInt16:
2306 case kAtomicExchangeUint16: {
2307 Register base = i.InputRegister(0);
2308 Register index = i.InputRegister(1);
2309 Register value = i.InputRegister(2);
2310 Register output = i.OutputRegister();
2311 __ la(r1, MemOperand(base, index));
2312 __ AtomicExchangeU16(r1, value, output, r0);
2313 if (opcode == kAtomicExchangeInt16) {
2314 __ lghr(output, output);
2315 } else {
2316 __ llghr(output, output);
2317 }
2318 break;
2319 }
2320 case kAtomicExchangeWord32: {
2321 Register base = i.InputRegister(0);
2322 Register index = i.InputRegister(1);
2323 Register value = i.InputRegister(2);
2324 Register output = i.OutputRegister();
2325 Label do_cs;
2326 __ lay(r1, MemOperand(base, index));
2327 __ LoadU32(output, MemOperand(r1));
2328 __ bind(&do_cs);
2329 __ cs(output, value, MemOperand(r1));
2330 __ bne(&do_cs, Label::kNear);
2331 break;
2332 }
2333 case kAtomicCompareExchangeInt8:
2334 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_BYTE(LoadS8);
2335 break;
2336 case kAtomicCompareExchangeUint8:
2337 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_BYTE(LoadU8);
2338 break;
2339 case kAtomicCompareExchangeInt16:
2340 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_HALFWORD(LoadS16);
2341 break;
2342 case kAtomicCompareExchangeUint16:
2343 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_HALFWORD(LoadU16);
2344 break;
2345 case kAtomicCompareExchangeWord32:
2346 ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_WORD();
2347 break;
2348 #define ATOMIC_BINOP_CASE(op, inst) \
2349 case kAtomic##op##Int8: \
2350 ASSEMBLE_ATOMIC_BINOP_BYTE(inst, [&]() { \
2351 intptr_t shift_right = static_cast<intptr_t>(shift_amount); \
2352 __ srlk(result, prev, Operand(shift_right)); \
2353 __ LoadS8(result, result); \
2354 }); \
2355 break; \
2356 case kAtomic##op##Uint8: \
2357 ASSEMBLE_ATOMIC_BINOP_BYTE(inst, [&]() { \
2358 int rotate_left = shift_amount == 0 ? 0 : 64 - shift_amount; \
2359 __ RotateInsertSelectBits(result, prev, Operand(56), Operand(63), \
2360 Operand(static_cast<intptr_t>(rotate_left)), \
2361 true); \
2362 }); \
2363 break; \
2364 case kAtomic##op##Int16: \
2365 ASSEMBLE_ATOMIC_BINOP_HALFWORD(inst, [&]() { \
2366 intptr_t shift_right = static_cast<intptr_t>(shift_amount); \
2367 __ srlk(result, prev, Operand(shift_right)); \
2368 __ LoadS16(result, result); \
2369 }); \
2370 break; \
2371 case kAtomic##op##Uint16: \
2372 ASSEMBLE_ATOMIC_BINOP_HALFWORD(inst, [&]() { \
2373 int rotate_left = shift_amount == 0 ? 0 : 64 - shift_amount; \
2374 __ RotateInsertSelectBits(result, prev, Operand(48), Operand(63), \
2375 Operand(static_cast<intptr_t>(rotate_left)), \
2376 true); \
2377 }); \
2378 break;
2379 ATOMIC_BINOP_CASE(Add, AddS32)
2380 ATOMIC_BINOP_CASE(Sub, SubS32)
2381 ATOMIC_BINOP_CASE(And, And)
2382 ATOMIC_BINOP_CASE(Or, Or)
2383 ATOMIC_BINOP_CASE(Xor, Xor)
2384 #undef ATOMIC_BINOP_CASE
2385 case kAtomicAddWord32:
2386 ASSEMBLE_ATOMIC_BINOP_WORD(laa);
2387 break;
2388 case kAtomicSubWord32:
2389 ASSEMBLE_ATOMIC_BINOP_WORD(LoadAndSub32);
2390 break;
2391 case kAtomicAndWord32:
2392 ASSEMBLE_ATOMIC_BINOP_WORD(lan);
2393 break;
2394 case kAtomicOrWord32:
2395 ASSEMBLE_ATOMIC_BINOP_WORD(lao);
2396 break;
2397 case kAtomicXorWord32:
2398 ASSEMBLE_ATOMIC_BINOP_WORD(lax);
2399 break;
2400 case kS390_Word64AtomicAddUint64:
2401 ASSEMBLE_ATOMIC_BINOP_WORD64(laag);
2402 break;
2403 case kS390_Word64AtomicSubUint64:
2404 ASSEMBLE_ATOMIC_BINOP_WORD64(LoadAndSub64);
2405 break;
2406 case kS390_Word64AtomicAndUint64:
2407 ASSEMBLE_ATOMIC_BINOP_WORD64(lang);
2408 break;
2409 case kS390_Word64AtomicOrUint64:
2410 ASSEMBLE_ATOMIC_BINOP_WORD64(laog);
2411 break;
2412 case kS390_Word64AtomicXorUint64:
2413 ASSEMBLE_ATOMIC_BINOP_WORD64(laxg);
2414 break;
2415 case kS390_Word64AtomicExchangeUint64: {
2416 Register base = i.InputRegister(0);
2417 Register index = i.InputRegister(1);
2418 Register value = i.InputRegister(2);
2419 Register output = i.OutputRegister();
2420 Label do_cs;
2421 __ la(r1, MemOperand(base, index));
2422 __ lg(output, MemOperand(r1));
2423 __ bind(&do_cs);
2424 __ csg(output, value, MemOperand(r1));
2425 __ bne(&do_cs, Label::kNear);
2426 break;
2427 }
2428 case kS390_Word64AtomicCompareExchangeUint64:
2429 ASSEMBLE_ATOMIC64_COMP_EXCHANGE_WORD64();
2430 break;
2431 // Simd Support.
2432 #define SIMD_BINOP_LIST(V) \
2433 V(F64x2Add, Simd128Register) \
2434 V(F64x2Sub, Simd128Register) \
2435 V(F64x2Mul, Simd128Register) \
2436 V(F64x2Div, Simd128Register) \
2437 V(F64x2Min, Simd128Register) \
2438 V(F64x2Max, Simd128Register) \
2439 V(F64x2Eq, Simd128Register) \
2440 V(F64x2Ne, Simd128Register) \
2441 V(F64x2Lt, Simd128Register) \
2442 V(F64x2Le, Simd128Register) \
2443 V(F32x4Add, Simd128Register) \
2444 V(F32x4Sub, Simd128Register) \
2445 V(F32x4Mul, Simd128Register) \
2446 V(F32x4Div, Simd128Register) \
2447 V(F32x4Min, Simd128Register) \
2448 V(F32x4Max, Simd128Register) \
2449 V(F32x4Eq, Simd128Register) \
2450 V(F32x4Ne, Simd128Register) \
2451 V(F32x4Lt, Simd128Register) \
2452 V(F32x4Le, Simd128Register) \
2453 V(I64x2Add, Simd128Register) \
2454 V(I64x2Sub, Simd128Register) \
2455 V(I64x2Mul, Simd128Register) \
2456 V(I64x2Eq, Simd128Register) \
2457 V(I64x2Ne, Simd128Register) \
2458 V(I64x2GtS, Simd128Register) \
2459 V(I64x2GeS, Simd128Register) \
2460 V(I64x2Shl, Register) \
2461 V(I64x2ShrS, Register) \
2462 V(I64x2ShrU, Register) \
2463 V(I32x4Add, Simd128Register) \
2464 V(I32x4Sub, Simd128Register) \
2465 V(I32x4Mul, Simd128Register) \
2466 V(I32x4Eq, Simd128Register) \
2467 V(I32x4Ne, Simd128Register) \
2468 V(I32x4GtS, Simd128Register) \
2469 V(I32x4GeS, Simd128Register) \
2470 V(I32x4GtU, Simd128Register) \
2471 V(I32x4GeU, Simd128Register) \
2472 V(I32x4MinS, Simd128Register) \
2473 V(I32x4MinU, Simd128Register) \
2474 V(I32x4MaxS, Simd128Register) \
2475 V(I32x4MaxU, Simd128Register) \
2476 V(I32x4Shl, Register) \
2477 V(I32x4ShrS, Register) \
2478 V(I32x4ShrU, Register) \
2479 V(I16x8Add, Simd128Register) \
2480 V(I16x8Sub, Simd128Register) \
2481 V(I16x8Mul, Simd128Register) \
2482 V(I16x8Eq, Simd128Register) \
2483 V(I16x8Ne, Simd128Register) \
2484 V(I16x8GtS, Simd128Register) \
2485 V(I16x8GeS, Simd128Register) \
2486 V(I16x8GtU, Simd128Register) \
2487 V(I16x8GeU, Simd128Register) \
2488 V(I16x8MinS, Simd128Register) \
2489 V(I16x8MinU, Simd128Register) \
2490 V(I16x8MaxS, Simd128Register) \
2491 V(I16x8MaxU, Simd128Register) \
2492 V(I16x8Shl, Register) \
2493 V(I16x8ShrS, Register) \
2494 V(I16x8ShrU, Register) \
2495 V(I8x16Add, Simd128Register) \
2496 V(I8x16Sub, Simd128Register) \
2497 V(I8x16Eq, Simd128Register) \
2498 V(I8x16Ne, Simd128Register) \
2499 V(I8x16GtS, Simd128Register) \
2500 V(I8x16GeS, Simd128Register) \
2501 V(I8x16GtU, Simd128Register) \
2502 V(I8x16GeU, Simd128Register) \
2503 V(I8x16MinS, Simd128Register) \
2504 V(I8x16MinU, Simd128Register) \
2505 V(I8x16MaxS, Simd128Register) \
2506 V(I8x16MaxU, Simd128Register) \
2507 V(I8x16Shl, Register) \
2508 V(I8x16ShrS, Register) \
2509 V(I8x16ShrU, Register)
2510
2511 #define EMIT_SIMD_BINOP(name, stype) \
2512 case kS390_##name: { \
2513 __ name(i.OutputSimd128Register(), i.InputSimd128Register(0), \
2514 i.Input##stype(1)); \
2515 break; \
2516 }
2517 SIMD_BINOP_LIST(EMIT_SIMD_BINOP)
2518 #undef EMIT_SIMD_BINOP
2519 #undef SIMD_BINOP_LIST
2520
2521 #define SIMD_UNOP_LIST(V) \
2522 V(F64x2Splat, F64x2Splat, Simd128Register, DoubleRegister) \
2523 V(F32x4Splat, F32x4Splat, Simd128Register, DoubleRegister) \
2524 V(I64x2Splat, I64x2Splat, Simd128Register, Register) \
2525 V(I32x4Splat, I32x4Splat, Simd128Register, Register) \
2526 V(I16x8Splat, I16x8Splat, Simd128Register, Register) \
2527 V(I8x16Splat, I8x16Splat, Simd128Register, Register)
2528
2529 #define EMIT_SIMD_UNOP(name, op, dtype, stype) \
2530 case kS390_##name: { \
2531 __ op(i.Output##dtype(), i.Input##stype(0)); \
2532 break; \
2533 }
2534 SIMD_UNOP_LIST(EMIT_SIMD_UNOP)
2535 #undef EMIT_SIMD_UNOP
2536 #undef SIMD_UNOP_LIST
2537
2538 #define SIMD_EXTRACT_LANE_LIST(V) \
2539 V(F64x2ExtractLane, DoubleRegister) \
2540 V(F32x4ExtractLane, DoubleRegister) \
2541 V(I64x2ExtractLane, Register) \
2542 V(I32x4ExtractLane, Register) \
2543 V(I16x8ExtractLaneU, Register) \
2544 V(I16x8ExtractLaneS, Register) \
2545 V(I8x16ExtractLaneU, Register) \
2546 V(I8x16ExtractLaneS, Register)
2547
2548 #define EMIT_SIMD_EXTRACT_LANE(name, dtype) \
2549 case kS390_##name: { \
2550 __ name(i.Output##dtype(), i.InputSimd128Register(0), i.InputInt8(1)); \
2551 break; \
2552 }
2553 SIMD_EXTRACT_LANE_LIST(EMIT_SIMD_EXTRACT_LANE)
2554 #undef EMIT_SIMD_EXTRACT_LANE
2555 #undef SIMD_EXTRACT_LANE_LIST
2556
2557 #define SIMD_REPLACE_LANE_LIST(V) \
2558 V(F64x2ReplaceLane, DoubleRegister) \
2559 V(F32x4ReplaceLane, DoubleRegister) \
2560 V(I64x2ReplaceLane, Register) \
2561 V(I32x4ReplaceLane, Register) \
2562 V(I16x8ReplaceLane, Register) \
2563 V(I8x16ReplaceLane, Register)
2564
2565 #define EMIT_SIMD_REPLACE_LANE(name, stype) \
2566 case kS390_##name: { \
2567 __ name(i.OutputSimd128Register(), i.InputSimd128Register(0), \
2568 i.Input##stype(2), i.InputInt8(1)); \
2569 break; \
2570 }
2571 SIMD_REPLACE_LANE_LIST(EMIT_SIMD_REPLACE_LANE)
2572 #undef EMIT_SIMD_REPLACE_LANE
2573 #undef SIMD_REPLACE_LANE_LIST
2574 // vector binops
2575 case kS390_F64x2Qfma: {
2576 Simd128Register src0 = i.InputSimd128Register(0);
2577 Simd128Register src1 = i.InputSimd128Register(1);
2578 Simd128Register src2 = i.InputSimd128Register(2);
2579 Simd128Register dst = i.OutputSimd128Register();
2580 __ vfma(dst, src1, src2, src0, Condition(3), Condition(0));
2581 break;
2582 }
2583 case kS390_F64x2Qfms: {
2584 Simd128Register src0 = i.InputSimd128Register(0);
2585 Simd128Register src1 = i.InputSimd128Register(1);
2586 Simd128Register src2 = i.InputSimd128Register(2);
2587 Simd128Register dst = i.OutputSimd128Register();
2588 __ vfnms(dst, src1, src2, src0, Condition(3), Condition(0));
2589 break;
2590 }
2591 case kS390_F32x4Qfma: {
2592 Simd128Register src0 = i.InputSimd128Register(0);
2593 Simd128Register src1 = i.InputSimd128Register(1);
2594 Simd128Register src2 = i.InputSimd128Register(2);
2595 Simd128Register dst = i.OutputSimd128Register();
2596 __ vfma(dst, src1, src2, src0, Condition(2), Condition(0));
2597 break;
2598 }
2599 case kS390_F32x4Qfms: {
2600 Simd128Register src0 = i.InputSimd128Register(0);
2601 Simd128Register src1 = i.InputSimd128Register(1);
2602 Simd128Register src2 = i.InputSimd128Register(2);
2603 Simd128Register dst = i.OutputSimd128Register();
2604 __ vfnms(dst, src1, src2, src0, Condition(2), Condition(0));
2605 break;
2606 }
2607 case kS390_I16x8RoundingAverageU: {
2608 __ vavgl(i.OutputSimd128Register(), i.InputSimd128Register(0),
2609 i.InputSimd128Register(1), Condition(0), Condition(0),
2610 Condition(1));
2611 break;
2612 }
2613 case kS390_I8x16RoundingAverageU: {
2614 __ vavgl(i.OutputSimd128Register(), i.InputSimd128Register(0),
2615 i.InputSimd128Register(1), Condition(0), Condition(0),
2616 Condition(0));
2617 break;
2618 }
2619 // vector unary ops
2620 case kS390_F64x2Abs: {
2621 __ vfpso(i.OutputSimd128Register(), i.InputSimd128Register(0),
2622 Condition(2), Condition(0), Condition(3));
2623 break;
2624 }
2625 case kS390_F64x2Neg: {
2626 __ vfpso(i.OutputSimd128Register(), i.InputSimd128Register(0),
2627 Condition(0), Condition(0), Condition(3));
2628 break;
2629 }
2630 case kS390_F64x2Sqrt: {
2631 __ vfsq(i.OutputSimd128Register(), i.InputSimd128Register(0),
2632 Condition(0), Condition(0), Condition(3));
2633 break;
2634 }
2635 case kS390_F32x4Abs: {
2636 __ vfpso(i.OutputSimd128Register(), i.InputSimd128Register(0),
2637 Condition(2), Condition(0), Condition(2));
2638 break;
2639 }
2640 case kS390_F32x4Neg: {
2641 __ vfpso(i.OutputSimd128Register(), i.InputSimd128Register(0),
2642 Condition(0), Condition(0), Condition(2));
2643 break;
2644 }
2645 case kS390_I64x2Neg: {
2646 __ vlc(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(0),
2647 Condition(0), Condition(3));
2648 break;
2649 }
2650 case kS390_I32x4Neg: {
2651 __ vlc(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(0),
2652 Condition(0), Condition(2));
2653 break;
2654 }
2655 case kS390_I16x8Neg: {
2656 __ vlc(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(0),
2657 Condition(0), Condition(1));
2658 break;
2659 }
2660 case kS390_I8x16Neg: {
2661 __ vlc(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(0),
2662 Condition(0), Condition(0));
2663 break;
2664 }
2665 case kS390_F32x4RecipApprox: {
2666 __ mov(kScratchReg, Operand(1));
2667 __ ConvertIntToFloat(kScratchDoubleReg, kScratchReg);
2668 __ vrep(kScratchDoubleReg, kScratchDoubleReg, Operand(0), Condition(2));
2669 __ vfd(i.OutputSimd128Register(), kScratchDoubleReg,
2670 i.InputSimd128Register(0), Condition(0), Condition(0),
2671 Condition(2));
2672 break;
2673 }
2674 case kS390_F32x4RecipSqrtApprox: {
2675 Simd128Register dst = i.OutputSimd128Register();
2676 __ vfsq(dst, i.InputSimd128Register(0), Condition(0), Condition(0),
2677 Condition(2));
2678 __ mov(kScratchReg, Operand(1));
2679 __ ConvertIntToFloat(kScratchDoubleReg, kScratchReg);
2680 __ vrep(kScratchDoubleReg, kScratchDoubleReg, Operand(0), Condition(2));
2681 __ vfd(dst, kScratchDoubleReg, dst, Condition(0), Condition(0),
2682 Condition(2));
2683 break;
2684 }
2685 case kS390_F32x4Sqrt: {
2686 __ vfsq(i.OutputSimd128Register(), i.InputSimd128Register(0),
2687 Condition(0), Condition(0), Condition(2));
2688 break;
2689 }
2690 case kS390_S128Not: {
2691 Simd128Register src = i.InputSimd128Register(0);
2692 Simd128Register dst = i.OutputSimd128Register();
2693 __ vno(dst, src, src, Condition(0), Condition(0), Condition(0));
2694 break;
2695 }
2696 case kS390_I8x16Abs: {
2697 __ vlp(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(0),
2698 Condition(0), Condition(0));
2699 break;
2700 }
2701 case kS390_I16x8Abs: {
2702 __ vlp(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(0),
2703 Condition(0), Condition(1));
2704 break;
2705 }
2706 case kS390_I32x4Abs: {
2707 __ vlp(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(0),
2708 Condition(0), Condition(2));
2709 break;
2710 }
2711 case kS390_I64x2Abs: {
2712 __ vlp(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(0),
2713 Condition(0), Condition(3));
2714 break;
2715 }
2716 // vector boolean unops
2717 case kS390_V128AnyTrue: {
2718 Simd128Register src = i.InputSimd128Register(0);
2719 Register dst = i.OutputRegister();
2720 __ mov(dst, Operand(1));
2721 __ xgr(kScratchReg, kScratchReg);
2722 __ vtm(src, src, Condition(0), Condition(0), Condition(0));
2723 __ locgr(Condition(8), dst, kScratchReg);
2724 break;
2725 }
2726 #define SIMD_ALL_TRUE(mode) \
2727 Simd128Register src = i.InputSimd128Register(0); \
2728 Register dst = i.OutputRegister(); \
2729 __ mov(kScratchReg, Operand(1)); \
2730 __ xgr(dst, dst); \
2731 __ vx(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg, Condition(0), \
2732 Condition(0), Condition(2)); \
2733 __ vceq(kScratchDoubleReg, src, kScratchDoubleReg, Condition(0), \
2734 Condition(mode)); \
2735 __ vtm(kScratchDoubleReg, kScratchDoubleReg, Condition(0), Condition(0), \
2736 Condition(0)); \
2737 __ locgr(Condition(8), dst, kScratchReg);
2738 case kS390_I64x2AllTrue: {
2739 SIMD_ALL_TRUE(3)
2740 break;
2741 }
2742 case kS390_I32x4AllTrue: {
2743 SIMD_ALL_TRUE(2)
2744 break;
2745 }
2746 case kS390_I16x8AllTrue: {
2747 SIMD_ALL_TRUE(1)
2748 break;
2749 }
2750 case kS390_I8x16AllTrue: {
2751 SIMD_ALL_TRUE(0)
2752 break;
2753 }
2754 #undef SIMD_ALL_TRUE
2755 // vector bitwise ops
2756 case kS390_S128And: {
2757 Simd128Register dst = i.OutputSimd128Register();
2758 Simd128Register src = i.InputSimd128Register(1);
2759 __ vn(dst, i.InputSimd128Register(0), src, Condition(0), Condition(0),
2760 Condition(0));
2761 break;
2762 }
2763 case kS390_S128Or: {
2764 Simd128Register dst = i.OutputSimd128Register();
2765 Simd128Register src = i.InputSimd128Register(1);
2766 __ vo(dst, i.InputSimd128Register(0), src, Condition(0), Condition(0),
2767 Condition(0));
2768 break;
2769 }
2770 case kS390_S128Xor: {
2771 Simd128Register dst = i.OutputSimd128Register();
2772 Simd128Register src = i.InputSimd128Register(1);
2773 __ vx(dst, i.InputSimd128Register(0), src, Condition(0), Condition(0),
2774 Condition(0));
2775 break;
2776 }
2777 case kS390_S128Const: {
2778 uint64_t low = make_uint64(i.InputUint32(1), i.InputUint32(0));
2779 uint64_t high = make_uint64(i.InputUint32(3), i.InputUint32(2));
2780 __ mov(r0, Operand(low));
2781 __ mov(ip, Operand(high));
2782 __ vlvgp(i.OutputSimd128Register(), ip, r0);
2783 break;
2784 }
2785 case kS390_S128Zero: {
2786 Simd128Register dst = i.OutputSimd128Register();
2787 __ vx(dst, dst, dst, Condition(0), Condition(0), Condition(0));
2788 break;
2789 }
2790 case kS390_S128AllOnes: {
2791 Simd128Register dst = i.OutputSimd128Register();
2792 __ vceq(dst, dst, dst, Condition(0), Condition(3));
2793 break;
2794 }
2795 case kS390_S128Select: {
2796 Simd128Register dst = i.OutputSimd128Register();
2797 Simd128Register mask = i.InputSimd128Register(0);
2798 Simd128Register src1 = i.InputSimd128Register(1);
2799 Simd128Register src2 = i.InputSimd128Register(2);
2800 __ vsel(dst, src1, src2, mask, Condition(0), Condition(0));
2801 break;
2802 }
2803 case kS390_S128AndNot: {
2804 Simd128Register dst = i.OutputSimd128Register();
2805 Simd128Register src = i.InputSimd128Register(1);
2806 __ vnc(dst, i.InputSimd128Register(0), src, Condition(0), Condition(0),
2807 Condition(0));
2808 break;
2809 }
2810 // vector conversions
2811 #define CONVERT_FLOAT_TO_INT32(convert) \
2812 for (int index = 0; index < 4; index++) { \
2813 __ vlgv(kScratchReg, kScratchDoubleReg, MemOperand(r0, index), \
2814 Condition(2)); \
2815 __ MovIntToFloat(tempFPReg1, kScratchReg); \
2816 __ convert(kScratchReg, tempFPReg1, kRoundToZero); \
2817 __ vlvg(dst, kScratchReg, MemOperand(r0, index), Condition(2)); \
2818 }
2819 case kS390_I32x4SConvertF32x4: {
2820 Simd128Register src = i.InputSimd128Register(0);
2821 Simd128Register dst = i.OutputSimd128Register();
2822 Simd128Register tempFPReg1 = i.ToDoubleRegister(instr->TempAt(0));
2823 DCHECK_NE(dst, tempFPReg1);
2824 // NaN to 0
2825 __ vlr(kScratchDoubleReg, src, Condition(0), Condition(0), Condition(0));
2826 __ vfce(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg,
2827 Condition(0), Condition(0), Condition(2));
2828 __ vn(kScratchDoubleReg, src, kScratchDoubleReg, Condition(0),
2829 Condition(0), Condition(0));
2830 if (CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_2)) {
2831 __ vcgd(i.OutputSimd128Register(), kScratchDoubleReg, Condition(5),
2832 Condition(0), Condition(2));
2833 } else {
2834 CONVERT_FLOAT_TO_INT32(ConvertFloat32ToInt32)
2835 }
2836 break;
2837 }
2838 case kS390_I32x4UConvertF32x4: {
2839 Simd128Register src = i.InputSimd128Register(0);
2840 Simd128Register dst = i.OutputSimd128Register();
2841 Simd128Register tempFPReg1 = i.ToDoubleRegister(instr->TempAt(0));
2842 DCHECK_NE(dst, tempFPReg1);
2843 // NaN to 0, negative to 0
2844 __ vx(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg,
2845 Condition(0), Condition(0), Condition(0));
2846 __ vfmax(kScratchDoubleReg, src, kScratchDoubleReg, Condition(1),
2847 Condition(0), Condition(2));
2848 if (CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_2)) {
2849 __ vclgd(i.OutputSimd128Register(), kScratchDoubleReg, Condition(5),
2850 Condition(0), Condition(2));
2851 } else {
2852 CONVERT_FLOAT_TO_INT32(ConvertFloat32ToUnsignedInt32)
2853 }
2854 break;
2855 }
2856 #undef CONVERT_FLOAT_TO_INT32
2857 #define CONVERT_INT32_TO_FLOAT(convert, double_index) \
2858 Simd128Register src = i.InputSimd128Register(0); \
2859 Simd128Register dst = i.OutputSimd128Register(); \
2860 for (int index = 0; index < 4; index++) { \
2861 __ vlgv(kScratchReg, src, MemOperand(r0, index), Condition(2)); \
2862 __ convert(kScratchDoubleReg, kScratchReg); \
2863 __ MovFloatToInt(kScratchReg, kScratchDoubleReg); \
2864 __ vlvg(dst, kScratchReg, MemOperand(r0, index), Condition(2)); \
2865 }
2866 case kS390_F32x4SConvertI32x4: {
2867 if (CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_2)) {
2868 __ vcdg(i.OutputSimd128Register(), i.InputSimd128Register(0),
2869 Condition(4), Condition(0), Condition(2));
2870 } else {
2871 CONVERT_INT32_TO_FLOAT(ConvertIntToFloat, 0)
2872 }
2873 break;
2874 }
2875 case kS390_F32x4UConvertI32x4: {
2876 if (CpuFeatures::IsSupported(VECTOR_ENHANCE_FACILITY_2)) {
2877 __ vcdlg(i.OutputSimd128Register(), i.InputSimd128Register(0),
2878 Condition(4), Condition(0), Condition(2));
2879 } else {
2880 CONVERT_INT32_TO_FLOAT(ConvertUnsignedIntToFloat, 0)
2881 }
2882 break;
2883 }
2884 #undef CONVERT_INT32_TO_FLOAT
2885 #define VECTOR_UNPACK(op, mode) \
2886 __ op(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(0), \
2887 Condition(0), Condition(mode));
2888 case kS390_I64x2SConvertI32x4Low: {
2889 VECTOR_UNPACK(vupl, 2)
2890 break;
2891 }
2892 case kS390_I64x2SConvertI32x4High: {
2893 VECTOR_UNPACK(vuph, 2)
2894 break;
2895 }
2896 case kS390_I64x2UConvertI32x4Low: {
2897 VECTOR_UNPACK(vupll, 2)
2898 break;
2899 }
2900 case kS390_I64x2UConvertI32x4High: {
2901 VECTOR_UNPACK(vuplh, 2)
2902 break;
2903 }
2904 case kS390_I32x4SConvertI16x8Low: {
2905 VECTOR_UNPACK(vupl, 1)
2906 break;
2907 }
2908 case kS390_I32x4SConvertI16x8High: {
2909 VECTOR_UNPACK(vuph, 1)
2910 break;
2911 }
2912 case kS390_I32x4UConvertI16x8Low: {
2913 VECTOR_UNPACK(vupll, 1)
2914 break;
2915 }
2916 case kS390_I32x4UConvertI16x8High: {
2917 VECTOR_UNPACK(vuplh, 1)
2918 break;
2919 }
2920 case kS390_I16x8SConvertI8x16Low: {
2921 VECTOR_UNPACK(vupl, 0)
2922 break;
2923 }
2924 case kS390_I16x8SConvertI8x16High: {
2925 VECTOR_UNPACK(vuph, 0)
2926 break;
2927 }
2928 case kS390_I16x8UConvertI8x16Low: {
2929 VECTOR_UNPACK(vupll, 0)
2930 break;
2931 }
2932 case kS390_I16x8UConvertI8x16High: {
2933 VECTOR_UNPACK(vuplh, 0)
2934 break;
2935 }
2936 #undef VECTOR_UNPACK
2937 case kS390_I16x8SConvertI32x4:
2938 __ vpks(i.OutputSimd128Register(), i.InputSimd128Register(1),
2939 i.InputSimd128Register(0), Condition(0), Condition(2));
2940 break;
2941 case kS390_I8x16SConvertI16x8:
2942 __ vpks(i.OutputSimd128Register(), i.InputSimd128Register(1),
2943 i.InputSimd128Register(0), Condition(0), Condition(1));
2944 break;
2945 #define VECTOR_PACK_UNSIGNED(mode) \
2946 Simd128Register tempFPReg = i.ToSimd128Register(instr->TempAt(0)); \
2947 __ vx(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg, Condition(0), \
2948 Condition(0), Condition(mode)); \
2949 __ vmx(tempFPReg, i.InputSimd128Register(0), kScratchDoubleReg, \
2950 Condition(0), Condition(0), Condition(mode)); \
2951 __ vmx(kScratchDoubleReg, i.InputSimd128Register(1), kScratchDoubleReg, \
2952 Condition(0), Condition(0), Condition(mode));
2953 case kS390_I16x8UConvertI32x4: {
2954 // treat inputs as signed, and saturate to unsigned (negative to 0)
2955 VECTOR_PACK_UNSIGNED(2)
2956 __ vpkls(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg,
2957 Condition(0), Condition(2));
2958 break;
2959 }
2960 case kS390_I8x16UConvertI16x8: {
2961 // treat inputs as signed, and saturate to unsigned (negative to 0)
2962 VECTOR_PACK_UNSIGNED(1)
2963 __ vpkls(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg,
2964 Condition(0), Condition(1));
2965 break;
2966 }
2967 #undef VECTOR_PACK_UNSIGNED
2968 #define BINOP_EXTRACT(op, extract_high, extract_low, mode) \
2969 Simd128Register src1 = i.InputSimd128Register(0); \
2970 Simd128Register src2 = i.InputSimd128Register(1); \
2971 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0)); \
2972 Simd128Register tempFPReg2 = i.ToSimd128Register(instr->TempAt(1)); \
2973 DCHECK_NE(src1, tempFPReg1); \
2974 DCHECK_NE(src2, tempFPReg1); \
2975 __ extract_high(kScratchDoubleReg, src1, Condition(0), Condition(0), \
2976 Condition(mode)); \
2977 __ extract_high(tempFPReg1, src2, Condition(0), Condition(0), \
2978 Condition(mode)); \
2979 __ op(kScratchDoubleReg, kScratchDoubleReg, tempFPReg1, Condition(0), \
2980 Condition(0), Condition(mode + 1)); \
2981 __ extract_low(tempFPReg1, src1, Condition(0), Condition(0), \
2982 Condition(mode)); \
2983 __ extract_low(tempFPReg2, src2, Condition(0), Condition(0), \
2984 Condition(mode)); \
2985 __ op(tempFPReg1, tempFPReg1, tempFPReg2, Condition(0), Condition(0), \
2986 Condition(mode + 1));
2987 case kS390_I16x8AddSatS: {
2988 BINOP_EXTRACT(va, vuph, vupl, 1)
2989 __ vpks(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg1,
2990 Condition(0), Condition(2));
2991 break;
2992 }
2993 case kS390_I16x8SubSatS: {
2994 BINOP_EXTRACT(vs, vuph, vupl, 1)
2995 __ vpks(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg1,
2996 Condition(0), Condition(2));
2997 break;
2998 }
2999 case kS390_I16x8AddSatU: {
3000 BINOP_EXTRACT(va, vuplh, vupll, 1)
3001 __ vpkls(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg1,
3002 Condition(0), Condition(2));
3003 break;
3004 }
3005 case kS390_I16x8SubSatU: {
3006 BINOP_EXTRACT(vs, vuplh, vupll, 1)
3007 // negative to 0
3008 __ vx(tempFPReg2, tempFPReg2, tempFPReg2, Condition(0), Condition(0),
3009 Condition(0));
3010 __ vmx(kScratchDoubleReg, tempFPReg2, kScratchDoubleReg, Condition(0),
3011 Condition(0), Condition(2));
3012 __ vmx(tempFPReg1, tempFPReg2, tempFPReg1, Condition(0), Condition(0),
3013 Condition(2));
3014 __ vpkls(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg1,
3015 Condition(0), Condition(2));
3016 break;
3017 }
3018 case kS390_I8x16AddSatS: {
3019 BINOP_EXTRACT(va, vuph, vupl, 0)
3020 __ vpks(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg1,
3021 Condition(0), Condition(1));
3022 break;
3023 }
3024 case kS390_I8x16SubSatS: {
3025 BINOP_EXTRACT(vs, vuph, vupl, 0)
3026 __ vpks(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg1,
3027 Condition(0), Condition(1));
3028 break;
3029 }
3030 case kS390_I8x16AddSatU: {
3031 BINOP_EXTRACT(va, vuplh, vupll, 0)
3032 __ vpkls(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg1,
3033 Condition(0), Condition(1));
3034 break;
3035 }
3036 case kS390_I8x16SubSatU: {
3037 BINOP_EXTRACT(vs, vuplh, vupll, 0)
3038 // negative to 0
3039 __ vx(tempFPReg2, tempFPReg2, tempFPReg2, Condition(0), Condition(0),
3040 Condition(0));
3041 __ vmx(kScratchDoubleReg, tempFPReg2, kScratchDoubleReg, Condition(0),
3042 Condition(0), Condition(1));
3043 __ vmx(tempFPReg1, tempFPReg2, tempFPReg1, Condition(0), Condition(0),
3044 Condition(1));
3045 __ vpkls(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg1,
3046 Condition(0), Condition(1));
3047 break;
3048 }
3049 #undef BINOP_EXTRACT
3050 case kS390_I8x16Shuffle: {
3051 Simd128Register dst = i.OutputSimd128Register(),
3052 src0 = i.InputSimd128Register(0),
3053 src1 = i.InputSimd128Register(1);
3054 uint64_t low = make_uint64(i.InputUint32(3), i.InputUint32(2));
3055 uint64_t high = make_uint64(i.InputUint32(5), i.InputUint32(4));
3056 __ mov(r0, Operand(low));
3057 __ mov(ip, Operand(high));
3058 __ vlvgp(kScratchDoubleReg, ip, r0);
3059 __ vperm(dst, src0, src1, kScratchDoubleReg, Condition(0), Condition(0));
3060 break;
3061 }
3062 case kS390_I8x16Swizzle: {
3063 Simd128Register dst = i.OutputSimd128Register(),
3064 src0 = i.InputSimd128Register(0),
3065 src1 = i.InputSimd128Register(1);
3066 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
3067 DCHECK_NE(src0, tempFPReg1);
3068 // Saturate the indices to 5 bits. Input indices more than 31 should
3069 // return 0.
3070 __ vrepi(kScratchDoubleReg, Operand(31), Condition(0));
3071 __ vmnl(tempFPReg1, src1, kScratchDoubleReg, Condition(0), Condition(0),
3072 Condition(0));
3073 // input needs to be reversed
3074 __ vlgv(r0, src0, MemOperand(r0, 0), Condition(3));
3075 __ vlgv(r1, src0, MemOperand(r0, 1), Condition(3));
3076 __ lrvgr(r0, r0);
3077 __ lrvgr(r1, r1);
3078 __ vlvgp(dst, r1, r0);
3079 // clear scratch
3080 __ vx(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg,
3081 Condition(0), Condition(0), Condition(0));
3082 __ vperm(dst, dst, kScratchDoubleReg, tempFPReg1, Condition(0),
3083 Condition(0));
3084 break;
3085 }
3086 case kS390_I64x2BitMask: {
3087 __ mov(kScratchReg, Operand(0x80800040));
3088 __ iihf(kScratchReg, Operand(0x80808080)); // Zeroing the high bits.
3089 __ vlvg(kScratchDoubleReg, kScratchReg, MemOperand(r0, 1), Condition(3));
3090 __ vbperm(kScratchDoubleReg, i.InputSimd128Register(0), kScratchDoubleReg,
3091 Condition(0), Condition(0), Condition(0));
3092 __ vlgv(i.OutputRegister(), kScratchDoubleReg, MemOperand(r0, 7),
3093 Condition(0));
3094 break;
3095 }
3096 case kS390_I32x4BitMask: {
3097 __ mov(kScratchReg, Operand(0x204060));
3098 __ iihf(kScratchReg, Operand(0x80808080)); // Zeroing the high bits.
3099 __ vlvg(kScratchDoubleReg, kScratchReg, MemOperand(r0, 1), Condition(3));
3100 __ vbperm(kScratchDoubleReg, i.InputSimd128Register(0), kScratchDoubleReg,
3101 Condition(0), Condition(0), Condition(0));
3102 __ vlgv(i.OutputRegister(), kScratchDoubleReg, MemOperand(r0, 7),
3103 Condition(0));
3104 break;
3105 }
3106 case kS390_I16x8BitMask: {
3107 __ mov(kScratchReg, Operand(0x40506070));
3108 __ iihf(kScratchReg, Operand(0x102030));
3109 __ vlvg(kScratchDoubleReg, kScratchReg, MemOperand(r0, 1), Condition(3));
3110 __ vbperm(kScratchDoubleReg, i.InputSimd128Register(0), kScratchDoubleReg,
3111 Condition(0), Condition(0), Condition(0));
3112 __ vlgv(i.OutputRegister(), kScratchDoubleReg, MemOperand(r0, 7),
3113 Condition(0));
3114 break;
3115 }
3116 case kS390_I8x16BitMask: {
3117 __ mov(r0, Operand(0x60687078));
3118 __ iihf(r0, Operand(0x40485058));
3119 __ mov(ip, Operand(0x20283038));
3120 __ iihf(ip, Operand(0x81018));
3121 __ vlvgp(kScratchDoubleReg, ip, r0);
3122 __ vbperm(kScratchDoubleReg, i.InputSimd128Register(0), kScratchDoubleReg,
3123 Condition(0), Condition(0), Condition(0));
3124 __ vlgv(i.OutputRegister(), kScratchDoubleReg, MemOperand(r0, 3),
3125 Condition(1));
3126 break;
3127 }
3128 case kS390_F32x4Pmin: {
3129 __ vfmin(i.OutputSimd128Register(), i.InputSimd128Register(0),
3130 i.InputSimd128Register(1), Condition(3), Condition(0),
3131 Condition(2));
3132 break;
3133 }
3134 case kS390_F32x4Pmax: {
3135 __ vfmax(i.OutputSimd128Register(), i.InputSimd128Register(0),
3136 i.InputSimd128Register(1), Condition(3), Condition(0),
3137 Condition(2));
3138 break;
3139 }
3140 case kS390_F64x2Pmin: {
3141 __ vfmin(i.OutputSimd128Register(), i.InputSimd128Register(0),
3142 i.InputSimd128Register(1), Condition(3), Condition(0),
3143 Condition(3));
3144 break;
3145 }
3146 case kS390_F64x2Pmax: {
3147 __ vfmax(i.OutputSimd128Register(), i.InputSimd128Register(0),
3148 i.InputSimd128Register(1), Condition(3), Condition(0),
3149 Condition(3));
3150 break;
3151 }
3152 case kS390_F64x2Ceil: {
3153 __ vfi(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(6),
3154 Condition(0), Condition(3));
3155 break;
3156 }
3157 case kS390_F64x2Floor: {
3158 __ vfi(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(7),
3159 Condition(0), Condition(3));
3160 break;
3161 }
3162 case kS390_F64x2Trunc: {
3163 __ vfi(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(5),
3164 Condition(0), Condition(3));
3165 break;
3166 }
3167 case kS390_F64x2NearestInt: {
3168 __ vfi(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(4),
3169 Condition(0), Condition(3));
3170 break;
3171 }
3172 case kS390_F32x4Ceil: {
3173 __ vfi(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(6),
3174 Condition(0), Condition(2));
3175 break;
3176 }
3177 case kS390_F32x4Floor: {
3178 __ vfi(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(7),
3179 Condition(0), Condition(2));
3180 break;
3181 }
3182 case kS390_F32x4Trunc: {
3183 __ vfi(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(5),
3184 Condition(0), Condition(2));
3185 break;
3186 }
3187 case kS390_F32x4NearestInt: {
3188 __ vfi(i.OutputSimd128Register(), i.InputSimd128Register(0), Condition(4),
3189 Condition(0), Condition(2));
3190 break;
3191 }
3192 case kS390_I32x4DotI16x8S: {
3193 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
3194 __ vme(kScratchDoubleReg, i.InputSimd128Register(0),
3195 i.InputSimd128Register(1), Condition(0), Condition(0),
3196 Condition(1));
3197 __ vmo(tempFPReg1, i.InputSimd128Register(0), i.InputSimd128Register(1),
3198 Condition(0), Condition(0), Condition(1));
3199 __ va(i.OutputSimd128Register(), kScratchDoubleReg, tempFPReg1,
3200 Condition(0), Condition(0), Condition(2));
3201 break;
3202 }
3203 #define EXT_MUL(mul_even, mul_odd, merge, mode) \
3204 Simd128Register dst = i.OutputSimd128Register(), \
3205 src0 = i.InputSimd128Register(0), \
3206 src1 = i.InputSimd128Register(1); \
3207 __ mul_even(kScratchDoubleReg, src0, src1, Condition(0), Condition(0), \
3208 Condition(mode)); \
3209 __ mul_odd(dst, src0, src1, Condition(0), Condition(0), Condition(mode)); \
3210 __ merge(dst, kScratchDoubleReg, dst, Condition(0), Condition(0), \
3211 Condition(mode + 1));
3212 case kS390_I64x2ExtMulLowI32x4S: {
3213 EXT_MUL(vme, vmo, vmrl, 2)
3214 break;
3215 }
3216 case kS390_I64x2ExtMulHighI32x4S: {
3217 EXT_MUL(vme, vmo, vmrh, 2)
3218 break;
3219 }
3220 case kS390_I64x2ExtMulLowI32x4U: {
3221 EXT_MUL(vmle, vmlo, vmrl, 2)
3222 break;
3223 }
3224 case kS390_I64x2ExtMulHighI32x4U: {
3225 EXT_MUL(vmle, vmlo, vmrh, 2)
3226 break;
3227 }
3228 case kS390_I32x4ExtMulLowI16x8S: {
3229 EXT_MUL(vme, vmo, vmrl, 1)
3230 break;
3231 }
3232 case kS390_I32x4ExtMulHighI16x8S: {
3233 EXT_MUL(vme, vmo, vmrh, 1)
3234 break;
3235 }
3236 case kS390_I32x4ExtMulLowI16x8U: {
3237 EXT_MUL(vmle, vmlo, vmrl, 1)
3238 break;
3239 }
3240 case kS390_I32x4ExtMulHighI16x8U: {
3241 EXT_MUL(vmle, vmlo, vmrh, 1)
3242 break;
3243 }
3244
3245 case kS390_I16x8ExtMulLowI8x16S: {
3246 EXT_MUL(vme, vmo, vmrl, 0)
3247 break;
3248 }
3249 case kS390_I16x8ExtMulHighI8x16S: {
3250 EXT_MUL(vme, vmo, vmrh, 0)
3251 break;
3252 }
3253 case kS390_I16x8ExtMulLowI8x16U: {
3254 EXT_MUL(vmle, vmlo, vmrl, 0)
3255 break;
3256 }
3257 case kS390_I16x8ExtMulHighI8x16U: {
3258 EXT_MUL(vmle, vmlo, vmrh, 0)
3259 break;
3260 }
3261 #undef EXT_MUL
3262 #define EXT_ADD_PAIRWISE(lane_size, mul_even, mul_odd) \
3263 Simd128Register src = i.InputSimd128Register(0); \
3264 Simd128Register dst = i.OutputSimd128Register(); \
3265 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0)); \
3266 DCHECK_NE(src, tempFPReg1); \
3267 __ vrepi(tempFPReg1, Operand(1), Condition(lane_size)); \
3268 __ mul_even(kScratchDoubleReg, src, tempFPReg1, Condition(0), Condition(0), \
3269 Condition(lane_size)); \
3270 __ mul_odd(tempFPReg1, src, tempFPReg1, Condition(0), Condition(0), \
3271 Condition(lane_size)); \
3272 __ va(dst, kScratchDoubleReg, tempFPReg1, Condition(0), Condition(0), \
3273 Condition(lane_size + 1));
3274 case kS390_I32x4ExtAddPairwiseI16x8S: {
3275 EXT_ADD_PAIRWISE(1, vme, vmo)
3276 break;
3277 }
3278 case kS390_I32x4ExtAddPairwiseI16x8U: {
3279 Simd128Register src0 = i.InputSimd128Register(0);
3280 Simd128Register dst = i.OutputSimd128Register();
3281 __ vx(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg,
3282 Condition(0), Condition(0), Condition(3));
3283 __ vsum(dst, src0, kScratchDoubleReg, Condition(0), Condition(0),
3284 Condition(1));
3285 break;
3286 }
3287 case kS390_I16x8ExtAddPairwiseI8x16S: {
3288 EXT_ADD_PAIRWISE(0, vme, vmo)
3289 break;
3290 }
3291 case kS390_I16x8ExtAddPairwiseI8x16U: {
3292 EXT_ADD_PAIRWISE(0, vmle, vmlo)
3293 break;
3294 }
3295 #undef EXT_ADD_PAIRWISE
3296 #define Q15_MUL_ROAUND(accumulator, unpack) \
3297 __ unpack(tempFPReg1, src0, Condition(0), Condition(0), Condition(1)); \
3298 __ unpack(accumulator, src1, Condition(0), Condition(0), Condition(1)); \
3299 __ vml(accumulator, tempFPReg1, accumulator, Condition(0), Condition(0), \
3300 Condition(2)); \
3301 __ va(accumulator, accumulator, tempFPReg2, Condition(0), Condition(0), \
3302 Condition(2)); \
3303 __ vrepi(tempFPReg1, Operand(15), Condition(2)); \
3304 __ vesrav(accumulator, accumulator, tempFPReg1, Condition(0), Condition(0), \
3305 Condition(2));
3306 case kS390_I16x8Q15MulRSatS: {
3307 Simd128Register dst = i.OutputSimd128Register();
3308 Simd128Register src0 = i.InputSimd128Register(0);
3309 Simd128Register src1 = i.InputSimd128Register(1);
3310 Simd128Register tempFPReg1 = i.ToSimd128Register(instr->TempAt(0));
3311 Simd128Register tempFPReg2 = i.ToSimd128Register(instr->TempAt(1));
3312 DCHECK_NE(src1, tempFPReg1);
3313 DCHECK_NE(src0, tempFPReg2);
3314 DCHECK_NE(src1, tempFPReg2);
3315 __ vrepi(tempFPReg2, Operand(0x4000), Condition(2));
3316 Q15_MUL_ROAUND(kScratchDoubleReg, vupl)
3317 Q15_MUL_ROAUND(dst, vuph)
3318 __ vpks(dst, dst, kScratchDoubleReg, Condition(0), Condition(2));
3319 break;
3320 }
3321 #undef Q15_MUL_ROAUND
3322 case kS390_I8x16Popcnt: {
3323 __ vpopct(i.OutputSimd128Register(), i.InputSimd128Register(0),
3324 Condition(0), Condition(0), Condition(0));
3325 break;
3326 }
3327 case kS390_F64x2ConvertLowI32x4S: {
3328 __ vupl(kScratchDoubleReg, i.InputSimd128Register(0), Condition(0),
3329 Condition(0), Condition(2));
3330 __ vcdg(i.OutputSimd128Register(), kScratchDoubleReg, Condition(4),
3331 Condition(0), Condition(3));
3332 break;
3333 }
3334 case kS390_F64x2ConvertLowI32x4U: {
3335 __ vupll(kScratchDoubleReg, i.InputSimd128Register(0), Condition(0),
3336 Condition(0), Condition(2));
3337 __ vcdlg(i.OutputSimd128Register(), kScratchDoubleReg, Condition(4),
3338 Condition(0), Condition(3));
3339 break;
3340 }
3341 case kS390_F64x2PromoteLowF32x4: {
3342 Register holder = r1;
3343 for (int index = 0; index < 2; ++index) {
3344 __ vlgv(r0, i.InputSimd128Register(0), MemOperand(r0, index + 2),
3345 Condition(2));
3346 __ MovIntToFloat(kScratchDoubleReg, r0);
3347 __ ldebr(kScratchDoubleReg, kScratchDoubleReg);
3348 __ MovDoubleToInt64(holder, kScratchDoubleReg);
3349 holder = ip;
3350 }
3351 __ vlvgp(i.OutputSimd128Register(), r1, ip);
3352 break;
3353 }
3354 case kS390_F32x4DemoteF64x2Zero: {
3355 Simd128Register dst = i.OutputSimd128Register();
3356 Register holder = r1;
3357 for (int index = 0; index < 2; ++index) {
3358 __ vlgv(r0, i.InputSimd128Register(0), MemOperand(r0, index),
3359 Condition(3));
3360 __ MovInt64ToDouble(kScratchDoubleReg, r0);
3361 __ ledbr(kScratchDoubleReg, kScratchDoubleReg);
3362 __ MovFloatToInt(holder, kScratchDoubleReg);
3363 holder = ip;
3364 }
3365 __ vx(dst, dst, dst, Condition(0), Condition(0), Condition(2));
3366 __ vlvg(dst, r1, MemOperand(r0, 2), Condition(2));
3367 __ vlvg(dst, ip, MemOperand(r0, 3), Condition(2));
3368 break;
3369 }
3370 case kS390_I32x4TruncSatF64x2SZero: {
3371 Simd128Register src = i.InputSimd128Register(0);
3372 Simd128Register dst = i.OutputSimd128Register();
3373 // NaN to 0
3374 __ vlr(kScratchDoubleReg, src, Condition(0), Condition(0), Condition(0));
3375 __ vfce(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg,
3376 Condition(0), Condition(0), Condition(3));
3377 __ vn(kScratchDoubleReg, src, kScratchDoubleReg, Condition(0),
3378 Condition(0), Condition(0));
3379 __ vcgd(kScratchDoubleReg, kScratchDoubleReg, Condition(5), Condition(0),
3380 Condition(3));
3381 __ vx(dst, dst, dst, Condition(0), Condition(0), Condition(2));
3382 __ vpks(dst, dst, kScratchDoubleReg, Condition(0), Condition(3));
3383 break;
3384 }
3385 case kS390_I32x4TruncSatF64x2UZero: {
3386 Simd128Register dst = i.OutputSimd128Register();
3387 __ vclgd(kScratchDoubleReg, i.InputSimd128Register(0), Condition(5),
3388 Condition(0), Condition(3));
3389 __ vx(dst, dst, dst, Condition(0), Condition(0), Condition(2));
3390 __ vpkls(dst, dst, kScratchDoubleReg, Condition(0), Condition(3));
3391 break;
3392 }
3393 #define LOAD_SPLAT(type) \
3394 AddressingMode mode = kMode_None; \
3395 MemOperand operand = i.MemoryOperand(&mode); \
3396 Simd128Register dst = i.OutputSimd128Register(); \
3397 __ LoadAndSplat##type##LE(dst, operand);
3398 case kS390_S128Load64Splat: {
3399 LOAD_SPLAT(64x2);
3400 break;
3401 }
3402 case kS390_S128Load32Splat: {
3403 LOAD_SPLAT(32x4);
3404 break;
3405 }
3406 case kS390_S128Load16Splat: {
3407 LOAD_SPLAT(16x8);
3408 break;
3409 }
3410 case kS390_S128Load8Splat: {
3411 LOAD_SPLAT(8x16);
3412 break;
3413 }
3414 #undef LOAD_SPLAT
3415 #define LOAD_EXTEND(type) \
3416 AddressingMode mode = kMode_None; \
3417 MemOperand operand = i.MemoryOperand(&mode); \
3418 Simd128Register dst = i.OutputSimd128Register(); \
3419 __ LoadAndExtend##type##LE(dst, operand);
3420 case kS390_S128Load32x2U: {
3421 LOAD_EXTEND(32x2U);
3422 break;
3423 }
3424 case kS390_S128Load32x2S: {
3425 LOAD_EXTEND(32x2S);
3426 break;
3427 }
3428 case kS390_S128Load16x4U: {
3429 LOAD_EXTEND(16x4U);
3430 break;
3431 }
3432 case kS390_S128Load16x4S: {
3433 LOAD_EXTEND(16x4S);
3434 break;
3435 }
3436 case kS390_S128Load8x8U: {
3437 LOAD_EXTEND(8x8U);
3438 break;
3439 }
3440 case kS390_S128Load8x8S: {
3441 LOAD_EXTEND(8x8S);
3442 break;
3443 }
3444 #undef LOAD_EXTEND
3445 #define LOAD_AND_ZERO(type) \
3446 AddressingMode mode = kMode_None; \
3447 MemOperand operand = i.MemoryOperand(&mode); \
3448 Simd128Register dst = i.OutputSimd128Register(); \
3449 __ LoadV##type##ZeroLE(dst, operand);
3450 case kS390_S128Load32Zero: {
3451 LOAD_AND_ZERO(32);
3452 break;
3453 }
3454 case kS390_S128Load64Zero: {
3455 LOAD_AND_ZERO(64);
3456 break;
3457 }
3458 #undef LOAD_AND_ZERO
3459 #undef LOAD_EXTEND
3460 #define LOAD_LANE(type, lane) \
3461 AddressingMode mode = kMode_None; \
3462 size_t index = 2; \
3463 MemOperand operand = i.MemoryOperand(&mode, &index); \
3464 Simd128Register dst = i.OutputSimd128Register(); \
3465 DCHECK_EQ(dst, i.InputSimd128Register(0)); \
3466 __ LoadLane##type##LE(dst, operand, lane);
3467 case kS390_S128Load8Lane: {
3468 LOAD_LANE(8, 15 - i.InputUint8(1));
3469 break;
3470 }
3471 case kS390_S128Load16Lane: {
3472 LOAD_LANE(16, 7 - i.InputUint8(1));
3473 break;
3474 }
3475 case kS390_S128Load32Lane: {
3476 LOAD_LANE(32, 3 - i.InputUint8(1));
3477 break;
3478 }
3479 case kS390_S128Load64Lane: {
3480 LOAD_LANE(64, 1 - i.InputUint8(1));
3481 break;
3482 }
3483 #undef LOAD_LANE
3484 #define STORE_LANE(type, lane) \
3485 AddressingMode mode = kMode_None; \
3486 size_t index = 2; \
3487 MemOperand operand = i.MemoryOperand(&mode, &index); \
3488 Simd128Register src = i.InputSimd128Register(0); \
3489 __ StoreLane##type##LE(src, operand, lane);
3490 case kS390_S128Store8Lane: {
3491 STORE_LANE(8, 15 - i.InputUint8(1));
3492 break;
3493 }
3494 case kS390_S128Store16Lane: {
3495 STORE_LANE(16, 7 - i.InputUint8(1));
3496 break;
3497 }
3498 case kS390_S128Store32Lane: {
3499 STORE_LANE(32, 3 - i.InputUint8(1));
3500 break;
3501 }
3502 case kS390_S128Store64Lane: {
3503 STORE_LANE(64, 1 - i.InputUint8(1));
3504 break;
3505 }
3506 #undef STORE_LANE
3507 case kS390_StoreCompressTagged: {
3508 CHECK(!instr->HasOutput());
3509 size_t index = 0;
3510 AddressingMode mode = kMode_None;
3511 MemOperand operand = i.MemoryOperand(&mode, &index);
3512 Register value = i.InputRegister(index);
3513 __ StoreTaggedField(value, operand, r1);
3514 break;
3515 }
3516 case kS390_LoadDecompressTaggedSigned: {
3517 CHECK(instr->HasOutput());
3518 __ DecompressTaggedSigned(i.OutputRegister(), i.MemoryOperand());
3519 break;
3520 }
3521 case kS390_LoadDecompressTaggedPointer: {
3522 CHECK(instr->HasOutput());
3523 __ DecompressTaggedPointer(i.OutputRegister(), i.MemoryOperand());
3524 break;
3525 }
3526 case kS390_LoadDecompressAnyTagged: {
3527 CHECK(instr->HasOutput());
3528 __ DecompressAnyTagged(i.OutputRegister(), i.MemoryOperand());
3529 break;
3530 }
3531 default:
3532 UNREACHABLE();
3533 }
3534 return kSuccess;
3535 }
3536
3537 // Assembles branches after an instruction.
AssembleArchBranch(Instruction * instr,BranchInfo * branch)3538 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
3539 S390OperandConverter i(this, instr);
3540 Label* tlabel = branch->true_label;
3541 Label* flabel = branch->false_label;
3542 ArchOpcode op = instr->arch_opcode();
3543 FlagsCondition condition = branch->condition;
3544
3545 Condition cond = FlagsConditionToCondition(condition, op);
3546 if (op == kS390_CmpFloat || op == kS390_CmpDouble) {
3547 // check for unordered if necessary
3548 // Branching to flabel/tlabel according to what's expected by tests
3549 if (cond == le || cond == eq || cond == lt) {
3550 __ bunordered(flabel);
3551 } else if (cond == gt || cond == ne || cond == ge) {
3552 __ bunordered(tlabel);
3553 }
3554 }
3555 __ b(cond, tlabel);
3556 if (!branch->fallthru) __ b(flabel); // no fallthru to flabel.
3557 }
3558
AssembleArchDeoptBranch(Instruction * instr,BranchInfo * branch)3559 void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr,
3560 BranchInfo* branch) {
3561 AssembleArchBranch(instr, branch);
3562 }
3563
AssembleArchJump(RpoNumber target)3564 void CodeGenerator::AssembleArchJump(RpoNumber target) {
3565 if (!IsNextInAssemblyOrder(target)) __ b(GetLabel(target));
3566 }
3567
3568 #if V8_ENABLE_WEBASSEMBLY
AssembleArchTrap(Instruction * instr,FlagsCondition condition)3569 void CodeGenerator::AssembleArchTrap(Instruction* instr,
3570 FlagsCondition condition) {
3571 class OutOfLineTrap final : public OutOfLineCode {
3572 public:
3573 OutOfLineTrap(CodeGenerator* gen, Instruction* instr)
3574 : OutOfLineCode(gen), instr_(instr), gen_(gen) {}
3575
3576 void Generate() final {
3577 S390OperandConverter i(gen_, instr_);
3578 TrapId trap_id =
3579 static_cast<TrapId>(i.InputInt32(instr_->InputCount() - 1));
3580 GenerateCallToTrap(trap_id);
3581 }
3582
3583 private:
3584 void GenerateCallToTrap(TrapId trap_id) {
3585 if (trap_id == TrapId::kInvalid) {
3586 // We cannot test calls to the runtime in cctest/test-run-wasm.
3587 // Therefore we emit a call to C here instead of a call to the runtime.
3588 // We use the context register as the scratch register, because we do
3589 // not have a context here.
3590 __ PrepareCallCFunction(0, 0, cp);
3591 __ CallCFunction(
3592 ExternalReference::wasm_call_trap_callback_for_testing(), 0);
3593 __ LeaveFrame(StackFrame::WASM);
3594 auto call_descriptor = gen_->linkage()->GetIncomingDescriptor();
3595 int pop_count = static_cast<int>(call_descriptor->ParameterSlotCount());
3596 __ Drop(pop_count);
3597 __ Ret();
3598 } else {
3599 gen_->AssembleSourcePosition(instr_);
3600 // A direct call to a wasm runtime stub defined in this module.
3601 // Just encode the stub index. This will be patched when the code
3602 // is added to the native module and copied into wasm code space.
3603 __ Call(static_cast<Address>(trap_id), RelocInfo::WASM_STUB_CALL);
3604 ReferenceMap* reference_map =
3605 gen_->zone()->New<ReferenceMap>(gen_->zone());
3606 gen_->RecordSafepoint(reference_map);
3607 if (FLAG_debug_code) {
3608 __ stop();
3609 }
3610 }
3611 }
3612
3613 Instruction* instr_;
3614 CodeGenerator* gen_;
3615 };
3616 auto ool = zone()->New<OutOfLineTrap>(this, instr);
3617 Label* tlabel = ool->entry();
3618 Label end;
3619
3620 ArchOpcode op = instr->arch_opcode();
3621 Condition cond = FlagsConditionToCondition(condition, op);
3622 if (op == kS390_CmpFloat || op == kS390_CmpDouble) {
3623 // check for unordered if necessary
3624 if (cond == le || cond == eq || cond == lt) {
3625 __ bunordered(&end);
3626 } else if (cond == gt || cond == ne || cond == ge) {
3627 __ bunordered(tlabel);
3628 }
3629 }
3630 __ b(cond, tlabel);
3631 __ bind(&end);
3632 }
3633 #endif // V8_ENABLE_WEBASSEMBLY
3634
3635 // Assembles boolean materializations after an instruction.
AssembleArchBoolean(Instruction * instr,FlagsCondition condition)3636 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
3637 FlagsCondition condition) {
3638 S390OperandConverter i(this, instr);
3639 ArchOpcode op = instr->arch_opcode();
3640 bool check_unordered = (op == kS390_CmpDouble || op == kS390_CmpFloat);
3641
3642 // Overflow checked for add/sub only.
3643 DCHECK((condition != kOverflow && condition != kNotOverflow) ||
3644 (op == kS390_Add32 || op == kS390_Add64 || op == kS390_Sub32 ||
3645 op == kS390_Sub64 || op == kS390_Mul32));
3646
3647 // Materialize a full 32-bit 1 or 0 value. The result register is always the
3648 // last output of the instruction.
3649 DCHECK_NE(0u, instr->OutputCount());
3650 Register reg = i.OutputRegister(instr->OutputCount() - 1);
3651 Condition cond = FlagsConditionToCondition(condition, op);
3652 Label done;
3653 if (check_unordered) {
3654 __ mov(reg, (cond == eq || cond == le || cond == lt) ? Operand::Zero()
3655 : Operand(1));
3656 __ bunordered(&done);
3657 }
3658
3659 // TODO(john.yan): use load imm high on condition here
3660 __ mov(reg, Operand::Zero());
3661 __ mov(kScratchReg, Operand(1));
3662 // locr is sufficient since reg's upper 32 is guarrantee to be 0
3663 __ locr(cond, reg, kScratchReg);
3664 __ bind(&done);
3665 }
3666
AssembleArchBinarySearchSwitch(Instruction * instr)3667 void CodeGenerator::AssembleArchBinarySearchSwitch(Instruction* instr) {
3668 S390OperandConverter i(this, instr);
3669 Register input = i.InputRegister(0);
3670 std::vector<std::pair<int32_t, Label*>> cases;
3671 for (size_t index = 2; index < instr->InputCount(); index += 2) {
3672 cases.push_back({i.InputInt32(index + 0), GetLabel(i.InputRpo(index + 1))});
3673 }
3674 AssembleArchBinarySearchSwitchRange(input, i.InputRpo(1), cases.data(),
3675 cases.data() + cases.size());
3676 }
3677
AssembleArchTableSwitch(Instruction * instr)3678 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
3679 S390OperandConverter i(this, instr);
3680 Register input = i.InputRegister(0);
3681 int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
3682 Label** cases = zone()->NewArray<Label*>(case_count);
3683 for (int32_t index = 0; index < case_count; ++index) {
3684 cases[index] = GetLabel(i.InputRpo(index + 2));
3685 }
3686 Label* const table = AddJumpTable(cases, case_count);
3687 __ CmpU64(input, Operand(case_count));
3688 __ bge(GetLabel(i.InputRpo(1)));
3689 __ larl(kScratchReg, table);
3690 __ ShiftLeftU64(r1, input, Operand(kSystemPointerSizeLog2));
3691 __ LoadU64(kScratchReg, MemOperand(kScratchReg, r1));
3692 __ Jump(kScratchReg);
3693 }
3694
AssembleArchSelect(Instruction * instr,FlagsCondition condition)3695 void CodeGenerator::AssembleArchSelect(Instruction* instr,
3696 FlagsCondition condition) {
3697 UNIMPLEMENTED();
3698 }
3699
FinishFrame(Frame * frame)3700 void CodeGenerator::FinishFrame(Frame* frame) {
3701 auto call_descriptor = linkage()->GetIncomingDescriptor();
3702 const RegList double_saves = call_descriptor->CalleeSavedFPRegisters();
3703
3704 // Save callee-saved Double registers.
3705 if (double_saves != 0) {
3706 frame->AlignSavedCalleeRegisterSlots();
3707 DCHECK_EQ(kNumCalleeSavedDoubles,
3708 base::bits::CountPopulation(double_saves));
3709 frame->AllocateSavedCalleeRegisterSlots(kNumCalleeSavedDoubles *
3710 (kDoubleSize / kSystemPointerSize));
3711 }
3712 // Save callee-saved registers.
3713 const RegList saves = call_descriptor->CalleeSavedRegisters();
3714 if (saves != 0) {
3715 // register save area does not include the fp or constant pool pointer.
3716 const int num_saves = kNumCalleeSaved - 1;
3717 frame->AllocateSavedCalleeRegisterSlots(num_saves);
3718 }
3719 }
3720
AssembleConstructFrame()3721 void CodeGenerator::AssembleConstructFrame() {
3722 auto call_descriptor = linkage()->GetIncomingDescriptor();
3723
3724 if (frame_access_state()->has_frame()) {
3725 if (call_descriptor->IsCFunctionCall()) {
3726 #if V8_ENABLE_WEBASSEMBLY
3727 if (info()->GetOutputStackFrameType() == StackFrame::C_WASM_ENTRY) {
3728 __ StubPrologue(StackFrame::C_WASM_ENTRY);
3729 // Reserve stack space for saving the c_entry_fp later.
3730 __ lay(sp, MemOperand(sp, -kSystemPointerSize));
3731 #else
3732 // For balance.
3733 if (false) {
3734 #endif // V8_ENABLE_WEBASSEMBLY
3735 } else {
3736 __ Push(r14, fp);
3737 __ mov(fp, sp);
3738 }
3739 } else if (call_descriptor->IsJSFunctionCall()) {
3740 __ Prologue(ip);
3741 } else {
3742 StackFrame::Type type = info()->GetOutputStackFrameType();
3743 // TODO(mbrandy): Detect cases where ip is the entrypoint (for
3744 // efficient intialization of the constant pool pointer register).
3745 __ StubPrologue(type);
3746 #if V8_ENABLE_WEBASSEMBLY
3747 if (call_descriptor->IsWasmFunctionCall()) {
3748 __ Push(kWasmInstanceRegister);
3749 } else if (call_descriptor->IsWasmImportWrapper() ||
3750 call_descriptor->IsWasmCapiFunction()) {
3751 // Wasm import wrappers are passed a tuple in the place of the instance.
3752 // Unpack the tuple into the instance and the target callable.
3753 // This must be done here in the codegen because it cannot be expressed
3754 // properly in the graph.
3755 __ LoadTaggedPointerField(
3756 kJSFunctionRegister,
3757 FieldMemOperand(kWasmInstanceRegister, Tuple2::kValue2Offset), r0);
3758 __ LoadTaggedPointerField(
3759 kWasmInstanceRegister,
3760 FieldMemOperand(kWasmInstanceRegister, Tuple2::kValue1Offset), r0);
3761 __ Push(kWasmInstanceRegister);
3762 if (call_descriptor->IsWasmCapiFunction()) {
3763 // Reserve space for saving the PC later.
3764 __ lay(sp, MemOperand(sp, -kSystemPointerSize));
3765 }
3766 }
3767 #endif // V8_ENABLE_WEBASSEMBLY
3768 }
3769 unwinding_info_writer_.MarkFrameConstructed(__ pc_offset());
3770 }
3771
3772 int required_slots =
3773 frame()->GetTotalFrameSlotCount() - frame()->GetFixedSlotCount();
3774 if (info()->is_osr()) {
3775 // TurboFan OSR-compiled functions cannot be entered directly.
3776 __ Abort(AbortReason::kShouldNotDirectlyEnterOsrFunction);
3777
3778 // Unoptimized code jumps directly to this entrypoint while the unoptimized
3779 // frame is still on the stack. Optimized code uses OSR values directly from
3780 // the unoptimized frame. Thus, all that needs to be done is to allocate the
3781 // remaining stack slots.
3782 __ RecordComment("-- OSR entrypoint --");
3783 osr_pc_offset_ = __ pc_offset();
3784 required_slots -= osr_helper()->UnoptimizedFrameSlots();
3785 }
3786
3787 const RegList saves_fp = call_descriptor->CalleeSavedFPRegisters();
3788 const RegList saves = call_descriptor->CalleeSavedRegisters();
3789
3790 if (required_slots > 0) {
3791 #if V8_ENABLE_WEBASSEMBLY
3792 if (info()->IsWasm() && required_slots * kSystemPointerSize > 4 * KB) {
3793 // For WebAssembly functions with big frames we have to do the stack
3794 // overflow check before we construct the frame. Otherwise we may not
3795 // have enough space on the stack to call the runtime for the stack
3796 // overflow.
3797 Label done;
3798
3799 // If the frame is bigger than the stack, we throw the stack overflow
3800 // exception unconditionally. Thereby we can avoid the integer overflow
3801 // check in the condition code.
3802 if (required_slots * kSystemPointerSize < FLAG_stack_size * KB) {
3803 Register scratch = r1;
3804 __ LoadU64(
3805 scratch,
3806 FieldMemOperand(kWasmInstanceRegister,
3807 WasmInstanceObject::kRealStackLimitAddressOffset));
3808 __ LoadU64(scratch, MemOperand(scratch));
3809 __ AddS64(scratch, scratch,
3810 Operand(required_slots * kSystemPointerSize));
3811 __ CmpU64(sp, scratch);
3812 __ bge(&done);
3813 }
3814
3815 __ Call(wasm::WasmCode::kWasmStackOverflow, RelocInfo::WASM_STUB_CALL);
3816 // The call does not return, hence we can ignore any references and just
3817 // define an empty safepoint.
3818 ReferenceMap* reference_map = zone()->New<ReferenceMap>(zone());
3819 RecordSafepoint(reference_map);
3820 if (FLAG_debug_code) __ stop();
3821
3822 __ bind(&done);
3823 }
3824 #endif // V8_ENABLE_WEBASSEMBLY
3825
3826 // Skip callee-saved and return slots, which are pushed below.
3827 required_slots -= base::bits::CountPopulation(saves);
3828 required_slots -= frame()->GetReturnSlotCount();
3829 required_slots -= (kDoubleSize / kSystemPointerSize) *
3830 base::bits::CountPopulation(saves_fp);
3831 __ lay(sp, MemOperand(sp, -required_slots * kSystemPointerSize));
3832 }
3833
3834 // Save callee-saved Double registers.
3835 if (saves_fp != 0) {
3836 __ MultiPushDoubles(saves_fp);
3837 DCHECK_EQ(kNumCalleeSavedDoubles, base::bits::CountPopulation(saves_fp));
3838 }
3839
3840 // Save callee-saved registers.
3841 if (saves != 0) {
3842 __ MultiPush(saves);
3843 // register save area does not include the fp or constant pool pointer.
3844 }
3845
3846 const int returns = frame()->GetReturnSlotCount();
3847 // Create space for returns.
3848 __ AllocateStackSpace(returns * kSystemPointerSize);
3849 }
3850
3851 void CodeGenerator::AssembleReturn(InstructionOperand* additional_pop_count) {
3852 auto call_descriptor = linkage()->GetIncomingDescriptor();
3853
3854 const int returns = frame()->GetReturnSlotCount();
3855 if (returns != 0) {
3856 // Create space for returns.
3857 __ lay(sp, MemOperand(sp, returns * kSystemPointerSize));
3858 }
3859
3860 // Restore registers.
3861 const RegList saves = call_descriptor->CalleeSavedRegisters();
3862 if (saves != 0) {
3863 __ MultiPop(saves);
3864 }
3865
3866 // Restore double registers.
3867 const RegList double_saves = call_descriptor->CalleeSavedFPRegisters();
3868 if (double_saves != 0) {
3869 __ MultiPopDoubles(double_saves);
3870 }
3871
3872 unwinding_info_writer_.MarkBlockWillExit();
3873
3874 // We might need r3 for scratch.
3875 DCHECK_EQ(0u, call_descriptor->CalleeSavedRegisters() & r5.bit());
3876 S390OperandConverter g(this, nullptr);
3877 const int parameter_slots =
3878 static_cast<int>(call_descriptor->ParameterSlotCount());
3879
3880 // {aditional_pop_count} is only greater than zero if {parameter_slots = 0}.
3881 // Check RawMachineAssembler::PopAndReturn.
3882 if (parameter_slots != 0) {
3883 if (additional_pop_count->IsImmediate()) {
3884 DCHECK_EQ(g.ToConstant(additional_pop_count).ToInt32(), 0);
3885 } else if (FLAG_debug_code) {
3886 __ CmpS64(g.ToRegister(additional_pop_count), Operand(0));
3887 __ Assert(eq, AbortReason::kUnexpectedAdditionalPopValue);
3888 }
3889 }
3890
3891 Register argc_reg = r5;
3892 // Functions with JS linkage have at least one parameter (the receiver).
3893 // If {parameter_slots} == 0, it means it is a builtin with
3894 // kDontAdaptArgumentsSentinel, which takes care of JS arguments popping
3895 // itself.
3896 const bool drop_jsargs = parameter_slots != 0 &&
3897 frame_access_state()->has_frame() &&
3898 call_descriptor->IsJSFunctionCall();
3899
3900 if (call_descriptor->IsCFunctionCall()) {
3901 AssembleDeconstructFrame();
3902 } else if (frame_access_state()->has_frame()) {
3903 // Canonicalize JSFunction return sites for now unless they have an variable
3904 // number of stack slot pops
3905 if (additional_pop_count->IsImmediate() &&
3906 g.ToConstant(additional_pop_count).ToInt32() == 0) {
3907 if (return_label_.is_bound()) {
3908 __ b(&return_label_);
3909 return;
3910 } else {
3911 __ bind(&return_label_);
3912 }
3913 }
3914 if (drop_jsargs) {
3915 // Get the actual argument count.
3916 DCHECK_EQ(0u, call_descriptor->CalleeSavedRegisters() & argc_reg.bit());
3917 __ LoadU64(argc_reg, MemOperand(fp, StandardFrameConstants::kArgCOffset));
3918 }
3919 AssembleDeconstructFrame();
3920 }
3921
3922 if (drop_jsargs) {
3923 // We must pop all arguments from the stack (including the receiver).
3924 // The number of arguments without the receiver is
3925 // max(argc_reg, parameter_slots-1), and the receiver is added in
3926 // DropArguments().
3927 if (parameter_slots > 1) {
3928 const int parameter_slots_without_receiver = parameter_slots - 1;
3929 Label skip;
3930 __ CmpS64(argc_reg, Operand(parameter_slots_without_receiver));
3931 __ bgt(&skip);
3932 __ mov(argc_reg, Operand(parameter_slots_without_receiver));
3933 __ bind(&skip);
3934 }
3935 __ DropArguments(argc_reg, TurboAssembler::kCountIsInteger,
3936 TurboAssembler::kCountExcludesReceiver);
3937 } else if (additional_pop_count->IsImmediate()) {
3938 int additional_count = g.ToConstant(additional_pop_count).ToInt32();
3939 __ Drop(parameter_slots + additional_count);
3940 } else if (parameter_slots == 0) {
3941 __ Drop(g.ToRegister(additional_pop_count));
3942 } else {
3943 // {additional_pop_count} is guaranteed to be zero if {parameter_slots !=
3944 // 0}. Check RawMachineAssembler::PopAndReturn.
3945 __ Drop(parameter_slots);
3946 }
3947 __ Ret();
3948 }
3949
3950 void CodeGenerator::FinishCode() {}
3951
3952 void CodeGenerator::PrepareForDeoptimizationExits(
3953 ZoneDeque<DeoptimizationExit*>* exits) {}
3954
3955 void CodeGenerator::AssembleMove(InstructionOperand* source,
3956 InstructionOperand* destination) {
3957 S390OperandConverter g(this, nullptr);
3958 // Dispatch on the source and destination operand kinds. Not all
3959 // combinations are possible.
3960 if (source->IsRegister()) {
3961 DCHECK(destination->IsRegister() || destination->IsStackSlot());
3962 Register src = g.ToRegister(source);
3963 if (destination->IsRegister()) {
3964 __ Move(g.ToRegister(destination), src);
3965 } else {
3966 __ StoreU64(src, g.ToMemOperand(destination));
3967 }
3968 } else if (source->IsStackSlot()) {
3969 DCHECK(destination->IsRegister() || destination->IsStackSlot());
3970 MemOperand src = g.ToMemOperand(source);
3971 if (destination->IsRegister()) {
3972 __ LoadU64(g.ToRegister(destination), src);
3973 } else {
3974 Register temp = kScratchReg;
3975 __ LoadU64(temp, src, r0);
3976 __ StoreU64(temp, g.ToMemOperand(destination));
3977 }
3978 } else if (source->IsConstant()) {
3979 Constant src = g.ToConstant(source);
3980 if (destination->IsRegister() || destination->IsStackSlot()) {
3981 Register dst =
3982 destination->IsRegister() ? g.ToRegister(destination) : kScratchReg;
3983 switch (src.type()) {
3984 case Constant::kInt32:
3985 __ mov(dst, Operand(src.ToInt32()));
3986 break;
3987 case Constant::kInt64:
3988 #if V8_ENABLE_WEBASSEMBLY
3989 if (RelocInfo::IsWasmReference(src.rmode())) {
3990 __ mov(dst, Operand(src.ToInt64(), src.rmode()));
3991 break;
3992 }
3993 #endif // V8_ENABLE_WEBASSEMBLY
3994 __ mov(dst, Operand(src.ToInt64()));
3995 break;
3996 case Constant::kFloat32:
3997 __ mov(dst, Operand::EmbeddedNumber(src.ToFloat32()));
3998 break;
3999 case Constant::kFloat64:
4000 __ mov(dst, Operand::EmbeddedNumber(src.ToFloat64().value()));
4001 break;
4002 case Constant::kExternalReference:
4003 __ Move(dst, src.ToExternalReference());
4004 break;
4005 case Constant::kDelayedStringConstant:
4006 __ mov(dst, Operand::EmbeddedStringConstant(
4007 src.ToDelayedStringConstant()));
4008 break;
4009 case Constant::kHeapObject: {
4010 Handle<HeapObject> src_object = src.ToHeapObject();
4011 RootIndex index;
4012 if (IsMaterializableFromRoot(src_object, &index)) {
4013 __ LoadRoot(dst, index);
4014 } else {
4015 __ Move(dst, src_object);
4016 }
4017 break;
4018 }
4019 case Constant::kCompressedHeapObject: {
4020 Handle<HeapObject> src_object = src.ToHeapObject();
4021 RootIndex index;
4022 if (IsMaterializableFromRoot(src_object, &index)) {
4023 __ LoadRoot(dst, index);
4024 } else {
4025 __ Move(dst, src_object, RelocInfo::COMPRESSED_EMBEDDED_OBJECT);
4026 }
4027 break;
4028 }
4029 case Constant::kRpoNumber:
4030 UNREACHABLE(); // TODO(dcarney): loading RPO constants on S390.
4031 }
4032 if (destination->IsStackSlot()) {
4033 __ StoreU64(dst, g.ToMemOperand(destination), r0);
4034 }
4035 } else {
4036 DoubleRegister dst = destination->IsFPRegister()
4037 ? g.ToDoubleRegister(destination)
4038 : kScratchDoubleReg;
4039 double value = (src.type() == Constant::kFloat32)
4040 ? src.ToFloat32()
4041 : src.ToFloat64().value();
4042 if (src.type() == Constant::kFloat32) {
4043 __ LoadF32<float>(dst, src.ToFloat32(), kScratchReg);
4044 } else {
4045 __ LoadF64<double>(dst, value, kScratchReg);
4046 }
4047
4048 if (destination->IsFloatStackSlot()) {
4049 __ StoreF32(dst, g.ToMemOperand(destination));
4050 } else if (destination->IsDoubleStackSlot()) {
4051 __ StoreF64(dst, g.ToMemOperand(destination));
4052 }
4053 }
4054 } else if (source->IsFPRegister()) {
4055 MachineRepresentation rep = LocationOperand::cast(source)->representation();
4056 if (rep == MachineRepresentation::kSimd128) {
4057 if (destination->IsSimd128Register()) {
4058 __ vlr(g.ToSimd128Register(destination), g.ToSimd128Register(source),
4059 Condition(0), Condition(0), Condition(0));
4060 } else {
4061 DCHECK(destination->IsSimd128StackSlot());
4062 __ StoreV128(g.ToSimd128Register(source), g.ToMemOperand(destination),
4063 kScratchReg);
4064 }
4065 } else {
4066 DoubleRegister src = g.ToDoubleRegister(source);
4067 if (destination->IsFPRegister()) {
4068 DoubleRegister dst = g.ToDoubleRegister(destination);
4069 __ Move(dst, src);
4070 } else {
4071 DCHECK(destination->IsFPStackSlot());
4072 LocationOperand* op = LocationOperand::cast(source);
4073 if (op->representation() == MachineRepresentation::kFloat64) {
4074 __ StoreF64(src, g.ToMemOperand(destination));
4075 } else {
4076 __ StoreF32(src, g.ToMemOperand(destination));
4077 }
4078 }
4079 }
4080 } else if (source->IsFPStackSlot()) {
4081 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
4082 MemOperand src = g.ToMemOperand(source);
4083 if (destination->IsFPRegister()) {
4084 LocationOperand* op = LocationOperand::cast(source);
4085 if (op->representation() == MachineRepresentation::kFloat64) {
4086 __ LoadF64(g.ToDoubleRegister(destination), src);
4087 } else if (op->representation() == MachineRepresentation::kFloat32) {
4088 __ LoadF32(g.ToDoubleRegister(destination), src);
4089 } else {
4090 DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
4091 __ LoadV128(g.ToSimd128Register(destination), g.ToMemOperand(source),
4092 kScratchReg);
4093 }
4094 } else {
4095 LocationOperand* op = LocationOperand::cast(source);
4096 DoubleRegister temp = kScratchDoubleReg;
4097 if (op->representation() == MachineRepresentation::kFloat64) {
4098 __ LoadF64(temp, src);
4099 __ StoreF64(temp, g.ToMemOperand(destination));
4100 } else if (op->representation() == MachineRepresentation::kFloat32) {
4101 __ LoadF32(temp, src);
4102 __ StoreF32(temp, g.ToMemOperand(destination));
4103 } else {
4104 DCHECK_EQ(MachineRepresentation::kSimd128, op->representation());
4105 __ LoadV128(kScratchDoubleReg, g.ToMemOperand(source), kScratchReg);
4106 __ StoreV128(kScratchDoubleReg, g.ToMemOperand(destination),
4107 kScratchReg);
4108 }
4109 }
4110 } else {
4111 UNREACHABLE();
4112 }
4113 }
4114
4115 // Swaping contents in source and destination.
4116 // source and destination could be:
4117 // Register,
4118 // FloatRegister,
4119 // DoubleRegister,
4120 // StackSlot,
4121 // FloatStackSlot,
4122 // or DoubleStackSlot
4123 void CodeGenerator::AssembleSwap(InstructionOperand* source,
4124 InstructionOperand* destination) {
4125 S390OperandConverter g(this, nullptr);
4126 if (source->IsRegister()) {
4127 Register src = g.ToRegister(source);
4128 if (destination->IsRegister()) {
4129 __ SwapP(src, g.ToRegister(destination), kScratchReg);
4130 } else {
4131 DCHECK(destination->IsStackSlot());
4132 __ SwapP(src, g.ToMemOperand(destination), kScratchReg);
4133 }
4134 } else if (source->IsStackSlot()) {
4135 DCHECK(destination->IsStackSlot());
4136 __ SwapP(g.ToMemOperand(source), g.ToMemOperand(destination), kScratchReg,
4137 r0);
4138 } else if (source->IsFloatRegister()) {
4139 DoubleRegister src = g.ToDoubleRegister(source);
4140 if (destination->IsFloatRegister()) {
4141 __ SwapFloat32(src, g.ToDoubleRegister(destination), kScratchDoubleReg);
4142 } else {
4143 DCHECK(destination->IsFloatStackSlot());
4144 __ SwapFloat32(src, g.ToMemOperand(destination), kScratchDoubleReg);
4145 }
4146 } else if (source->IsDoubleRegister()) {
4147 DoubleRegister src = g.ToDoubleRegister(source);
4148 if (destination->IsDoubleRegister()) {
4149 __ SwapDouble(src, g.ToDoubleRegister(destination), kScratchDoubleReg);
4150 } else {
4151 DCHECK(destination->IsDoubleStackSlot());
4152 __ SwapDouble(src, g.ToMemOperand(destination), kScratchDoubleReg);
4153 }
4154 } else if (source->IsFloatStackSlot()) {
4155 DCHECK(destination->IsFloatStackSlot());
4156 __ SwapFloat32(g.ToMemOperand(source), g.ToMemOperand(destination),
4157 kScratchDoubleReg);
4158 } else if (source->IsDoubleStackSlot()) {
4159 DCHECK(destination->IsDoubleStackSlot());
4160 __ SwapDouble(g.ToMemOperand(source), g.ToMemOperand(destination),
4161 kScratchDoubleReg);
4162 } else if (source->IsSimd128Register()) {
4163 Simd128Register src = g.ToSimd128Register(source);
4164 if (destination->IsSimd128Register()) {
4165 __ SwapSimd128(src, g.ToSimd128Register(destination), kScratchDoubleReg);
4166 } else {
4167 DCHECK(destination->IsSimd128StackSlot());
4168 __ SwapSimd128(src, g.ToMemOperand(destination), kScratchDoubleReg);
4169 }
4170 } else if (source->IsSimd128StackSlot()) {
4171 DCHECK(destination->IsSimd128StackSlot());
4172 __ SwapSimd128(g.ToMemOperand(source), g.ToMemOperand(destination),
4173 kScratchDoubleReg);
4174 } else {
4175 UNREACHABLE();
4176 }
4177 }
4178
4179 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
4180 for (size_t index = 0; index < target_count; ++index) {
4181 __ emit_label_addr(targets[index]);
4182 }
4183 }
4184
4185 #undef __
4186
4187 } // namespace compiler
4188 } // namespace internal
4189 } // namespace v8
4190