1 /*
2 * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "code/compiledIC.hpp"
38 #include "gc/shared/barrierSet.hpp"
39 #include "gc/shared/cardTableBarrierSet.hpp"
40 #include "gc/shared/collectedHeap.hpp"
41 #include "nativeInst_aarch64.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "vmreg_aarch64.inline.hpp"
46
47
48
49 #ifndef PRODUCT
50 #define COMMENT(x) do { __ block_comment(x); } while (0)
51 #else
52 #define COMMENT(x)
53 #endif
54
55 NEEDS_CLEANUP // remove this definitions ?
56 const Register IC_Klass = rscratch2; // where the IC klass is cached
57 const Register SYNC_header = r0; // synchronization header
58 const Register SHIFT_count = r0; // where count for shift operations must be
59
60 #define __ _masm->
61
62
select_different_registers(Register preserve,Register extra,Register & tmp1,Register & tmp2)63 static void select_different_registers(Register preserve,
64 Register extra,
65 Register &tmp1,
66 Register &tmp2) {
67 if (tmp1 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp1 = extra;
70 } else if (tmp2 == preserve) {
71 assert_different_registers(tmp1, tmp2, extra);
72 tmp2 = extra;
73 }
74 assert_different_registers(preserve, tmp1, tmp2);
75 }
76
77
78
select_different_registers(Register preserve,Register extra,Register & tmp1,Register & tmp2,Register & tmp3)79 static void select_different_registers(Register preserve,
80 Register extra,
81 Register &tmp1,
82 Register &tmp2,
83 Register &tmp3) {
84 if (tmp1 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp1 = extra;
87 } else if (tmp2 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp2 = extra;
90 } else if (tmp3 == preserve) {
91 assert_different_registers(tmp1, tmp2, tmp3, extra);
92 tmp3 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2, tmp3);
95 }
96
97
is_small_constant(LIR_Opr opr)98 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
99
100
receiverOpr()101 LIR_Opr LIR_Assembler::receiverOpr() {
102 return FrameMap::receiver_opr;
103 }
104
osrBufferPointer()105 LIR_Opr LIR_Assembler::osrBufferPointer() {
106 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
107 }
108
109 //--------------fpu register translations-----------------------
110
111
float_constant(float f)112 address LIR_Assembler::float_constant(float f) {
113 address const_addr = __ float_constant(f);
114 if (const_addr == NULL) {
115 bailout("const section overflow");
116 return __ code()->consts()->start();
117 } else {
118 return const_addr;
119 }
120 }
121
122
double_constant(double d)123 address LIR_Assembler::double_constant(double d) {
124 address const_addr = __ double_constant(d);
125 if (const_addr == NULL) {
126 bailout("const section overflow");
127 return __ code()->consts()->start();
128 } else {
129 return const_addr;
130 }
131 }
132
int_constant(jlong n)133 address LIR_Assembler::int_constant(jlong n) {
134 address const_addr = __ long_constant(n);
135 if (const_addr == NULL) {
136 bailout("const section overflow");
137 return __ code()->consts()->start();
138 } else {
139 return const_addr;
140 }
141 }
142
set_24bit_FPU()143 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
144
reset_FPU()145 void LIR_Assembler::reset_FPU() { Unimplemented(); }
146
fpop()147 void LIR_Assembler::fpop() { Unimplemented(); }
148
fxch(int i)149 void LIR_Assembler::fxch(int i) { Unimplemented(); }
150
fld(int i)151 void LIR_Assembler::fld(int i) { Unimplemented(); }
152
ffree(int i)153 void LIR_Assembler::ffree(int i) { Unimplemented(); }
154
breakpoint()155 void LIR_Assembler::breakpoint() { Unimplemented(); }
156
push(LIR_Opr opr)157 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
158
pop(LIR_Opr opr)159 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
160
is_literal_address(LIR_Address * addr)161 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
162 //-------------------------------------------
163
as_reg(LIR_Opr op)164 static Register as_reg(LIR_Opr op) {
165 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
166 }
167
as_long(LIR_Opr data)168 static jlong as_long(LIR_Opr data) {
169 jlong result;
170 switch (data->type()) {
171 case T_INT:
172 result = (data->as_jint());
173 break;
174 case T_LONG:
175 result = (data->as_jlong());
176 break;
177 default:
178 ShouldNotReachHere();
179 result = 0; // unreachable
180 }
181 return result;
182 }
183
as_Address(LIR_Address * addr,Register tmp)184 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
185 Register base = addr->base()->as_pointer_register();
186 LIR_Opr opr = addr->index();
187 if (opr->is_cpu_register()) {
188 Register index;
189 if (opr->is_single_cpu())
190 index = opr->as_register();
191 else
192 index = opr->as_register_lo();
193 assert(addr->disp() == 0, "must be");
194 switch(opr->type()) {
195 case T_INT:
196 return Address(base, index, Address::sxtw(addr->scale()));
197 case T_LONG:
198 return Address(base, index, Address::lsl(addr->scale()));
199 default:
200 ShouldNotReachHere();
201 }
202 } else {
203 intptr_t addr_offset = intptr_t(addr->disp());
204 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
205 return Address(base, addr_offset, Address::lsl(addr->scale()));
206 else {
207 __ mov(tmp, addr_offset);
208 return Address(base, tmp, Address::lsl(addr->scale()));
209 }
210 }
211 return Address();
212 }
213
as_Address_hi(LIR_Address * addr)214 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
215 ShouldNotReachHere();
216 return Address();
217 }
218
as_Address(LIR_Address * addr)219 Address LIR_Assembler::as_Address(LIR_Address* addr) {
220 return as_Address(addr, rscratch1);
221 }
222
as_Address_lo(LIR_Address * addr)223 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
224 return as_Address(addr, rscratch1); // Ouch
225 // FIXME: This needs to be much more clever. See x86.
226 }
227
228
osr_entry()229 void LIR_Assembler::osr_entry() {
230 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
231 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
232 ValueStack* entry_state = osr_entry->state();
233 int number_of_locks = entry_state->locks_size();
234
235 // we jump here if osr happens with the interpreter
236 // state set up to continue at the beginning of the
237 // loop that triggered osr - in particular, we have
238 // the following registers setup:
239 //
240 // r2: osr buffer
241 //
242
243 // build frame
244 ciMethod* m = compilation()->method();
245 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
246
247 // OSR buffer is
248 //
249 // locals[nlocals-1..0]
250 // monitors[0..number_of_locks]
251 //
252 // locals is a direct copy of the interpreter frame so in the osr buffer
253 // so first slot in the local array is the last local from the interpreter
254 // and last slot is local[0] (receiver) from the interpreter
255 //
256 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
257 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
258 // in the interpreter frame (the method lock if a sync method)
259
260 // Initialize monitors in the compiled activation.
261 // r2: pointer to osr buffer
262 //
263 // All other registers are dead at this point and the locals will be
264 // copied into place by code emitted in the IR.
265
266 Register OSR_buf = osrBufferPointer()->as_pointer_register();
267 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
268 int monitor_offset = BytesPerWord * method()->max_locals() +
269 (2 * BytesPerWord) * (number_of_locks - 1);
270 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
271 // the OSR buffer using 2 word entries: first the lock and then
272 // the oop.
273 for (int i = 0; i < number_of_locks; i++) {
274 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
275 #ifdef ASSERT
276 // verify the interpreter's monitor has a non-null object
277 {
278 Label L;
279 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
280 __ cbnz(rscratch1, L);
281 __ stop("locked object is NULL");
282 __ bind(L);
283 }
284 #endif
285 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
286 __ str(r19, frame_map()->address_for_monitor_lock(i));
287 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
288 __ str(r19, frame_map()->address_for_monitor_object(i));
289 }
290 }
291 }
292
293
294 // inline cache check; done before the frame is built.
check_icache()295 int LIR_Assembler::check_icache() {
296 Register receiver = FrameMap::receiver_opr->as_register();
297 Register ic_klass = IC_Klass;
298 int start_offset = __ offset();
299 __ inline_cache_check(receiver, ic_klass);
300
301 // if icache check fails, then jump to runtime routine
302 // Note: RECEIVER must still contain the receiver!
303 Label dont;
304 __ br(Assembler::EQ, dont);
305 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
306
307 // We align the verified entry point unless the method body
308 // (including its inline cache check) will fit in a single 64-byte
309 // icache line.
310 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
311 // force alignment after the cache check.
312 __ align(CodeEntryAlignment);
313 }
314
315 __ bind(dont);
316 return start_offset;
317 }
318
clinit_barrier(ciMethod * method)319 void LIR_Assembler::clinit_barrier(ciMethod* method) {
320 assert(VM_Version::supports_fast_class_init_checks(), "sanity");
321 assert(!method->holder()->is_not_initialized(), "initialization should have been started");
322
323 Label L_skip_barrier;
324
325 __ mov_metadata(rscratch2, method->holder()->constant_encoding());
326 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/);
327 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
328 __ bind(L_skip_barrier);
329 }
330
jobject2reg(jobject o,Register reg)331 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
332 if (o == NULL) {
333 __ mov(reg, zr);
334 } else {
335 __ movoop(reg, o, /*immediate*/true);
336 }
337 }
338
deoptimize_trap(CodeEmitInfo * info)339 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
340 address target = NULL;
341 relocInfo::relocType reloc_type = relocInfo::none;
342
343 switch (patching_id(info)) {
344 case PatchingStub::access_field_id:
345 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
346 reloc_type = relocInfo::section_word_type;
347 break;
348 case PatchingStub::load_klass_id:
349 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
350 reloc_type = relocInfo::metadata_type;
351 break;
352 case PatchingStub::load_mirror_id:
353 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
354 reloc_type = relocInfo::oop_type;
355 break;
356 case PatchingStub::load_appendix_id:
357 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
358 reloc_type = relocInfo::oop_type;
359 break;
360 default: ShouldNotReachHere();
361 }
362
363 __ far_call(RuntimeAddress(target));
364 add_call_info_here(info);
365 }
366
jobject2reg_with_patching(Register reg,CodeEmitInfo * info)367 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
368 deoptimize_trap(info);
369 }
370
371
372 // This specifies the rsp decrement needed to build the frame
initial_frame_size_in_bytes() const373 int LIR_Assembler::initial_frame_size_in_bytes() const {
374 // if rounding, must let FrameMap know!
375
376 // The frame_map records size in slots (32bit word)
377
378 // subtract two words to account for return address and link
379 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
380 }
381
382
emit_exception_handler()383 int LIR_Assembler::emit_exception_handler() {
384 // if the last instruction is a call (typically to do a throw which
385 // is coming at the end after block reordering) the return address
386 // must still point into the code area in order to avoid assertion
387 // failures when searching for the corresponding bci => add a nop
388 // (was bug 5/14/1999 - gri)
389 __ nop();
390
391 // generate code for exception handler
392 address handler_base = __ start_a_stub(exception_handler_size());
393 if (handler_base == NULL) {
394 // not enough space left for the handler
395 bailout("exception handler overflow");
396 return -1;
397 }
398
399 int offset = code_offset();
400
401 // the exception oop and pc are in r0, and r3
402 // no other registers need to be preserved, so invalidate them
403 __ invalidate_registers(false, true, true, false, true, true);
404
405 // check that there is really an exception
406 __ verify_not_null_oop(r0);
407
408 // search an exception handler (r0: exception oop, r3: throwing pc)
409 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
410 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
411 __ end_a_stub();
412
413 return offset;
414 }
415
416
417 // Emit the code to remove the frame from the stack in the exception
418 // unwind path.
emit_unwind_handler()419 int LIR_Assembler::emit_unwind_handler() {
420 #ifndef PRODUCT
421 if (CommentedAssembly) {
422 _masm->block_comment("Unwind handler");
423 }
424 #endif
425
426 int offset = code_offset();
427
428 // Fetch the exception from TLS and clear out exception related thread state
429 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
430 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
431 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
432
433 __ bind(_unwind_handler_entry);
434 __ verify_not_null_oop(r0);
435 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
436 __ mov(r19, r0); // Preserve the exception
437 }
438
439 // Preform needed unlocking
440 MonitorExitStub* stub = NULL;
441 if (method()->is_synchronized()) {
442 monitor_address(0, FrameMap::r0_opr);
443 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
444 __ unlock_object(r5, r4, r0, *stub->entry());
445 __ bind(*stub->continuation());
446 }
447
448 if (compilation()->env()->dtrace_method_probes()) {
449 __ call_Unimplemented();
450 #if 0
451 __ movptr(Address(rsp, 0), rax);
452 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
453 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
454 #endif
455 }
456
457 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
458 __ mov(r0, r19); // Restore the exception
459 }
460
461 // remove the activation and dispatch to the unwind handler
462 __ block_comment("remove_frame and dispatch to the unwind handler");
463 __ remove_frame(initial_frame_size_in_bytes());
464 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
465
466 // Emit the slow path assembly
467 if (stub != NULL) {
468 stub->emit_code(this);
469 }
470
471 return offset;
472 }
473
474
emit_deopt_handler()475 int LIR_Assembler::emit_deopt_handler() {
476 // if the last instruction is a call (typically to do a throw which
477 // is coming at the end after block reordering) the return address
478 // must still point into the code area in order to avoid assertion
479 // failures when searching for the corresponding bci => add a nop
480 // (was bug 5/14/1999 - gri)
481 __ nop();
482
483 // generate code for exception handler
484 address handler_base = __ start_a_stub(deopt_handler_size());
485 if (handler_base == NULL) {
486 // not enough space left for the handler
487 bailout("deopt handler overflow");
488 return -1;
489 }
490
491 int offset = code_offset();
492
493 __ adr(lr, pc());
494 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
495 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
496 __ end_a_stub();
497
498 return offset;
499 }
500
add_debug_info_for_branch(address adr,CodeEmitInfo * info)501 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
502 _masm->code_section()->relocate(adr, relocInfo::poll_type);
503 int pc_offset = code_offset();
504 flush_debug_info(pc_offset);
505 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
506 if (info->exception_handlers() != NULL) {
507 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
508 }
509 }
510
return_op(LIR_Opr result)511 void LIR_Assembler::return_op(LIR_Opr result) {
512 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
513
514 // Pop the stack before the safepoint code
515 __ remove_frame(initial_frame_size_in_bytes());
516
517 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
518 __ reserved_stack_check();
519 }
520
521 address polling_page(os::get_polling_page());
522 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
523 __ ret(lr);
524 }
525
safepoint_poll(LIR_Opr tmp,CodeEmitInfo * info)526 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
527 address polling_page(os::get_polling_page());
528 guarantee(info != NULL, "Shouldn't be NULL");
529 assert(os::is_poll_address(polling_page), "should be");
530 __ get_polling_page(rscratch1, polling_page, relocInfo::poll_type);
531 add_debug_info_for_branch(info); // This isn't just debug info:
532 // it's the oop map
533 __ read_polling_page(rscratch1, relocInfo::poll_type);
534 return __ offset();
535 }
536
537
move_regs(Register from_reg,Register to_reg)538 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
539 if (from_reg == r31_sp)
540 from_reg = sp;
541 if (to_reg == r31_sp)
542 to_reg = sp;
543 __ mov(to_reg, from_reg);
544 }
545
swap_reg(Register a,Register b)546 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
547
548
const2reg(LIR_Opr src,LIR_Opr dest,LIR_PatchCode patch_code,CodeEmitInfo * info)549 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
550 assert(src->is_constant(), "should not call otherwise");
551 assert(dest->is_register(), "should not call otherwise");
552 LIR_Const* c = src->as_constant_ptr();
553
554 switch (c->type()) {
555 case T_INT: {
556 assert(patch_code == lir_patch_none, "no patching handled here");
557 __ movw(dest->as_register(), c->as_jint());
558 break;
559 }
560
561 case T_ADDRESS: {
562 assert(patch_code == lir_patch_none, "no patching handled here");
563 __ mov(dest->as_register(), c->as_jint());
564 break;
565 }
566
567 case T_LONG: {
568 assert(patch_code == lir_patch_none, "no patching handled here");
569 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
570 break;
571 }
572
573 case T_OBJECT: {
574 if (patch_code == lir_patch_none) {
575 jobject2reg(c->as_jobject(), dest->as_register());
576 } else {
577 jobject2reg_with_patching(dest->as_register(), info);
578 }
579 break;
580 }
581
582 case T_METADATA: {
583 if (patch_code != lir_patch_none) {
584 klass2reg_with_patching(dest->as_register(), info);
585 } else {
586 __ mov_metadata(dest->as_register(), c->as_metadata());
587 }
588 break;
589 }
590
591 case T_FLOAT: {
592 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
593 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
594 } else {
595 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
596 __ ldrs(dest->as_float_reg(), Address(rscratch1));
597 }
598 break;
599 }
600
601 case T_DOUBLE: {
602 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
603 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
604 } else {
605 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
606 __ ldrd(dest->as_double_reg(), Address(rscratch1));
607 }
608 break;
609 }
610
611 default:
612 ShouldNotReachHere();
613 }
614 }
615
const2stack(LIR_Opr src,LIR_Opr dest)616 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
617 LIR_Const* c = src->as_constant_ptr();
618 switch (c->type()) {
619 case T_OBJECT:
620 {
621 if (! c->as_jobject())
622 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
623 else {
624 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
625 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
626 }
627 }
628 break;
629 case T_ADDRESS:
630 {
631 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
632 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
633 }
634 case T_INT:
635 case T_FLOAT:
636 {
637 Register reg = zr;
638 if (c->as_jint_bits() == 0)
639 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
640 else {
641 __ movw(rscratch1, c->as_jint_bits());
642 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
643 }
644 }
645 break;
646 case T_LONG:
647 case T_DOUBLE:
648 {
649 Register reg = zr;
650 if (c->as_jlong_bits() == 0)
651 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
652 lo_word_offset_in_bytes));
653 else {
654 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
655 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
656 lo_word_offset_in_bytes));
657 }
658 }
659 break;
660 default:
661 ShouldNotReachHere();
662 }
663 }
664
const2mem(LIR_Opr src,LIR_Opr dest,BasicType type,CodeEmitInfo * info,bool wide)665 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
666 assert(src->is_constant(), "should not call otherwise");
667 LIR_Const* c = src->as_constant_ptr();
668 LIR_Address* to_addr = dest->as_address_ptr();
669
670 void (Assembler::* insn)(Register Rt, const Address &adr);
671
672 switch (type) {
673 case T_ADDRESS:
674 assert(c->as_jint() == 0, "should be");
675 insn = &Assembler::str;
676 break;
677 case T_LONG:
678 assert(c->as_jlong() == 0, "should be");
679 insn = &Assembler::str;
680 break;
681 case T_INT:
682 assert(c->as_jint() == 0, "should be");
683 insn = &Assembler::strw;
684 break;
685 case T_OBJECT:
686 case T_ARRAY:
687 assert(c->as_jobject() == 0, "should be");
688 if (UseCompressedOops && !wide) {
689 insn = &Assembler::strw;
690 } else {
691 insn = &Assembler::str;
692 }
693 break;
694 case T_CHAR:
695 case T_SHORT:
696 assert(c->as_jint() == 0, "should be");
697 insn = &Assembler::strh;
698 break;
699 case T_BOOLEAN:
700 case T_BYTE:
701 assert(c->as_jint() == 0, "should be");
702 insn = &Assembler::strb;
703 break;
704 default:
705 ShouldNotReachHere();
706 insn = &Assembler::str; // unreachable
707 }
708
709 if (info) add_debug_info_for_null_check_here(info);
710 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
711 }
712
reg2reg(LIR_Opr src,LIR_Opr dest)713 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
714 assert(src->is_register(), "should not call otherwise");
715 assert(dest->is_register(), "should not call otherwise");
716
717 // move between cpu-registers
718 if (dest->is_single_cpu()) {
719 if (src->type() == T_LONG) {
720 // Can do LONG -> OBJECT
721 move_regs(src->as_register_lo(), dest->as_register());
722 return;
723 }
724 assert(src->is_single_cpu(), "must match");
725 if (src->type() == T_OBJECT) {
726 __ verify_oop(src->as_register());
727 }
728 move_regs(src->as_register(), dest->as_register());
729
730 } else if (dest->is_double_cpu()) {
731 if (is_reference_type(src->type())) {
732 // Surprising to me but we can see move of a long to t_object
733 __ verify_oop(src->as_register());
734 move_regs(src->as_register(), dest->as_register_lo());
735 return;
736 }
737 assert(src->is_double_cpu(), "must match");
738 Register f_lo = src->as_register_lo();
739 Register f_hi = src->as_register_hi();
740 Register t_lo = dest->as_register_lo();
741 Register t_hi = dest->as_register_hi();
742 assert(f_hi == f_lo, "must be same");
743 assert(t_hi == t_lo, "must be same");
744 move_regs(f_lo, t_lo);
745
746 } else if (dest->is_single_fpu()) {
747 __ fmovs(dest->as_float_reg(), src->as_float_reg());
748
749 } else if (dest->is_double_fpu()) {
750 __ fmovd(dest->as_double_reg(), src->as_double_reg());
751
752 } else {
753 ShouldNotReachHere();
754 }
755 }
756
reg2stack(LIR_Opr src,LIR_Opr dest,BasicType type,bool pop_fpu_stack)757 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
758 if (src->is_single_cpu()) {
759 if (is_reference_type(type)) {
760 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
761 __ verify_oop(src->as_register());
762 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
763 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
764 } else {
765 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
766 }
767
768 } else if (src->is_double_cpu()) {
769 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
770 __ str(src->as_register_lo(), dest_addr_LO);
771
772 } else if (src->is_single_fpu()) {
773 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
774 __ strs(src->as_float_reg(), dest_addr);
775
776 } else if (src->is_double_fpu()) {
777 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
778 __ strd(src->as_double_reg(), dest_addr);
779
780 } else {
781 ShouldNotReachHere();
782 }
783
784 }
785
786
reg2mem(LIR_Opr src,LIR_Opr dest,BasicType type,LIR_PatchCode patch_code,CodeEmitInfo * info,bool pop_fpu_stack,bool wide,bool)787 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
788 LIR_Address* to_addr = dest->as_address_ptr();
789 PatchingStub* patch = NULL;
790 Register compressed_src = rscratch1;
791
792 if (patch_code != lir_patch_none) {
793 deoptimize_trap(info);
794 return;
795 }
796
797 if (is_reference_type(type)) {
798 __ verify_oop(src->as_register());
799
800 if (UseCompressedOops && !wide) {
801 __ encode_heap_oop(compressed_src, src->as_register());
802 } else {
803 compressed_src = src->as_register();
804 }
805 }
806
807 int null_check_here = code_offset();
808 switch (type) {
809 case T_FLOAT: {
810 __ strs(src->as_float_reg(), as_Address(to_addr));
811 break;
812 }
813
814 case T_DOUBLE: {
815 __ strd(src->as_double_reg(), as_Address(to_addr));
816 break;
817 }
818
819 case T_ARRAY: // fall through
820 case T_OBJECT: // fall through
821 if (UseCompressedOops && !wide) {
822 __ strw(compressed_src, as_Address(to_addr, rscratch2));
823 } else {
824 __ str(compressed_src, as_Address(to_addr));
825 }
826 break;
827 case T_METADATA:
828 // We get here to store a method pointer to the stack to pass to
829 // a dtrace runtime call. This can't work on 64 bit with
830 // compressed klass ptrs: T_METADATA can be a compressed klass
831 // ptr or a 64 bit method pointer.
832 ShouldNotReachHere();
833 __ str(src->as_register(), as_Address(to_addr));
834 break;
835 case T_ADDRESS:
836 __ str(src->as_register(), as_Address(to_addr));
837 break;
838 case T_INT:
839 __ strw(src->as_register(), as_Address(to_addr));
840 break;
841
842 case T_LONG: {
843 __ str(src->as_register_lo(), as_Address_lo(to_addr));
844 break;
845 }
846
847 case T_BYTE: // fall through
848 case T_BOOLEAN: {
849 __ strb(src->as_register(), as_Address(to_addr));
850 break;
851 }
852
853 case T_CHAR: // fall through
854 case T_SHORT:
855 __ strh(src->as_register(), as_Address(to_addr));
856 break;
857
858 default:
859 ShouldNotReachHere();
860 }
861 if (info != NULL) {
862 add_debug_info_for_null_check(null_check_here, info);
863 }
864 }
865
866
stack2reg(LIR_Opr src,LIR_Opr dest,BasicType type)867 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
868 assert(src->is_stack(), "should not call otherwise");
869 assert(dest->is_register(), "should not call otherwise");
870
871 if (dest->is_single_cpu()) {
872 if (is_reference_type(type)) {
873 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
874 __ verify_oop(dest->as_register());
875 } else if (type == T_METADATA || type == T_ADDRESS) {
876 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
877 } else {
878 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
879 }
880
881 } else if (dest->is_double_cpu()) {
882 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
883 __ ldr(dest->as_register_lo(), src_addr_LO);
884
885 } else if (dest->is_single_fpu()) {
886 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
887 __ ldrs(dest->as_float_reg(), src_addr);
888
889 } else if (dest->is_double_fpu()) {
890 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
891 __ ldrd(dest->as_double_reg(), src_addr);
892
893 } else {
894 ShouldNotReachHere();
895 }
896 }
897
898
klass2reg_with_patching(Register reg,CodeEmitInfo * info)899 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
900 address target = NULL;
901 relocInfo::relocType reloc_type = relocInfo::none;
902
903 switch (patching_id(info)) {
904 case PatchingStub::access_field_id:
905 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
906 reloc_type = relocInfo::section_word_type;
907 break;
908 case PatchingStub::load_klass_id:
909 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
910 reloc_type = relocInfo::metadata_type;
911 break;
912 case PatchingStub::load_mirror_id:
913 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
914 reloc_type = relocInfo::oop_type;
915 break;
916 case PatchingStub::load_appendix_id:
917 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
918 reloc_type = relocInfo::oop_type;
919 break;
920 default: ShouldNotReachHere();
921 }
922
923 __ far_call(RuntimeAddress(target));
924 add_call_info_here(info);
925 }
926
stack2stack(LIR_Opr src,LIR_Opr dest,BasicType type)927 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
928
929 LIR_Opr temp;
930 if (type == T_LONG || type == T_DOUBLE)
931 temp = FrameMap::rscratch1_long_opr;
932 else
933 temp = FrameMap::rscratch1_opr;
934
935 stack2reg(src, temp, src->type());
936 reg2stack(temp, dest, dest->type(), false);
937 }
938
939
mem2reg(LIR_Opr src,LIR_Opr dest,BasicType type,LIR_PatchCode patch_code,CodeEmitInfo * info,bool wide,bool)940 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
941 LIR_Address* addr = src->as_address_ptr();
942 LIR_Address* from_addr = src->as_address_ptr();
943
944 if (addr->base()->type() == T_OBJECT) {
945 __ verify_oop(addr->base()->as_pointer_register());
946 }
947
948 if (patch_code != lir_patch_none) {
949 deoptimize_trap(info);
950 return;
951 }
952
953 if (info != NULL) {
954 add_debug_info_for_null_check_here(info);
955 }
956 int null_check_here = code_offset();
957 switch (type) {
958 case T_FLOAT: {
959 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
960 break;
961 }
962
963 case T_DOUBLE: {
964 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
965 break;
966 }
967
968 case T_ARRAY: // fall through
969 case T_OBJECT: // fall through
970 if (UseCompressedOops && !wide) {
971 __ ldrw(dest->as_register(), as_Address(from_addr));
972 } else {
973 __ ldr(dest->as_register(), as_Address(from_addr));
974 }
975 break;
976 case T_METADATA:
977 // We get here to store a method pointer to the stack to pass to
978 // a dtrace runtime call. This can't work on 64 bit with
979 // compressed klass ptrs: T_METADATA can be a compressed klass
980 // ptr or a 64 bit method pointer.
981 ShouldNotReachHere();
982 __ ldr(dest->as_register(), as_Address(from_addr));
983 break;
984 case T_ADDRESS:
985 // FIXME: OMG this is a horrible kludge. Any offset from an
986 // address that matches klass_offset_in_bytes() will be loaded
987 // as a word, not a long.
988 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
989 __ ldrw(dest->as_register(), as_Address(from_addr));
990 } else {
991 __ ldr(dest->as_register(), as_Address(from_addr));
992 }
993 break;
994 case T_INT:
995 __ ldrw(dest->as_register(), as_Address(from_addr));
996 break;
997
998 case T_LONG: {
999 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1000 break;
1001 }
1002
1003 case T_BYTE:
1004 __ ldrsb(dest->as_register(), as_Address(from_addr));
1005 break;
1006 case T_BOOLEAN: {
1007 __ ldrb(dest->as_register(), as_Address(from_addr));
1008 break;
1009 }
1010
1011 case T_CHAR:
1012 __ ldrh(dest->as_register(), as_Address(from_addr));
1013 break;
1014 case T_SHORT:
1015 __ ldrsh(dest->as_register(), as_Address(from_addr));
1016 break;
1017
1018 default:
1019 ShouldNotReachHere();
1020 }
1021
1022 if (is_reference_type(type)) {
1023 if (UseCompressedOops && !wide) {
1024 __ decode_heap_oop(dest->as_register());
1025 }
1026
1027 if (!UseZGC) {
1028 // Load barrier has not yet been applied, so ZGC can't verify the oop here
1029 __ verify_oop(dest->as_register());
1030 }
1031 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1032 if (UseCompressedClassPointers) {
1033 __ decode_klass_not_null(dest->as_register());
1034 }
1035 }
1036 }
1037
1038
array_element_size(BasicType type) const1039 int LIR_Assembler::array_element_size(BasicType type) const {
1040 int elem_size = type2aelembytes(type);
1041 return exact_log2(elem_size);
1042 }
1043
1044
emit_op3(LIR_Op3 * op)1045 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1046 switch (op->code()) {
1047 case lir_idiv:
1048 case lir_irem:
1049 arithmetic_idiv(op->code(),
1050 op->in_opr1(),
1051 op->in_opr2(),
1052 op->in_opr3(),
1053 op->result_opr(),
1054 op->info());
1055 break;
1056 case lir_fmad:
1057 __ fmaddd(op->result_opr()->as_double_reg(),
1058 op->in_opr1()->as_double_reg(),
1059 op->in_opr2()->as_double_reg(),
1060 op->in_opr3()->as_double_reg());
1061 break;
1062 case lir_fmaf:
1063 __ fmadds(op->result_opr()->as_float_reg(),
1064 op->in_opr1()->as_float_reg(),
1065 op->in_opr2()->as_float_reg(),
1066 op->in_opr3()->as_float_reg());
1067 break;
1068 default: ShouldNotReachHere(); break;
1069 }
1070 }
1071
emit_opBranch(LIR_OpBranch * op)1072 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1073 #ifdef ASSERT
1074 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1075 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1076 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1077 #endif
1078
1079 if (op->cond() == lir_cond_always) {
1080 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1081 __ b(*(op->label()));
1082 } else {
1083 Assembler::Condition acond;
1084 if (op->code() == lir_cond_float_branch) {
1085 bool is_unordered = (op->ublock() == op->block());
1086 // Assembler::EQ does not permit unordered branches, so we add
1087 // another branch here. Likewise, Assembler::NE does not permit
1088 // ordered branches.
1089 if ((is_unordered && op->cond() == lir_cond_equal)
1090 || (!is_unordered && op->cond() == lir_cond_notEqual))
1091 __ br(Assembler::VS, *(op->ublock()->label()));
1092 switch(op->cond()) {
1093 case lir_cond_equal: acond = Assembler::EQ; break;
1094 case lir_cond_notEqual: acond = Assembler::NE; break;
1095 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1096 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1097 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1098 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1099 default: ShouldNotReachHere();
1100 acond = Assembler::EQ; // unreachable
1101 }
1102 } else {
1103 switch (op->cond()) {
1104 case lir_cond_equal: acond = Assembler::EQ; break;
1105 case lir_cond_notEqual: acond = Assembler::NE; break;
1106 case lir_cond_less: acond = Assembler::LT; break;
1107 case lir_cond_lessEqual: acond = Assembler::LE; break;
1108 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1109 case lir_cond_greater: acond = Assembler::GT; break;
1110 case lir_cond_belowEqual: acond = Assembler::LS; break;
1111 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1112 default: ShouldNotReachHere();
1113 acond = Assembler::EQ; // unreachable
1114 }
1115 }
1116 __ br(acond,*(op->label()));
1117 }
1118 }
1119
1120
1121
emit_opConvert(LIR_OpConvert * op)1122 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1123 LIR_Opr src = op->in_opr();
1124 LIR_Opr dest = op->result_opr();
1125
1126 switch (op->bytecode()) {
1127 case Bytecodes::_i2f:
1128 {
1129 __ scvtfws(dest->as_float_reg(), src->as_register());
1130 break;
1131 }
1132 case Bytecodes::_i2d:
1133 {
1134 __ scvtfwd(dest->as_double_reg(), src->as_register());
1135 break;
1136 }
1137 case Bytecodes::_l2d:
1138 {
1139 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1140 break;
1141 }
1142 case Bytecodes::_l2f:
1143 {
1144 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1145 break;
1146 }
1147 case Bytecodes::_f2d:
1148 {
1149 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1150 break;
1151 }
1152 case Bytecodes::_d2f:
1153 {
1154 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1155 break;
1156 }
1157 case Bytecodes::_i2c:
1158 {
1159 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1160 break;
1161 }
1162 case Bytecodes::_i2l:
1163 {
1164 __ sxtw(dest->as_register_lo(), src->as_register());
1165 break;
1166 }
1167 case Bytecodes::_i2s:
1168 {
1169 __ sxth(dest->as_register(), src->as_register());
1170 break;
1171 }
1172 case Bytecodes::_i2b:
1173 {
1174 __ sxtb(dest->as_register(), src->as_register());
1175 break;
1176 }
1177 case Bytecodes::_l2i:
1178 {
1179 _masm->block_comment("FIXME: This could be a no-op");
1180 __ uxtw(dest->as_register(), src->as_register_lo());
1181 break;
1182 }
1183 case Bytecodes::_d2l:
1184 {
1185 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1186 break;
1187 }
1188 case Bytecodes::_f2i:
1189 {
1190 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1191 break;
1192 }
1193 case Bytecodes::_f2l:
1194 {
1195 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1196 break;
1197 }
1198 case Bytecodes::_d2i:
1199 {
1200 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1201 break;
1202 }
1203 default: ShouldNotReachHere();
1204 }
1205 }
1206
emit_alloc_obj(LIR_OpAllocObj * op)1207 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1208 if (op->init_check()) {
1209 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1210 InstanceKlass::init_state_offset()));
1211 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1212 add_debug_info_for_null_check_here(op->stub()->info());
1213 __ br(Assembler::NE, *op->stub()->entry());
1214 }
1215 __ allocate_object(op->obj()->as_register(),
1216 op->tmp1()->as_register(),
1217 op->tmp2()->as_register(),
1218 op->header_size(),
1219 op->object_size(),
1220 op->klass()->as_register(),
1221 *op->stub()->entry());
1222 __ bind(*op->stub()->continuation());
1223 }
1224
emit_alloc_array(LIR_OpAllocArray * op)1225 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1226 Register len = op->len()->as_register();
1227 __ uxtw(len, len);
1228
1229 if (UseSlowPath ||
1230 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1231 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1232 __ b(*op->stub()->entry());
1233 } else {
1234 Register tmp1 = op->tmp1()->as_register();
1235 Register tmp2 = op->tmp2()->as_register();
1236 Register tmp3 = op->tmp3()->as_register();
1237 if (len == tmp1) {
1238 tmp1 = tmp3;
1239 } else if (len == tmp2) {
1240 tmp2 = tmp3;
1241 } else if (len == tmp3) {
1242 // everything is ok
1243 } else {
1244 __ mov(tmp3, len);
1245 }
1246 __ allocate_array(op->obj()->as_register(),
1247 len,
1248 tmp1,
1249 tmp2,
1250 arrayOopDesc::header_size(op->type()),
1251 array_element_size(op->type()),
1252 op->klass()->as_register(),
1253 *op->stub()->entry());
1254 }
1255 __ bind(*op->stub()->continuation());
1256 }
1257
type_profile_helper(Register mdo,ciMethodData * md,ciProfileData * data,Register recv,Label * update_done)1258 void LIR_Assembler::type_profile_helper(Register mdo,
1259 ciMethodData *md, ciProfileData *data,
1260 Register recv, Label* update_done) {
1261 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1262 Label next_test;
1263 // See if the receiver is receiver[n].
1264 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1265 __ ldr(rscratch1, Address(rscratch2));
1266 __ cmp(recv, rscratch1);
1267 __ br(Assembler::NE, next_test);
1268 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1269 __ addptr(data_addr, DataLayout::counter_increment);
1270 __ b(*update_done);
1271 __ bind(next_test);
1272 }
1273
1274 // Didn't find receiver; find next empty slot and fill it in
1275 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1276 Label next_test;
1277 __ lea(rscratch2,
1278 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1279 Address recv_addr(rscratch2);
1280 __ ldr(rscratch1, recv_addr);
1281 __ cbnz(rscratch1, next_test);
1282 __ str(recv, recv_addr);
1283 __ mov(rscratch1, DataLayout::counter_increment);
1284 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1285 __ str(rscratch1, Address(rscratch2));
1286 __ b(*update_done);
1287 __ bind(next_test);
1288 }
1289 }
1290
emit_typecheck_helper(LIR_OpTypeCheck * op,Label * success,Label * failure,Label * obj_is_null)1291 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1292 // we always need a stub for the failure case.
1293 CodeStub* stub = op->stub();
1294 Register obj = op->object()->as_register();
1295 Register k_RInfo = op->tmp1()->as_register();
1296 Register klass_RInfo = op->tmp2()->as_register();
1297 Register dst = op->result_opr()->as_register();
1298 ciKlass* k = op->klass();
1299 Register Rtmp1 = noreg;
1300
1301 // check if it needs to be profiled
1302 ciMethodData* md;
1303 ciProfileData* data;
1304
1305 const bool should_profile = op->should_profile();
1306
1307 if (should_profile) {
1308 ciMethod* method = op->profiled_method();
1309 assert(method != NULL, "Should have method");
1310 int bci = op->profiled_bci();
1311 md = method->method_data_or_null();
1312 assert(md != NULL, "Sanity");
1313 data = md->bci_to_data(bci);
1314 assert(data != NULL, "need data for type check");
1315 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1316 }
1317 Label profile_cast_success, profile_cast_failure;
1318 Label *success_target = should_profile ? &profile_cast_success : success;
1319 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1320
1321 if (obj == k_RInfo) {
1322 k_RInfo = dst;
1323 } else if (obj == klass_RInfo) {
1324 klass_RInfo = dst;
1325 }
1326 if (k->is_loaded() && !UseCompressedClassPointers) {
1327 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1328 } else {
1329 Rtmp1 = op->tmp3()->as_register();
1330 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1331 }
1332
1333 assert_different_registers(obj, k_RInfo, klass_RInfo);
1334
1335 if (should_profile) {
1336 Label not_null;
1337 __ cbnz(obj, not_null);
1338 // Object is null; update MDO and exit
1339 Register mdo = klass_RInfo;
1340 __ mov_metadata(mdo, md->constant_encoding());
1341 Address data_addr
1342 = __ form_address(rscratch2, mdo,
1343 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1344 0);
1345 __ ldrb(rscratch1, data_addr);
1346 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1347 __ strb(rscratch1, data_addr);
1348 __ b(*obj_is_null);
1349 __ bind(not_null);
1350 } else {
1351 __ cbz(obj, *obj_is_null);
1352 }
1353
1354 if (!k->is_loaded()) {
1355 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1356 } else {
1357 __ mov_metadata(k_RInfo, k->constant_encoding());
1358 }
1359 __ verify_oop(obj);
1360
1361 if (op->fast_check()) {
1362 // get object class
1363 // not a safepoint as obj null check happens earlier
1364 __ load_klass(rscratch1, obj);
1365 __ cmp( rscratch1, k_RInfo);
1366
1367 __ br(Assembler::NE, *failure_target);
1368 // successful cast, fall through to profile or jump
1369 } else {
1370 // get object class
1371 // not a safepoint as obj null check happens earlier
1372 __ load_klass(klass_RInfo, obj);
1373 if (k->is_loaded()) {
1374 // See if we get an immediate positive hit
1375 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1376 __ cmp(k_RInfo, rscratch1);
1377 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1378 __ br(Assembler::NE, *failure_target);
1379 // successful cast, fall through to profile or jump
1380 } else {
1381 // See if we get an immediate positive hit
1382 __ br(Assembler::EQ, *success_target);
1383 // check for self
1384 __ cmp(klass_RInfo, k_RInfo);
1385 __ br(Assembler::EQ, *success_target);
1386
1387 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1388 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1389 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1390 // result is a boolean
1391 __ cbzw(klass_RInfo, *failure_target);
1392 // successful cast, fall through to profile or jump
1393 }
1394 } else {
1395 // perform the fast part of the checking logic
1396 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1397 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1398 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1399 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1400 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1401 // result is a boolean
1402 __ cbz(k_RInfo, *failure_target);
1403 // successful cast, fall through to profile or jump
1404 }
1405 }
1406 if (should_profile) {
1407 Register mdo = klass_RInfo, recv = k_RInfo;
1408 __ bind(profile_cast_success);
1409 __ mov_metadata(mdo, md->constant_encoding());
1410 __ load_klass(recv, obj);
1411 Label update_done;
1412 type_profile_helper(mdo, md, data, recv, success);
1413 __ b(*success);
1414
1415 __ bind(profile_cast_failure);
1416 __ mov_metadata(mdo, md->constant_encoding());
1417 Address counter_addr
1418 = __ form_address(rscratch2, mdo,
1419 md->byte_offset_of_slot(data, CounterData::count_offset()),
1420 0);
1421 __ ldr(rscratch1, counter_addr);
1422 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1423 __ str(rscratch1, counter_addr);
1424 __ b(*failure);
1425 }
1426 __ b(*success);
1427 }
1428
1429
emit_opTypeCheck(LIR_OpTypeCheck * op)1430 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1431 const bool should_profile = op->should_profile();
1432
1433 LIR_Code code = op->code();
1434 if (code == lir_store_check) {
1435 Register value = op->object()->as_register();
1436 Register array = op->array()->as_register();
1437 Register k_RInfo = op->tmp1()->as_register();
1438 Register klass_RInfo = op->tmp2()->as_register();
1439 Register Rtmp1 = op->tmp3()->as_register();
1440
1441 CodeStub* stub = op->stub();
1442
1443 // check if it needs to be profiled
1444 ciMethodData* md;
1445 ciProfileData* data;
1446
1447 if (should_profile) {
1448 ciMethod* method = op->profiled_method();
1449 assert(method != NULL, "Should have method");
1450 int bci = op->profiled_bci();
1451 md = method->method_data_or_null();
1452 assert(md != NULL, "Sanity");
1453 data = md->bci_to_data(bci);
1454 assert(data != NULL, "need data for type check");
1455 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1456 }
1457 Label profile_cast_success, profile_cast_failure, done;
1458 Label *success_target = should_profile ? &profile_cast_success : &done;
1459 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1460
1461 if (should_profile) {
1462 Label not_null;
1463 __ cbnz(value, not_null);
1464 // Object is null; update MDO and exit
1465 Register mdo = klass_RInfo;
1466 __ mov_metadata(mdo, md->constant_encoding());
1467 Address data_addr
1468 = __ form_address(rscratch2, mdo,
1469 md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1470 0);
1471 __ ldrb(rscratch1, data_addr);
1472 __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1473 __ strb(rscratch1, data_addr);
1474 __ b(done);
1475 __ bind(not_null);
1476 } else {
1477 __ cbz(value, done);
1478 }
1479
1480 add_debug_info_for_null_check_here(op->info_for_exception());
1481 __ load_klass(k_RInfo, array);
1482 __ load_klass(klass_RInfo, value);
1483
1484 // get instance klass (it's already uncompressed)
1485 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1486 // perform the fast part of the checking logic
1487 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1488 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1489 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1490 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1491 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1492 // result is a boolean
1493 __ cbzw(k_RInfo, *failure_target);
1494 // fall through to the success case
1495
1496 if (should_profile) {
1497 Register mdo = klass_RInfo, recv = k_RInfo;
1498 __ bind(profile_cast_success);
1499 __ mov_metadata(mdo, md->constant_encoding());
1500 __ load_klass(recv, value);
1501 Label update_done;
1502 type_profile_helper(mdo, md, data, recv, &done);
1503 __ b(done);
1504
1505 __ bind(profile_cast_failure);
1506 __ mov_metadata(mdo, md->constant_encoding());
1507 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1508 __ lea(rscratch2, counter_addr);
1509 __ ldr(rscratch1, Address(rscratch2));
1510 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1511 __ str(rscratch1, Address(rscratch2));
1512 __ b(*stub->entry());
1513 }
1514
1515 __ bind(done);
1516 } else if (code == lir_checkcast) {
1517 Register obj = op->object()->as_register();
1518 Register dst = op->result_opr()->as_register();
1519 Label success;
1520 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1521 __ bind(success);
1522 if (dst != obj) {
1523 __ mov(dst, obj);
1524 }
1525 } else if (code == lir_instanceof) {
1526 Register obj = op->object()->as_register();
1527 Register dst = op->result_opr()->as_register();
1528 Label success, failure, done;
1529 emit_typecheck_helper(op, &success, &failure, &failure);
1530 __ bind(failure);
1531 __ mov(dst, zr);
1532 __ b(done);
1533 __ bind(success);
1534 __ mov(dst, 1);
1535 __ bind(done);
1536 } else {
1537 ShouldNotReachHere();
1538 }
1539 }
1540
casw(Register addr,Register newval,Register cmpval)1541 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1542 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1543 __ cset(rscratch1, Assembler::NE);
1544 __ membar(__ AnyAny);
1545 }
1546
casl(Register addr,Register newval,Register cmpval)1547 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1548 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1549 __ cset(rscratch1, Assembler::NE);
1550 __ membar(__ AnyAny);
1551 }
1552
1553
emit_compare_and_swap(LIR_OpCompareAndSwap * op)1554 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1555 assert(VM_Version::supports_cx8(), "wrong machine");
1556 Register addr;
1557 if (op->addr()->is_register()) {
1558 addr = as_reg(op->addr());
1559 } else {
1560 assert(op->addr()->is_address(), "what else?");
1561 LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1562 assert(addr_ptr->disp() == 0, "need 0 disp");
1563 assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index");
1564 addr = as_reg(addr_ptr->base());
1565 }
1566 Register newval = as_reg(op->new_value());
1567 Register cmpval = as_reg(op->cmp_value());
1568
1569 if (op->code() == lir_cas_obj) {
1570 if (UseCompressedOops) {
1571 Register t1 = op->tmp1()->as_register();
1572 assert(op->tmp1()->is_valid(), "must be");
1573 __ encode_heap_oop(t1, cmpval);
1574 cmpval = t1;
1575 __ encode_heap_oop(rscratch2, newval);
1576 newval = rscratch2;
1577 casw(addr, newval, cmpval);
1578 } else {
1579 casl(addr, newval, cmpval);
1580 }
1581 } else if (op->code() == lir_cas_int) {
1582 casw(addr, newval, cmpval);
1583 } else {
1584 casl(addr, newval, cmpval);
1585 }
1586 }
1587
1588
cmove(LIR_Condition condition,LIR_Opr opr1,LIR_Opr opr2,LIR_Opr result,BasicType type)1589 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1590
1591 Assembler::Condition acond, ncond;
1592 switch (condition) {
1593 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1594 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1595 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1596 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1597 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1598 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1599 case lir_cond_belowEqual:
1600 case lir_cond_aboveEqual:
1601 default: ShouldNotReachHere();
1602 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable
1603 }
1604
1605 assert(result->is_single_cpu() || result->is_double_cpu(),
1606 "expect single register for result");
1607 if (opr1->is_constant() && opr2->is_constant()
1608 && opr1->type() == T_INT && opr2->type() == T_INT) {
1609 jint val1 = opr1->as_jint();
1610 jint val2 = opr2->as_jint();
1611 if (val1 == 0 && val2 == 1) {
1612 __ cset(result->as_register(), ncond);
1613 return;
1614 } else if (val1 == 1 && val2 == 0) {
1615 __ cset(result->as_register(), acond);
1616 return;
1617 }
1618 }
1619
1620 if (opr1->is_constant() && opr2->is_constant()
1621 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1622 jlong val1 = opr1->as_jlong();
1623 jlong val2 = opr2->as_jlong();
1624 if (val1 == 0 && val2 == 1) {
1625 __ cset(result->as_register_lo(), ncond);
1626 return;
1627 } else if (val1 == 1 && val2 == 0) {
1628 __ cset(result->as_register_lo(), acond);
1629 return;
1630 }
1631 }
1632
1633 if (opr1->is_stack()) {
1634 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1635 opr1 = FrameMap::rscratch1_opr;
1636 } else if (opr1->is_constant()) {
1637 LIR_Opr tmp
1638 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1639 const2reg(opr1, tmp, lir_patch_none, NULL);
1640 opr1 = tmp;
1641 }
1642
1643 if (opr2->is_stack()) {
1644 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1645 opr2 = FrameMap::rscratch2_opr;
1646 } else if (opr2->is_constant()) {
1647 LIR_Opr tmp
1648 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1649 const2reg(opr2, tmp, lir_patch_none, NULL);
1650 opr2 = tmp;
1651 }
1652
1653 if (result->type() == T_LONG)
1654 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1655 else
1656 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1657 }
1658
arith_op(LIR_Code code,LIR_Opr left,LIR_Opr right,LIR_Opr dest,CodeEmitInfo * info,bool pop_fpu_stack)1659 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1660 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1661
1662 if (left->is_single_cpu()) {
1663 Register lreg = left->as_register();
1664 Register dreg = as_reg(dest);
1665
1666 if (right->is_single_cpu()) {
1667 // cpu register - cpu register
1668
1669 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1670 "should be");
1671 Register rreg = right->as_register();
1672 switch (code) {
1673 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1674 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1675 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1676 default: ShouldNotReachHere();
1677 }
1678
1679 } else if (right->is_double_cpu()) {
1680 Register rreg = right->as_register_lo();
1681 // single_cpu + double_cpu: can happen with obj+long
1682 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1683 switch (code) {
1684 case lir_add: __ add(dreg, lreg, rreg); break;
1685 case lir_sub: __ sub(dreg, lreg, rreg); break;
1686 default: ShouldNotReachHere();
1687 }
1688 } else if (right->is_constant()) {
1689 // cpu register - constant
1690 jlong c;
1691
1692 // FIXME. This is fugly: we really need to factor all this logic.
1693 switch(right->type()) {
1694 case T_LONG:
1695 c = right->as_constant_ptr()->as_jlong();
1696 break;
1697 case T_INT:
1698 case T_ADDRESS:
1699 c = right->as_constant_ptr()->as_jint();
1700 break;
1701 default:
1702 ShouldNotReachHere();
1703 c = 0; // unreachable
1704 break;
1705 }
1706
1707 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1708 if (c == 0 && dreg == lreg) {
1709 COMMENT("effective nop elided");
1710 return;
1711 }
1712 switch(left->type()) {
1713 case T_INT:
1714 switch (code) {
1715 case lir_add: __ addw(dreg, lreg, c); break;
1716 case lir_sub: __ subw(dreg, lreg, c); break;
1717 default: ShouldNotReachHere();
1718 }
1719 break;
1720 case T_OBJECT:
1721 case T_ADDRESS:
1722 switch (code) {
1723 case lir_add: __ add(dreg, lreg, c); break;
1724 case lir_sub: __ sub(dreg, lreg, c); break;
1725 default: ShouldNotReachHere();
1726 }
1727 break;
1728 default:
1729 ShouldNotReachHere();
1730 }
1731 } else {
1732 ShouldNotReachHere();
1733 }
1734
1735 } else if (left->is_double_cpu()) {
1736 Register lreg_lo = left->as_register_lo();
1737
1738 if (right->is_double_cpu()) {
1739 // cpu register - cpu register
1740 Register rreg_lo = right->as_register_lo();
1741 switch (code) {
1742 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1743 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1744 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1745 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1746 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1747 default:
1748 ShouldNotReachHere();
1749 }
1750
1751 } else if (right->is_constant()) {
1752 jlong c = right->as_constant_ptr()->as_jlong();
1753 Register dreg = as_reg(dest);
1754 switch (code) {
1755 case lir_add:
1756 case lir_sub:
1757 if (c == 0 && dreg == lreg_lo) {
1758 COMMENT("effective nop elided");
1759 return;
1760 }
1761 code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1762 break;
1763 case lir_div:
1764 assert(c > 0 && is_power_of_2_long(c), "divisor must be power-of-2 constant");
1765 if (c == 1) {
1766 // move lreg_lo to dreg if divisor is 1
1767 __ mov(dreg, lreg_lo);
1768 } else {
1769 unsigned int shift = exact_log2_long(c);
1770 // use rscratch1 as intermediate result register
1771 __ asr(rscratch1, lreg_lo, 63);
1772 __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1773 __ asr(dreg, rscratch1, shift);
1774 }
1775 break;
1776 case lir_rem:
1777 assert(c > 0 && is_power_of_2_long(c), "divisor must be power-of-2 constant");
1778 if (c == 1) {
1779 // move 0 to dreg if divisor is 1
1780 __ mov(dreg, zr);
1781 } else {
1782 // use rscratch1 as intermediate result register
1783 __ negs(rscratch1, lreg_lo);
1784 __ andr(dreg, lreg_lo, c - 1);
1785 __ andr(rscratch1, rscratch1, c - 1);
1786 __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1787 }
1788 break;
1789 default:
1790 ShouldNotReachHere();
1791 }
1792 } else {
1793 ShouldNotReachHere();
1794 }
1795 } else if (left->is_single_fpu()) {
1796 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1797 switch (code) {
1798 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1799 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1800 case lir_mul_strictfp: // fall through
1801 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1802 case lir_div_strictfp: // fall through
1803 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1804 default:
1805 ShouldNotReachHere();
1806 }
1807 } else if (left->is_double_fpu()) {
1808 if (right->is_double_fpu()) {
1809 // fpu register - fpu register
1810 switch (code) {
1811 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1812 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1813 case lir_mul_strictfp: // fall through
1814 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1815 case lir_div_strictfp: // fall through
1816 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1817 default:
1818 ShouldNotReachHere();
1819 }
1820 } else {
1821 if (right->is_constant()) {
1822 ShouldNotReachHere();
1823 }
1824 ShouldNotReachHere();
1825 }
1826 } else if (left->is_single_stack() || left->is_address()) {
1827 assert(left == dest, "left and dest must be equal");
1828 ShouldNotReachHere();
1829 } else {
1830 ShouldNotReachHere();
1831 }
1832 }
1833
arith_fpu_implementation(LIR_Code code,int left_index,int right_index,int dest_index,bool pop_fpu_stack)1834 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1835
1836
intrinsic_op(LIR_Code code,LIR_Opr value,LIR_Opr unused,LIR_Opr dest,LIR_Op * op)1837 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1838 switch(code) {
1839 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1840 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1841 default : ShouldNotReachHere();
1842 }
1843 }
1844
logic_op(LIR_Code code,LIR_Opr left,LIR_Opr right,LIR_Opr dst)1845 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1846
1847 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1848 Register Rleft = left->is_single_cpu() ? left->as_register() :
1849 left->as_register_lo();
1850 if (dst->is_single_cpu()) {
1851 Register Rdst = dst->as_register();
1852 if (right->is_constant()) {
1853 switch (code) {
1854 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1855 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1856 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1857 default: ShouldNotReachHere(); break;
1858 }
1859 } else {
1860 Register Rright = right->is_single_cpu() ? right->as_register() :
1861 right->as_register_lo();
1862 switch (code) {
1863 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1864 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1865 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1866 default: ShouldNotReachHere(); break;
1867 }
1868 }
1869 } else {
1870 Register Rdst = dst->as_register_lo();
1871 if (right->is_constant()) {
1872 switch (code) {
1873 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1874 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1875 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1876 default: ShouldNotReachHere(); break;
1877 }
1878 } else {
1879 Register Rright = right->is_single_cpu() ? right->as_register() :
1880 right->as_register_lo();
1881 switch (code) {
1882 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1883 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1884 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1885 default: ShouldNotReachHere(); break;
1886 }
1887 }
1888 }
1889 }
1890
1891
1892
arithmetic_idiv(LIR_Code code,LIR_Opr left,LIR_Opr right,LIR_Opr illegal,LIR_Opr result,CodeEmitInfo * info)1893 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
1894
1895 // opcode check
1896 assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
1897 bool is_irem = (code == lir_irem);
1898
1899 // operand check
1900 assert(left->is_single_cpu(), "left must be register");
1901 assert(right->is_single_cpu() || right->is_constant(), "right must be register or constant");
1902 assert(result->is_single_cpu(), "result must be register");
1903 Register lreg = left->as_register();
1904 Register dreg = result->as_register();
1905
1906 // power-of-2 constant check and codegen
1907 if (right->is_constant()) {
1908 int c = right->as_constant_ptr()->as_jint();
1909 assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1910 if (is_irem) {
1911 if (c == 1) {
1912 // move 0 to dreg if divisor is 1
1913 __ movw(dreg, zr);
1914 } else {
1915 // use rscratch1 as intermediate result register
1916 __ negsw(rscratch1, lreg);
1917 __ andw(dreg, lreg, c - 1);
1918 __ andw(rscratch1, rscratch1, c - 1);
1919 __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
1920 }
1921 } else {
1922 if (c == 1) {
1923 // move lreg to dreg if divisor is 1
1924 __ movw(dreg, lreg);
1925 } else {
1926 unsigned int shift = exact_log2(c);
1927 // use rscratch1 as intermediate result register
1928 __ asrw(rscratch1, lreg, 31);
1929 __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
1930 __ asrw(dreg, rscratch1, shift);
1931 }
1932 }
1933 } else {
1934 Register rreg = right->as_register();
1935 __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
1936 }
1937 }
1938
1939
comp_op(LIR_Condition condition,LIR_Opr opr1,LIR_Opr opr2,LIR_Op2 * op)1940 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1941 if (opr1->is_constant() && opr2->is_single_cpu()) {
1942 // tableswitch
1943 Register reg = as_reg(opr2);
1944 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1945 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1946 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1947 Register reg1 = as_reg(opr1);
1948 if (opr2->is_single_cpu()) {
1949 // cpu register - cpu register
1950 Register reg2 = opr2->as_register();
1951 if (is_reference_type(opr1->type())) {
1952 __ cmpoop(reg1, reg2);
1953 } else {
1954 assert(!is_reference_type(opr2->type()), "cmp int, oop?");
1955 __ cmpw(reg1, reg2);
1956 }
1957 return;
1958 }
1959 if (opr2->is_double_cpu()) {
1960 // cpu register - cpu register
1961 Register reg2 = opr2->as_register_lo();
1962 __ cmp(reg1, reg2);
1963 return;
1964 }
1965
1966 if (opr2->is_constant()) {
1967 bool is_32bit = false; // width of register operand
1968 jlong imm;
1969
1970 switch(opr2->type()) {
1971 case T_INT:
1972 imm = opr2->as_constant_ptr()->as_jint();
1973 is_32bit = true;
1974 break;
1975 case T_LONG:
1976 imm = opr2->as_constant_ptr()->as_jlong();
1977 break;
1978 case T_ADDRESS:
1979 imm = opr2->as_constant_ptr()->as_jint();
1980 break;
1981 case T_METADATA:
1982 imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
1983 break;
1984 case T_OBJECT:
1985 case T_ARRAY:
1986 jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1987 __ cmpoop(reg1, rscratch1);
1988 return;
1989 default:
1990 ShouldNotReachHere();
1991 imm = 0; // unreachable
1992 break;
1993 }
1994
1995 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1996 if (is_32bit)
1997 __ cmpw(reg1, imm);
1998 else
1999 __ subs(zr, reg1, imm);
2000 return;
2001 } else {
2002 __ mov(rscratch1, imm);
2003 if (is_32bit)
2004 __ cmpw(reg1, rscratch1);
2005 else
2006 __ cmp(reg1, rscratch1);
2007 return;
2008 }
2009 } else
2010 ShouldNotReachHere();
2011 } else if (opr1->is_single_fpu()) {
2012 FloatRegister reg1 = opr1->as_float_reg();
2013 assert(opr2->is_single_fpu(), "expect single float register");
2014 FloatRegister reg2 = opr2->as_float_reg();
2015 __ fcmps(reg1, reg2);
2016 } else if (opr1->is_double_fpu()) {
2017 FloatRegister reg1 = opr1->as_double_reg();
2018 assert(opr2->is_double_fpu(), "expect double float register");
2019 FloatRegister reg2 = opr2->as_double_reg();
2020 __ fcmpd(reg1, reg2);
2021 } else {
2022 ShouldNotReachHere();
2023 }
2024 }
2025
comp_fl2i(LIR_Code code,LIR_Opr left,LIR_Opr right,LIR_Opr dst,LIR_Op2 * op)2026 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2027 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2028 bool is_unordered_less = (code == lir_ucmp_fd2i);
2029 if (left->is_single_fpu()) {
2030 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2031 } else if (left->is_double_fpu()) {
2032 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2033 } else {
2034 ShouldNotReachHere();
2035 }
2036 } else if (code == lir_cmp_l2i) {
2037 Label done;
2038 __ cmp(left->as_register_lo(), right->as_register_lo());
2039 __ mov(dst->as_register(), (u_int64_t)-1L);
2040 __ br(Assembler::LT, done);
2041 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2042 __ bind(done);
2043 } else {
2044 ShouldNotReachHere();
2045 }
2046 }
2047
2048
align_call(LIR_Code code)2049 void LIR_Assembler::align_call(LIR_Code code) { }
2050
2051
call(LIR_OpJavaCall * op,relocInfo::relocType rtype)2052 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2053 address call = __ trampoline_call(Address(op->addr(), rtype));
2054 if (call == NULL) {
2055 bailout("trampoline stub overflow");
2056 return;
2057 }
2058 add_call_info(code_offset(), op->info());
2059 }
2060
2061
ic_call(LIR_OpJavaCall * op)2062 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2063 address call = __ ic_call(op->addr());
2064 if (call == NULL) {
2065 bailout("trampoline stub overflow");
2066 return;
2067 }
2068 add_call_info(code_offset(), op->info());
2069 }
2070
2071
2072 /* Currently, vtable-dispatch is only enabled for sparc platforms */
vtable_call(LIR_OpJavaCall * op)2073 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2074 ShouldNotReachHere();
2075 }
2076
2077
emit_static_call_stub()2078 void LIR_Assembler::emit_static_call_stub() {
2079 address call_pc = __ pc();
2080 address stub = __ start_a_stub(call_stub_size());
2081 if (stub == NULL) {
2082 bailout("static call stub overflow");
2083 return;
2084 }
2085
2086 int start = __ offset();
2087
2088 __ relocate(static_stub_Relocation::spec(call_pc));
2089 __ emit_static_call_stub();
2090
2091 assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
2092 <= call_stub_size(), "stub too big");
2093 __ end_a_stub();
2094 }
2095
2096
throw_op(LIR_Opr exceptionPC,LIR_Opr exceptionOop,CodeEmitInfo * info)2097 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2098 assert(exceptionOop->as_register() == r0, "must match");
2099 assert(exceptionPC->as_register() == r3, "must match");
2100
2101 // exception object is not added to oop map by LinearScan
2102 // (LinearScan assumes that no oops are in fixed registers)
2103 info->add_register_oop(exceptionOop);
2104 Runtime1::StubID unwind_id;
2105
2106 // get current pc information
2107 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2108 int pc_for_athrow_offset = __ offset();
2109 InternalAddress pc_for_athrow(__ pc());
2110 __ adr(exceptionPC->as_register(), pc_for_athrow);
2111 add_call_info(pc_for_athrow_offset, info); // for exception handler
2112
2113 __ verify_not_null_oop(r0);
2114 // search an exception handler (r0: exception oop, r3: throwing pc)
2115 if (compilation()->has_fpu_code()) {
2116 unwind_id = Runtime1::handle_exception_id;
2117 } else {
2118 unwind_id = Runtime1::handle_exception_nofpu_id;
2119 }
2120 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2121
2122 // FIXME: enough room for two byte trap ????
2123 __ nop();
2124 }
2125
2126
unwind_op(LIR_Opr exceptionOop)2127 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2128 assert(exceptionOop->as_register() == r0, "must match");
2129
2130 __ b(_unwind_handler_entry);
2131 }
2132
2133
shift_op(LIR_Code code,LIR_Opr left,LIR_Opr count,LIR_Opr dest,LIR_Opr tmp)2134 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2135 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2136 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2137
2138 switch (left->type()) {
2139 case T_INT: {
2140 switch (code) {
2141 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2142 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2143 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2144 default:
2145 ShouldNotReachHere();
2146 break;
2147 }
2148 break;
2149 case T_LONG:
2150 case T_ADDRESS:
2151 case T_OBJECT:
2152 switch (code) {
2153 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2154 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2155 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2156 default:
2157 ShouldNotReachHere();
2158 break;
2159 }
2160 break;
2161 default:
2162 ShouldNotReachHere();
2163 break;
2164 }
2165 }
2166 }
2167
2168
shift_op(LIR_Code code,LIR_Opr left,jint count,LIR_Opr dest)2169 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2170 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2171 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2172
2173 switch (left->type()) {
2174 case T_INT: {
2175 switch (code) {
2176 case lir_shl: __ lslw (dreg, lreg, count); break;
2177 case lir_shr: __ asrw (dreg, lreg, count); break;
2178 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2179 default:
2180 ShouldNotReachHere();
2181 break;
2182 }
2183 break;
2184 case T_LONG:
2185 case T_ADDRESS:
2186 case T_OBJECT:
2187 switch (code) {
2188 case lir_shl: __ lsl (dreg, lreg, count); break;
2189 case lir_shr: __ asr (dreg, lreg, count); break;
2190 case lir_ushr: __ lsr (dreg, lreg, count); break;
2191 default:
2192 ShouldNotReachHere();
2193 break;
2194 }
2195 break;
2196 default:
2197 ShouldNotReachHere();
2198 break;
2199 }
2200 }
2201 }
2202
2203
store_parameter(Register r,int offset_from_rsp_in_words)2204 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2205 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2206 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2207 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2208 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2209 }
2210
2211
store_parameter(jint c,int offset_from_rsp_in_words)2212 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2213 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2214 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2215 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2216 __ mov (rscratch1, c);
2217 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2218 }
2219
2220
store_parameter(jobject o,int offset_from_rsp_in_words)2221 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2222 ShouldNotReachHere();
2223 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2224 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2225 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2226 __ lea(rscratch1, __ constant_oop_address(o));
2227 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2228 }
2229
2230
2231 // This code replaces a call to arraycopy; no exception may
2232 // be thrown in this code, they must be thrown in the System.arraycopy
2233 // activation frame; we could save some checks if this would not be the case
emit_arraycopy(LIR_OpArrayCopy * op)2234 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2235 ciArrayKlass* default_type = op->expected_type();
2236 Register src = op->src()->as_register();
2237 Register dst = op->dst()->as_register();
2238 Register src_pos = op->src_pos()->as_register();
2239 Register dst_pos = op->dst_pos()->as_register();
2240 Register length = op->length()->as_register();
2241 Register tmp = op->tmp()->as_register();
2242
2243 __ resolve(ACCESS_READ, src);
2244 __ resolve(ACCESS_WRITE, dst);
2245
2246 CodeStub* stub = op->stub();
2247 int flags = op->flags();
2248 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2249 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2250
2251 // if we don't know anything, just go through the generic arraycopy
2252 if (default_type == NULL // || basic_type == T_OBJECT
2253 ) {
2254 Label done;
2255 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2256
2257 // Save the arguments in case the generic arraycopy fails and we
2258 // have to fall back to the JNI stub
2259 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2260 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2261 __ str(src, Address(sp, 4*BytesPerWord));
2262
2263 address copyfunc_addr = StubRoutines::generic_arraycopy();
2264 assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2265
2266 // The arguments are in java calling convention so we shift them
2267 // to C convention
2268 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2269 __ mov(c_rarg0, j_rarg0);
2270 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2271 __ mov(c_rarg1, j_rarg1);
2272 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2273 __ mov(c_rarg2, j_rarg2);
2274 assert_different_registers(c_rarg3, j_rarg4);
2275 __ mov(c_rarg3, j_rarg3);
2276 __ mov(c_rarg4, j_rarg4);
2277 #ifndef PRODUCT
2278 if (PrintC1Statistics) {
2279 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2280 }
2281 #endif
2282 __ far_call(RuntimeAddress(copyfunc_addr));
2283
2284 __ cbz(r0, *stub->continuation());
2285
2286 // Reload values from the stack so they are where the stub
2287 // expects them.
2288 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2289 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2290 __ ldr(src, Address(sp, 4*BytesPerWord));
2291
2292 // r0 is -1^K where K == partial copied count
2293 __ eonw(rscratch1, r0, zr);
2294 // adjust length down and src/end pos up by partial copied count
2295 __ subw(length, length, rscratch1);
2296 __ addw(src_pos, src_pos, rscratch1);
2297 __ addw(dst_pos, dst_pos, rscratch1);
2298 __ b(*stub->entry());
2299
2300 __ bind(*stub->continuation());
2301 return;
2302 }
2303
2304 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2305
2306 int elem_size = type2aelembytes(basic_type);
2307 int shift_amount;
2308 int scale = exact_log2(elem_size);
2309
2310 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2311 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2312 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2313 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2314
2315 // test for NULL
2316 if (flags & LIR_OpArrayCopy::src_null_check) {
2317 __ cbz(src, *stub->entry());
2318 }
2319 if (flags & LIR_OpArrayCopy::dst_null_check) {
2320 __ cbz(dst, *stub->entry());
2321 }
2322
2323 // If the compiler was not able to prove that exact type of the source or the destination
2324 // of the arraycopy is an array type, check at runtime if the source or the destination is
2325 // an instance type.
2326 if (flags & LIR_OpArrayCopy::type_check) {
2327 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2328 __ load_klass(tmp, dst);
2329 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2330 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2331 __ br(Assembler::GE, *stub->entry());
2332 }
2333
2334 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2335 __ load_klass(tmp, src);
2336 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2337 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2338 __ br(Assembler::GE, *stub->entry());
2339 }
2340 }
2341
2342 // check if negative
2343 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2344 __ cmpw(src_pos, 0);
2345 __ br(Assembler::LT, *stub->entry());
2346 }
2347 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2348 __ cmpw(dst_pos, 0);
2349 __ br(Assembler::LT, *stub->entry());
2350 }
2351
2352 if (flags & LIR_OpArrayCopy::length_positive_check) {
2353 __ cmpw(length, 0);
2354 __ br(Assembler::LT, *stub->entry());
2355 }
2356
2357 if (flags & LIR_OpArrayCopy::src_range_check) {
2358 __ addw(tmp, src_pos, length);
2359 __ ldrw(rscratch1, src_length_addr);
2360 __ cmpw(tmp, rscratch1);
2361 __ br(Assembler::HI, *stub->entry());
2362 }
2363 if (flags & LIR_OpArrayCopy::dst_range_check) {
2364 __ addw(tmp, dst_pos, length);
2365 __ ldrw(rscratch1, dst_length_addr);
2366 __ cmpw(tmp, rscratch1);
2367 __ br(Assembler::HI, *stub->entry());
2368 }
2369
2370 if (flags & LIR_OpArrayCopy::type_check) {
2371 // We don't know the array types are compatible
2372 if (basic_type != T_OBJECT) {
2373 // Simple test for basic type arrays
2374 if (UseCompressedClassPointers) {
2375 __ ldrw(tmp, src_klass_addr);
2376 __ ldrw(rscratch1, dst_klass_addr);
2377 __ cmpw(tmp, rscratch1);
2378 } else {
2379 __ ldr(tmp, src_klass_addr);
2380 __ ldr(rscratch1, dst_klass_addr);
2381 __ cmp(tmp, rscratch1);
2382 }
2383 __ br(Assembler::NE, *stub->entry());
2384 } else {
2385 // For object arrays, if src is a sub class of dst then we can
2386 // safely do the copy.
2387 Label cont, slow;
2388
2389 #define PUSH(r1, r2) \
2390 stp(r1, r2, __ pre(sp, -2 * wordSize));
2391
2392 #define POP(r1, r2) \
2393 ldp(r1, r2, __ post(sp, 2 * wordSize));
2394
2395 __ PUSH(src, dst);
2396
2397 __ load_klass(src, src);
2398 __ load_klass(dst, dst);
2399
2400 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2401
2402 __ PUSH(src, dst);
2403 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2404 __ POP(src, dst);
2405
2406 __ cbnz(src, cont);
2407
2408 __ bind(slow);
2409 __ POP(src, dst);
2410
2411 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2412 if (copyfunc_addr != NULL) { // use stub if available
2413 // src is not a sub class of dst so we have to do a
2414 // per-element check.
2415
2416 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2417 if ((flags & mask) != mask) {
2418 // Check that at least both of them object arrays.
2419 assert(flags & mask, "one of the two should be known to be an object array");
2420
2421 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2422 __ load_klass(tmp, src);
2423 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2424 __ load_klass(tmp, dst);
2425 }
2426 int lh_offset = in_bytes(Klass::layout_helper_offset());
2427 Address klass_lh_addr(tmp, lh_offset);
2428 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2429 __ ldrw(rscratch1, klass_lh_addr);
2430 __ mov(rscratch2, objArray_lh);
2431 __ eorw(rscratch1, rscratch1, rscratch2);
2432 __ cbnzw(rscratch1, *stub->entry());
2433 }
2434
2435 // Spill because stubs can use any register they like and it's
2436 // easier to restore just those that we care about.
2437 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2438 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2439 __ str(src, Address(sp, 4*BytesPerWord));
2440
2441 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2442 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2443 assert_different_registers(c_rarg0, dst, dst_pos, length);
2444 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2445 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2446 assert_different_registers(c_rarg1, dst, length);
2447 __ uxtw(c_rarg2, length);
2448 assert_different_registers(c_rarg2, dst);
2449
2450 __ load_klass(c_rarg4, dst);
2451 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2452 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2453 __ far_call(RuntimeAddress(copyfunc_addr));
2454
2455 #ifndef PRODUCT
2456 if (PrintC1Statistics) {
2457 Label failed;
2458 __ cbnz(r0, failed);
2459 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2460 __ bind(failed);
2461 }
2462 #endif
2463
2464 __ cbz(r0, *stub->continuation());
2465
2466 #ifndef PRODUCT
2467 if (PrintC1Statistics) {
2468 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2469 }
2470 #endif
2471 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2472
2473 // Restore previously spilled arguments
2474 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2475 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2476 __ ldr(src, Address(sp, 4*BytesPerWord));
2477
2478 // return value is -1^K where K is partial copied count
2479 __ eonw(rscratch1, r0, zr);
2480 // adjust length down and src/end pos up by partial copied count
2481 __ subw(length, length, rscratch1);
2482 __ addw(src_pos, src_pos, rscratch1);
2483 __ addw(dst_pos, dst_pos, rscratch1);
2484 }
2485
2486 __ b(*stub->entry());
2487
2488 __ bind(cont);
2489 __ POP(src, dst);
2490 }
2491 }
2492
2493 #ifdef ASSERT
2494 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2495 // Sanity check the known type with the incoming class. For the
2496 // primitive case the types must match exactly with src.klass and
2497 // dst.klass each exactly matching the default type. For the
2498 // object array case, if no type check is needed then either the
2499 // dst type is exactly the expected type and the src type is a
2500 // subtype which we can't check or src is the same array as dst
2501 // but not necessarily exactly of type default_type.
2502 Label known_ok, halt;
2503 __ mov_metadata(tmp, default_type->constant_encoding());
2504 if (UseCompressedClassPointers) {
2505 __ encode_klass_not_null(tmp);
2506 }
2507
2508 if (basic_type != T_OBJECT) {
2509
2510 if (UseCompressedClassPointers) {
2511 __ ldrw(rscratch1, dst_klass_addr);
2512 __ cmpw(tmp, rscratch1);
2513 } else {
2514 __ ldr(rscratch1, dst_klass_addr);
2515 __ cmp(tmp, rscratch1);
2516 }
2517 __ br(Assembler::NE, halt);
2518 if (UseCompressedClassPointers) {
2519 __ ldrw(rscratch1, src_klass_addr);
2520 __ cmpw(tmp, rscratch1);
2521 } else {
2522 __ ldr(rscratch1, src_klass_addr);
2523 __ cmp(tmp, rscratch1);
2524 }
2525 __ br(Assembler::EQ, known_ok);
2526 } else {
2527 if (UseCompressedClassPointers) {
2528 __ ldrw(rscratch1, dst_klass_addr);
2529 __ cmpw(tmp, rscratch1);
2530 } else {
2531 __ ldr(rscratch1, dst_klass_addr);
2532 __ cmp(tmp, rscratch1);
2533 }
2534 __ br(Assembler::EQ, known_ok);
2535 __ cmp(src, dst);
2536 __ br(Assembler::EQ, known_ok);
2537 }
2538 __ bind(halt);
2539 __ stop("incorrect type information in arraycopy");
2540 __ bind(known_ok);
2541 }
2542 #endif
2543
2544 #ifndef PRODUCT
2545 if (PrintC1Statistics) {
2546 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2547 }
2548 #endif
2549
2550 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2551 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2552 assert_different_registers(c_rarg0, dst, dst_pos, length);
2553 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2554 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2555 assert_different_registers(c_rarg1, dst, length);
2556 __ uxtw(c_rarg2, length);
2557 assert_different_registers(c_rarg2, dst);
2558
2559 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2560 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2561 const char *name;
2562 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2563
2564 CodeBlob *cb = CodeCache::find_blob(entry);
2565 if (cb) {
2566 __ far_call(RuntimeAddress(entry));
2567 } else {
2568 __ call_VM_leaf(entry, 3);
2569 }
2570
2571 __ bind(*stub->continuation());
2572 }
2573
2574
2575
2576
emit_lock(LIR_OpLock * op)2577 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2578 Register obj = op->obj_opr()->as_register(); // may not be an oop
2579 Register hdr = op->hdr_opr()->as_register();
2580 Register lock = op->lock_opr()->as_register();
2581 if (!UseFastLocking) {
2582 __ b(*op->stub()->entry());
2583 } else if (op->code() == lir_lock) {
2584 Register scratch = noreg;
2585 if (UseBiasedLocking) {
2586 scratch = op->scratch_opr()->as_register();
2587 }
2588 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2589 __ resolve(ACCESS_READ | ACCESS_WRITE, obj);
2590 // add debug info for NullPointerException only if one is possible
2591 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2592 if (op->info() != NULL) {
2593 add_debug_info_for_null_check(null_check_offset, op->info());
2594 }
2595 // done
2596 } else if (op->code() == lir_unlock) {
2597 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2598 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2599 } else {
2600 Unimplemented();
2601 }
2602 __ bind(*op->stub()->continuation());
2603 }
2604
2605
emit_profile_call(LIR_OpProfileCall * op)2606 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2607 ciMethod* method = op->profiled_method();
2608 int bci = op->profiled_bci();
2609 ciMethod* callee = op->profiled_callee();
2610
2611 // Update counter for all call types
2612 ciMethodData* md = method->method_data_or_null();
2613 assert(md != NULL, "Sanity");
2614 ciProfileData* data = md->bci_to_data(bci);
2615 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2616 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2617 Register mdo = op->mdo()->as_register();
2618 __ mov_metadata(mdo, md->constant_encoding());
2619 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2620 // Perform additional virtual call profiling for invokevirtual and
2621 // invokeinterface bytecodes
2622 if (op->should_profile_receiver_type()) {
2623 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2624 Register recv = op->recv()->as_register();
2625 assert_different_registers(mdo, recv);
2626 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2627 ciKlass* known_klass = op->known_holder();
2628 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2629 // We know the type that will be seen at this call site; we can
2630 // statically update the MethodData* rather than needing to do
2631 // dynamic tests on the receiver type
2632
2633 // NOTE: we should probably put a lock around this search to
2634 // avoid collisions by concurrent compilations
2635 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2636 uint i;
2637 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2638 ciKlass* receiver = vc_data->receiver(i);
2639 if (known_klass->equals(receiver)) {
2640 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2641 __ addptr(data_addr, DataLayout::counter_increment);
2642 return;
2643 }
2644 }
2645
2646 // Receiver type not found in profile data; select an empty slot
2647
2648 // Note that this is less efficient than it should be because it
2649 // always does a write to the receiver part of the
2650 // VirtualCallData rather than just the first time
2651 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2652 ciKlass* receiver = vc_data->receiver(i);
2653 if (receiver == NULL) {
2654 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2655 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2656 __ lea(rscratch2, recv_addr);
2657 __ str(rscratch1, Address(rscratch2));
2658 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2659 __ addptr(data_addr, DataLayout::counter_increment);
2660 return;
2661 }
2662 }
2663 } else {
2664 __ load_klass(recv, recv);
2665 Label update_done;
2666 type_profile_helper(mdo, md, data, recv, &update_done);
2667 // Receiver did not match any saved receiver and there is no empty row for it.
2668 // Increment total counter to indicate polymorphic case.
2669 __ addptr(counter_addr, DataLayout::counter_increment);
2670
2671 __ bind(update_done);
2672 }
2673 } else {
2674 // Static call
2675 __ addptr(counter_addr, DataLayout::counter_increment);
2676 }
2677 }
2678
2679
emit_delay(LIR_OpDelay *)2680 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2681 Unimplemented();
2682 }
2683
2684
monitor_address(int monitor_no,LIR_Opr dst)2685 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2686 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2687 }
2688
emit_updatecrc32(LIR_OpUpdateCRC32 * op)2689 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2690 assert(op->crc()->is_single_cpu(), "crc must be register");
2691 assert(op->val()->is_single_cpu(), "byte value must be register");
2692 assert(op->result_opr()->is_single_cpu(), "result must be register");
2693 Register crc = op->crc()->as_register();
2694 Register val = op->val()->as_register();
2695 Register res = op->result_opr()->as_register();
2696
2697 assert_different_registers(val, crc, res);
2698 unsigned long offset;
2699 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2700 if (offset) __ add(res, res, offset);
2701
2702 __ mvnw(crc, crc); // ~crc
2703 __ update_byte_crc32(crc, val, res);
2704 __ mvnw(res, crc); // ~crc
2705 }
2706
emit_profile_type(LIR_OpProfileType * op)2707 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2708 COMMENT("emit_profile_type {");
2709 Register obj = op->obj()->as_register();
2710 Register tmp = op->tmp()->as_pointer_register();
2711 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2712 ciKlass* exact_klass = op->exact_klass();
2713 intptr_t current_klass = op->current_klass();
2714 bool not_null = op->not_null();
2715 bool no_conflict = op->no_conflict();
2716
2717 Label update, next, none;
2718
2719 bool do_null = !not_null;
2720 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2721 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2722
2723 assert(do_null || do_update, "why are we here?");
2724 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2725 assert(mdo_addr.base() != rscratch1, "wrong register");
2726
2727 __ verify_oop(obj);
2728
2729 if (tmp != obj) {
2730 __ mov(tmp, obj);
2731 }
2732 if (do_null) {
2733 __ cbnz(tmp, update);
2734 if (!TypeEntries::was_null_seen(current_klass)) {
2735 __ ldr(rscratch2, mdo_addr);
2736 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2737 __ str(rscratch2, mdo_addr);
2738 }
2739 if (do_update) {
2740 #ifndef ASSERT
2741 __ b(next);
2742 }
2743 #else
2744 __ b(next);
2745 }
2746 } else {
2747 __ cbnz(tmp, update);
2748 __ stop("unexpected null obj");
2749 #endif
2750 }
2751
2752 __ bind(update);
2753
2754 if (do_update) {
2755 #ifdef ASSERT
2756 if (exact_klass != NULL) {
2757 Label ok;
2758 __ load_klass(tmp, tmp);
2759 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2760 __ eor(rscratch1, tmp, rscratch1);
2761 __ cbz(rscratch1, ok);
2762 __ stop("exact klass and actual klass differ");
2763 __ bind(ok);
2764 }
2765 #endif
2766 if (!no_conflict) {
2767 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2768 if (exact_klass != NULL) {
2769 __ mov_metadata(tmp, exact_klass->constant_encoding());
2770 } else {
2771 __ load_klass(tmp, tmp);
2772 }
2773
2774 __ ldr(rscratch2, mdo_addr);
2775 __ eor(tmp, tmp, rscratch2);
2776 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2777 // klass seen before, nothing to do. The unknown bit may have been
2778 // set already but no need to check.
2779 __ cbz(rscratch1, next);
2780
2781 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2782
2783 if (TypeEntries::is_type_none(current_klass)) {
2784 __ cbz(rscratch2, none);
2785 __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2786 __ br(Assembler::EQ, none);
2787 // There is a chance that the checks above (re-reading profiling
2788 // data from memory) fail if another thread has just set the
2789 // profiling to this obj's klass
2790 __ dmb(Assembler::ISHLD);
2791 __ ldr(rscratch2, mdo_addr);
2792 __ eor(tmp, tmp, rscratch2);
2793 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2794 __ cbz(rscratch1, next);
2795 }
2796 } else {
2797 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2798 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2799
2800 __ ldr(tmp, mdo_addr);
2801 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2802 }
2803
2804 // different than before. Cannot keep accurate profile.
2805 __ ldr(rscratch2, mdo_addr);
2806 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2807 __ str(rscratch2, mdo_addr);
2808
2809 if (TypeEntries::is_type_none(current_klass)) {
2810 __ b(next);
2811
2812 __ bind(none);
2813 // first time here. Set profile type.
2814 __ str(tmp, mdo_addr);
2815 }
2816 } else {
2817 // There's a single possible klass at this profile point
2818 assert(exact_klass != NULL, "should be");
2819 if (TypeEntries::is_type_none(current_klass)) {
2820 __ mov_metadata(tmp, exact_klass->constant_encoding());
2821 __ ldr(rscratch2, mdo_addr);
2822 __ eor(tmp, tmp, rscratch2);
2823 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2824 __ cbz(rscratch1, next);
2825 #ifdef ASSERT
2826 {
2827 Label ok;
2828 __ ldr(rscratch1, mdo_addr);
2829 __ cbz(rscratch1, ok);
2830 __ cmp(rscratch1, (u1)TypeEntries::null_seen);
2831 __ br(Assembler::EQ, ok);
2832 // may have been set by another thread
2833 __ dmb(Assembler::ISHLD);
2834 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2835 __ ldr(rscratch2, mdo_addr);
2836 __ eor(rscratch2, rscratch1, rscratch2);
2837 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2838 __ cbz(rscratch2, ok);
2839
2840 __ stop("unexpected profiling mismatch");
2841 __ bind(ok);
2842 }
2843 #endif
2844 // first time here. Set profile type.
2845 __ ldr(tmp, mdo_addr);
2846 } else {
2847 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2848 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2849
2850 __ ldr(tmp, mdo_addr);
2851 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2852
2853 __ orr(tmp, tmp, TypeEntries::type_unknown);
2854 __ str(tmp, mdo_addr);
2855 // FIXME: Write barrier needed here?
2856 }
2857 }
2858
2859 __ bind(next);
2860 }
2861 COMMENT("} emit_profile_type");
2862 }
2863
2864
align_backward_branch_target()2865 void LIR_Assembler::align_backward_branch_target() {
2866 }
2867
2868
negate(LIR_Opr left,LIR_Opr dest,LIR_Opr tmp)2869 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2870 // tmp must be unused
2871 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2872
2873 if (left->is_single_cpu()) {
2874 assert(dest->is_single_cpu(), "expect single result reg");
2875 __ negw(dest->as_register(), left->as_register());
2876 } else if (left->is_double_cpu()) {
2877 assert(dest->is_double_cpu(), "expect double result reg");
2878 __ neg(dest->as_register_lo(), left->as_register_lo());
2879 } else if (left->is_single_fpu()) {
2880 assert(dest->is_single_fpu(), "expect single float result reg");
2881 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2882 } else {
2883 assert(left->is_double_fpu(), "expect double float operand reg");
2884 assert(dest->is_double_fpu(), "expect double float result reg");
2885 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2886 }
2887 }
2888
2889
leal(LIR_Opr addr,LIR_Opr dest,LIR_PatchCode patch_code,CodeEmitInfo * info)2890 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2891 if (patch_code != lir_patch_none) {
2892 deoptimize_trap(info);
2893 return;
2894 }
2895
2896 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2897 }
2898
2899
rt_call(LIR_Opr result,address dest,const LIR_OprList * args,LIR_Opr tmp,CodeEmitInfo * info)2900 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2901 assert(!tmp->is_valid(), "don't need temporary");
2902
2903 CodeBlob *cb = CodeCache::find_blob(dest);
2904 if (cb) {
2905 __ far_call(RuntimeAddress(dest));
2906 } else {
2907 __ mov(rscratch1, RuntimeAddress(dest));
2908 __ blr(rscratch1);
2909 }
2910
2911 if (info != NULL) {
2912 add_call_info_here(info);
2913 }
2914 __ maybe_isb();
2915 }
2916
volatile_move_op(LIR_Opr src,LIR_Opr dest,BasicType type,CodeEmitInfo * info)2917 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2918 if (dest->is_address() || src->is_address()) {
2919 move_op(src, dest, type, lir_patch_none, info,
2920 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2921 } else {
2922 ShouldNotReachHere();
2923 }
2924 }
2925
2926 #ifdef ASSERT
2927 // emit run-time assertion
emit_assert(LIR_OpAssert * op)2928 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2929 assert(op->code() == lir_assert, "must be");
2930
2931 if (op->in_opr1()->is_valid()) {
2932 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2933 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2934 } else {
2935 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2936 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2937 }
2938
2939 Label ok;
2940 if (op->condition() != lir_cond_always) {
2941 Assembler::Condition acond = Assembler::AL;
2942 switch (op->condition()) {
2943 case lir_cond_equal: acond = Assembler::EQ; break;
2944 case lir_cond_notEqual: acond = Assembler::NE; break;
2945 case lir_cond_less: acond = Assembler::LT; break;
2946 case lir_cond_lessEqual: acond = Assembler::LE; break;
2947 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2948 case lir_cond_greater: acond = Assembler::GT; break;
2949 case lir_cond_belowEqual: acond = Assembler::LS; break;
2950 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2951 default: ShouldNotReachHere();
2952 }
2953 __ br(acond, ok);
2954 }
2955 if (op->halt()) {
2956 const char* str = __ code_string(op->msg());
2957 __ stop(str);
2958 } else {
2959 breakpoint();
2960 }
2961 __ bind(ok);
2962 }
2963 #endif
2964
2965 #ifndef PRODUCT
2966 #define COMMENT(x) do { __ block_comment(x); } while (0)
2967 #else
2968 #define COMMENT(x)
2969 #endif
2970
membar()2971 void LIR_Assembler::membar() {
2972 COMMENT("membar");
2973 __ membar(MacroAssembler::AnyAny);
2974 }
2975
membar_acquire()2976 void LIR_Assembler::membar_acquire() {
2977 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2978 }
2979
membar_release()2980 void LIR_Assembler::membar_release() {
2981 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2982 }
2983
membar_loadload()2984 void LIR_Assembler::membar_loadload() {
2985 __ membar(Assembler::LoadLoad);
2986 }
2987
membar_storestore()2988 void LIR_Assembler::membar_storestore() {
2989 __ membar(MacroAssembler::StoreStore);
2990 }
2991
membar_loadstore()2992 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2993
membar_storeload()2994 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2995
on_spin_wait()2996 void LIR_Assembler::on_spin_wait() {
2997 Unimplemented();
2998 }
2999
get_thread(LIR_Opr result_reg)3000 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3001 __ mov(result_reg->as_register(), rthread);
3002 }
3003
3004
peephole(LIR_List * lir)3005 void LIR_Assembler::peephole(LIR_List *lir) {
3006 #if 0
3007 if (tableswitch_count >= max_tableswitches)
3008 return;
3009
3010 /*
3011 This finite-state automaton recognizes sequences of compare-and-
3012 branch instructions. We will turn them into a tableswitch. You
3013 could argue that C1 really shouldn't be doing this sort of
3014 optimization, but without it the code is really horrible.
3015 */
3016
3017 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3018 int first_key, last_key = -2147483648;
3019 int next_key = 0;
3020 int start_insn = -1;
3021 int last_insn = -1;
3022 Register reg = noreg;
3023 LIR_Opr reg_opr;
3024 state = start_s;
3025
3026 LIR_OpList* inst = lir->instructions_list();
3027 for (int i = 0; i < inst->length(); i++) {
3028 LIR_Op* op = inst->at(i);
3029 switch (state) {
3030 case start_s:
3031 first_key = -1;
3032 start_insn = i;
3033 switch (op->code()) {
3034 case lir_cmp:
3035 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3036 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3037 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3038 && opr2->is_constant()
3039 && opr2->type() == T_INT) {
3040 reg_opr = opr1;
3041 reg = opr1->as_register();
3042 first_key = opr2->as_constant_ptr()->as_jint();
3043 next_key = first_key + 1;
3044 state = cmp_s;
3045 goto next_state;
3046 }
3047 break;
3048 }
3049 break;
3050 case cmp_s:
3051 switch (op->code()) {
3052 case lir_branch:
3053 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3054 state = beq_s;
3055 last_insn = i;
3056 goto next_state;
3057 }
3058 }
3059 state = start_s;
3060 break;
3061 case beq_s:
3062 switch (op->code()) {
3063 case lir_cmp: {
3064 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3065 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3066 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3067 && opr1->as_register() == reg
3068 && opr2->is_constant()
3069 && opr2->type() == T_INT
3070 && opr2->as_constant_ptr()->as_jint() == next_key) {
3071 last_key = next_key;
3072 next_key++;
3073 state = cmp_s;
3074 goto next_state;
3075 }
3076 }
3077 }
3078 last_key = next_key;
3079 state = start_s;
3080 break;
3081 default:
3082 assert(false, "impossible state");
3083 }
3084 if (state == start_s) {
3085 if (first_key < last_key - 5L && reg != noreg) {
3086 {
3087 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3088 // reg->encoding(),
3089 // start_insn, first_key, last_key);
3090 // for (int i = 0; i < inst->length(); i++) {
3091 // inst->at(i)->print();
3092 // tty->print("\n");
3093 // }
3094 // tty->print("\n");
3095 }
3096
3097 struct tableswitch *sw = &switches[tableswitch_count];
3098 sw->_insn_index = start_insn, sw->_first_key = first_key,
3099 sw->_last_key = last_key, sw->_reg = reg;
3100 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3101 {
3102 // Insert the new table of branches
3103 int offset = last_insn;
3104 for (int n = first_key; n < last_key; n++) {
3105 inst->insert_before
3106 (last_insn + 1,
3107 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3108 inst->at(offset)->as_OpBranch()->label()));
3109 offset -= 2, i++;
3110 }
3111 }
3112 // Delete all the old compare-and-branch instructions
3113 for (int n = first_key; n < last_key; n++) {
3114 inst->remove_at(start_insn);
3115 inst->remove_at(start_insn);
3116 }
3117 // Insert the tableswitch instruction
3118 inst->insert_before(start_insn,
3119 new LIR_Op2(lir_cmp, lir_cond_always,
3120 LIR_OprFact::intConst(tableswitch_count),
3121 reg_opr));
3122 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3123 tableswitch_count++;
3124 }
3125 reg = noreg;
3126 last_key = -2147483648;
3127 }
3128 next_state:
3129 ;
3130 }
3131 #endif
3132 }
3133
atomic_op(LIR_Code code,LIR_Opr src,LIR_Opr data,LIR_Opr dest,LIR_Opr tmp_op)3134 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3135 Address addr = as_Address(src->as_address_ptr());
3136 BasicType type = src->type();
3137 bool is_oop = is_reference_type(type);
3138
3139 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3140 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3141
3142 switch(type) {
3143 case T_INT:
3144 xchg = &MacroAssembler::atomic_xchgalw;
3145 add = &MacroAssembler::atomic_addalw;
3146 break;
3147 case T_LONG:
3148 xchg = &MacroAssembler::atomic_xchgal;
3149 add = &MacroAssembler::atomic_addal;
3150 break;
3151 case T_OBJECT:
3152 case T_ARRAY:
3153 if (UseCompressedOops) {
3154 xchg = &MacroAssembler::atomic_xchgalw;
3155 add = &MacroAssembler::atomic_addalw;
3156 } else {
3157 xchg = &MacroAssembler::atomic_xchgal;
3158 add = &MacroAssembler::atomic_addal;
3159 }
3160 break;
3161 default:
3162 ShouldNotReachHere();
3163 xchg = &MacroAssembler::atomic_xchgal;
3164 add = &MacroAssembler::atomic_addal; // unreachable
3165 }
3166
3167 switch (code) {
3168 case lir_xadd:
3169 {
3170 RegisterOrConstant inc;
3171 Register tmp = as_reg(tmp_op);
3172 Register dst = as_reg(dest);
3173 if (data->is_constant()) {
3174 inc = RegisterOrConstant(as_long(data));
3175 assert_different_registers(dst, addr.base(), tmp,
3176 rscratch1, rscratch2);
3177 } else {
3178 inc = RegisterOrConstant(as_reg(data));
3179 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3180 rscratch1, rscratch2);
3181 }
3182 __ lea(tmp, addr);
3183 (_masm->*add)(dst, inc, tmp);
3184 break;
3185 }
3186 case lir_xchg:
3187 {
3188 Register tmp = tmp_op->as_register();
3189 Register obj = as_reg(data);
3190 Register dst = as_reg(dest);
3191 if (is_oop && UseCompressedOops) {
3192 __ encode_heap_oop(rscratch2, obj);
3193 obj = rscratch2;
3194 }
3195 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3196 __ lea(tmp, addr);
3197 (_masm->*xchg)(dst, obj, tmp);
3198 if (is_oop && UseCompressedOops) {
3199 __ decode_heap_oop(dst);
3200 }
3201 }
3202 break;
3203 default:
3204 ShouldNotReachHere();
3205 }
3206 __ membar(__ AnyAny);
3207 }
3208
3209 #undef __
3210