1 /*
2 * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "ci/ciEnv.hpp"
30 #include "code/nativeInst.hpp"
31 #include "compiler/disassembler.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/cardTable.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/collectedHeap.inline.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "memory/resourceArea.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/klass.inline.hpp"
41 #include "prims/methodHandles.hpp"
42 #include "runtime/biasedLocking.hpp"
43 #include "runtime/interfaceSupport.inline.hpp"
44 #include "runtime/objectMonitor.hpp"
45 #include "runtime/os.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "utilities/macros.hpp"
49 #include "utilities/powerOfTwo.hpp"
50
51 // Implementation of AddressLiteral
52
set_rspec(relocInfo::relocType rtype)53 void AddressLiteral::set_rspec(relocInfo::relocType rtype) {
54 switch (rtype) {
55 case relocInfo::oop_type:
56 // Oops are a special case. Normally they would be their own section
57 // but in cases like icBuffer they are literals in the code stream that
58 // we don't have a section for. We use none so that we get a literal address
59 // which is always patchable.
60 break;
61 case relocInfo::external_word_type:
62 _rspec = external_word_Relocation::spec(_target);
63 break;
64 case relocInfo::internal_word_type:
65 _rspec = internal_word_Relocation::spec(_target);
66 break;
67 case relocInfo::opt_virtual_call_type:
68 _rspec = opt_virtual_call_Relocation::spec();
69 break;
70 case relocInfo::static_call_type:
71 _rspec = static_call_Relocation::spec();
72 break;
73 case relocInfo::runtime_call_type:
74 _rspec = runtime_call_Relocation::spec();
75 break;
76 case relocInfo::poll_type:
77 case relocInfo::poll_return_type:
78 _rspec = Relocation::spec_simple(rtype);
79 break;
80 case relocInfo::none:
81 break;
82 default:
83 ShouldNotReachHere();
84 break;
85 }
86 }
87
88
89 // virtual method calling
lookup_virtual_method(Register recv_klass,Register vtable_index,Register method_result)90 void MacroAssembler::lookup_virtual_method(Register recv_klass,
91 Register vtable_index,
92 Register method_result) {
93 const int base_offset = in_bytes(Klass::vtable_start_offset()) + vtableEntry::method_offset_in_bytes();
94 assert(vtableEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
95 add(recv_klass, recv_klass, AsmOperand(vtable_index, lsl, LogBytesPerWord));
96 ldr(method_result, Address(recv_klass, base_offset));
97 }
98
99
100 // Simplified, combined version, good for typical uses.
101 // Falls through on failure.
check_klass_subtype(Register sub_klass,Register super_klass,Register temp_reg,Register temp_reg2,Register temp_reg3,Label & L_success)102 void MacroAssembler::check_klass_subtype(Register sub_klass,
103 Register super_klass,
104 Register temp_reg,
105 Register temp_reg2,
106 Register temp_reg3,
107 Label& L_success) {
108 Label L_failure;
109 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, temp_reg2, &L_success, &L_failure, NULL);
110 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, temp_reg2, temp_reg3, &L_success, NULL);
111 bind(L_failure);
112 };
113
check_klass_subtype_fast_path(Register sub_klass,Register super_klass,Register temp_reg,Register temp_reg2,Label * L_success,Label * L_failure,Label * L_slow_path)114 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
115 Register super_klass,
116 Register temp_reg,
117 Register temp_reg2,
118 Label* L_success,
119 Label* L_failure,
120 Label* L_slow_path) {
121
122 assert_different_registers(sub_klass, super_klass, temp_reg, temp_reg2, noreg);
123 const Register super_check_offset = temp_reg2;
124
125 Label L_fallthrough;
126 int label_nulls = 0;
127 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
128 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
129 if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
130 assert(label_nulls <= 1, "at most one NULL in the batch");
131
132 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
133 int sco_offset = in_bytes(Klass::super_check_offset_offset());
134 Address super_check_offset_addr(super_klass, sco_offset);
135
136 // If the pointers are equal, we are done (e.g., String[] elements).
137 // This self-check enables sharing of secondary supertype arrays among
138 // non-primary types such as array-of-interface. Otherwise, each such
139 // type would need its own customized SSA.
140 // We move this check to the front of the fast path because many
141 // type checks are in fact trivially successful in this manner,
142 // so we get a nicely predicted branch right at the start of the check.
143 cmp(sub_klass, super_klass);
144 b(*L_success, eq);
145
146 // Check the supertype display:
147 ldr_u32(super_check_offset, super_check_offset_addr);
148
149 Address super_check_addr(sub_klass, super_check_offset);
150 ldr(temp_reg, super_check_addr);
151 cmp(super_klass, temp_reg); // load displayed supertype
152
153 // This check has worked decisively for primary supers.
154 // Secondary supers are sought in the super_cache ('super_cache_addr').
155 // (Secondary supers are interfaces and very deeply nested subtypes.)
156 // This works in the same check above because of a tricky aliasing
157 // between the super_cache and the primary super display elements.
158 // (The 'super_check_addr' can address either, as the case requires.)
159 // Note that the cache is updated below if it does not help us find
160 // what we need immediately.
161 // So if it was a primary super, we can just fail immediately.
162 // Otherwise, it's the slow path for us (no success at this point).
163
164 b(*L_success, eq);
165 cmp_32(super_check_offset, sc_offset);
166 if (L_failure == &L_fallthrough) {
167 b(*L_slow_path, eq);
168 } else {
169 b(*L_failure, ne);
170 if (L_slow_path != &L_fallthrough) {
171 b(*L_slow_path);
172 }
173 }
174
175 bind(L_fallthrough);
176 }
177
178
check_klass_subtype_slow_path(Register sub_klass,Register super_klass,Register temp_reg,Register temp2_reg,Register temp3_reg,Label * L_success,Label * L_failure,bool set_cond_codes)179 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
180 Register super_klass,
181 Register temp_reg,
182 Register temp2_reg,
183 Register temp3_reg,
184 Label* L_success,
185 Label* L_failure,
186 bool set_cond_codes) {
187 // Note: if used by code that expects a register to be 0 on success,
188 // this register must be temp_reg and set_cond_codes must be true
189
190 Register saved_reg = noreg;
191
192 // get additional tmp registers
193 if (temp3_reg == noreg) {
194 saved_reg = temp3_reg = LR;
195 push(saved_reg);
196 }
197
198 assert(temp2_reg != noreg, "need all the temporary registers");
199 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg);
200
201 Register cmp_temp = temp_reg;
202 Register scan_temp = temp3_reg;
203 Register count_temp = temp2_reg;
204
205 Label L_fallthrough;
206 int label_nulls = 0;
207 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
208 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
209 assert(label_nulls <= 1, "at most one NULL in the batch");
210
211 // a couple of useful fields in sub_klass:
212 int ss_offset = in_bytes(Klass::secondary_supers_offset());
213 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
214 Address secondary_supers_addr(sub_klass, ss_offset);
215 Address super_cache_addr( sub_klass, sc_offset);
216
217 #ifndef PRODUCT
218 inc_counter((address)&SharedRuntime::_partial_subtype_ctr, scan_temp, count_temp);
219 #endif
220
221 // We will consult the secondary-super array.
222 ldr(scan_temp, Address(sub_klass, ss_offset));
223
224 assert(! UseCompressedOops, "search_key must be the compressed super_klass");
225 // else search_key is the
226 Register search_key = super_klass;
227
228 // Load the array length.
229 ldr(count_temp, Address(scan_temp, Array<Klass*>::length_offset_in_bytes()));
230 add(scan_temp, scan_temp, Array<Klass*>::base_offset_in_bytes());
231
232 add(count_temp, count_temp, 1);
233
234 Label L_loop, L_fail;
235
236 // Top of search loop
237 bind(L_loop);
238 // Notes:
239 // scan_temp starts at the array elements
240 // count_temp is 1+size
241 subs(count_temp, count_temp, 1);
242 if ((L_failure != &L_fallthrough) && (! set_cond_codes) && (saved_reg == noreg)) {
243 // direct jump to L_failure if failed and no cleanup needed
244 b(*L_failure, eq); // not found and
245 } else {
246 b(L_fail, eq); // not found in the array
247 }
248
249 // Load next super to check
250 // In the array of super classes elements are pointer sized.
251 int element_size = wordSize;
252 ldr(cmp_temp, Address(scan_temp, element_size, post_indexed));
253
254 // Look for Rsuper_klass on Rsub_klass's secondary super-class-overflow list
255 subs(cmp_temp, cmp_temp, search_key);
256
257 // A miss means we are NOT a subtype and need to keep looping
258 b(L_loop, ne);
259
260 // Falling out the bottom means we found a hit; we ARE a subtype
261
262 // Note: temp_reg/cmp_temp is already 0 and flag Z is set
263
264 // Success. Cache the super we found and proceed in triumph.
265 str(super_klass, Address(sub_klass, sc_offset));
266
267 if (saved_reg != noreg) {
268 // Return success
269 pop(saved_reg);
270 }
271
272 b(*L_success);
273
274 bind(L_fail);
275 // Note1: check "b(*L_failure, eq)" above if adding extra instructions here
276 if (set_cond_codes) {
277 movs(temp_reg, sub_klass); // clears Z and sets temp_reg to non-0 if needed
278 }
279 if (saved_reg != noreg) {
280 pop(saved_reg);
281 }
282 if (L_failure != &L_fallthrough) {
283 b(*L_failure);
284 }
285
286 bind(L_fallthrough);
287 }
288
289 // Returns address of receiver parameter, using tmp as base register. tmp and params_count can be the same.
receiver_argument_address(Register params_base,Register params_count,Register tmp)290 Address MacroAssembler::receiver_argument_address(Register params_base, Register params_count, Register tmp) {
291 assert_different_registers(params_base, params_count);
292 add(tmp, params_base, AsmOperand(params_count, lsl, Interpreter::logStackElementSize));
293 return Address(tmp, -Interpreter::stackElementSize);
294 }
295
296
align(int modulus)297 void MacroAssembler::align(int modulus) {
298 while (offset() % modulus != 0) {
299 nop();
300 }
301 }
302
set_last_Java_frame(Register last_java_sp,Register last_java_fp,bool save_last_java_pc,Register tmp)303 int MacroAssembler::set_last_Java_frame(Register last_java_sp,
304 Register last_java_fp,
305 bool save_last_java_pc,
306 Register tmp) {
307 int pc_offset;
308 if (last_java_fp != noreg) {
309 // optional
310 str(last_java_fp, Address(Rthread, JavaThread::last_Java_fp_offset()));
311 _fp_saved = true;
312 } else {
313 _fp_saved = false;
314 }
315 if (save_last_java_pc) {
316 str(PC, Address(Rthread, JavaThread::last_Java_pc_offset()));
317 pc_offset = offset() + VM_Version::stored_pc_adjustment();
318 _pc_saved = true;
319 } else {
320 _pc_saved = false;
321 pc_offset = -1;
322 }
323 // According to comment in javaFrameAnchorm SP must be saved last, so that other
324 // entries are valid when SP is set.
325
326 // However, this is probably not a strong constrainst since for instance PC is
327 // sometimes read from the stack at SP... but is pushed later (by the call). Hence,
328 // we now write the fields in the expected order but we have not added a StoreStore
329 // barrier.
330
331 // XXX: if the ordering is really important, PC should always be saved (without forgetting
332 // to update oop_map offsets) and a StoreStore barrier might be needed.
333
334 if (last_java_sp == noreg) {
335 last_java_sp = SP; // always saved
336 }
337 str(last_java_sp, Address(Rthread, JavaThread::last_Java_sp_offset()));
338
339 return pc_offset; // for oopmaps
340 }
341
reset_last_Java_frame(Register tmp)342 void MacroAssembler::reset_last_Java_frame(Register tmp) {
343 const Register Rzero = zero_register(tmp);
344 str(Rzero, Address(Rthread, JavaThread::last_Java_sp_offset()));
345 if (_fp_saved) {
346 str(Rzero, Address(Rthread, JavaThread::last_Java_fp_offset()));
347 }
348 if (_pc_saved) {
349 str(Rzero, Address(Rthread, JavaThread::last_Java_pc_offset()));
350 }
351 }
352
353
354 // Implementation of call_VM versions
355
call_VM_leaf_helper(address entry_point,int number_of_arguments)356 void MacroAssembler::call_VM_leaf_helper(address entry_point, int number_of_arguments) {
357 assert(number_of_arguments >= 0, "cannot have negative number of arguments");
358 assert(number_of_arguments <= 4, "cannot have more than 4 arguments");
359
360 // Safer to save R9 here since callers may have been written
361 // assuming R9 survives. This is suboptimal but is not worth
362 // optimizing for the few platforms where R9 is scratched.
363 push(RegisterSet(R4) | R9ifScratched);
364 mov(R4, SP);
365 bic(SP, SP, StackAlignmentInBytes - 1);
366 call(entry_point, relocInfo::runtime_call_type);
367 mov(SP, R4);
368 pop(RegisterSet(R4) | R9ifScratched);
369 }
370
371
call_VM_helper(Register oop_result,address entry_point,int number_of_arguments,bool check_exceptions)372 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
373 assert(number_of_arguments >= 0, "cannot have negative number of arguments");
374 assert(number_of_arguments <= 3, "cannot have more than 3 arguments");
375
376 const Register tmp = Rtemp;
377 assert_different_registers(oop_result, tmp);
378
379 set_last_Java_frame(SP, FP, true, tmp);
380
381 #if R9_IS_SCRATCHED
382 // Safer to save R9 here since callers may have been written
383 // assuming R9 survives. This is suboptimal but is not worth
384 // optimizing for the few platforms where R9 is scratched.
385
386 // Note: cannot save R9 above the saved SP (some calls expect for
387 // instance the Java stack top at the saved SP)
388 // => once saved (with set_last_Java_frame), decrease SP before rounding to
389 // ensure the slot at SP will be free for R9).
390 sub(SP, SP, 4);
391 bic(SP, SP, StackAlignmentInBytes - 1);
392 str(R9, Address(SP, 0));
393 #else
394 bic(SP, SP, StackAlignmentInBytes - 1);
395 #endif // R9_IS_SCRATCHED
396
397 mov(R0, Rthread);
398 call(entry_point, relocInfo::runtime_call_type);
399
400 #if R9_IS_SCRATCHED
401 ldr(R9, Address(SP, 0));
402 #endif
403 ldr(SP, Address(Rthread, JavaThread::last_Java_sp_offset()));
404
405 reset_last_Java_frame(tmp);
406
407 // C++ interp handles this in the interpreter
408 check_and_handle_popframe();
409 check_and_handle_earlyret();
410
411 if (check_exceptions) {
412 // check for pending exceptions
413 ldr(tmp, Address(Rthread, Thread::pending_exception_offset()));
414 cmp(tmp, 0);
415 mov(Rexception_pc, PC, ne);
416 b(StubRoutines::forward_exception_entry(), ne);
417 }
418
419 // get oop result if there is one and reset the value in the thread
420 if (oop_result->is_valid()) {
421 get_vm_result(oop_result, tmp);
422 }
423 }
424
call_VM(Register oop_result,address entry_point,bool check_exceptions)425 void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) {
426 call_VM_helper(oop_result, entry_point, 0, check_exceptions);
427 }
428
429
call_VM(Register oop_result,address entry_point,Register arg_1,bool check_exceptions)430 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, bool check_exceptions) {
431 assert (arg_1 == R1, "fixed register for arg_1");
432 call_VM_helper(oop_result, entry_point, 1, check_exceptions);
433 }
434
435
call_VM(Register oop_result,address entry_point,Register arg_1,Register arg_2,bool check_exceptions)436 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions) {
437 assert (arg_1 == R1, "fixed register for arg_1");
438 assert (arg_2 == R2, "fixed register for arg_2");
439 call_VM_helper(oop_result, entry_point, 2, check_exceptions);
440 }
441
442
call_VM(Register oop_result,address entry_point,Register arg_1,Register arg_2,Register arg_3,bool check_exceptions)443 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions) {
444 assert (arg_1 == R1, "fixed register for arg_1");
445 assert (arg_2 == R2, "fixed register for arg_2");
446 assert (arg_3 == R3, "fixed register for arg_3");
447 call_VM_helper(oop_result, entry_point, 3, check_exceptions);
448 }
449
450
call_VM(Register oop_result,Register last_java_sp,address entry_point,int number_of_arguments,bool check_exceptions)451 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments, bool check_exceptions) {
452 // Not used on ARM
453 Unimplemented();
454 }
455
456
call_VM(Register oop_result,Register last_java_sp,address entry_point,Register arg_1,bool check_exceptions)457 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions) {
458 // Not used on ARM
459 Unimplemented();
460 }
461
462
call_VM(Register oop_result,Register last_java_sp,address entry_point,Register arg_1,Register arg_2,bool check_exceptions)463 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions) {
464 // Not used on ARM
465 Unimplemented();
466 }
467
468
call_VM(Register oop_result,Register last_java_sp,address entry_point,Register arg_1,Register arg_2,Register arg_3,bool check_exceptions)469 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions) {
470 // Not used on ARM
471 Unimplemented();
472 }
473
474 // Raw call, without saving/restoring registers, exception handling, etc.
475 // Mainly used from various stubs.
call_VM(address entry_point,bool save_R9_if_scratched)476 void MacroAssembler::call_VM(address entry_point, bool save_R9_if_scratched) {
477 const Register tmp = Rtemp; // Rtemp free since scratched by call
478 set_last_Java_frame(SP, FP, true, tmp);
479 #if R9_IS_SCRATCHED
480 if (save_R9_if_scratched) {
481 // Note: Saving also R10 for alignment.
482 push(RegisterSet(R9, R10));
483 }
484 #endif
485 mov(R0, Rthread);
486 call(entry_point, relocInfo::runtime_call_type);
487 #if R9_IS_SCRATCHED
488 if (save_R9_if_scratched) {
489 pop(RegisterSet(R9, R10));
490 }
491 #endif
492 reset_last_Java_frame(tmp);
493 }
494
call_VM_leaf(address entry_point)495 void MacroAssembler::call_VM_leaf(address entry_point) {
496 call_VM_leaf_helper(entry_point, 0);
497 }
498
call_VM_leaf(address entry_point,Register arg_1)499 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
500 assert (arg_1 == R0, "fixed register for arg_1");
501 call_VM_leaf_helper(entry_point, 1);
502 }
503
call_VM_leaf(address entry_point,Register arg_1,Register arg_2)504 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2) {
505 assert (arg_1 == R0, "fixed register for arg_1");
506 assert (arg_2 == R1, "fixed register for arg_2");
507 call_VM_leaf_helper(entry_point, 2);
508 }
509
call_VM_leaf(address entry_point,Register arg_1,Register arg_2,Register arg_3)510 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3) {
511 assert (arg_1 == R0, "fixed register for arg_1");
512 assert (arg_2 == R1, "fixed register for arg_2");
513 assert (arg_3 == R2, "fixed register for arg_3");
514 call_VM_leaf_helper(entry_point, 3);
515 }
516
call_VM_leaf(address entry_point,Register arg_1,Register arg_2,Register arg_3,Register arg_4)517 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4) {
518 assert (arg_1 == R0, "fixed register for arg_1");
519 assert (arg_2 == R1, "fixed register for arg_2");
520 assert (arg_3 == R2, "fixed register for arg_3");
521 assert (arg_4 == R3, "fixed register for arg_4");
522 call_VM_leaf_helper(entry_point, 4);
523 }
524
get_vm_result(Register oop_result,Register tmp)525 void MacroAssembler::get_vm_result(Register oop_result, Register tmp) {
526 assert_different_registers(oop_result, tmp);
527 ldr(oop_result, Address(Rthread, JavaThread::vm_result_offset()));
528 str(zero_register(tmp), Address(Rthread, JavaThread::vm_result_offset()));
529 verify_oop(oop_result);
530 }
531
get_vm_result_2(Register metadata_result,Register tmp)532 void MacroAssembler::get_vm_result_2(Register metadata_result, Register tmp) {
533 assert_different_registers(metadata_result, tmp);
534 ldr(metadata_result, Address(Rthread, JavaThread::vm_result_2_offset()));
535 str(zero_register(tmp), Address(Rthread, JavaThread::vm_result_2_offset()));
536 }
537
add_rc(Register dst,Register arg1,RegisterOrConstant arg2)538 void MacroAssembler::add_rc(Register dst, Register arg1, RegisterOrConstant arg2) {
539 if (arg2.is_register()) {
540 add(dst, arg1, arg2.as_register());
541 } else {
542 add(dst, arg1, arg2.as_constant());
543 }
544 }
545
add_slow(Register rd,Register rn,int c)546 void MacroAssembler::add_slow(Register rd, Register rn, int c) {
547 // This function is used in compiler for handling large frame offsets
548 if ((c < 0) && (((-c) & ~0x3fc) == 0)) {
549 return sub(rd, rn, (-c));
550 }
551 int low = c & 0x3fc;
552 if (low != 0) {
553 add(rd, rn, low);
554 rn = rd;
555 }
556 if (c & ~0x3fc) {
557 assert(AsmOperand::is_rotated_imm(c & ~0x3fc), "unsupported add_slow offset %d", c);
558 add(rd, rn, c & ~0x3fc);
559 } else if (rd != rn) {
560 assert(c == 0, "");
561 mov(rd, rn); // need to generate at least one move!
562 }
563 }
564
sub_slow(Register rd,Register rn,int c)565 void MacroAssembler::sub_slow(Register rd, Register rn, int c) {
566 // This function is used in compiler for handling large frame offsets
567 if ((c < 0) && (((-c) & ~0x3fc) == 0)) {
568 return add(rd, rn, (-c));
569 }
570 int low = c & 0x3fc;
571 if (low != 0) {
572 sub(rd, rn, low);
573 rn = rd;
574 }
575 if (c & ~0x3fc) {
576 assert(AsmOperand::is_rotated_imm(c & ~0x3fc), "unsupported sub_slow offset %d", c);
577 sub(rd, rn, c & ~0x3fc);
578 } else if (rd != rn) {
579 assert(c == 0, "");
580 mov(rd, rn); // need to generate at least one move!
581 }
582 }
583
mov_slow(Register rd,address addr)584 void MacroAssembler::mov_slow(Register rd, address addr) {
585 // do *not* call the non relocated mov_related_address
586 mov_slow(rd, (intptr_t)addr);
587 }
588
mov_slow(Register rd,const char * str)589 void MacroAssembler::mov_slow(Register rd, const char *str) {
590 mov_slow(rd, (intptr_t)str);
591 }
592
593
mov_slow(Register rd,intptr_t c,AsmCondition cond)594 void MacroAssembler::mov_slow(Register rd, intptr_t c, AsmCondition cond) {
595 if (AsmOperand::is_rotated_imm(c)) {
596 mov(rd, c, cond);
597 } else if (AsmOperand::is_rotated_imm(~c)) {
598 mvn(rd, ~c, cond);
599 } else if (VM_Version::supports_movw()) {
600 movw(rd, c & 0xffff, cond);
601 if ((unsigned int)c >> 16) {
602 movt(rd, (unsigned int)c >> 16, cond);
603 }
604 } else {
605 // Find first non-zero bit
606 int shift = 0;
607 while ((c & (3 << shift)) == 0) {
608 shift += 2;
609 }
610 // Put the least significant part of the constant
611 int mask = 0xff << shift;
612 mov(rd, c & mask, cond);
613 // Add up to 3 other parts of the constant;
614 // each of them can be represented as rotated_imm
615 if (c & (mask << 8)) {
616 orr(rd, rd, c & (mask << 8), cond);
617 }
618 if (c & (mask << 16)) {
619 orr(rd, rd, c & (mask << 16), cond);
620 }
621 if (c & (mask << 24)) {
622 orr(rd, rd, c & (mask << 24), cond);
623 }
624 }
625 }
626
627
mov_oop(Register rd,jobject o,int oop_index,AsmCondition cond)628 void MacroAssembler::mov_oop(Register rd, jobject o, int oop_index,
629 AsmCondition cond
630 ) {
631
632 if (o == NULL) {
633 mov(rd, 0, cond);
634 return;
635 }
636
637 if (oop_index == 0) {
638 oop_index = oop_recorder()->allocate_oop_index(o);
639 }
640 relocate(oop_Relocation::spec(oop_index));
641
642 if (VM_Version::supports_movw()) {
643 movw(rd, 0, cond);
644 movt(rd, 0, cond);
645 } else {
646 ldr(rd, Address(PC), cond);
647 // Extra nop to handle case of large offset of oop placeholder (see NativeMovConstReg::set_data).
648 nop();
649 }
650 }
651
mov_metadata(Register rd,Metadata * o,int metadata_index)652 void MacroAssembler::mov_metadata(Register rd, Metadata* o, int metadata_index) {
653 if (o == NULL) {
654 mov(rd, 0);
655 return;
656 }
657
658 if (metadata_index == 0) {
659 metadata_index = oop_recorder()->allocate_metadata_index(o);
660 }
661 relocate(metadata_Relocation::spec(metadata_index));
662
663 if (VM_Version::supports_movw()) {
664 movw(rd, ((int)o) & 0xffff);
665 movt(rd, (unsigned int)o >> 16);
666 } else {
667 ldr(rd, Address(PC));
668 // Extra nop to handle case of large offset of metadata placeholder (see NativeMovConstReg::set_data).
669 nop();
670 }
671 }
672
mov_float(FloatRegister fd,jfloat c,AsmCondition cond)673 void MacroAssembler::mov_float(FloatRegister fd, jfloat c, AsmCondition cond) {
674 Label skip_constant;
675 union {
676 jfloat f;
677 jint i;
678 } accessor;
679 accessor.f = c;
680
681 flds(fd, Address(PC), cond);
682 b(skip_constant);
683 emit_int32(accessor.i);
684 bind(skip_constant);
685 }
686
mov_double(FloatRegister fd,jdouble c,AsmCondition cond)687 void MacroAssembler::mov_double(FloatRegister fd, jdouble c, AsmCondition cond) {
688 Label skip_constant;
689 union {
690 jdouble d;
691 jint i[2];
692 } accessor;
693 accessor.d = c;
694
695 fldd(fd, Address(PC), cond);
696 b(skip_constant);
697 emit_int32(accessor.i[0]);
698 emit_int32(accessor.i[1]);
699 bind(skip_constant);
700 }
701
ldr_global_s32(Register reg,address address_of_global)702 void MacroAssembler::ldr_global_s32(Register reg, address address_of_global) {
703 intptr_t addr = (intptr_t) address_of_global;
704 mov_slow(reg, addr & ~0xfff);
705 ldr(reg, Address(reg, addr & 0xfff));
706 }
707
ldr_global_ptr(Register reg,address address_of_global)708 void MacroAssembler::ldr_global_ptr(Register reg, address address_of_global) {
709 ldr_global_s32(reg, address_of_global);
710 }
711
ldrb_global(Register reg,address address_of_global)712 void MacroAssembler::ldrb_global(Register reg, address address_of_global) {
713 intptr_t addr = (intptr_t) address_of_global;
714 mov_slow(reg, addr & ~0xfff);
715 ldrb(reg, Address(reg, addr & 0xfff));
716 }
717
zero_extend(Register rd,Register rn,int bits)718 void MacroAssembler::zero_extend(Register rd, Register rn, int bits) {
719 if (bits <= 8) {
720 andr(rd, rn, (1 << bits) - 1);
721 } else if (bits >= 24) {
722 bic(rd, rn, -1 << bits);
723 } else {
724 mov(rd, AsmOperand(rn, lsl, 32 - bits));
725 mov(rd, AsmOperand(rd, lsr, 32 - bits));
726 }
727 }
728
sign_extend(Register rd,Register rn,int bits)729 void MacroAssembler::sign_extend(Register rd, Register rn, int bits) {
730 mov(rd, AsmOperand(rn, lsl, 32 - bits));
731 mov(rd, AsmOperand(rd, asr, 32 - bits));
732 }
733
734
cmpoop(Register obj1,Register obj2)735 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
736 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
737 bs->obj_equals(this, obj1, obj2);
738 }
739
long_move(Register rd_lo,Register rd_hi,Register rn_lo,Register rn_hi,AsmCondition cond)740 void MacroAssembler::long_move(Register rd_lo, Register rd_hi,
741 Register rn_lo, Register rn_hi,
742 AsmCondition cond) {
743 if (rd_lo != rn_hi) {
744 if (rd_lo != rn_lo) { mov(rd_lo, rn_lo, cond); }
745 if (rd_hi != rn_hi) { mov(rd_hi, rn_hi, cond); }
746 } else if (rd_hi != rn_lo) {
747 if (rd_hi != rn_hi) { mov(rd_hi, rn_hi, cond); }
748 if (rd_lo != rn_lo) { mov(rd_lo, rn_lo, cond); }
749 } else {
750 eor(rd_lo, rd_hi, rd_lo, cond);
751 eor(rd_hi, rd_lo, rd_hi, cond);
752 eor(rd_lo, rd_hi, rd_lo, cond);
753 }
754 }
755
long_shift(Register rd_lo,Register rd_hi,Register rn_lo,Register rn_hi,AsmShift shift,Register count)756 void MacroAssembler::long_shift(Register rd_lo, Register rd_hi,
757 Register rn_lo, Register rn_hi,
758 AsmShift shift, Register count) {
759 Register tmp;
760 if (rd_lo != rn_lo && rd_lo != rn_hi && rd_lo != count) {
761 tmp = rd_lo;
762 } else {
763 tmp = rd_hi;
764 }
765 assert_different_registers(tmp, count, rn_lo, rn_hi);
766
767 subs(tmp, count, 32);
768 if (shift == lsl) {
769 assert_different_registers(rd_hi, rn_lo);
770 assert_different_registers(count, rd_hi);
771 mov(rd_hi, AsmOperand(rn_lo, shift, tmp), pl);
772 rsb(tmp, count, 32, mi);
773 if (rd_hi == rn_hi) {
774 mov(rd_hi, AsmOperand(rn_hi, lsl, count), mi);
775 orr(rd_hi, rd_hi, AsmOperand(rn_lo, lsr, tmp), mi);
776 } else {
777 mov(rd_hi, AsmOperand(rn_lo, lsr, tmp), mi);
778 orr(rd_hi, rd_hi, AsmOperand(rn_hi, lsl, count), mi);
779 }
780 mov(rd_lo, AsmOperand(rn_lo, shift, count));
781 } else {
782 assert_different_registers(rd_lo, rn_hi);
783 assert_different_registers(rd_lo, count);
784 mov(rd_lo, AsmOperand(rn_hi, shift, tmp), pl);
785 rsb(tmp, count, 32, mi);
786 if (rd_lo == rn_lo) {
787 mov(rd_lo, AsmOperand(rn_lo, lsr, count), mi);
788 orr(rd_lo, rd_lo, AsmOperand(rn_hi, lsl, tmp), mi);
789 } else {
790 mov(rd_lo, AsmOperand(rn_hi, lsl, tmp), mi);
791 orr(rd_lo, rd_lo, AsmOperand(rn_lo, lsr, count), mi);
792 }
793 mov(rd_hi, AsmOperand(rn_hi, shift, count));
794 }
795 }
796
long_shift(Register rd_lo,Register rd_hi,Register rn_lo,Register rn_hi,AsmShift shift,int count)797 void MacroAssembler::long_shift(Register rd_lo, Register rd_hi,
798 Register rn_lo, Register rn_hi,
799 AsmShift shift, int count) {
800 assert(count != 0 && (count & ~63) == 0, "must be");
801
802 if (shift == lsl) {
803 assert_different_registers(rd_hi, rn_lo);
804 if (count >= 32) {
805 mov(rd_hi, AsmOperand(rn_lo, lsl, count - 32));
806 mov(rd_lo, 0);
807 } else {
808 mov(rd_hi, AsmOperand(rn_hi, lsl, count));
809 orr(rd_hi, rd_hi, AsmOperand(rn_lo, lsr, 32 - count));
810 mov(rd_lo, AsmOperand(rn_lo, lsl, count));
811 }
812 } else {
813 assert_different_registers(rd_lo, rn_hi);
814 if (count >= 32) {
815 if (count == 32) {
816 mov(rd_lo, rn_hi);
817 } else {
818 mov(rd_lo, AsmOperand(rn_hi, shift, count - 32));
819 }
820 if (shift == asr) {
821 mov(rd_hi, AsmOperand(rn_hi, asr, 0));
822 } else {
823 mov(rd_hi, 0);
824 }
825 } else {
826 mov(rd_lo, AsmOperand(rn_lo, lsr, count));
827 orr(rd_lo, rd_lo, AsmOperand(rn_hi, lsl, 32 - count));
828 mov(rd_hi, AsmOperand(rn_hi, shift, count));
829 }
830 }
831 }
832
_verify_oop(Register reg,const char * s,const char * file,int line)833 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
834 // This code pattern is matched in NativeIntruction::skip_verify_oop.
835 // Update it at modifications.
836 if (!VerifyOops) return;
837
838 char buffer[64];
839 #ifdef COMPILER1
840 if (CommentedAssembly) {
841 snprintf(buffer, sizeof(buffer), "verify_oop at %d", offset());
842 block_comment(buffer);
843 }
844 #endif
845 const char* msg_buffer = NULL;
846 {
847 ResourceMark rm;
848 stringStream ss;
849 ss.print("%s at offset %d (%s:%d)", s, offset(), file, line);
850 msg_buffer = code_string(ss.as_string());
851 }
852
853 save_all_registers();
854
855 if (reg != R2) {
856 mov(R2, reg); // oop to verify
857 }
858 mov(R1, SP); // register save area
859
860 Label done;
861 InlinedString Lmsg(msg_buffer);
862 ldr_literal(R0, Lmsg); // message
863
864 // call indirectly to solve generation ordering problem
865 ldr_global_ptr(Rtemp, StubRoutines::verify_oop_subroutine_entry_address());
866 call(Rtemp);
867
868 restore_all_registers();
869
870 b(done);
871 #ifdef COMPILER2
872 int off = offset();
873 #endif
874 bind_literal(Lmsg);
875 #ifdef COMPILER2
876 if (offset() - off == 1 * wordSize) {
877 // no padding, so insert nop for worst-case sizing
878 nop();
879 }
880 #endif
881 bind(done);
882 }
883
_verify_oop_addr(Address addr,const char * s,const char * file,int line)884 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
885 if (!VerifyOops) return;
886
887 const char* msg_buffer = NULL;
888 {
889 ResourceMark rm;
890 stringStream ss;
891 if ((addr.base() == SP) && (addr.index()==noreg)) {
892 ss.print("verify_oop_addr SP[%d]: %s", (int)addr.disp(), s);
893 } else {
894 ss.print("verify_oop_addr: %s", s);
895 }
896 ss.print(" (%s:%d)", file, line);
897 msg_buffer = code_string(ss.as_string());
898 }
899
900 int push_size = save_all_registers();
901
902 if (addr.base() == SP) {
903 // computes an addr that takes into account the push
904 if (addr.index() != noreg) {
905 Register new_base = addr.index() == R2 ? R1 : R2; // avoid corrupting the index
906 add(new_base, SP, push_size);
907 addr = addr.rebase(new_base);
908 } else {
909 addr = addr.plus_disp(push_size);
910 }
911 }
912
913 ldr(R2, addr); // oop to verify
914 mov(R1, SP); // register save area
915
916 Label done;
917 InlinedString Lmsg(msg_buffer);
918 ldr_literal(R0, Lmsg); // message
919
920 // call indirectly to solve generation ordering problem
921 ldr_global_ptr(Rtemp, StubRoutines::verify_oop_subroutine_entry_address());
922 call(Rtemp);
923
924 restore_all_registers();
925
926 b(done);
927 bind_literal(Lmsg);
928 bind(done);
929 }
930
c2bool(Register x)931 void MacroAssembler::c2bool(Register x)
932 {
933 tst(x, 0xff); // Only look at the lowest byte
934 mov(x, 1, ne);
935 }
936
null_check(Register reg,Register tmp,int offset)937 void MacroAssembler::null_check(Register reg, Register tmp, int offset) {
938 if (needs_explicit_null_check(offset)) {
939 assert_different_registers(reg, tmp);
940 if (tmp == noreg) {
941 tmp = Rtemp;
942 assert((! Thread::current()->is_Compiler_thread()) ||
943 (! (ciEnv::current()->task() == NULL)) ||
944 (! (ciEnv::current()->comp_level() == CompLevel_full_optimization)),
945 "Rtemp not available in C2"); // explicit tmp register required
946 // XXX: could we mark the code buffer as not compatible with C2 ?
947 }
948 ldr(tmp, Address(reg));
949 }
950 }
951
952 // Puts address of allocated object into register `obj` and end of allocated object into register `obj_end`.
eden_allocate(Register obj,Register obj_end,Register tmp1,Register tmp2,RegisterOrConstant size_expression,Label & slow_case)953 void MacroAssembler::eden_allocate(Register obj, Register obj_end, Register tmp1, Register tmp2,
954 RegisterOrConstant size_expression, Label& slow_case) {
955 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
956 bs->eden_allocate(this, obj, obj_end, tmp1, tmp2, size_expression, slow_case);
957 }
958
959 // Puts address of allocated object into register `obj` and end of allocated object into register `obj_end`.
tlab_allocate(Register obj,Register obj_end,Register tmp1,RegisterOrConstant size_expression,Label & slow_case)960 void MacroAssembler::tlab_allocate(Register obj, Register obj_end, Register tmp1,
961 RegisterOrConstant size_expression, Label& slow_case) {
962 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
963 bs->tlab_allocate(this, obj, obj_end, tmp1, size_expression, slow_case);
964 }
965
966 // Fills memory regions [start..end] with zeroes. Clobbers `start` and `tmp` registers.
zero_memory(Register start,Register end,Register tmp)967 void MacroAssembler::zero_memory(Register start, Register end, Register tmp) {
968 Label loop;
969 const Register ptr = start;
970
971 mov(tmp, 0);
972 bind(loop);
973 cmp(ptr, end);
974 str(tmp, Address(ptr, wordSize, post_indexed), lo);
975 b(loop, lo);
976 }
977
arm_stack_overflow_check(int frame_size_in_bytes,Register tmp)978 void MacroAssembler::arm_stack_overflow_check(int frame_size_in_bytes, Register tmp) {
979 // Version of AbstractAssembler::generate_stack_overflow_check optimized for ARM
980 if (UseStackBanging) {
981 const int page_size = os::vm_page_size();
982
983 sub_slow(tmp, SP, StackOverflow::stack_shadow_zone_size());
984 strb(R0, Address(tmp));
985 for (; frame_size_in_bytes >= page_size; frame_size_in_bytes -= 0xff0) {
986 strb(R0, Address(tmp, -0xff0, pre_indexed));
987 }
988 }
989 }
990
arm_stack_overflow_check(Register Rsize,Register tmp)991 void MacroAssembler::arm_stack_overflow_check(Register Rsize, Register tmp) {
992 if (UseStackBanging) {
993 Label loop;
994
995 mov(tmp, SP);
996 add_slow(Rsize, Rsize, StackOverflow::stack_shadow_zone_size() - os::vm_page_size());
997 bind(loop);
998 subs(Rsize, Rsize, 0xff0);
999 strb(R0, Address(tmp, -0xff0, pre_indexed));
1000 b(loop, hi);
1001 }
1002 }
1003
stop(const char * msg)1004 void MacroAssembler::stop(const char* msg) {
1005 // This code pattern is matched in NativeIntruction::is_stop.
1006 // Update it at modifications.
1007 #ifdef COMPILER1
1008 if (CommentedAssembly) {
1009 block_comment("stop");
1010 }
1011 #endif
1012
1013 InlinedAddress Ldebug(CAST_FROM_FN_PTR(address, MacroAssembler::debug));
1014 InlinedString Lmsg(msg);
1015
1016 // save all registers for further inspection
1017 save_all_registers();
1018
1019 ldr_literal(R0, Lmsg); // message
1020 mov(R1, SP); // register save area
1021
1022 ldr_literal(PC, Ldebug); // call MacroAssembler::debug
1023
1024 bind_literal(Lmsg);
1025 bind_literal(Ldebug);
1026 }
1027
warn(const char * msg)1028 void MacroAssembler::warn(const char* msg) {
1029 #ifdef COMPILER1
1030 if (CommentedAssembly) {
1031 block_comment("warn");
1032 }
1033 #endif
1034
1035 InlinedAddress Lwarn(CAST_FROM_FN_PTR(address, warning));
1036 InlinedString Lmsg(msg);
1037 Label done;
1038
1039 int push_size = save_caller_save_registers();
1040
1041
1042 ldr_literal(R0, Lmsg); // message
1043 ldr_literal(LR, Lwarn); // call warning
1044
1045 call(LR);
1046
1047 restore_caller_save_registers();
1048
1049 b(done);
1050 bind_literal(Lmsg);
1051 bind_literal(Lwarn);
1052 bind(done);
1053 }
1054
1055
save_all_registers()1056 int MacroAssembler::save_all_registers() {
1057 // This code pattern is matched in NativeIntruction::is_save_all_registers.
1058 // Update it at modifications.
1059 push(RegisterSet(R0, R12) | RegisterSet(LR) | RegisterSet(PC));
1060 return 15*wordSize;
1061 }
1062
restore_all_registers()1063 void MacroAssembler::restore_all_registers() {
1064 pop(RegisterSet(R0, R12) | RegisterSet(LR)); // restore registers
1065 add(SP, SP, wordSize); // discard saved PC
1066 }
1067
save_caller_save_registers()1068 int MacroAssembler::save_caller_save_registers() {
1069 #if R9_IS_SCRATCHED
1070 // Save also R10 to preserve alignment
1071 push(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR) | RegisterSet(R9,R10));
1072 return 8*wordSize;
1073 #else
1074 push(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR));
1075 return 6*wordSize;
1076 #endif
1077 }
1078
restore_caller_save_registers()1079 void MacroAssembler::restore_caller_save_registers() {
1080 #if R9_IS_SCRATCHED
1081 pop(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR) | RegisterSet(R9,R10));
1082 #else
1083 pop(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR));
1084 #endif
1085 }
1086
debug(const char * msg,const intx * registers)1087 void MacroAssembler::debug(const char* msg, const intx* registers) {
1088 // In order to get locks to work, we need to fake a in_VM state
1089 JavaThread* thread = JavaThread::current();
1090 thread->set_thread_state(_thread_in_vm);
1091
1092 if (ShowMessageBoxOnError) {
1093 ttyLocker ttyl;
1094 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
1095 BytecodeCounter::print();
1096 }
1097 if (os::message_box(msg, "Execution stopped, print registers?")) {
1098 // saved registers: R0-R12, LR, PC
1099 const int nregs = 15;
1100 const Register regs[nregs] = {R0, R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, LR, PC};
1101
1102 for (int i = 0; i < nregs; i++) {
1103 tty->print_cr("%s = " INTPTR_FORMAT, regs[i]->name(), registers[i]);
1104 }
1105
1106 // derive original SP value from the address of register save area
1107 tty->print_cr("%s = " INTPTR_FORMAT, SP->name(), p2i(®isters[nregs]));
1108 }
1109 BREAKPOINT;
1110 } else {
1111 ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
1112 }
1113 assert(false, "DEBUG MESSAGE: %s", msg);
1114 fatal("%s", msg); // returning from MacroAssembler::debug is not supported
1115 }
1116
unimplemented(const char * what)1117 void MacroAssembler::unimplemented(const char* what) {
1118 const char* buf = NULL;
1119 {
1120 ResourceMark rm;
1121 stringStream ss;
1122 ss.print("unimplemented: %s", what);
1123 buf = code_string(ss.as_string());
1124 }
1125 stop(buf);
1126 }
1127
1128
1129 // Implementation of FixedSizeCodeBlock
1130
FixedSizeCodeBlock(MacroAssembler * masm,int size_in_instrs,bool enabled)1131 FixedSizeCodeBlock::FixedSizeCodeBlock(MacroAssembler* masm, int size_in_instrs, bool enabled) :
1132 _masm(masm), _start(masm->pc()), _size_in_instrs(size_in_instrs), _enabled(enabled) {
1133 }
1134
~FixedSizeCodeBlock()1135 FixedSizeCodeBlock::~FixedSizeCodeBlock() {
1136 if (_enabled) {
1137 address curr_pc = _masm->pc();
1138
1139 assert(_start < curr_pc, "invalid current pc");
1140 guarantee(curr_pc <= _start + _size_in_instrs * Assembler::InstructionSize, "code block is too long");
1141
1142 int nops_count = (_start - curr_pc) / Assembler::InstructionSize + _size_in_instrs;
1143 for (int i = 0; i < nops_count; i++) {
1144 _masm->nop();
1145 }
1146 }
1147 }
1148
1149
1150 // Serializes memory. Potentially blows flags and reg.
1151 // tmp is a scratch for v6 co-processor write op (could be noreg for other architecure versions)
1152 // preserve_flags takes a longer path in LoadStore case (dmb rather then control dependency) to preserve status flags. Optional.
1153 // load_tgt is an ordered load target in a LoadStore case only, to create dependency between the load operation and conditional branch. Optional.
membar(Membar_mask_bits order_constraint,Register tmp,bool preserve_flags,Register load_tgt)1154 void MacroAssembler::membar(Membar_mask_bits order_constraint,
1155 Register tmp,
1156 bool preserve_flags,
1157 Register load_tgt) {
1158
1159 if (order_constraint == StoreStore) {
1160 dmb(DMB_st, tmp);
1161 } else if ((order_constraint & StoreLoad) ||
1162 (order_constraint & LoadLoad) ||
1163 (order_constraint & StoreStore) ||
1164 (load_tgt == noreg) ||
1165 preserve_flags) {
1166 dmb(DMB_all, tmp);
1167 } else {
1168 // LoadStore: speculative stores reordeing is prohibited
1169
1170 // By providing an ordered load target register, we avoid an extra memory load reference
1171 Label not_taken;
1172 bind(not_taken);
1173 cmp(load_tgt, load_tgt);
1174 b(not_taken, ne);
1175 }
1176 }
1177
1178
1179 // If "allow_fallthrough_on_failure" is false, we always branch to "slow_case"
1180 // on failure, so fall-through can only mean success.
1181 // "one_shot" controls whether we loop and retry to mitigate spurious failures.
1182 // This is only needed for C2, which for some reason does not rety,
1183 // while C1/interpreter does.
1184 // TODO: measure if it makes a difference
1185
cas_for_lock_acquire(Register oldval,Register newval,Register base,Register tmp,Label & slow_case,bool allow_fallthrough_on_failure,bool one_shot)1186 void MacroAssembler::cas_for_lock_acquire(Register oldval, Register newval,
1187 Register base, Register tmp, Label &slow_case,
1188 bool allow_fallthrough_on_failure, bool one_shot)
1189 {
1190
1191 bool fallthrough_is_success = false;
1192
1193 // ARM Litmus Test example does prefetching here.
1194 // TODO: investigate if it helps performance
1195
1196 // The last store was to the displaced header, so to prevent
1197 // reordering we must issue a StoreStore or Release barrier before
1198 // the CAS store.
1199
1200 membar(MacroAssembler::StoreStore, noreg);
1201
1202 if (one_shot) {
1203 ldrex(tmp, Address(base, oopDesc::mark_offset_in_bytes()));
1204 cmp(tmp, oldval);
1205 strex(tmp, newval, Address(base, oopDesc::mark_offset_in_bytes()), eq);
1206 cmp(tmp, 0, eq);
1207 } else {
1208 atomic_cas_bool(oldval, newval, base, oopDesc::mark_offset_in_bytes(), tmp);
1209 }
1210
1211 // MemBarAcquireLock barrier
1212 // According to JSR-133 Cookbook, this should be LoadLoad | LoadStore,
1213 // but that doesn't prevent a load or store from floating up between
1214 // the load and store in the CAS sequence, so play it safe and
1215 // do a full fence.
1216 membar(Membar_mask_bits(LoadLoad | LoadStore | StoreStore | StoreLoad), noreg);
1217 if (!fallthrough_is_success && !allow_fallthrough_on_failure) {
1218 b(slow_case, ne);
1219 }
1220 }
1221
cas_for_lock_release(Register oldval,Register newval,Register base,Register tmp,Label & slow_case,bool allow_fallthrough_on_failure,bool one_shot)1222 void MacroAssembler::cas_for_lock_release(Register oldval, Register newval,
1223 Register base, Register tmp, Label &slow_case,
1224 bool allow_fallthrough_on_failure, bool one_shot)
1225 {
1226
1227 bool fallthrough_is_success = false;
1228
1229 assert_different_registers(oldval,newval,base,tmp);
1230
1231 // MemBarReleaseLock barrier
1232 // According to JSR-133 Cookbook, this should be StoreStore | LoadStore,
1233 // but that doesn't prevent a load or store from floating down between
1234 // the load and store in the CAS sequence, so play it safe and
1235 // do a full fence.
1236 membar(Membar_mask_bits(LoadLoad | LoadStore | StoreStore | StoreLoad), tmp);
1237
1238 if (one_shot) {
1239 ldrex(tmp, Address(base, oopDesc::mark_offset_in_bytes()));
1240 cmp(tmp, oldval);
1241 strex(tmp, newval, Address(base, oopDesc::mark_offset_in_bytes()), eq);
1242 cmp(tmp, 0, eq);
1243 } else {
1244 atomic_cas_bool(oldval, newval, base, oopDesc::mark_offset_in_bytes(), tmp);
1245 }
1246 if (!fallthrough_is_success && !allow_fallthrough_on_failure) {
1247 b(slow_case, ne);
1248 }
1249
1250 // ExitEnter
1251 // According to JSR-133 Cookbook, this should be StoreLoad, the same
1252 // barrier that follows volatile store.
1253 // TODO: Should be able to remove on armv8 if volatile loads
1254 // use the load-acquire instruction.
1255 membar(StoreLoad, noreg);
1256 }
1257
1258 #ifndef PRODUCT
1259
1260 // Preserves flags and all registers.
1261 // On SMP the updated value might not be visible to external observers without a sychronization barrier
cond_atomic_inc32(AsmCondition cond,int * counter_addr)1262 void MacroAssembler::cond_atomic_inc32(AsmCondition cond, int* counter_addr) {
1263 if (counter_addr != NULL) {
1264 InlinedAddress counter_addr_literal((address)counter_addr);
1265 Label done, retry;
1266 if (cond != al) {
1267 b(done, inverse(cond));
1268 }
1269
1270 push(RegisterSet(R0, R3) | RegisterSet(Rtemp));
1271 ldr_literal(R0, counter_addr_literal);
1272
1273 mrs(CPSR, Rtemp);
1274
1275 bind(retry);
1276 ldr_s32(R1, Address(R0));
1277 add(R2, R1, 1);
1278 atomic_cas_bool(R1, R2, R0, 0, R3);
1279 b(retry, ne);
1280
1281 msr(CPSR_fsxc, Rtemp);
1282
1283 pop(RegisterSet(R0, R3) | RegisterSet(Rtemp));
1284
1285 b(done);
1286 bind_literal(counter_addr_literal);
1287
1288 bind(done);
1289 }
1290 }
1291
1292 #endif // !PRODUCT
1293
1294
1295 // Building block for CAS cases of biased locking: makes CAS and records statistics.
1296 // The slow_case label is used to transfer control if CAS fails. Otherwise leaves condition codes set.
biased_locking_enter_with_cas(Register obj_reg,Register old_mark_reg,Register new_mark_reg,Register tmp,Label & slow_case,int * counter_addr)1297 void MacroAssembler::biased_locking_enter_with_cas(Register obj_reg, Register old_mark_reg, Register new_mark_reg,
1298 Register tmp, Label& slow_case, int* counter_addr) {
1299
1300 cas_for_lock_acquire(old_mark_reg, new_mark_reg, obj_reg, tmp, slow_case);
1301 #ifdef ASSERT
1302 breakpoint(ne); // Fallthrough only on success
1303 #endif
1304 #ifndef PRODUCT
1305 if (counter_addr != NULL) {
1306 cond_atomic_inc32(al, counter_addr);
1307 }
1308 #endif // !PRODUCT
1309 }
1310
biased_locking_enter(Register obj_reg,Register swap_reg,Register tmp_reg,bool swap_reg_contains_mark,Register tmp2,Label & done,Label & slow_case,BiasedLockingCounters * counters)1311 void MacroAssembler::biased_locking_enter(Register obj_reg, Register swap_reg, Register tmp_reg,
1312 bool swap_reg_contains_mark,
1313 Register tmp2,
1314 Label& done, Label& slow_case,
1315 BiasedLockingCounters* counters) {
1316 // obj_reg must be preserved (at least) if the bias locking fails
1317 // tmp_reg is a temporary register
1318 // swap_reg was used as a temporary but contained a value
1319 // that was used afterwards in some call pathes. Callers
1320 // have been fixed so that swap_reg no longer needs to be
1321 // saved.
1322 // Rtemp in no longer scratched
1323
1324 assert(UseBiasedLocking, "why call this otherwise?");
1325 assert_different_registers(obj_reg, swap_reg, tmp_reg, tmp2);
1326 guarantee(swap_reg!=tmp_reg, "invariant");
1327 assert(tmp_reg != noreg, "must supply tmp_reg");
1328
1329 #ifndef PRODUCT
1330 if (PrintBiasedLockingStatistics && (counters == NULL)) {
1331 counters = BiasedLocking::counters();
1332 }
1333 #endif
1334
1335 assert(markWord::age_shift == markWord::lock_bits + markWord::biased_lock_bits, "biased locking makes assumptions about bit layout");
1336 Address mark_addr(obj_reg, oopDesc::mark_offset_in_bytes());
1337
1338 // Biased locking
1339 // See whether the lock is currently biased toward our thread and
1340 // whether the epoch is still valid
1341 // Note that the runtime guarantees sufficient alignment of JavaThread
1342 // pointers to allow age to be placed into low bits
1343 // First check to see whether biasing is even enabled for this object
1344 Label cas_label;
1345
1346 if (!swap_reg_contains_mark) {
1347 ldr(swap_reg, mark_addr);
1348 }
1349
1350 // On MP platform loads could return 'stale' values in some cases.
1351 // That is acceptable since either CAS or slow case path is taken in the worst case.
1352
1353 andr(tmp_reg, swap_reg, markWord::biased_lock_mask_in_place);
1354 cmp(tmp_reg, markWord::biased_lock_pattern);
1355
1356 b(cas_label, ne);
1357
1358 // The bias pattern is present in the object's header. Need to check
1359 // whether the bias owner and the epoch are both still current.
1360 load_klass(tmp_reg, obj_reg);
1361 ldr(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset()));
1362 orr(tmp_reg, tmp_reg, Rthread);
1363 eor(tmp_reg, tmp_reg, swap_reg);
1364
1365 bics(tmp_reg, tmp_reg, ((int) markWord::age_mask_in_place));
1366
1367 #ifndef PRODUCT
1368 if (counters != NULL) {
1369 cond_atomic_inc32(eq, counters->biased_lock_entry_count_addr());
1370 }
1371 #endif // !PRODUCT
1372
1373 b(done, eq);
1374
1375 Label try_revoke_bias;
1376 Label try_rebias;
1377
1378 // At this point we know that the header has the bias pattern and
1379 // that we are not the bias owner in the current epoch. We need to
1380 // figure out more details about the state of the header in order to
1381 // know what operations can be legally performed on the object's
1382 // header.
1383
1384 // If the low three bits in the xor result aren't clear, that means
1385 // the prototype header is no longer biased and we have to revoke
1386 // the bias on this object.
1387 tst(tmp_reg, markWord::biased_lock_mask_in_place);
1388 b(try_revoke_bias, ne);
1389
1390 // Biasing is still enabled for this data type. See whether the
1391 // epoch of the current bias is still valid, meaning that the epoch
1392 // bits of the mark word are equal to the epoch bits of the
1393 // prototype header. (Note that the prototype header's epoch bits
1394 // only change at a safepoint.) If not, attempt to rebias the object
1395 // toward the current thread. Note that we must be absolutely sure
1396 // that the current epoch is invalid in order to do this because
1397 // otherwise the manipulations it performs on the mark word are
1398 // illegal.
1399 tst(tmp_reg, markWord::epoch_mask_in_place);
1400 b(try_rebias, ne);
1401
1402 // tmp_reg has the age, epoch and pattern bits cleared
1403 // The remaining (owner) bits are (Thread ^ current_owner)
1404
1405 // The epoch of the current bias is still valid but we know nothing
1406 // about the owner; it might be set or it might be clear. Try to
1407 // acquire the bias of the object using an atomic operation. If this
1408 // fails we will go in to the runtime to revoke the object's bias.
1409 // Note that we first construct the presumed unbiased header so we
1410 // don't accidentally blow away another thread's valid bias.
1411
1412 // Note that we know the owner is not ourself. Hence, success can
1413 // only happen when the owner bits is 0
1414
1415 // until the assembler can be made smarter, we need to make some assumptions about the values
1416 // so we can optimize this:
1417 assert((markWord::biased_lock_mask_in_place | markWord::age_mask_in_place | markWord::epoch_mask_in_place) == 0x1ff, "biased bitmasks changed");
1418
1419 mov(swap_reg, AsmOperand(swap_reg, lsl, 23));
1420 mov(swap_reg, AsmOperand(swap_reg, lsr, 23)); // markWord with thread bits cleared (for CAS)
1421
1422 orr(tmp_reg, swap_reg, Rthread); // new mark
1423
1424 biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, slow_case,
1425 (counters != NULL) ? counters->anonymously_biased_lock_entry_count_addr() : NULL);
1426
1427 // If the biasing toward our thread failed, this means that
1428 // another thread succeeded in biasing it toward itself and we
1429 // need to revoke that bias. The revocation will occur in the
1430 // interpreter runtime in the slow case.
1431
1432 b(done);
1433
1434 bind(try_rebias);
1435
1436 // At this point we know the epoch has expired, meaning that the
1437 // current "bias owner", if any, is actually invalid. Under these
1438 // circumstances _only_, we are allowed to use the current header's
1439 // value as the comparison value when doing the cas to acquire the
1440 // bias in the current epoch. In other words, we allow transfer of
1441 // the bias from one thread to another directly in this situation.
1442
1443 // tmp_reg low (not owner) bits are (age: 0 | pattern&epoch: prototype^swap_reg)
1444
1445 eor(tmp_reg, tmp_reg, swap_reg); // OK except for owner bits (age preserved !)
1446
1447 // owner bits 'random'. Set them to Rthread.
1448 mov(tmp_reg, AsmOperand(tmp_reg, lsl, 23));
1449 mov(tmp_reg, AsmOperand(tmp_reg, lsr, 23));
1450
1451 orr(tmp_reg, tmp_reg, Rthread); // new mark
1452
1453 biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, slow_case,
1454 (counters != NULL) ? counters->rebiased_lock_entry_count_addr() : NULL);
1455
1456 // If the biasing toward our thread failed, then another thread
1457 // succeeded in biasing it toward itself and we need to revoke that
1458 // bias. The revocation will occur in the runtime in the slow case.
1459
1460 b(done);
1461
1462 bind(try_revoke_bias);
1463
1464 // The prototype mark in the klass doesn't have the bias bit set any
1465 // more, indicating that objects of this data type are not supposed
1466 // to be biased any more. We are going to try to reset the mark of
1467 // this object to the prototype value and fall through to the
1468 // CAS-based locking scheme. Note that if our CAS fails, it means
1469 // that another thread raced us for the privilege of revoking the
1470 // bias of this particular object, so it's okay to continue in the
1471 // normal locking code.
1472
1473 // tmp_reg low (not owner) bits are (age: 0 | pattern&epoch: prototype^swap_reg)
1474
1475 eor(tmp_reg, tmp_reg, swap_reg); // OK except for owner bits (age preserved !)
1476
1477 // owner bits 'random'. Clear them
1478 mov(tmp_reg, AsmOperand(tmp_reg, lsl, 23));
1479 mov(tmp_reg, AsmOperand(tmp_reg, lsr, 23));
1480
1481 biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, cas_label,
1482 (counters != NULL) ? counters->revoked_lock_entry_count_addr() : NULL);
1483
1484 // Fall through to the normal CAS-based lock, because no matter what
1485 // the result of the above CAS, some thread must have succeeded in
1486 // removing the bias bit from the object's header.
1487
1488 bind(cas_label);
1489 }
1490
1491
biased_locking_exit(Register obj_reg,Register tmp_reg,Label & done)1492 void MacroAssembler::biased_locking_exit(Register obj_reg, Register tmp_reg, Label& done) {
1493 assert(UseBiasedLocking, "why call this otherwise?");
1494
1495 // Check for biased locking unlock case, which is a no-op
1496 // Note: we do not have to check the thread ID for two reasons.
1497 // First, the interpreter checks for IllegalMonitorStateException at
1498 // a higher level. Second, if the bias was revoked while we held the
1499 // lock, the object could not be rebiased toward another thread, so
1500 // the bias bit would be clear.
1501 ldr(tmp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1502
1503 andr(tmp_reg, tmp_reg, markWord::biased_lock_mask_in_place);
1504 cmp(tmp_reg, markWord::biased_lock_pattern);
1505 b(done, eq);
1506 }
1507
1508
resolve_jobject(Register value,Register tmp1,Register tmp2)1509 void MacroAssembler::resolve_jobject(Register value,
1510 Register tmp1,
1511 Register tmp2) {
1512 assert_different_registers(value, tmp1, tmp2);
1513 Label done, not_weak;
1514 cbz(value, done); // Use NULL as-is.
1515 STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u);
1516 tbz(value, 0, not_weak); // Test for jweak tag.
1517
1518 // Resolve jweak.
1519 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
1520 Address(value, -JNIHandles::weak_tag_value), value, tmp1, tmp2, noreg);
1521 b(done);
1522 bind(not_weak);
1523 // Resolve (untagged) jobject.
1524 access_load_at(T_OBJECT, IN_NATIVE,
1525 Address(value, 0), value, tmp1, tmp2, noreg);
1526 verify_oop(value);
1527 bind(done);
1528 }
1529
1530
1531 //////////////////////////////////////////////////////////////////////////////////
1532
1533
load_sized_value(Register dst,Address src,size_t size_in_bytes,bool is_signed,AsmCondition cond)1534 void MacroAssembler::load_sized_value(Register dst, Address src,
1535 size_t size_in_bytes, bool is_signed, AsmCondition cond) {
1536 switch (size_in_bytes) {
1537 case 4: ldr(dst, src, cond); break;
1538 case 2: is_signed ? ldrsh(dst, src, cond) : ldrh(dst, src, cond); break;
1539 case 1: is_signed ? ldrsb(dst, src, cond) : ldrb(dst, src, cond); break;
1540 default: ShouldNotReachHere();
1541 }
1542 }
1543
1544
store_sized_value(Register src,Address dst,size_t size_in_bytes,AsmCondition cond)1545 void MacroAssembler::store_sized_value(Register src, Address dst, size_t size_in_bytes, AsmCondition cond) {
1546 switch (size_in_bytes) {
1547 case 4: str(src, dst, cond); break;
1548 case 2: strh(src, dst, cond); break;
1549 case 1: strb(src, dst, cond); break;
1550 default: ShouldNotReachHere();
1551 }
1552 }
1553
1554 // Look up the method for a megamorphic invokeinterface call.
1555 // The target method is determined by <Rinterf, Rindex>.
1556 // The receiver klass is in Rklass.
1557 // On success, the result will be in method_result, and execution falls through.
1558 // On failure, execution transfers to the given label.
lookup_interface_method(Register Rklass,Register Rintf,RegisterOrConstant itable_index,Register method_result,Register Rscan,Register Rtmp,Label & L_no_such_interface)1559 void MacroAssembler::lookup_interface_method(Register Rklass,
1560 Register Rintf,
1561 RegisterOrConstant itable_index,
1562 Register method_result,
1563 Register Rscan,
1564 Register Rtmp,
1565 Label& L_no_such_interface) {
1566
1567 assert_different_registers(Rklass, Rintf, Rscan, Rtmp);
1568
1569 const int entry_size = itableOffsetEntry::size() * HeapWordSize;
1570 assert(itableOffsetEntry::interface_offset_in_bytes() == 0, "not added for convenience");
1571
1572 // Compute start of first itableOffsetEntry (which is at the end of the vtable)
1573 const int base = in_bytes(Klass::vtable_start_offset());
1574 const int scale = exact_log2(vtableEntry::size_in_bytes());
1575 ldr_s32(Rtmp, Address(Rklass, Klass::vtable_length_offset())); // Get length of vtable
1576 add(Rscan, Rklass, base);
1577 add(Rscan, Rscan, AsmOperand(Rtmp, lsl, scale));
1578
1579 // Search through the itable for an interface equal to incoming Rintf
1580 // itable looks like [intface][offset][intface][offset][intface][offset]
1581
1582 Label loop;
1583 bind(loop);
1584 ldr(Rtmp, Address(Rscan, entry_size, post_indexed));
1585 cmp(Rtmp, Rintf); // set ZF and CF if interface is found
1586 cmn(Rtmp, 0, ne); // check if tmp == 0 and clear CF if it is
1587 b(loop, ne);
1588
1589 // CF == 0 means we reached the end of itable without finding icklass
1590 b(L_no_such_interface, cc);
1591
1592 if (method_result != noreg) {
1593 // Interface found at previous position of Rscan, now load the method
1594 ldr_s32(Rtmp, Address(Rscan, itableOffsetEntry::offset_offset_in_bytes() - entry_size));
1595 if (itable_index.is_register()) {
1596 add(Rtmp, Rtmp, Rklass); // Add offset to Klass*
1597 assert(itableMethodEntry::size() * HeapWordSize == wordSize, "adjust the scaling in the code below");
1598 assert(itableMethodEntry::method_offset_in_bytes() == 0, "adjust the offset in the code below");
1599 ldr(method_result, Address::indexed_ptr(Rtmp, itable_index.as_register()));
1600 } else {
1601 int method_offset = itableMethodEntry::size() * HeapWordSize * itable_index.as_constant() +
1602 itableMethodEntry::method_offset_in_bytes();
1603 add_slow(method_result, Rklass, method_offset);
1604 ldr(method_result, Address(method_result, Rtmp));
1605 }
1606 }
1607 }
1608
1609
inc_counter(address counter_addr,Register tmpreg1,Register tmpreg2)1610 void MacroAssembler::inc_counter(address counter_addr, Register tmpreg1, Register tmpreg2) {
1611 mov_slow(tmpreg1, counter_addr);
1612 ldr_s32(tmpreg2, tmpreg1);
1613 add_32(tmpreg2, tmpreg2, 1);
1614 str_32(tmpreg2, tmpreg1);
1615 }
1616
floating_cmp(Register dst)1617 void MacroAssembler::floating_cmp(Register dst) {
1618 vmrs(dst, FPSCR);
1619 orr(dst, dst, 0x08000000);
1620 eor(dst, dst, AsmOperand(dst, lsl, 3));
1621 mov(dst, AsmOperand(dst, asr, 30));
1622 }
1623
restore_default_fp_mode()1624 void MacroAssembler::restore_default_fp_mode() {
1625 #ifndef __SOFTFP__
1626 // Round to Near mode, IEEE compatible, masked exceptions
1627 mov(Rtemp, 0);
1628 vmsr(FPSCR, Rtemp);
1629 #endif // !__SOFTFP__
1630 }
1631
1632 // 24-bit word range == 26-bit byte range
check26(int offset)1633 bool check26(int offset) {
1634 // this could be simplified, but it mimics encoding and decoding
1635 // an actual branch insrtuction
1636 int off1 = offset << 6 >> 8;
1637 int encoded = off1 & ((1<<24)-1);
1638 int decoded = encoded << 8 >> 6;
1639 return offset == decoded;
1640 }
1641
1642 // Perform some slight adjustments so the default 32MB code cache
1643 // is fully reachable.
first_cache_address()1644 static inline address first_cache_address() {
1645 return CodeCache::low_bound() + sizeof(HeapBlock::Header);
1646 }
last_cache_address()1647 static inline address last_cache_address() {
1648 return CodeCache::high_bound() - Assembler::InstructionSize;
1649 }
1650
1651
1652 // Can we reach target using unconditional branch or call from anywhere
1653 // in the code cache (because code can be relocated)?
_reachable_from_cache(address target)1654 bool MacroAssembler::_reachable_from_cache(address target) {
1655 #ifdef __thumb__
1656 if ((1 & (intptr_t)target) != 0) {
1657 // Return false to avoid 'b' if we need switching to THUMB mode.
1658 return false;
1659 }
1660 #endif
1661
1662 address cl = first_cache_address();
1663 address ch = last_cache_address();
1664
1665 if (ForceUnreachable) {
1666 // Only addresses from CodeCache can be treated as reachable.
1667 if (target < CodeCache::low_bound() || CodeCache::high_bound() < target) {
1668 return false;
1669 }
1670 }
1671
1672 intptr_t loffset = (intptr_t)target - (intptr_t)cl;
1673 intptr_t hoffset = (intptr_t)target - (intptr_t)ch;
1674
1675 return check26(loffset - 8) && check26(hoffset - 8);
1676 }
1677
reachable_from_cache(address target)1678 bool MacroAssembler::reachable_from_cache(address target) {
1679 assert(CodeCache::contains(pc()), "not supported");
1680 return _reachable_from_cache(target);
1681 }
1682
1683 // Can we reach the entire code cache from anywhere else in the code cache?
_cache_fully_reachable()1684 bool MacroAssembler::_cache_fully_reachable() {
1685 address cl = first_cache_address();
1686 address ch = last_cache_address();
1687 return _reachable_from_cache(cl) && _reachable_from_cache(ch);
1688 }
1689
cache_fully_reachable()1690 bool MacroAssembler::cache_fully_reachable() {
1691 assert(CodeCache::contains(pc()), "not supported");
1692 return _cache_fully_reachable();
1693 }
1694
jump(address target,relocInfo::relocType rtype,Register scratch,AsmCondition cond)1695 void MacroAssembler::jump(address target, relocInfo::relocType rtype, Register scratch, AsmCondition cond) {
1696 assert((rtype == relocInfo::runtime_call_type) || (rtype == relocInfo::none), "not supported");
1697 if (reachable_from_cache(target)) {
1698 relocate(rtype);
1699 b(target, cond);
1700 return;
1701 }
1702
1703 // Note: relocate is not needed for the code below,
1704 // encoding targets in absolute format.
1705 if (ignore_non_patchable_relocations()) {
1706 rtype = relocInfo::none;
1707 }
1708
1709 if (VM_Version::supports_movw() && (scratch != noreg) && (rtype == relocInfo::none)) {
1710 // Note: this version cannot be (atomically) patched
1711 mov_slow(scratch, (intptr_t)target, cond);
1712 bx(scratch, cond);
1713 } else {
1714 Label skip;
1715 InlinedAddress address_literal(target);
1716 if (cond != al) {
1717 b(skip, inverse(cond));
1718 }
1719 relocate(rtype);
1720 ldr_literal(PC, address_literal);
1721 bind_literal(address_literal);
1722 bind(skip);
1723 }
1724 }
1725
1726 // Similar to jump except that:
1727 // - near calls are valid only if any destination in the cache is near
1728 // - no movt/movw (not atomically patchable)
patchable_jump(address target,relocInfo::relocType rtype,Register scratch,AsmCondition cond)1729 void MacroAssembler::patchable_jump(address target, relocInfo::relocType rtype, Register scratch, AsmCondition cond) {
1730 assert((rtype == relocInfo::runtime_call_type) || (rtype == relocInfo::none), "not supported");
1731 if (cache_fully_reachable()) {
1732 // Note: this assumes that all possible targets (the initial one
1733 // and the addressed patched to) are all in the code cache.
1734 assert(CodeCache::contains(target), "target might be too far");
1735 relocate(rtype);
1736 b(target, cond);
1737 return;
1738 }
1739
1740 // Discard the relocation information if not needed for CacheCompiledCode
1741 // since the next encodings are all in absolute format.
1742 if (ignore_non_patchable_relocations()) {
1743 rtype = relocInfo::none;
1744 }
1745
1746 {
1747 Label skip;
1748 InlinedAddress address_literal(target);
1749 if (cond != al) {
1750 b(skip, inverse(cond));
1751 }
1752 relocate(rtype);
1753 ldr_literal(PC, address_literal);
1754 bind_literal(address_literal);
1755 bind(skip);
1756 }
1757 }
1758
call(address target,RelocationHolder rspec,AsmCondition cond)1759 void MacroAssembler::call(address target, RelocationHolder rspec, AsmCondition cond) {
1760 Register scratch = LR;
1761 assert(rspec.type() == relocInfo::runtime_call_type || rspec.type() == relocInfo::none, "not supported");
1762 if (reachable_from_cache(target)) {
1763 relocate(rspec);
1764 bl(target, cond);
1765 return;
1766 }
1767
1768 // Note: relocate is not needed for the code below,
1769 // encoding targets in absolute format.
1770 if (ignore_non_patchable_relocations()) {
1771 // This assumes the information was needed only for relocating the code.
1772 rspec = RelocationHolder::none;
1773 }
1774
1775 if (VM_Version::supports_movw() && (rspec.type() == relocInfo::none)) {
1776 // Note: this version cannot be (atomically) patched
1777 mov_slow(scratch, (intptr_t)target, cond);
1778 blx(scratch, cond);
1779 return;
1780 }
1781
1782 {
1783 Label ret_addr;
1784 if (cond != al) {
1785 b(ret_addr, inverse(cond));
1786 }
1787
1788
1789 InlinedAddress address_literal(target);
1790 relocate(rspec);
1791 adr(LR, ret_addr);
1792 ldr_literal(PC, address_literal);
1793
1794 bind_literal(address_literal);
1795 bind(ret_addr);
1796 }
1797 }
1798
1799
patchable_call(address target,RelocationHolder const & rspec,bool c2)1800 int MacroAssembler::patchable_call(address target, RelocationHolder const& rspec, bool c2) {
1801 assert(rspec.type() == relocInfo::static_call_type ||
1802 rspec.type() == relocInfo::none ||
1803 rspec.type() == relocInfo::opt_virtual_call_type, "not supported");
1804
1805 // Always generate the relocation information, needed for patching
1806 relocate(rspec); // used by NativeCall::is_call_before()
1807 if (cache_fully_reachable()) {
1808 // Note: this assumes that all possible targets (the initial one
1809 // and the addresses patched to) are all in the code cache.
1810 assert(CodeCache::contains(target), "target might be too far");
1811 bl(target);
1812 } else {
1813 Label ret_addr;
1814 InlinedAddress address_literal(target);
1815 adr(LR, ret_addr);
1816 ldr_literal(PC, address_literal);
1817 bind_literal(address_literal);
1818 bind(ret_addr);
1819 }
1820 return offset();
1821 }
1822
1823 // ((OopHandle)result).resolve();
resolve_oop_handle(Register result)1824 void MacroAssembler::resolve_oop_handle(Register result) {
1825 // OopHandle::resolve is an indirection.
1826 ldr(result, Address(result, 0));
1827 }
1828
load_mirror(Register mirror,Register method,Register tmp)1829 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
1830 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
1831 ldr(tmp, Address(method, Method::const_offset()));
1832 ldr(tmp, Address(tmp, ConstMethod::constants_offset()));
1833 ldr(tmp, Address(tmp, ConstantPool::pool_holder_offset_in_bytes()));
1834 ldr(mirror, Address(tmp, mirror_offset));
1835 resolve_oop_handle(mirror);
1836 }
1837
1838
1839 ///////////////////////////////////////////////////////////////////////////////
1840
1841 // Compressed pointers
1842
1843
load_klass(Register dst_klass,Register src_oop,AsmCondition cond)1844 void MacroAssembler::load_klass(Register dst_klass, Register src_oop, AsmCondition cond) {
1845 ldr(dst_klass, Address(src_oop, oopDesc::klass_offset_in_bytes()), cond);
1846 }
1847
1848
1849 // Blows src_klass.
store_klass(Register src_klass,Register dst_oop)1850 void MacroAssembler::store_klass(Register src_klass, Register dst_oop) {
1851 str(src_klass, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
1852 }
1853
1854
1855
load_heap_oop(Register dst,Address src,Register tmp1,Register tmp2,Register tmp3,DecoratorSet decorators)1856 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
1857 access_load_at(T_OBJECT, IN_HEAP | decorators, src, dst, tmp1, tmp2, tmp3);
1858 }
1859
1860 // Blows src and flags.
store_heap_oop(Address obj,Register new_val,Register tmp1,Register tmp2,Register tmp3,DecoratorSet decorators)1861 void MacroAssembler::store_heap_oop(Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
1862 access_store_at(T_OBJECT, IN_HEAP | decorators, obj, new_val, tmp1, tmp2, tmp3, false);
1863 }
1864
store_heap_oop_null(Address obj,Register new_val,Register tmp1,Register tmp2,Register tmp3,DecoratorSet decorators)1865 void MacroAssembler::store_heap_oop_null(Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
1866 access_store_at(T_OBJECT, IN_HEAP, obj, new_val, tmp1, tmp2, tmp3, true);
1867 }
1868
access_load_at(BasicType type,DecoratorSet decorators,Address src,Register dst,Register tmp1,Register tmp2,Register tmp3)1869 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
1870 Address src, Register dst, Register tmp1, Register tmp2, Register tmp3) {
1871 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1872 decorators = AccessInternal::decorator_fixup(decorators);
1873 bool as_raw = (decorators & AS_RAW) != 0;
1874 if (as_raw) {
1875 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
1876 } else {
1877 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
1878 }
1879 }
1880
access_store_at(BasicType type,DecoratorSet decorators,Address obj,Register new_val,Register tmp1,Register tmp2,Register tmp3,bool is_null)1881 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
1882 Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, bool is_null) {
1883 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1884 decorators = AccessInternal::decorator_fixup(decorators);
1885 bool as_raw = (decorators & AS_RAW) != 0;
1886 if (as_raw) {
1887 bs->BarrierSetAssembler::store_at(this, decorators, type, obj, new_val, tmp1, tmp2, tmp3, is_null);
1888 } else {
1889 bs->store_at(this, decorators, type, obj, new_val, tmp1, tmp2, tmp3, is_null);
1890 }
1891 }
1892
resolve(DecoratorSet decorators,Register obj)1893 void MacroAssembler::resolve(DecoratorSet decorators, Register obj) {
1894 // Use stronger ACCESS_WRITE|ACCESS_READ by default.
1895 if ((decorators & (ACCESS_READ | ACCESS_WRITE)) == 0) {
1896 decorators |= ACCESS_READ | ACCESS_WRITE;
1897 }
1898 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1899 return bs->resolve(this, decorators, obj);
1900 }
1901
safepoint_poll(Register tmp1,Label & slow_path)1902 void MacroAssembler::safepoint_poll(Register tmp1, Label& slow_path) {
1903 ldr_u32(tmp1, Address(Rthread, Thread::polling_word_offset()));
1904 tst(tmp1, exact_log2(SafepointMechanism::poll_bit()));
1905 b(slow_path, eq);
1906 }
1907
get_polling_page(Register dest)1908 void MacroAssembler::get_polling_page(Register dest) {
1909 ldr(dest, Address(Rthread, Thread::polling_page_offset()));
1910 }
1911
read_polling_page(Register dest,relocInfo::relocType rtype)1912 void MacroAssembler::read_polling_page(Register dest, relocInfo::relocType rtype) {
1913 get_polling_page(dest);
1914 relocate(rtype);
1915 ldr(dest, Address(dest));
1916 }
1917
1918