1 /*
2 * Copyright (c) 2008, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "asm/macroAssembler.hpp"
29 #include "ci/ciEnv.hpp"
30 #include "code/nativeInst.hpp"
31 #include "compiler/disassembler.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/cardTable.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/collectedHeap.inline.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "memory/resourceArea.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/klass.inline.hpp"
41 #include "prims/methodHandles.hpp"
42 #include "runtime/biasedLocking.hpp"
43 #include "runtime/interfaceSupport.inline.hpp"
44 #include "runtime/objectMonitor.hpp"
45 #include "runtime/os.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubRoutines.hpp"
48 #include "utilities/macros.hpp"
49 #include "utilities/powerOfTwo.hpp"
50
51 // Implementation of AddressLiteral
52
set_rspec(relocInfo::relocType rtype)53 void AddressLiteral::set_rspec(relocInfo::relocType rtype) {
54 switch (rtype) {
55 case relocInfo::oop_type:
56 // Oops are a special case. Normally they would be their own section
57 // but in cases like icBuffer they are literals in the code stream that
58 // we don't have a section for. We use none so that we get a literal address
59 // which is always patchable.
60 break;
61 case relocInfo::external_word_type:
62 _rspec = external_word_Relocation::spec(_target);
63 break;
64 case relocInfo::internal_word_type:
65 _rspec = internal_word_Relocation::spec(_target);
66 break;
67 case relocInfo::opt_virtual_call_type:
68 _rspec = opt_virtual_call_Relocation::spec();
69 break;
70 case relocInfo::static_call_type:
71 _rspec = static_call_Relocation::spec();
72 break;
73 case relocInfo::runtime_call_type:
74 _rspec = runtime_call_Relocation::spec();
75 break;
76 case relocInfo::poll_type:
77 case relocInfo::poll_return_type:
78 _rspec = Relocation::spec_simple(rtype);
79 break;
80 case relocInfo::none:
81 break;
82 default:
83 ShouldNotReachHere();
84 break;
85 }
86 }
87
88 // Initially added to the Assembler interface as a pure virtual:
89 // RegisterConstant delayed_value(..)
90 // for:
91 // 6812678 macro assembler needs delayed binding of a few constants (for 6655638)
92 // this was subsequently modified to its present name and return type
delayed_value_impl(intptr_t * delayed_value_addr,Register tmp,int offset)93 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
94 Register tmp,
95 int offset) {
96 ShouldNotReachHere();
97 return RegisterOrConstant(-1);
98 }
99
100
101
102
103 // virtual method calling
lookup_virtual_method(Register recv_klass,Register vtable_index,Register method_result)104 void MacroAssembler::lookup_virtual_method(Register recv_klass,
105 Register vtable_index,
106 Register method_result) {
107 const int base_offset = in_bytes(Klass::vtable_start_offset()) + vtableEntry::method_offset_in_bytes();
108 assert(vtableEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
109 add(recv_klass, recv_klass, AsmOperand(vtable_index, lsl, LogBytesPerWord));
110 ldr(method_result, Address(recv_klass, base_offset));
111 }
112
113
114 // Simplified, combined version, good for typical uses.
115 // Falls through on failure.
check_klass_subtype(Register sub_klass,Register super_klass,Register temp_reg,Register temp_reg2,Register temp_reg3,Label & L_success)116 void MacroAssembler::check_klass_subtype(Register sub_klass,
117 Register super_klass,
118 Register temp_reg,
119 Register temp_reg2,
120 Register temp_reg3,
121 Label& L_success) {
122 Label L_failure;
123 check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, temp_reg2, &L_success, &L_failure, NULL);
124 check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, temp_reg2, temp_reg3, &L_success, NULL);
125 bind(L_failure);
126 };
127
check_klass_subtype_fast_path(Register sub_klass,Register super_klass,Register temp_reg,Register temp_reg2,Label * L_success,Label * L_failure,Label * L_slow_path)128 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
129 Register super_klass,
130 Register temp_reg,
131 Register temp_reg2,
132 Label* L_success,
133 Label* L_failure,
134 Label* L_slow_path) {
135
136 assert_different_registers(sub_klass, super_klass, temp_reg, temp_reg2, noreg);
137 const Register super_check_offset = temp_reg2;
138
139 Label L_fallthrough;
140 int label_nulls = 0;
141 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
142 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
143 if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
144 assert(label_nulls <= 1, "at most one NULL in the batch");
145
146 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
147 int sco_offset = in_bytes(Klass::super_check_offset_offset());
148 Address super_check_offset_addr(super_klass, sco_offset);
149
150 // If the pointers are equal, we are done (e.g., String[] elements).
151 // This self-check enables sharing of secondary supertype arrays among
152 // non-primary types such as array-of-interface. Otherwise, each such
153 // type would need its own customized SSA.
154 // We move this check to the front of the fast path because many
155 // type checks are in fact trivially successful in this manner,
156 // so we get a nicely predicted branch right at the start of the check.
157 cmp(sub_klass, super_klass);
158 b(*L_success, eq);
159
160 // Check the supertype display:
161 ldr_u32(super_check_offset, super_check_offset_addr);
162
163 Address super_check_addr(sub_klass, super_check_offset);
164 ldr(temp_reg, super_check_addr);
165 cmp(super_klass, temp_reg); // load displayed supertype
166
167 // This check has worked decisively for primary supers.
168 // Secondary supers are sought in the super_cache ('super_cache_addr').
169 // (Secondary supers are interfaces and very deeply nested subtypes.)
170 // This works in the same check above because of a tricky aliasing
171 // between the super_cache and the primary super display elements.
172 // (The 'super_check_addr' can address either, as the case requires.)
173 // Note that the cache is updated below if it does not help us find
174 // what we need immediately.
175 // So if it was a primary super, we can just fail immediately.
176 // Otherwise, it's the slow path for us (no success at this point).
177
178 b(*L_success, eq);
179 cmp_32(super_check_offset, sc_offset);
180 if (L_failure == &L_fallthrough) {
181 b(*L_slow_path, eq);
182 } else {
183 b(*L_failure, ne);
184 if (L_slow_path != &L_fallthrough) {
185 b(*L_slow_path);
186 }
187 }
188
189 bind(L_fallthrough);
190 }
191
192
check_klass_subtype_slow_path(Register sub_klass,Register super_klass,Register temp_reg,Register temp2_reg,Register temp3_reg,Label * L_success,Label * L_failure,bool set_cond_codes)193 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
194 Register super_klass,
195 Register temp_reg,
196 Register temp2_reg,
197 Register temp3_reg,
198 Label* L_success,
199 Label* L_failure,
200 bool set_cond_codes) {
201 // Note: if used by code that expects a register to be 0 on success,
202 // this register must be temp_reg and set_cond_codes must be true
203
204 Register saved_reg = noreg;
205
206 // get additional tmp registers
207 if (temp3_reg == noreg) {
208 saved_reg = temp3_reg = LR;
209 push(saved_reg);
210 }
211
212 assert(temp2_reg != noreg, "need all the temporary registers");
213 assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg);
214
215 Register cmp_temp = temp_reg;
216 Register scan_temp = temp3_reg;
217 Register count_temp = temp2_reg;
218
219 Label L_fallthrough;
220 int label_nulls = 0;
221 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
222 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
223 assert(label_nulls <= 1, "at most one NULL in the batch");
224
225 // a couple of useful fields in sub_klass:
226 int ss_offset = in_bytes(Klass::secondary_supers_offset());
227 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
228 Address secondary_supers_addr(sub_klass, ss_offset);
229 Address super_cache_addr( sub_klass, sc_offset);
230
231 #ifndef PRODUCT
232 inc_counter((address)&SharedRuntime::_partial_subtype_ctr, scan_temp, count_temp);
233 #endif
234
235 // We will consult the secondary-super array.
236 ldr(scan_temp, Address(sub_klass, ss_offset));
237
238 assert(! UseCompressedOops, "search_key must be the compressed super_klass");
239 // else search_key is the
240 Register search_key = super_klass;
241
242 // Load the array length.
243 ldr(count_temp, Address(scan_temp, Array<Klass*>::length_offset_in_bytes()));
244 add(scan_temp, scan_temp, Array<Klass*>::base_offset_in_bytes());
245
246 add(count_temp, count_temp, 1);
247
248 Label L_loop, L_fail;
249
250 // Top of search loop
251 bind(L_loop);
252 // Notes:
253 // scan_temp starts at the array elements
254 // count_temp is 1+size
255 subs(count_temp, count_temp, 1);
256 if ((L_failure != &L_fallthrough) && (! set_cond_codes) && (saved_reg == noreg)) {
257 // direct jump to L_failure if failed and no cleanup needed
258 b(*L_failure, eq); // not found and
259 } else {
260 b(L_fail, eq); // not found in the array
261 }
262
263 // Load next super to check
264 // In the array of super classes elements are pointer sized.
265 int element_size = wordSize;
266 ldr(cmp_temp, Address(scan_temp, element_size, post_indexed));
267
268 // Look for Rsuper_klass on Rsub_klass's secondary super-class-overflow list
269 subs(cmp_temp, cmp_temp, search_key);
270
271 // A miss means we are NOT a subtype and need to keep looping
272 b(L_loop, ne);
273
274 // Falling out the bottom means we found a hit; we ARE a subtype
275
276 // Note: temp_reg/cmp_temp is already 0 and flag Z is set
277
278 // Success. Cache the super we found and proceed in triumph.
279 str(super_klass, Address(sub_klass, sc_offset));
280
281 if (saved_reg != noreg) {
282 // Return success
283 pop(saved_reg);
284 }
285
286 b(*L_success);
287
288 bind(L_fail);
289 // Note1: check "b(*L_failure, eq)" above if adding extra instructions here
290 if (set_cond_codes) {
291 movs(temp_reg, sub_klass); // clears Z and sets temp_reg to non-0 if needed
292 }
293 if (saved_reg != noreg) {
294 pop(saved_reg);
295 }
296 if (L_failure != &L_fallthrough) {
297 b(*L_failure);
298 }
299
300 bind(L_fallthrough);
301 }
302
303 // Returns address of receiver parameter, using tmp as base register. tmp and params_count can be the same.
receiver_argument_address(Register params_base,Register params_count,Register tmp)304 Address MacroAssembler::receiver_argument_address(Register params_base, Register params_count, Register tmp) {
305 assert_different_registers(params_base, params_count);
306 add(tmp, params_base, AsmOperand(params_count, lsl, Interpreter::logStackElementSize));
307 return Address(tmp, -Interpreter::stackElementSize);
308 }
309
310
align(int modulus)311 void MacroAssembler::align(int modulus) {
312 while (offset() % modulus != 0) {
313 nop();
314 }
315 }
316
set_last_Java_frame(Register last_java_sp,Register last_java_fp,bool save_last_java_pc,Register tmp)317 int MacroAssembler::set_last_Java_frame(Register last_java_sp,
318 Register last_java_fp,
319 bool save_last_java_pc,
320 Register tmp) {
321 int pc_offset;
322 if (last_java_fp != noreg) {
323 // optional
324 str(last_java_fp, Address(Rthread, JavaThread::last_Java_fp_offset()));
325 _fp_saved = true;
326 } else {
327 _fp_saved = false;
328 }
329 if (save_last_java_pc) {
330 str(PC, Address(Rthread, JavaThread::last_Java_pc_offset()));
331 pc_offset = offset() + VM_Version::stored_pc_adjustment();
332 _pc_saved = true;
333 } else {
334 _pc_saved = false;
335 pc_offset = -1;
336 }
337 // According to comment in javaFrameAnchorm SP must be saved last, so that other
338 // entries are valid when SP is set.
339
340 // However, this is probably not a strong constrainst since for instance PC is
341 // sometimes read from the stack at SP... but is pushed later (by the call). Hence,
342 // we now write the fields in the expected order but we have not added a StoreStore
343 // barrier.
344
345 // XXX: if the ordering is really important, PC should always be saved (without forgetting
346 // to update oop_map offsets) and a StoreStore barrier might be needed.
347
348 if (last_java_sp == noreg) {
349 last_java_sp = SP; // always saved
350 }
351 str(last_java_sp, Address(Rthread, JavaThread::last_Java_sp_offset()));
352
353 return pc_offset; // for oopmaps
354 }
355
reset_last_Java_frame(Register tmp)356 void MacroAssembler::reset_last_Java_frame(Register tmp) {
357 const Register Rzero = zero_register(tmp);
358 str(Rzero, Address(Rthread, JavaThread::last_Java_sp_offset()));
359 if (_fp_saved) {
360 str(Rzero, Address(Rthread, JavaThread::last_Java_fp_offset()));
361 }
362 if (_pc_saved) {
363 str(Rzero, Address(Rthread, JavaThread::last_Java_pc_offset()));
364 }
365 }
366
367
368 // Implementation of call_VM versions
369
call_VM_leaf_helper(address entry_point,int number_of_arguments)370 void MacroAssembler::call_VM_leaf_helper(address entry_point, int number_of_arguments) {
371 assert(number_of_arguments >= 0, "cannot have negative number of arguments");
372 assert(number_of_arguments <= 4, "cannot have more than 4 arguments");
373
374 // Safer to save R9 here since callers may have been written
375 // assuming R9 survives. This is suboptimal but is not worth
376 // optimizing for the few platforms where R9 is scratched.
377 push(RegisterSet(R4) | R9ifScratched);
378 mov(R4, SP);
379 bic(SP, SP, StackAlignmentInBytes - 1);
380 call(entry_point, relocInfo::runtime_call_type);
381 mov(SP, R4);
382 pop(RegisterSet(R4) | R9ifScratched);
383 }
384
385
call_VM_helper(Register oop_result,address entry_point,int number_of_arguments,bool check_exceptions)386 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
387 assert(number_of_arguments >= 0, "cannot have negative number of arguments");
388 assert(number_of_arguments <= 3, "cannot have more than 3 arguments");
389
390 const Register tmp = Rtemp;
391 assert_different_registers(oop_result, tmp);
392
393 set_last_Java_frame(SP, FP, true, tmp);
394
395 #if R9_IS_SCRATCHED
396 // Safer to save R9 here since callers may have been written
397 // assuming R9 survives. This is suboptimal but is not worth
398 // optimizing for the few platforms where R9 is scratched.
399
400 // Note: cannot save R9 above the saved SP (some calls expect for
401 // instance the Java stack top at the saved SP)
402 // => once saved (with set_last_Java_frame), decrease SP before rounding to
403 // ensure the slot at SP will be free for R9).
404 sub(SP, SP, 4);
405 bic(SP, SP, StackAlignmentInBytes - 1);
406 str(R9, Address(SP, 0));
407 #else
408 bic(SP, SP, StackAlignmentInBytes - 1);
409 #endif // R9_IS_SCRATCHED
410
411 mov(R0, Rthread);
412 call(entry_point, relocInfo::runtime_call_type);
413
414 #if R9_IS_SCRATCHED
415 ldr(R9, Address(SP, 0));
416 #endif
417 ldr(SP, Address(Rthread, JavaThread::last_Java_sp_offset()));
418
419 reset_last_Java_frame(tmp);
420
421 // C++ interp handles this in the interpreter
422 check_and_handle_popframe();
423 check_and_handle_earlyret();
424
425 if (check_exceptions) {
426 // check for pending exceptions
427 ldr(tmp, Address(Rthread, Thread::pending_exception_offset()));
428 cmp(tmp, 0);
429 mov(Rexception_pc, PC, ne);
430 b(StubRoutines::forward_exception_entry(), ne);
431 }
432
433 // get oop result if there is one and reset the value in the thread
434 if (oop_result->is_valid()) {
435 get_vm_result(oop_result, tmp);
436 }
437 }
438
call_VM(Register oop_result,address entry_point,bool check_exceptions)439 void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) {
440 call_VM_helper(oop_result, entry_point, 0, check_exceptions);
441 }
442
443
call_VM(Register oop_result,address entry_point,Register arg_1,bool check_exceptions)444 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, bool check_exceptions) {
445 assert (arg_1 == R1, "fixed register for arg_1");
446 call_VM_helper(oop_result, entry_point, 1, check_exceptions);
447 }
448
449
call_VM(Register oop_result,address entry_point,Register arg_1,Register arg_2,bool check_exceptions)450 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions) {
451 assert (arg_1 == R1, "fixed register for arg_1");
452 assert (arg_2 == R2, "fixed register for arg_2");
453 call_VM_helper(oop_result, entry_point, 2, check_exceptions);
454 }
455
456
call_VM(Register oop_result,address entry_point,Register arg_1,Register arg_2,Register arg_3,bool check_exceptions)457 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions) {
458 assert (arg_1 == R1, "fixed register for arg_1");
459 assert (arg_2 == R2, "fixed register for arg_2");
460 assert (arg_3 == R3, "fixed register for arg_3");
461 call_VM_helper(oop_result, entry_point, 3, check_exceptions);
462 }
463
464
call_VM(Register oop_result,Register last_java_sp,address entry_point,int number_of_arguments,bool check_exceptions)465 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments, bool check_exceptions) {
466 // Not used on ARM
467 Unimplemented();
468 }
469
470
call_VM(Register oop_result,Register last_java_sp,address entry_point,Register arg_1,bool check_exceptions)471 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions) {
472 // Not used on ARM
473 Unimplemented();
474 }
475
476
call_VM(Register oop_result,Register last_java_sp,address entry_point,Register arg_1,Register arg_2,bool check_exceptions)477 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions) {
478 // Not used on ARM
479 Unimplemented();
480 }
481
482
call_VM(Register oop_result,Register last_java_sp,address entry_point,Register arg_1,Register arg_2,Register arg_3,bool check_exceptions)483 void MacroAssembler::call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions) {
484 // Not used on ARM
485 Unimplemented();
486 }
487
488 // Raw call, without saving/restoring registers, exception handling, etc.
489 // Mainly used from various stubs.
call_VM(address entry_point,bool save_R9_if_scratched)490 void MacroAssembler::call_VM(address entry_point, bool save_R9_if_scratched) {
491 const Register tmp = Rtemp; // Rtemp free since scratched by call
492 set_last_Java_frame(SP, FP, true, tmp);
493 #if R9_IS_SCRATCHED
494 if (save_R9_if_scratched) {
495 // Note: Saving also R10 for alignment.
496 push(RegisterSet(R9, R10));
497 }
498 #endif
499 mov(R0, Rthread);
500 call(entry_point, relocInfo::runtime_call_type);
501 #if R9_IS_SCRATCHED
502 if (save_R9_if_scratched) {
503 pop(RegisterSet(R9, R10));
504 }
505 #endif
506 reset_last_Java_frame(tmp);
507 }
508
call_VM_leaf(address entry_point)509 void MacroAssembler::call_VM_leaf(address entry_point) {
510 call_VM_leaf_helper(entry_point, 0);
511 }
512
call_VM_leaf(address entry_point,Register arg_1)513 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
514 assert (arg_1 == R0, "fixed register for arg_1");
515 call_VM_leaf_helper(entry_point, 1);
516 }
517
call_VM_leaf(address entry_point,Register arg_1,Register arg_2)518 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2) {
519 assert (arg_1 == R0, "fixed register for arg_1");
520 assert (arg_2 == R1, "fixed register for arg_2");
521 call_VM_leaf_helper(entry_point, 2);
522 }
523
call_VM_leaf(address entry_point,Register arg_1,Register arg_2,Register arg_3)524 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3) {
525 assert (arg_1 == R0, "fixed register for arg_1");
526 assert (arg_2 == R1, "fixed register for arg_2");
527 assert (arg_3 == R2, "fixed register for arg_3");
528 call_VM_leaf_helper(entry_point, 3);
529 }
530
call_VM_leaf(address entry_point,Register arg_1,Register arg_2,Register arg_3,Register arg_4)531 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4) {
532 assert (arg_1 == R0, "fixed register for arg_1");
533 assert (arg_2 == R1, "fixed register for arg_2");
534 assert (arg_3 == R2, "fixed register for arg_3");
535 assert (arg_4 == R3, "fixed register for arg_4");
536 call_VM_leaf_helper(entry_point, 4);
537 }
538
get_vm_result(Register oop_result,Register tmp)539 void MacroAssembler::get_vm_result(Register oop_result, Register tmp) {
540 assert_different_registers(oop_result, tmp);
541 ldr(oop_result, Address(Rthread, JavaThread::vm_result_offset()));
542 str(zero_register(tmp), Address(Rthread, JavaThread::vm_result_offset()));
543 verify_oop(oop_result);
544 }
545
get_vm_result_2(Register metadata_result,Register tmp)546 void MacroAssembler::get_vm_result_2(Register metadata_result, Register tmp) {
547 assert_different_registers(metadata_result, tmp);
548 ldr(metadata_result, Address(Rthread, JavaThread::vm_result_2_offset()));
549 str(zero_register(tmp), Address(Rthread, JavaThread::vm_result_2_offset()));
550 }
551
add_rc(Register dst,Register arg1,RegisterOrConstant arg2)552 void MacroAssembler::add_rc(Register dst, Register arg1, RegisterOrConstant arg2) {
553 if (arg2.is_register()) {
554 add(dst, arg1, arg2.as_register());
555 } else {
556 add(dst, arg1, arg2.as_constant());
557 }
558 }
559
add_slow(Register rd,Register rn,int c)560 void MacroAssembler::add_slow(Register rd, Register rn, int c) {
561 // This function is used in compiler for handling large frame offsets
562 if ((c < 0) && (((-c) & ~0x3fc) == 0)) {
563 return sub(rd, rn, (-c));
564 }
565 int low = c & 0x3fc;
566 if (low != 0) {
567 add(rd, rn, low);
568 rn = rd;
569 }
570 if (c & ~0x3fc) {
571 assert(AsmOperand::is_rotated_imm(c & ~0x3fc), "unsupported add_slow offset %d", c);
572 add(rd, rn, c & ~0x3fc);
573 } else if (rd != rn) {
574 assert(c == 0, "");
575 mov(rd, rn); // need to generate at least one move!
576 }
577 }
578
sub_slow(Register rd,Register rn,int c)579 void MacroAssembler::sub_slow(Register rd, Register rn, int c) {
580 // This function is used in compiler for handling large frame offsets
581 if ((c < 0) && (((-c) & ~0x3fc) == 0)) {
582 return add(rd, rn, (-c));
583 }
584 int low = c & 0x3fc;
585 if (low != 0) {
586 sub(rd, rn, low);
587 rn = rd;
588 }
589 if (c & ~0x3fc) {
590 assert(AsmOperand::is_rotated_imm(c & ~0x3fc), "unsupported sub_slow offset %d", c);
591 sub(rd, rn, c & ~0x3fc);
592 } else if (rd != rn) {
593 assert(c == 0, "");
594 mov(rd, rn); // need to generate at least one move!
595 }
596 }
597
mov_slow(Register rd,address addr)598 void MacroAssembler::mov_slow(Register rd, address addr) {
599 // do *not* call the non relocated mov_related_address
600 mov_slow(rd, (intptr_t)addr);
601 }
602
mov_slow(Register rd,const char * str)603 void MacroAssembler::mov_slow(Register rd, const char *str) {
604 mov_slow(rd, (intptr_t)str);
605 }
606
607
mov_slow(Register rd,intptr_t c,AsmCondition cond)608 void MacroAssembler::mov_slow(Register rd, intptr_t c, AsmCondition cond) {
609 if (AsmOperand::is_rotated_imm(c)) {
610 mov(rd, c, cond);
611 } else if (AsmOperand::is_rotated_imm(~c)) {
612 mvn(rd, ~c, cond);
613 } else if (VM_Version::supports_movw()) {
614 movw(rd, c & 0xffff, cond);
615 if ((unsigned int)c >> 16) {
616 movt(rd, (unsigned int)c >> 16, cond);
617 }
618 } else {
619 // Find first non-zero bit
620 int shift = 0;
621 while ((c & (3 << shift)) == 0) {
622 shift += 2;
623 }
624 // Put the least significant part of the constant
625 int mask = 0xff << shift;
626 mov(rd, c & mask, cond);
627 // Add up to 3 other parts of the constant;
628 // each of them can be represented as rotated_imm
629 if (c & (mask << 8)) {
630 orr(rd, rd, c & (mask << 8), cond);
631 }
632 if (c & (mask << 16)) {
633 orr(rd, rd, c & (mask << 16), cond);
634 }
635 if (c & (mask << 24)) {
636 orr(rd, rd, c & (mask << 24), cond);
637 }
638 }
639 }
640
641
mov_oop(Register rd,jobject o,int oop_index,AsmCondition cond)642 void MacroAssembler::mov_oop(Register rd, jobject o, int oop_index,
643 AsmCondition cond
644 ) {
645
646 if (o == NULL) {
647 mov(rd, 0, cond);
648 return;
649 }
650
651 if (oop_index == 0) {
652 oop_index = oop_recorder()->allocate_oop_index(o);
653 }
654 relocate(oop_Relocation::spec(oop_index));
655
656 if (VM_Version::supports_movw()) {
657 movw(rd, 0, cond);
658 movt(rd, 0, cond);
659 } else {
660 ldr(rd, Address(PC), cond);
661 // Extra nop to handle case of large offset of oop placeholder (see NativeMovConstReg::set_data).
662 nop();
663 }
664 }
665
mov_metadata(Register rd,Metadata * o,int metadata_index)666 void MacroAssembler::mov_metadata(Register rd, Metadata* o, int metadata_index) {
667 if (o == NULL) {
668 mov(rd, 0);
669 return;
670 }
671
672 if (metadata_index == 0) {
673 metadata_index = oop_recorder()->allocate_metadata_index(o);
674 }
675 relocate(metadata_Relocation::spec(metadata_index));
676
677 if (VM_Version::supports_movw()) {
678 movw(rd, ((int)o) & 0xffff);
679 movt(rd, (unsigned int)o >> 16);
680 } else {
681 ldr(rd, Address(PC));
682 // Extra nop to handle case of large offset of metadata placeholder (see NativeMovConstReg::set_data).
683 nop();
684 }
685 }
686
mov_float(FloatRegister fd,jfloat c,AsmCondition cond)687 void MacroAssembler::mov_float(FloatRegister fd, jfloat c, AsmCondition cond) {
688 Label skip_constant;
689 union {
690 jfloat f;
691 jint i;
692 } accessor;
693 accessor.f = c;
694
695 flds(fd, Address(PC), cond);
696 b(skip_constant);
697 emit_int32(accessor.i);
698 bind(skip_constant);
699 }
700
mov_double(FloatRegister fd,jdouble c,AsmCondition cond)701 void MacroAssembler::mov_double(FloatRegister fd, jdouble c, AsmCondition cond) {
702 Label skip_constant;
703 union {
704 jdouble d;
705 jint i[2];
706 } accessor;
707 accessor.d = c;
708
709 fldd(fd, Address(PC), cond);
710 b(skip_constant);
711 emit_int32(accessor.i[0]);
712 emit_int32(accessor.i[1]);
713 bind(skip_constant);
714 }
715
ldr_global_s32(Register reg,address address_of_global)716 void MacroAssembler::ldr_global_s32(Register reg, address address_of_global) {
717 intptr_t addr = (intptr_t) address_of_global;
718 mov_slow(reg, addr & ~0xfff);
719 ldr(reg, Address(reg, addr & 0xfff));
720 }
721
ldr_global_ptr(Register reg,address address_of_global)722 void MacroAssembler::ldr_global_ptr(Register reg, address address_of_global) {
723 ldr_global_s32(reg, address_of_global);
724 }
725
ldrb_global(Register reg,address address_of_global)726 void MacroAssembler::ldrb_global(Register reg, address address_of_global) {
727 intptr_t addr = (intptr_t) address_of_global;
728 mov_slow(reg, addr & ~0xfff);
729 ldrb(reg, Address(reg, addr & 0xfff));
730 }
731
zero_extend(Register rd,Register rn,int bits)732 void MacroAssembler::zero_extend(Register rd, Register rn, int bits) {
733 if (bits <= 8) {
734 andr(rd, rn, (1 << bits) - 1);
735 } else if (bits >= 24) {
736 bic(rd, rn, -1 << bits);
737 } else {
738 mov(rd, AsmOperand(rn, lsl, 32 - bits));
739 mov(rd, AsmOperand(rd, lsr, 32 - bits));
740 }
741 }
742
sign_extend(Register rd,Register rn,int bits)743 void MacroAssembler::sign_extend(Register rd, Register rn, int bits) {
744 mov(rd, AsmOperand(rn, lsl, 32 - bits));
745 mov(rd, AsmOperand(rd, asr, 32 - bits));
746 }
747
748
cmpoop(Register obj1,Register obj2)749 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
750 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
751 bs->obj_equals(this, obj1, obj2);
752 }
753
long_move(Register rd_lo,Register rd_hi,Register rn_lo,Register rn_hi,AsmCondition cond)754 void MacroAssembler::long_move(Register rd_lo, Register rd_hi,
755 Register rn_lo, Register rn_hi,
756 AsmCondition cond) {
757 if (rd_lo != rn_hi) {
758 if (rd_lo != rn_lo) { mov(rd_lo, rn_lo, cond); }
759 if (rd_hi != rn_hi) { mov(rd_hi, rn_hi, cond); }
760 } else if (rd_hi != rn_lo) {
761 if (rd_hi != rn_hi) { mov(rd_hi, rn_hi, cond); }
762 if (rd_lo != rn_lo) { mov(rd_lo, rn_lo, cond); }
763 } else {
764 eor(rd_lo, rd_hi, rd_lo, cond);
765 eor(rd_hi, rd_lo, rd_hi, cond);
766 eor(rd_lo, rd_hi, rd_lo, cond);
767 }
768 }
769
long_shift(Register rd_lo,Register rd_hi,Register rn_lo,Register rn_hi,AsmShift shift,Register count)770 void MacroAssembler::long_shift(Register rd_lo, Register rd_hi,
771 Register rn_lo, Register rn_hi,
772 AsmShift shift, Register count) {
773 Register tmp;
774 if (rd_lo != rn_lo && rd_lo != rn_hi && rd_lo != count) {
775 tmp = rd_lo;
776 } else {
777 tmp = rd_hi;
778 }
779 assert_different_registers(tmp, count, rn_lo, rn_hi);
780
781 subs(tmp, count, 32);
782 if (shift == lsl) {
783 assert_different_registers(rd_hi, rn_lo);
784 assert_different_registers(count, rd_hi);
785 mov(rd_hi, AsmOperand(rn_lo, shift, tmp), pl);
786 rsb(tmp, count, 32, mi);
787 if (rd_hi == rn_hi) {
788 mov(rd_hi, AsmOperand(rn_hi, lsl, count), mi);
789 orr(rd_hi, rd_hi, AsmOperand(rn_lo, lsr, tmp), mi);
790 } else {
791 mov(rd_hi, AsmOperand(rn_lo, lsr, tmp), mi);
792 orr(rd_hi, rd_hi, AsmOperand(rn_hi, lsl, count), mi);
793 }
794 mov(rd_lo, AsmOperand(rn_lo, shift, count));
795 } else {
796 assert_different_registers(rd_lo, rn_hi);
797 assert_different_registers(rd_lo, count);
798 mov(rd_lo, AsmOperand(rn_hi, shift, tmp), pl);
799 rsb(tmp, count, 32, mi);
800 if (rd_lo == rn_lo) {
801 mov(rd_lo, AsmOperand(rn_lo, lsr, count), mi);
802 orr(rd_lo, rd_lo, AsmOperand(rn_hi, lsl, tmp), mi);
803 } else {
804 mov(rd_lo, AsmOperand(rn_hi, lsl, tmp), mi);
805 orr(rd_lo, rd_lo, AsmOperand(rn_lo, lsr, count), mi);
806 }
807 mov(rd_hi, AsmOperand(rn_hi, shift, count));
808 }
809 }
810
long_shift(Register rd_lo,Register rd_hi,Register rn_lo,Register rn_hi,AsmShift shift,int count)811 void MacroAssembler::long_shift(Register rd_lo, Register rd_hi,
812 Register rn_lo, Register rn_hi,
813 AsmShift shift, int count) {
814 assert(count != 0 && (count & ~63) == 0, "must be");
815
816 if (shift == lsl) {
817 assert_different_registers(rd_hi, rn_lo);
818 if (count >= 32) {
819 mov(rd_hi, AsmOperand(rn_lo, lsl, count - 32));
820 mov(rd_lo, 0);
821 } else {
822 mov(rd_hi, AsmOperand(rn_hi, lsl, count));
823 orr(rd_hi, rd_hi, AsmOperand(rn_lo, lsr, 32 - count));
824 mov(rd_lo, AsmOperand(rn_lo, lsl, count));
825 }
826 } else {
827 assert_different_registers(rd_lo, rn_hi);
828 if (count >= 32) {
829 if (count == 32) {
830 mov(rd_lo, rn_hi);
831 } else {
832 mov(rd_lo, AsmOperand(rn_hi, shift, count - 32));
833 }
834 if (shift == asr) {
835 mov(rd_hi, AsmOperand(rn_hi, asr, 0));
836 } else {
837 mov(rd_hi, 0);
838 }
839 } else {
840 mov(rd_lo, AsmOperand(rn_lo, lsr, count));
841 orr(rd_lo, rd_lo, AsmOperand(rn_hi, lsl, 32 - count));
842 mov(rd_hi, AsmOperand(rn_hi, shift, count));
843 }
844 }
845 }
846
_verify_oop(Register reg,const char * s,const char * file,int line)847 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
848 // This code pattern is matched in NativeIntruction::skip_verify_oop.
849 // Update it at modifications.
850 if (!VerifyOops) return;
851
852 char buffer[64];
853 #ifdef COMPILER1
854 if (CommentedAssembly) {
855 snprintf(buffer, sizeof(buffer), "verify_oop at %d", offset());
856 block_comment(buffer);
857 }
858 #endif
859 const char* msg_buffer = NULL;
860 {
861 ResourceMark rm;
862 stringStream ss;
863 ss.print("%s at offset %d (%s:%d)", s, offset(), file, line);
864 msg_buffer = code_string(ss.as_string());
865 }
866
867 save_all_registers();
868
869 if (reg != R2) {
870 mov(R2, reg); // oop to verify
871 }
872 mov(R1, SP); // register save area
873
874 Label done;
875 InlinedString Lmsg(msg_buffer);
876 ldr_literal(R0, Lmsg); // message
877
878 // call indirectly to solve generation ordering problem
879 ldr_global_ptr(Rtemp, StubRoutines::verify_oop_subroutine_entry_address());
880 call(Rtemp);
881
882 restore_all_registers();
883
884 b(done);
885 #ifdef COMPILER2
886 int off = offset();
887 #endif
888 bind_literal(Lmsg);
889 #ifdef COMPILER2
890 if (offset() - off == 1 * wordSize) {
891 // no padding, so insert nop for worst-case sizing
892 nop();
893 }
894 #endif
895 bind(done);
896 }
897
_verify_oop_addr(Address addr,const char * s,const char * file,int line)898 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
899 if (!VerifyOops) return;
900
901 const char* msg_buffer = NULL;
902 {
903 ResourceMark rm;
904 stringStream ss;
905 if ((addr.base() == SP) && (addr.index()==noreg)) {
906 ss.print("verify_oop_addr SP[%d]: %s", (int)addr.disp(), s);
907 } else {
908 ss.print("verify_oop_addr: %s", s);
909 }
910 ss.print(" (%s:%d)", file, line);
911 msg_buffer = code_string(ss.as_string());
912 }
913
914 int push_size = save_all_registers();
915
916 if (addr.base() == SP) {
917 // computes an addr that takes into account the push
918 if (addr.index() != noreg) {
919 Register new_base = addr.index() == R2 ? R1 : R2; // avoid corrupting the index
920 add(new_base, SP, push_size);
921 addr = addr.rebase(new_base);
922 } else {
923 addr = addr.plus_disp(push_size);
924 }
925 }
926
927 ldr(R2, addr); // oop to verify
928 mov(R1, SP); // register save area
929
930 Label done;
931 InlinedString Lmsg(msg_buffer);
932 ldr_literal(R0, Lmsg); // message
933
934 // call indirectly to solve generation ordering problem
935 ldr_global_ptr(Rtemp, StubRoutines::verify_oop_subroutine_entry_address());
936 call(Rtemp);
937
938 restore_all_registers();
939
940 b(done);
941 bind_literal(Lmsg);
942 bind(done);
943 }
944
c2bool(Register x)945 void MacroAssembler::c2bool(Register x)
946 {
947 tst(x, 0xff); // Only look at the lowest byte
948 mov(x, 1, ne);
949 }
950
null_check(Register reg,Register tmp,int offset)951 void MacroAssembler::null_check(Register reg, Register tmp, int offset) {
952 if (needs_explicit_null_check(offset)) {
953 assert_different_registers(reg, tmp);
954 if (tmp == noreg) {
955 tmp = Rtemp;
956 assert((! Thread::current()->is_Compiler_thread()) ||
957 (! (ciEnv::current()->task() == NULL)) ||
958 (! (ciEnv::current()->comp_level() == CompLevel_full_optimization)),
959 "Rtemp not available in C2"); // explicit tmp register required
960 // XXX: could we mark the code buffer as not compatible with C2 ?
961 }
962 ldr(tmp, Address(reg));
963 }
964 }
965
966 // Puts address of allocated object into register `obj` and end of allocated object into register `obj_end`.
eden_allocate(Register obj,Register obj_end,Register tmp1,Register tmp2,RegisterOrConstant size_expression,Label & slow_case)967 void MacroAssembler::eden_allocate(Register obj, Register obj_end, Register tmp1, Register tmp2,
968 RegisterOrConstant size_expression, Label& slow_case) {
969 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
970 bs->eden_allocate(this, obj, obj_end, tmp1, tmp2, size_expression, slow_case);
971 }
972
973 // Puts address of allocated object into register `obj` and end of allocated object into register `obj_end`.
tlab_allocate(Register obj,Register obj_end,Register tmp1,RegisterOrConstant size_expression,Label & slow_case)974 void MacroAssembler::tlab_allocate(Register obj, Register obj_end, Register tmp1,
975 RegisterOrConstant size_expression, Label& slow_case) {
976 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
977 bs->tlab_allocate(this, obj, obj_end, tmp1, size_expression, slow_case);
978 }
979
980 // Fills memory regions [start..end] with zeroes. Clobbers `start` and `tmp` registers.
zero_memory(Register start,Register end,Register tmp)981 void MacroAssembler::zero_memory(Register start, Register end, Register tmp) {
982 Label loop;
983 const Register ptr = start;
984
985 mov(tmp, 0);
986 bind(loop);
987 cmp(ptr, end);
988 str(tmp, Address(ptr, wordSize, post_indexed), lo);
989 b(loop, lo);
990 }
991
arm_stack_overflow_check(int frame_size_in_bytes,Register tmp)992 void MacroAssembler::arm_stack_overflow_check(int frame_size_in_bytes, Register tmp) {
993 // Version of AbstractAssembler::generate_stack_overflow_check optimized for ARM
994 if (UseStackBanging) {
995 const int page_size = os::vm_page_size();
996
997 sub_slow(tmp, SP, JavaThread::stack_shadow_zone_size());
998 strb(R0, Address(tmp));
999 for (; frame_size_in_bytes >= page_size; frame_size_in_bytes -= 0xff0) {
1000 strb(R0, Address(tmp, -0xff0, pre_indexed));
1001 }
1002 }
1003 }
1004
arm_stack_overflow_check(Register Rsize,Register tmp)1005 void MacroAssembler::arm_stack_overflow_check(Register Rsize, Register tmp) {
1006 if (UseStackBanging) {
1007 Label loop;
1008
1009 mov(tmp, SP);
1010 add_slow(Rsize, Rsize, JavaThread::stack_shadow_zone_size() - os::vm_page_size());
1011 bind(loop);
1012 subs(Rsize, Rsize, 0xff0);
1013 strb(R0, Address(tmp, -0xff0, pre_indexed));
1014 b(loop, hi);
1015 }
1016 }
1017
stop(const char * msg)1018 void MacroAssembler::stop(const char* msg) {
1019 // This code pattern is matched in NativeIntruction::is_stop.
1020 // Update it at modifications.
1021 #ifdef COMPILER1
1022 if (CommentedAssembly) {
1023 block_comment("stop");
1024 }
1025 #endif
1026
1027 InlinedAddress Ldebug(CAST_FROM_FN_PTR(address, MacroAssembler::debug));
1028 InlinedString Lmsg(msg);
1029
1030 // save all registers for further inspection
1031 save_all_registers();
1032
1033 ldr_literal(R0, Lmsg); // message
1034 mov(R1, SP); // register save area
1035
1036 ldr_literal(PC, Ldebug); // call MacroAssembler::debug
1037
1038 bind_literal(Lmsg);
1039 bind_literal(Ldebug);
1040 }
1041
warn(const char * msg)1042 void MacroAssembler::warn(const char* msg) {
1043 #ifdef COMPILER1
1044 if (CommentedAssembly) {
1045 block_comment("warn");
1046 }
1047 #endif
1048
1049 InlinedAddress Lwarn(CAST_FROM_FN_PTR(address, warning));
1050 InlinedString Lmsg(msg);
1051 Label done;
1052
1053 int push_size = save_caller_save_registers();
1054
1055
1056 ldr_literal(R0, Lmsg); // message
1057 ldr_literal(LR, Lwarn); // call warning
1058
1059 call(LR);
1060
1061 restore_caller_save_registers();
1062
1063 b(done);
1064 bind_literal(Lmsg);
1065 bind_literal(Lwarn);
1066 bind(done);
1067 }
1068
1069
save_all_registers()1070 int MacroAssembler::save_all_registers() {
1071 // This code pattern is matched in NativeIntruction::is_save_all_registers.
1072 // Update it at modifications.
1073 push(RegisterSet(R0, R12) | RegisterSet(LR) | RegisterSet(PC));
1074 return 15*wordSize;
1075 }
1076
restore_all_registers()1077 void MacroAssembler::restore_all_registers() {
1078 pop(RegisterSet(R0, R12) | RegisterSet(LR)); // restore registers
1079 add(SP, SP, wordSize); // discard saved PC
1080 }
1081
save_caller_save_registers()1082 int MacroAssembler::save_caller_save_registers() {
1083 #if R9_IS_SCRATCHED
1084 // Save also R10 to preserve alignment
1085 push(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR) | RegisterSet(R9,R10));
1086 return 8*wordSize;
1087 #else
1088 push(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR));
1089 return 6*wordSize;
1090 #endif
1091 }
1092
restore_caller_save_registers()1093 void MacroAssembler::restore_caller_save_registers() {
1094 #if R9_IS_SCRATCHED
1095 pop(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR) | RegisterSet(R9,R10));
1096 #else
1097 pop(RegisterSet(R0, R3) | RegisterSet(R12) | RegisterSet(LR));
1098 #endif
1099 }
1100
debug(const char * msg,const intx * registers)1101 void MacroAssembler::debug(const char* msg, const intx* registers) {
1102 // In order to get locks to work, we need to fake a in_VM state
1103 JavaThread* thread = JavaThread::current();
1104 thread->set_thread_state(_thread_in_vm);
1105
1106 if (ShowMessageBoxOnError) {
1107 ttyLocker ttyl;
1108 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
1109 BytecodeCounter::print();
1110 }
1111 if (os::message_box(msg, "Execution stopped, print registers?")) {
1112 // saved registers: R0-R12, LR, PC
1113 const int nregs = 15;
1114 const Register regs[nregs] = {R0, R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, R12, LR, PC};
1115
1116 for (int i = 0; i < nregs; i++) {
1117 tty->print_cr("%s = " INTPTR_FORMAT, regs[i]->name(), registers[i]);
1118 }
1119
1120 // derive original SP value from the address of register save area
1121 tty->print_cr("%s = " INTPTR_FORMAT, SP->name(), p2i(®isters[nregs]));
1122 }
1123 BREAKPOINT;
1124 } else {
1125 ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
1126 }
1127 assert(false, "DEBUG MESSAGE: %s", msg);
1128 fatal("%s", msg); // returning from MacroAssembler::debug is not supported
1129 }
1130
unimplemented(const char * what)1131 void MacroAssembler::unimplemented(const char* what) {
1132 const char* buf = NULL;
1133 {
1134 ResourceMark rm;
1135 stringStream ss;
1136 ss.print("unimplemented: %s", what);
1137 buf = code_string(ss.as_string());
1138 }
1139 stop(buf);
1140 }
1141
1142
1143 // Implementation of FixedSizeCodeBlock
1144
FixedSizeCodeBlock(MacroAssembler * masm,int size_in_instrs,bool enabled)1145 FixedSizeCodeBlock::FixedSizeCodeBlock(MacroAssembler* masm, int size_in_instrs, bool enabled) :
1146 _masm(masm), _start(masm->pc()), _size_in_instrs(size_in_instrs), _enabled(enabled) {
1147 }
1148
~FixedSizeCodeBlock()1149 FixedSizeCodeBlock::~FixedSizeCodeBlock() {
1150 if (_enabled) {
1151 address curr_pc = _masm->pc();
1152
1153 assert(_start < curr_pc, "invalid current pc");
1154 guarantee(curr_pc <= _start + _size_in_instrs * Assembler::InstructionSize, "code block is too long");
1155
1156 int nops_count = (_start - curr_pc) / Assembler::InstructionSize + _size_in_instrs;
1157 for (int i = 0; i < nops_count; i++) {
1158 _masm->nop();
1159 }
1160 }
1161 }
1162
1163
1164 // Serializes memory. Potentially blows flags and reg.
1165 // tmp is a scratch for v6 co-processor write op (could be noreg for other architecure versions)
1166 // preserve_flags takes a longer path in LoadStore case (dmb rather then control dependency) to preserve status flags. Optional.
1167 // load_tgt is an ordered load target in a LoadStore case only, to create dependency between the load operation and conditional branch. Optional.
membar(Membar_mask_bits order_constraint,Register tmp,bool preserve_flags,Register load_tgt)1168 void MacroAssembler::membar(Membar_mask_bits order_constraint,
1169 Register tmp,
1170 bool preserve_flags,
1171 Register load_tgt) {
1172
1173 if (order_constraint == StoreStore) {
1174 dmb(DMB_st, tmp);
1175 } else if ((order_constraint & StoreLoad) ||
1176 (order_constraint & LoadLoad) ||
1177 (order_constraint & StoreStore) ||
1178 (load_tgt == noreg) ||
1179 preserve_flags) {
1180 dmb(DMB_all, tmp);
1181 } else {
1182 // LoadStore: speculative stores reordeing is prohibited
1183
1184 // By providing an ordered load target register, we avoid an extra memory load reference
1185 Label not_taken;
1186 bind(not_taken);
1187 cmp(load_tgt, load_tgt);
1188 b(not_taken, ne);
1189 }
1190 }
1191
1192
1193 // If "allow_fallthrough_on_failure" is false, we always branch to "slow_case"
1194 // on failure, so fall-through can only mean success.
1195 // "one_shot" controls whether we loop and retry to mitigate spurious failures.
1196 // This is only needed for C2, which for some reason does not rety,
1197 // while C1/interpreter does.
1198 // TODO: measure if it makes a difference
1199
cas_for_lock_acquire(Register oldval,Register newval,Register base,Register tmp,Label & slow_case,bool allow_fallthrough_on_failure,bool one_shot)1200 void MacroAssembler::cas_for_lock_acquire(Register oldval, Register newval,
1201 Register base, Register tmp, Label &slow_case,
1202 bool allow_fallthrough_on_failure, bool one_shot)
1203 {
1204
1205 bool fallthrough_is_success = false;
1206
1207 // ARM Litmus Test example does prefetching here.
1208 // TODO: investigate if it helps performance
1209
1210 // The last store was to the displaced header, so to prevent
1211 // reordering we must issue a StoreStore or Release barrier before
1212 // the CAS store.
1213
1214 membar(MacroAssembler::StoreStore, noreg);
1215
1216 if (one_shot) {
1217 ldrex(tmp, Address(base, oopDesc::mark_offset_in_bytes()));
1218 cmp(tmp, oldval);
1219 strex(tmp, newval, Address(base, oopDesc::mark_offset_in_bytes()), eq);
1220 cmp(tmp, 0, eq);
1221 } else {
1222 atomic_cas_bool(oldval, newval, base, oopDesc::mark_offset_in_bytes(), tmp);
1223 }
1224
1225 // MemBarAcquireLock barrier
1226 // According to JSR-133 Cookbook, this should be LoadLoad | LoadStore,
1227 // but that doesn't prevent a load or store from floating up between
1228 // the load and store in the CAS sequence, so play it safe and
1229 // do a full fence.
1230 membar(Membar_mask_bits(LoadLoad | LoadStore | StoreStore | StoreLoad), noreg);
1231 if (!fallthrough_is_success && !allow_fallthrough_on_failure) {
1232 b(slow_case, ne);
1233 }
1234 }
1235
cas_for_lock_release(Register oldval,Register newval,Register base,Register tmp,Label & slow_case,bool allow_fallthrough_on_failure,bool one_shot)1236 void MacroAssembler::cas_for_lock_release(Register oldval, Register newval,
1237 Register base, Register tmp, Label &slow_case,
1238 bool allow_fallthrough_on_failure, bool one_shot)
1239 {
1240
1241 bool fallthrough_is_success = false;
1242
1243 assert_different_registers(oldval,newval,base,tmp);
1244
1245 // MemBarReleaseLock barrier
1246 // According to JSR-133 Cookbook, this should be StoreStore | LoadStore,
1247 // but that doesn't prevent a load or store from floating down between
1248 // the load and store in the CAS sequence, so play it safe and
1249 // do a full fence.
1250 membar(Membar_mask_bits(LoadLoad | LoadStore | StoreStore | StoreLoad), tmp);
1251
1252 if (one_shot) {
1253 ldrex(tmp, Address(base, oopDesc::mark_offset_in_bytes()));
1254 cmp(tmp, oldval);
1255 strex(tmp, newval, Address(base, oopDesc::mark_offset_in_bytes()), eq);
1256 cmp(tmp, 0, eq);
1257 } else {
1258 atomic_cas_bool(oldval, newval, base, oopDesc::mark_offset_in_bytes(), tmp);
1259 }
1260 if (!fallthrough_is_success && !allow_fallthrough_on_failure) {
1261 b(slow_case, ne);
1262 }
1263
1264 // ExitEnter
1265 // According to JSR-133 Cookbook, this should be StoreLoad, the same
1266 // barrier that follows volatile store.
1267 // TODO: Should be able to remove on armv8 if volatile loads
1268 // use the load-acquire instruction.
1269 membar(StoreLoad, noreg);
1270 }
1271
1272 #ifndef PRODUCT
1273
1274 // Preserves flags and all registers.
1275 // On SMP the updated value might not be visible to external observers without a sychronization barrier
cond_atomic_inc32(AsmCondition cond,int * counter_addr)1276 void MacroAssembler::cond_atomic_inc32(AsmCondition cond, int* counter_addr) {
1277 if (counter_addr != NULL) {
1278 InlinedAddress counter_addr_literal((address)counter_addr);
1279 Label done, retry;
1280 if (cond != al) {
1281 b(done, inverse(cond));
1282 }
1283
1284 push(RegisterSet(R0, R3) | RegisterSet(Rtemp));
1285 ldr_literal(R0, counter_addr_literal);
1286
1287 mrs(CPSR, Rtemp);
1288
1289 bind(retry);
1290 ldr_s32(R1, Address(R0));
1291 add(R2, R1, 1);
1292 atomic_cas_bool(R1, R2, R0, 0, R3);
1293 b(retry, ne);
1294
1295 msr(CPSR_fsxc, Rtemp);
1296
1297 pop(RegisterSet(R0, R3) | RegisterSet(Rtemp));
1298
1299 b(done);
1300 bind_literal(counter_addr_literal);
1301
1302 bind(done);
1303 }
1304 }
1305
1306 #endif // !PRODUCT
1307
1308
1309 // Building block for CAS cases of biased locking: makes CAS and records statistics.
1310 // The slow_case label is used to transfer control if CAS fails. Otherwise leaves condition codes set.
biased_locking_enter_with_cas(Register obj_reg,Register old_mark_reg,Register new_mark_reg,Register tmp,Label & slow_case,int * counter_addr)1311 void MacroAssembler::biased_locking_enter_with_cas(Register obj_reg, Register old_mark_reg, Register new_mark_reg,
1312 Register tmp, Label& slow_case, int* counter_addr) {
1313
1314 cas_for_lock_acquire(old_mark_reg, new_mark_reg, obj_reg, tmp, slow_case);
1315 #ifdef ASSERT
1316 breakpoint(ne); // Fallthrough only on success
1317 #endif
1318 #ifndef PRODUCT
1319 if (counter_addr != NULL) {
1320 cond_atomic_inc32(al, counter_addr);
1321 }
1322 #endif // !PRODUCT
1323 }
1324
biased_locking_enter(Register obj_reg,Register swap_reg,Register tmp_reg,bool swap_reg_contains_mark,Register tmp2,Label & done,Label & slow_case,BiasedLockingCounters * counters)1325 int MacroAssembler::biased_locking_enter(Register obj_reg, Register swap_reg, Register tmp_reg,
1326 bool swap_reg_contains_mark,
1327 Register tmp2,
1328 Label& done, Label& slow_case,
1329 BiasedLockingCounters* counters) {
1330 // obj_reg must be preserved (at least) if the bias locking fails
1331 // tmp_reg is a temporary register
1332 // swap_reg was used as a temporary but contained a value
1333 // that was used afterwards in some call pathes. Callers
1334 // have been fixed so that swap_reg no longer needs to be
1335 // saved.
1336 // Rtemp in no longer scratched
1337
1338 assert(UseBiasedLocking, "why call this otherwise?");
1339 assert_different_registers(obj_reg, swap_reg, tmp_reg, tmp2);
1340 guarantee(swap_reg!=tmp_reg, "invariant");
1341 assert(tmp_reg != noreg, "must supply tmp_reg");
1342
1343 #ifndef PRODUCT
1344 if (PrintBiasedLockingStatistics && (counters == NULL)) {
1345 counters = BiasedLocking::counters();
1346 }
1347 #endif
1348
1349 assert(markWord::age_shift == markWord::lock_bits + markWord::biased_lock_bits, "biased locking makes assumptions about bit layout");
1350 Address mark_addr(obj_reg, oopDesc::mark_offset_in_bytes());
1351
1352 // Biased locking
1353 // See whether the lock is currently biased toward our thread and
1354 // whether the epoch is still valid
1355 // Note that the runtime guarantees sufficient alignment of JavaThread
1356 // pointers to allow age to be placed into low bits
1357 // First check to see whether biasing is even enabled for this object
1358 Label cas_label;
1359
1360 // The null check applies to the mark loading, if we need to load it.
1361 // If the mark has already been loaded in swap_reg then it has already
1362 // been performed and the offset is irrelevant.
1363 int null_check_offset = offset();
1364 if (!swap_reg_contains_mark) {
1365 ldr(swap_reg, mark_addr);
1366 }
1367
1368 // On MP platform loads could return 'stale' values in some cases.
1369 // That is acceptable since either CAS or slow case path is taken in the worst case.
1370
1371 andr(tmp_reg, swap_reg, markWord::biased_lock_mask_in_place);
1372 cmp(tmp_reg, markWord::biased_lock_pattern);
1373
1374 b(cas_label, ne);
1375
1376 // The bias pattern is present in the object's header. Need to check
1377 // whether the bias owner and the epoch are both still current.
1378 load_klass(tmp_reg, obj_reg);
1379 ldr(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset()));
1380 orr(tmp_reg, tmp_reg, Rthread);
1381 eor(tmp_reg, tmp_reg, swap_reg);
1382
1383 bics(tmp_reg, tmp_reg, ((int) markWord::age_mask_in_place));
1384
1385 #ifndef PRODUCT
1386 if (counters != NULL) {
1387 cond_atomic_inc32(eq, counters->biased_lock_entry_count_addr());
1388 }
1389 #endif // !PRODUCT
1390
1391 b(done, eq);
1392
1393 Label try_revoke_bias;
1394 Label try_rebias;
1395
1396 // At this point we know that the header has the bias pattern and
1397 // that we are not the bias owner in the current epoch. We need to
1398 // figure out more details about the state of the header in order to
1399 // know what operations can be legally performed on the object's
1400 // header.
1401
1402 // If the low three bits in the xor result aren't clear, that means
1403 // the prototype header is no longer biased and we have to revoke
1404 // the bias on this object.
1405 tst(tmp_reg, markWord::biased_lock_mask_in_place);
1406 b(try_revoke_bias, ne);
1407
1408 // Biasing is still enabled for this data type. See whether the
1409 // epoch of the current bias is still valid, meaning that the epoch
1410 // bits of the mark word are equal to the epoch bits of the
1411 // prototype header. (Note that the prototype header's epoch bits
1412 // only change at a safepoint.) If not, attempt to rebias the object
1413 // toward the current thread. Note that we must be absolutely sure
1414 // that the current epoch is invalid in order to do this because
1415 // otherwise the manipulations it performs on the mark word are
1416 // illegal.
1417 tst(tmp_reg, markWord::epoch_mask_in_place);
1418 b(try_rebias, ne);
1419
1420 // tmp_reg has the age, epoch and pattern bits cleared
1421 // The remaining (owner) bits are (Thread ^ current_owner)
1422
1423 // The epoch of the current bias is still valid but we know nothing
1424 // about the owner; it might be set or it might be clear. Try to
1425 // acquire the bias of the object using an atomic operation. If this
1426 // fails we will go in to the runtime to revoke the object's bias.
1427 // Note that we first construct the presumed unbiased header so we
1428 // don't accidentally blow away another thread's valid bias.
1429
1430 // Note that we know the owner is not ourself. Hence, success can
1431 // only happen when the owner bits is 0
1432
1433 // until the assembler can be made smarter, we need to make some assumptions about the values
1434 // so we can optimize this:
1435 assert((markWord::biased_lock_mask_in_place | markWord::age_mask_in_place | markWord::epoch_mask_in_place) == 0x1ff, "biased bitmasks changed");
1436
1437 mov(swap_reg, AsmOperand(swap_reg, lsl, 23));
1438 mov(swap_reg, AsmOperand(swap_reg, lsr, 23)); // markWord with thread bits cleared (for CAS)
1439
1440 orr(tmp_reg, swap_reg, Rthread); // new mark
1441
1442 biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, slow_case,
1443 (counters != NULL) ? counters->anonymously_biased_lock_entry_count_addr() : NULL);
1444
1445 // If the biasing toward our thread failed, this means that
1446 // another thread succeeded in biasing it toward itself and we
1447 // need to revoke that bias. The revocation will occur in the
1448 // interpreter runtime in the slow case.
1449
1450 b(done);
1451
1452 bind(try_rebias);
1453
1454 // At this point we know the epoch has expired, meaning that the
1455 // current "bias owner", if any, is actually invalid. Under these
1456 // circumstances _only_, we are allowed to use the current header's
1457 // value as the comparison value when doing the cas to acquire the
1458 // bias in the current epoch. In other words, we allow transfer of
1459 // the bias from one thread to another directly in this situation.
1460
1461 // tmp_reg low (not owner) bits are (age: 0 | pattern&epoch: prototype^swap_reg)
1462
1463 eor(tmp_reg, tmp_reg, swap_reg); // OK except for owner bits (age preserved !)
1464
1465 // owner bits 'random'. Set them to Rthread.
1466 mov(tmp_reg, AsmOperand(tmp_reg, lsl, 23));
1467 mov(tmp_reg, AsmOperand(tmp_reg, lsr, 23));
1468
1469 orr(tmp_reg, tmp_reg, Rthread); // new mark
1470
1471 biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, slow_case,
1472 (counters != NULL) ? counters->rebiased_lock_entry_count_addr() : NULL);
1473
1474 // If the biasing toward our thread failed, then another thread
1475 // succeeded in biasing it toward itself and we need to revoke that
1476 // bias. The revocation will occur in the runtime in the slow case.
1477
1478 b(done);
1479
1480 bind(try_revoke_bias);
1481
1482 // The prototype mark in the klass doesn't have the bias bit set any
1483 // more, indicating that objects of this data type are not supposed
1484 // to be biased any more. We are going to try to reset the mark of
1485 // this object to the prototype value and fall through to the
1486 // CAS-based locking scheme. Note that if our CAS fails, it means
1487 // that another thread raced us for the privilege of revoking the
1488 // bias of this particular object, so it's okay to continue in the
1489 // normal locking code.
1490
1491 // tmp_reg low (not owner) bits are (age: 0 | pattern&epoch: prototype^swap_reg)
1492
1493 eor(tmp_reg, tmp_reg, swap_reg); // OK except for owner bits (age preserved !)
1494
1495 // owner bits 'random'. Clear them
1496 mov(tmp_reg, AsmOperand(tmp_reg, lsl, 23));
1497 mov(tmp_reg, AsmOperand(tmp_reg, lsr, 23));
1498
1499 biased_locking_enter_with_cas(obj_reg, swap_reg, tmp_reg, tmp2, cas_label,
1500 (counters != NULL) ? counters->revoked_lock_entry_count_addr() : NULL);
1501
1502 // Fall through to the normal CAS-based lock, because no matter what
1503 // the result of the above CAS, some thread must have succeeded in
1504 // removing the bias bit from the object's header.
1505
1506 bind(cas_label);
1507
1508 return null_check_offset;
1509 }
1510
1511
biased_locking_exit(Register obj_reg,Register tmp_reg,Label & done)1512 void MacroAssembler::biased_locking_exit(Register obj_reg, Register tmp_reg, Label& done) {
1513 assert(UseBiasedLocking, "why call this otherwise?");
1514
1515 // Check for biased locking unlock case, which is a no-op
1516 // Note: we do not have to check the thread ID for two reasons.
1517 // First, the interpreter checks for IllegalMonitorStateException at
1518 // a higher level. Second, if the bias was revoked while we held the
1519 // lock, the object could not be rebiased toward another thread, so
1520 // the bias bit would be clear.
1521 ldr(tmp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1522
1523 andr(tmp_reg, tmp_reg, markWord::biased_lock_mask_in_place);
1524 cmp(tmp_reg, markWord::biased_lock_pattern);
1525 b(done, eq);
1526 }
1527
1528
resolve_jobject(Register value,Register tmp1,Register tmp2)1529 void MacroAssembler::resolve_jobject(Register value,
1530 Register tmp1,
1531 Register tmp2) {
1532 assert_different_registers(value, tmp1, tmp2);
1533 Label done, not_weak;
1534 cbz(value, done); // Use NULL as-is.
1535 STATIC_ASSERT(JNIHandles::weak_tag_mask == 1u);
1536 tbz(value, 0, not_weak); // Test for jweak tag.
1537
1538 // Resolve jweak.
1539 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
1540 Address(value, -JNIHandles::weak_tag_value), value, tmp1, tmp2, noreg);
1541 b(done);
1542 bind(not_weak);
1543 // Resolve (untagged) jobject.
1544 access_load_at(T_OBJECT, IN_NATIVE,
1545 Address(value, 0), value, tmp1, tmp2, noreg);
1546 verify_oop(value);
1547 bind(done);
1548 }
1549
1550
1551 //////////////////////////////////////////////////////////////////////////////////
1552
1553
load_sized_value(Register dst,Address src,size_t size_in_bytes,bool is_signed,AsmCondition cond)1554 void MacroAssembler::load_sized_value(Register dst, Address src,
1555 size_t size_in_bytes, bool is_signed, AsmCondition cond) {
1556 switch (size_in_bytes) {
1557 case 4: ldr(dst, src, cond); break;
1558 case 2: is_signed ? ldrsh(dst, src, cond) : ldrh(dst, src, cond); break;
1559 case 1: is_signed ? ldrsb(dst, src, cond) : ldrb(dst, src, cond); break;
1560 default: ShouldNotReachHere();
1561 }
1562 }
1563
1564
store_sized_value(Register src,Address dst,size_t size_in_bytes,AsmCondition cond)1565 void MacroAssembler::store_sized_value(Register src, Address dst, size_t size_in_bytes, AsmCondition cond) {
1566 switch (size_in_bytes) {
1567 case 4: str(src, dst, cond); break;
1568 case 2: strh(src, dst, cond); break;
1569 case 1: strb(src, dst, cond); break;
1570 default: ShouldNotReachHere();
1571 }
1572 }
1573
1574 // Look up the method for a megamorphic invokeinterface call.
1575 // The target method is determined by <Rinterf, Rindex>.
1576 // The receiver klass is in Rklass.
1577 // On success, the result will be in method_result, and execution falls through.
1578 // On failure, execution transfers to the given label.
lookup_interface_method(Register Rklass,Register Rintf,RegisterOrConstant itable_index,Register method_result,Register Rscan,Register Rtmp,Label & L_no_such_interface)1579 void MacroAssembler::lookup_interface_method(Register Rklass,
1580 Register Rintf,
1581 RegisterOrConstant itable_index,
1582 Register method_result,
1583 Register Rscan,
1584 Register Rtmp,
1585 Label& L_no_such_interface) {
1586
1587 assert_different_registers(Rklass, Rintf, Rscan, Rtmp);
1588
1589 const int entry_size = itableOffsetEntry::size() * HeapWordSize;
1590 assert(itableOffsetEntry::interface_offset_in_bytes() == 0, "not added for convenience");
1591
1592 // Compute start of first itableOffsetEntry (which is at the end of the vtable)
1593 const int base = in_bytes(Klass::vtable_start_offset());
1594 const int scale = exact_log2(vtableEntry::size_in_bytes());
1595 ldr_s32(Rtmp, Address(Rklass, Klass::vtable_length_offset())); // Get length of vtable
1596 add(Rscan, Rklass, base);
1597 add(Rscan, Rscan, AsmOperand(Rtmp, lsl, scale));
1598
1599 // Search through the itable for an interface equal to incoming Rintf
1600 // itable looks like [intface][offset][intface][offset][intface][offset]
1601
1602 Label loop;
1603 bind(loop);
1604 ldr(Rtmp, Address(Rscan, entry_size, post_indexed));
1605 cmp(Rtmp, Rintf); // set ZF and CF if interface is found
1606 cmn(Rtmp, 0, ne); // check if tmp == 0 and clear CF if it is
1607 b(loop, ne);
1608
1609 // CF == 0 means we reached the end of itable without finding icklass
1610 b(L_no_such_interface, cc);
1611
1612 if (method_result != noreg) {
1613 // Interface found at previous position of Rscan, now load the method
1614 ldr_s32(Rtmp, Address(Rscan, itableOffsetEntry::offset_offset_in_bytes() - entry_size));
1615 if (itable_index.is_register()) {
1616 add(Rtmp, Rtmp, Rklass); // Add offset to Klass*
1617 assert(itableMethodEntry::size() * HeapWordSize == wordSize, "adjust the scaling in the code below");
1618 assert(itableMethodEntry::method_offset_in_bytes() == 0, "adjust the offset in the code below");
1619 ldr(method_result, Address::indexed_ptr(Rtmp, itable_index.as_register()));
1620 } else {
1621 int method_offset = itableMethodEntry::size() * HeapWordSize * itable_index.as_constant() +
1622 itableMethodEntry::method_offset_in_bytes();
1623 add_slow(method_result, Rklass, method_offset);
1624 ldr(method_result, Address(method_result, Rtmp));
1625 }
1626 }
1627 }
1628
1629
inc_counter(address counter_addr,Register tmpreg1,Register tmpreg2)1630 void MacroAssembler::inc_counter(address counter_addr, Register tmpreg1, Register tmpreg2) {
1631 mov_slow(tmpreg1, counter_addr);
1632 ldr_s32(tmpreg2, tmpreg1);
1633 add_32(tmpreg2, tmpreg2, 1);
1634 str_32(tmpreg2, tmpreg1);
1635 }
1636
floating_cmp(Register dst)1637 void MacroAssembler::floating_cmp(Register dst) {
1638 vmrs(dst, FPSCR);
1639 orr(dst, dst, 0x08000000);
1640 eor(dst, dst, AsmOperand(dst, lsl, 3));
1641 mov(dst, AsmOperand(dst, asr, 30));
1642 }
1643
restore_default_fp_mode()1644 void MacroAssembler::restore_default_fp_mode() {
1645 #ifndef __SOFTFP__
1646 // Round to Near mode, IEEE compatible, masked exceptions
1647 mov(Rtemp, 0);
1648 vmsr(FPSCR, Rtemp);
1649 #endif // !__SOFTFP__
1650 }
1651
1652 // 24-bit word range == 26-bit byte range
check26(int offset)1653 bool check26(int offset) {
1654 // this could be simplified, but it mimics encoding and decoding
1655 // an actual branch insrtuction
1656 int off1 = offset << 6 >> 8;
1657 int encoded = off1 & ((1<<24)-1);
1658 int decoded = encoded << 8 >> 6;
1659 return offset == decoded;
1660 }
1661
1662 // Perform some slight adjustments so the default 32MB code cache
1663 // is fully reachable.
first_cache_address()1664 static inline address first_cache_address() {
1665 return CodeCache::low_bound() + sizeof(HeapBlock::Header);
1666 }
last_cache_address()1667 static inline address last_cache_address() {
1668 return CodeCache::high_bound() - Assembler::InstructionSize;
1669 }
1670
1671
1672 // Can we reach target using unconditional branch or call from anywhere
1673 // in the code cache (because code can be relocated)?
_reachable_from_cache(address target)1674 bool MacroAssembler::_reachable_from_cache(address target) {
1675 #ifdef __thumb__
1676 if ((1 & (intptr_t)target) != 0) {
1677 // Return false to avoid 'b' if we need switching to THUMB mode.
1678 return false;
1679 }
1680 #endif
1681
1682 address cl = first_cache_address();
1683 address ch = last_cache_address();
1684
1685 if (ForceUnreachable) {
1686 // Only addresses from CodeCache can be treated as reachable.
1687 if (target < CodeCache::low_bound() || CodeCache::high_bound() < target) {
1688 return false;
1689 }
1690 }
1691
1692 intptr_t loffset = (intptr_t)target - (intptr_t)cl;
1693 intptr_t hoffset = (intptr_t)target - (intptr_t)ch;
1694
1695 return check26(loffset - 8) && check26(hoffset - 8);
1696 }
1697
reachable_from_cache(address target)1698 bool MacroAssembler::reachable_from_cache(address target) {
1699 assert(CodeCache::contains(pc()), "not supported");
1700 return _reachable_from_cache(target);
1701 }
1702
1703 // Can we reach the entire code cache from anywhere else in the code cache?
_cache_fully_reachable()1704 bool MacroAssembler::_cache_fully_reachable() {
1705 address cl = first_cache_address();
1706 address ch = last_cache_address();
1707 return _reachable_from_cache(cl) && _reachable_from_cache(ch);
1708 }
1709
cache_fully_reachable()1710 bool MacroAssembler::cache_fully_reachable() {
1711 assert(CodeCache::contains(pc()), "not supported");
1712 return _cache_fully_reachable();
1713 }
1714
jump(address target,relocInfo::relocType rtype,Register scratch,AsmCondition cond)1715 void MacroAssembler::jump(address target, relocInfo::relocType rtype, Register scratch, AsmCondition cond) {
1716 assert((rtype == relocInfo::runtime_call_type) || (rtype == relocInfo::none), "not supported");
1717 if (reachable_from_cache(target)) {
1718 relocate(rtype);
1719 b(target, cond);
1720 return;
1721 }
1722
1723 // Note: relocate is not needed for the code below,
1724 // encoding targets in absolute format.
1725 if (ignore_non_patchable_relocations()) {
1726 rtype = relocInfo::none;
1727 }
1728
1729 if (VM_Version::supports_movw() && (scratch != noreg) && (rtype == relocInfo::none)) {
1730 // Note: this version cannot be (atomically) patched
1731 mov_slow(scratch, (intptr_t)target, cond);
1732 bx(scratch, cond);
1733 } else {
1734 Label skip;
1735 InlinedAddress address_literal(target);
1736 if (cond != al) {
1737 b(skip, inverse(cond));
1738 }
1739 relocate(rtype);
1740 ldr_literal(PC, address_literal);
1741 bind_literal(address_literal);
1742 bind(skip);
1743 }
1744 }
1745
1746 // Similar to jump except that:
1747 // - near calls are valid only if any destination in the cache is near
1748 // - no movt/movw (not atomically patchable)
patchable_jump(address target,relocInfo::relocType rtype,Register scratch,AsmCondition cond)1749 void MacroAssembler::patchable_jump(address target, relocInfo::relocType rtype, Register scratch, AsmCondition cond) {
1750 assert((rtype == relocInfo::runtime_call_type) || (rtype == relocInfo::none), "not supported");
1751 if (cache_fully_reachable()) {
1752 // Note: this assumes that all possible targets (the initial one
1753 // and the addressed patched to) are all in the code cache.
1754 assert(CodeCache::contains(target), "target might be too far");
1755 relocate(rtype);
1756 b(target, cond);
1757 return;
1758 }
1759
1760 // Discard the relocation information if not needed for CacheCompiledCode
1761 // since the next encodings are all in absolute format.
1762 if (ignore_non_patchable_relocations()) {
1763 rtype = relocInfo::none;
1764 }
1765
1766 {
1767 Label skip;
1768 InlinedAddress address_literal(target);
1769 if (cond != al) {
1770 b(skip, inverse(cond));
1771 }
1772 relocate(rtype);
1773 ldr_literal(PC, address_literal);
1774 bind_literal(address_literal);
1775 bind(skip);
1776 }
1777 }
1778
call(address target,RelocationHolder rspec,AsmCondition cond)1779 void MacroAssembler::call(address target, RelocationHolder rspec, AsmCondition cond) {
1780 Register scratch = LR;
1781 assert(rspec.type() == relocInfo::runtime_call_type || rspec.type() == relocInfo::none, "not supported");
1782 if (reachable_from_cache(target)) {
1783 relocate(rspec);
1784 bl(target, cond);
1785 return;
1786 }
1787
1788 // Note: relocate is not needed for the code below,
1789 // encoding targets in absolute format.
1790 if (ignore_non_patchable_relocations()) {
1791 // This assumes the information was needed only for relocating the code.
1792 rspec = RelocationHolder::none;
1793 }
1794
1795 if (VM_Version::supports_movw() && (rspec.type() == relocInfo::none)) {
1796 // Note: this version cannot be (atomically) patched
1797 mov_slow(scratch, (intptr_t)target, cond);
1798 blx(scratch, cond);
1799 return;
1800 }
1801
1802 {
1803 Label ret_addr;
1804 if (cond != al) {
1805 b(ret_addr, inverse(cond));
1806 }
1807
1808
1809 InlinedAddress address_literal(target);
1810 relocate(rspec);
1811 adr(LR, ret_addr);
1812 ldr_literal(PC, address_literal);
1813
1814 bind_literal(address_literal);
1815 bind(ret_addr);
1816 }
1817 }
1818
1819
patchable_call(address target,RelocationHolder const & rspec,bool c2)1820 int MacroAssembler::patchable_call(address target, RelocationHolder const& rspec, bool c2) {
1821 assert(rspec.type() == relocInfo::static_call_type ||
1822 rspec.type() == relocInfo::none ||
1823 rspec.type() == relocInfo::opt_virtual_call_type, "not supported");
1824
1825 // Always generate the relocation information, needed for patching
1826 relocate(rspec); // used by NativeCall::is_call_before()
1827 if (cache_fully_reachable()) {
1828 // Note: this assumes that all possible targets (the initial one
1829 // and the addresses patched to) are all in the code cache.
1830 assert(CodeCache::contains(target), "target might be too far");
1831 bl(target);
1832 } else {
1833 Label ret_addr;
1834 InlinedAddress address_literal(target);
1835 adr(LR, ret_addr);
1836 ldr_literal(PC, address_literal);
1837 bind_literal(address_literal);
1838 bind(ret_addr);
1839 }
1840 return offset();
1841 }
1842
1843 // ((OopHandle)result).resolve();
resolve_oop_handle(Register result)1844 void MacroAssembler::resolve_oop_handle(Register result) {
1845 // OopHandle::resolve is an indirection.
1846 ldr(result, Address(result, 0));
1847 }
1848
load_mirror(Register mirror,Register method,Register tmp)1849 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
1850 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
1851 ldr(tmp, Address(method, Method::const_offset()));
1852 ldr(tmp, Address(tmp, ConstMethod::constants_offset()));
1853 ldr(tmp, Address(tmp, ConstantPool::pool_holder_offset_in_bytes()));
1854 ldr(mirror, Address(tmp, mirror_offset));
1855 resolve_oop_handle(mirror);
1856 }
1857
1858
1859 ///////////////////////////////////////////////////////////////////////////////
1860
1861 // Compressed pointers
1862
1863
load_klass(Register dst_klass,Register src_oop,AsmCondition cond)1864 void MacroAssembler::load_klass(Register dst_klass, Register src_oop, AsmCondition cond) {
1865 ldr(dst_klass, Address(src_oop, oopDesc::klass_offset_in_bytes()), cond);
1866 }
1867
1868
1869 // Blows src_klass.
store_klass(Register src_klass,Register dst_oop)1870 void MacroAssembler::store_klass(Register src_klass, Register dst_oop) {
1871 str(src_klass, Address(dst_oop, oopDesc::klass_offset_in_bytes()));
1872 }
1873
1874
1875
load_heap_oop(Register dst,Address src,Register tmp1,Register tmp2,Register tmp3,DecoratorSet decorators)1876 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
1877 access_load_at(T_OBJECT, IN_HEAP | decorators, src, dst, tmp1, tmp2, tmp3);
1878 }
1879
1880 // Blows src and flags.
store_heap_oop(Address obj,Register new_val,Register tmp1,Register tmp2,Register tmp3,DecoratorSet decorators)1881 void MacroAssembler::store_heap_oop(Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
1882 access_store_at(T_OBJECT, IN_HEAP | decorators, obj, new_val, tmp1, tmp2, tmp3, false);
1883 }
1884
store_heap_oop_null(Address obj,Register new_val,Register tmp1,Register tmp2,Register tmp3,DecoratorSet decorators)1885 void MacroAssembler::store_heap_oop_null(Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, DecoratorSet decorators) {
1886 access_store_at(T_OBJECT, IN_HEAP, obj, new_val, tmp1, tmp2, tmp3, true);
1887 }
1888
access_load_at(BasicType type,DecoratorSet decorators,Address src,Register dst,Register tmp1,Register tmp2,Register tmp3)1889 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
1890 Address src, Register dst, Register tmp1, Register tmp2, Register tmp3) {
1891 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1892 decorators = AccessInternal::decorator_fixup(decorators);
1893 bool as_raw = (decorators & AS_RAW) != 0;
1894 if (as_raw) {
1895 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
1896 } else {
1897 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
1898 }
1899 }
1900
access_store_at(BasicType type,DecoratorSet decorators,Address obj,Register new_val,Register tmp1,Register tmp2,Register tmp3,bool is_null)1901 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
1902 Address obj, Register new_val, Register tmp1, Register tmp2, Register tmp3, bool is_null) {
1903 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1904 decorators = AccessInternal::decorator_fixup(decorators);
1905 bool as_raw = (decorators & AS_RAW) != 0;
1906 if (as_raw) {
1907 bs->BarrierSetAssembler::store_at(this, decorators, type, obj, new_val, tmp1, tmp2, tmp3, is_null);
1908 } else {
1909 bs->store_at(this, decorators, type, obj, new_val, tmp1, tmp2, tmp3, is_null);
1910 }
1911 }
1912
resolve(DecoratorSet decorators,Register obj)1913 void MacroAssembler::resolve(DecoratorSet decorators, Register obj) {
1914 // Use stronger ACCESS_WRITE|ACCESS_READ by default.
1915 if ((decorators & (ACCESS_READ | ACCESS_WRITE)) == 0) {
1916 decorators |= ACCESS_READ | ACCESS_WRITE;
1917 }
1918 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1919 return bs->resolve(this, decorators, obj);
1920 }
1921
safepoint_poll(Register tmp1,Label & slow_path)1922 void MacroAssembler::safepoint_poll(Register tmp1, Label& slow_path) {
1923 ldr_u32(tmp1, Address(Rthread, Thread::polling_page_offset()));
1924 tst(tmp1, exact_log2(SafepointMechanism::poll_bit()));
1925 b(slow_path, eq);
1926 }
1927
get_polling_page(Register dest)1928 void MacroAssembler::get_polling_page(Register dest) {
1929 ldr(dest, Address(Rthread, Thread::polling_page_offset()));
1930 }
1931
read_polling_page(Register dest,relocInfo::relocType rtype)1932 void MacroAssembler::read_polling_page(Register dest, relocInfo::relocType rtype) {
1933 get_polling_page(dest);
1934 relocate(rtype);
1935 ldr(dest, Address(dest));
1936 }
1937
1938