1 /*
2 * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "opto/arraycopynode.hpp"
28 #include "oops/objArrayKlass.hpp"
29 #include "opto/convertnode.hpp"
30 #include "opto/vectornode.hpp"
31 #include "opto/graphKit.hpp"
32 #include "opto/macro.hpp"
33 #include "opto/runtime.hpp"
34 #include "utilities/align.hpp"
35 #include "utilities/powerOfTwo.hpp"
36
insert_mem_bar(Node ** ctrl,Node ** mem,int opcode,Node * precedent)37 void PhaseMacroExpand::insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent) {
38 MemBarNode* mb = MemBarNode::make(C, opcode, Compile::AliasIdxBot, precedent);
39 mb->init_req(TypeFunc::Control, *ctrl);
40 mb->init_req(TypeFunc::Memory, *mem);
41 transform_later(mb);
42 *ctrl = new ProjNode(mb,TypeFunc::Control);
43 transform_later(*ctrl);
44 Node* mem_proj = new ProjNode(mb,TypeFunc::Memory);
45 transform_later(mem_proj);
46 *mem = mem_proj;
47 }
48
array_element_address(Node * ary,Node * idx,BasicType elembt)49 Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt) {
50 uint shift = exact_log2(type2aelembytes(elembt));
51 uint header = arrayOopDesc::base_offset_in_bytes(elembt);
52 Node* base = basic_plus_adr(ary, header);
53 #ifdef _LP64
54 // see comment in GraphKit::array_element_address
55 int index_max = max_jint - 1; // array size is max_jint, index is one less
56 const TypeLong* lidxtype = TypeLong::make(CONST64(0), index_max, Type::WidenMax);
57 idx = transform_later( new ConvI2LNode(idx, lidxtype) );
58 #endif
59 Node* scale = new LShiftXNode(idx, intcon(shift));
60 transform_later(scale);
61 return basic_plus_adr(ary, base, scale);
62 }
63
ConvI2L(Node * offset)64 Node* PhaseMacroExpand::ConvI2L(Node* offset) {
65 return transform_later(new ConvI2LNode(offset));
66 }
67
make_leaf_call(Node * ctrl,Node * mem,const TypeFunc * call_type,address call_addr,const char * call_name,const TypePtr * adr_type,Node * parm0,Node * parm1,Node * parm2,Node * parm3,Node * parm4,Node * parm5,Node * parm6,Node * parm7)68 Node* PhaseMacroExpand::make_leaf_call(Node* ctrl, Node* mem,
69 const TypeFunc* call_type, address call_addr,
70 const char* call_name,
71 const TypePtr* adr_type,
72 Node* parm0, Node* parm1,
73 Node* parm2, Node* parm3,
74 Node* parm4, Node* parm5,
75 Node* parm6, Node* parm7) {
76 Node* call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
77 call->init_req(TypeFunc::Control, ctrl);
78 call->init_req(TypeFunc::I_O , top());
79 call->init_req(TypeFunc::Memory , mem);
80 call->init_req(TypeFunc::ReturnAdr, top());
81 call->init_req(TypeFunc::FramePtr, top());
82
83 // Hook each parm in order. Stop looking at the first NULL.
84 if (parm0 != NULL) { call->init_req(TypeFunc::Parms+0, parm0);
85 if (parm1 != NULL) { call->init_req(TypeFunc::Parms+1, parm1);
86 if (parm2 != NULL) { call->init_req(TypeFunc::Parms+2, parm2);
87 if (parm3 != NULL) { call->init_req(TypeFunc::Parms+3, parm3);
88 if (parm4 != NULL) { call->init_req(TypeFunc::Parms+4, parm4);
89 if (parm5 != NULL) { call->init_req(TypeFunc::Parms+5, parm5);
90 if (parm6 != NULL) { call->init_req(TypeFunc::Parms+6, parm6);
91 if (parm7 != NULL) { call->init_req(TypeFunc::Parms+7, parm7);
92 /* close each nested if ===> */ } } } } } } } }
93 assert(call->in(call->req()-1) != NULL, "must initialize all parms");
94
95 return call;
96 }
97
98
99 //------------------------------generate_guard---------------------------
100 // Helper function for generating guarded fast-slow graph structures.
101 // The given 'test', if true, guards a slow path. If the test fails
102 // then a fast path can be taken. (We generally hope it fails.)
103 // In all cases, GraphKit::control() is updated to the fast path.
104 // The returned value represents the control for the slow path.
105 // The return value is never 'top'; it is either a valid control
106 // or NULL if it is obvious that the slow path can never be taken.
107 // Also, if region and the slow control are not NULL, the slow edge
108 // is appended to the region.
generate_guard(Node ** ctrl,Node * test,RegionNode * region,float true_prob)109 Node* PhaseMacroExpand::generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob) {
110 if ((*ctrl)->is_top()) {
111 // Already short circuited.
112 return NULL;
113 }
114 // Build an if node and its projections.
115 // If test is true we take the slow path, which we assume is uncommon.
116 if (_igvn.type(test) == TypeInt::ZERO) {
117 // The slow branch is never taken. No need to build this guard.
118 return NULL;
119 }
120
121 IfNode* iff = new IfNode(*ctrl, test, true_prob, COUNT_UNKNOWN);
122 transform_later(iff);
123
124 Node* if_slow = new IfTrueNode(iff);
125 transform_later(if_slow);
126
127 if (region != NULL) {
128 region->add_req(if_slow);
129 }
130
131 Node* if_fast = new IfFalseNode(iff);
132 transform_later(if_fast);
133
134 *ctrl = if_fast;
135
136 return if_slow;
137 }
138
generate_slow_guard(Node ** ctrl,Node * test,RegionNode * region)139 inline Node* PhaseMacroExpand::generate_slow_guard(Node** ctrl, Node* test, RegionNode* region) {
140 return generate_guard(ctrl, test, region, PROB_UNLIKELY_MAG(3));
141 }
142
generate_negative_guard(Node ** ctrl,Node * index,RegionNode * region)143 void PhaseMacroExpand::generate_negative_guard(Node** ctrl, Node* index, RegionNode* region) {
144 if ((*ctrl)->is_top())
145 return; // already stopped
146 if (_igvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
147 return; // index is already adequately typed
148 Node* cmp_lt = new CmpINode(index, intcon(0));
149 transform_later(cmp_lt);
150 Node* bol_lt = new BoolNode(cmp_lt, BoolTest::lt);
151 transform_later(bol_lt);
152 generate_guard(ctrl, bol_lt, region, PROB_MIN);
153 }
154
generate_limit_guard(Node ** ctrl,Node * offset,Node * subseq_length,Node * array_length,RegionNode * region)155 void PhaseMacroExpand::generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region) {
156 if ((*ctrl)->is_top())
157 return; // already stopped
158 bool zero_offset = _igvn.type(offset) == TypeInt::ZERO;
159 if (zero_offset && subseq_length->eqv_uncast(array_length))
160 return; // common case of whole-array copy
161 Node* last = subseq_length;
162 if (!zero_offset) { // last += offset
163 last = new AddINode(last, offset);
164 transform_later(last);
165 }
166 Node* cmp_lt = new CmpUNode(array_length, last);
167 transform_later(cmp_lt);
168 Node* bol_lt = new BoolNode(cmp_lt, BoolTest::lt);
169 transform_later(bol_lt);
170 generate_guard(ctrl, bol_lt, region, PROB_MIN);
171 }
172
173 //
174 // Partial in-lining handling for smaller conjoint/disjoint array copies having
175 // length(in bytes) less than ArrayCopyPartialInlineSize.
176 // if (length <= ArrayCopyPartialInlineSize) {
177 // partial_inlining_block:
178 // mask = Mask_Gen
179 // vload = LoadVectorMasked src , mask
180 // StoreVectorMasked dst, mask, vload
181 // } else {
182 // stub_block:
183 // callstub array_copy
184 // }
185 // exit_block:
186 // Phi = label partial_inlining_block:mem , label stub_block:mem (filled by caller)
187 // mem = MergeMem (Phi)
188 // control = stub_block
189 //
190 // Exit_block and associated phi(memory) are partially initialized for partial_in-lining_block
191 // edges. Remaining edges for exit_block coming from stub_block are connected by the caller
192 // post stub nodes creation.
193 //
194
generate_partial_inlining_block(Node ** ctrl,MergeMemNode ** mem,const TypePtr * adr_type,RegionNode ** exit_block,Node ** result_memory,Node * length,Node * src_start,Node * dst_start,BasicType type)195 void PhaseMacroExpand::generate_partial_inlining_block(Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type,
196 RegionNode** exit_block, Node** result_memory, Node* length,
197 Node* src_start, Node* dst_start, BasicType type) {
198 const TypePtr *src_adr_type = _igvn.type(src_start)->isa_ptr();
199 Node* inline_block = NULL;
200 Node* stub_block = NULL;
201
202 int const_len = -1;
203 const TypeInt* lty = NULL;
204 uint shift = exact_log2(type2aelembytes(type));
205 if (length->Opcode() == Op_ConvI2L) {
206 lty = _igvn.type(length->in(1))->isa_int();
207 } else {
208 lty = _igvn.type(length)->isa_int();
209 }
210 if (lty && lty->is_con()) {
211 const_len = lty->get_con() << shift;
212 }
213
214 // Return if copy length is greater than partial inline size limit or
215 // target does not supports masked load/stores.
216 int lane_count = ArrayCopyNode::get_partial_inline_vector_lane_count(type, const_len);
217 if ( const_len > ArrayCopyPartialInlineSize ||
218 !Matcher::match_rule_supported_vector(Op_LoadVectorMasked, lane_count, type) ||
219 !Matcher::match_rule_supported_vector(Op_StoreVectorMasked, lane_count, type) ||
220 !Matcher::match_rule_supported_vector(Op_VectorMaskGen, lane_count, type)) {
221 return;
222 }
223
224 Node* copy_bytes = new LShiftXNode(length, intcon(shift));
225 transform_later(copy_bytes);
226
227 Node* cmp_le = new CmpULNode(copy_bytes, longcon(ArrayCopyPartialInlineSize));
228 transform_later(cmp_le);
229 Node* bol_le = new BoolNode(cmp_le, BoolTest::le);
230 transform_later(bol_le);
231 inline_block = generate_guard(ctrl, bol_le, NULL, PROB_FAIR);
232 stub_block = *ctrl;
233
234 Node* mask_gen = new VectorMaskGenNode(length, TypeLong::LONG, Type::get_const_basic_type(type));
235 transform_later(mask_gen);
236
237 unsigned vec_size = lane_count * type2aelembytes(type);
238 if (C->max_vector_size() < vec_size) {
239 C->set_max_vector_size(vec_size);
240 }
241
242 const TypeVect * vt = TypeVect::make(type, lane_count);
243 Node* mm = (*mem)->memory_at(C->get_alias_index(src_adr_type));
244 Node* masked_load = new LoadVectorMaskedNode(inline_block, mm, src_start,
245 src_adr_type, vt, mask_gen);
246 transform_later(masked_load);
247
248 mm = (*mem)->memory_at(C->get_alias_index(adr_type));
249 Node* masked_store = new StoreVectorMaskedNode(inline_block, mm, dst_start,
250 masked_load, adr_type, mask_gen);
251 transform_later(masked_store);
252
253 // Convergence region for inline_block and stub_block.
254 *exit_block = new RegionNode(3);
255 transform_later(*exit_block);
256 (*exit_block)->init_req(1, inline_block);
257 *result_memory = new PhiNode(*exit_block, Type::MEMORY, adr_type);
258 transform_later(*result_memory);
259 (*result_memory)->init_req(1, masked_store);
260
261 *ctrl = stub_block;
262 }
263
264
generate_nonpositive_guard(Node ** ctrl,Node * index,bool never_negative)265 Node* PhaseMacroExpand::generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative) {
266 if ((*ctrl)->is_top()) return NULL;
267
268 if (_igvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
269 return NULL; // index is already adequately typed
270 Node* cmp_le = new CmpINode(index, intcon(0));
271 transform_later(cmp_le);
272 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
273 Node* bol_le = new BoolNode(cmp_le, le_or_eq);
274 transform_later(bol_le);
275 Node* is_notp = generate_guard(ctrl, bol_le, NULL, PROB_MIN);
276
277 return is_notp;
278 }
279
finish_arraycopy_call(Node * call,Node ** ctrl,MergeMemNode ** mem,const TypePtr * adr_type)280 void PhaseMacroExpand::finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type) {
281 transform_later(call);
282
283 *ctrl = new ProjNode(call,TypeFunc::Control);
284 transform_later(*ctrl);
285 Node* newmem = new ProjNode(call, TypeFunc::Memory);
286 transform_later(newmem);
287
288 uint alias_idx = C->get_alias_index(adr_type);
289 if (alias_idx != Compile::AliasIdxBot) {
290 *mem = MergeMemNode::make(*mem);
291 (*mem)->set_memory_at(alias_idx, newmem);
292 } else {
293 *mem = MergeMemNode::make(newmem);
294 }
295 transform_later(*mem);
296 }
297
basictype2arraycopy(BasicType t,Node * src_offset,Node * dest_offset,bool disjoint_bases,const char * & name,bool dest_uninitialized)298 address PhaseMacroExpand::basictype2arraycopy(BasicType t,
299 Node* src_offset,
300 Node* dest_offset,
301 bool disjoint_bases,
302 const char* &name,
303 bool dest_uninitialized) {
304 const TypeInt* src_offset_inttype = _igvn.find_int_type(src_offset);;
305 const TypeInt* dest_offset_inttype = _igvn.find_int_type(dest_offset);;
306
307 bool aligned = false;
308 bool disjoint = disjoint_bases;
309
310 // if the offsets are the same, we can treat the memory regions as
311 // disjoint, because either the memory regions are in different arrays,
312 // or they are identical (which we can treat as disjoint.) We can also
313 // treat a copy with a destination index less that the source index
314 // as disjoint since a low->high copy will work correctly in this case.
315 if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
316 dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
317 // both indices are constants
318 int s_offs = src_offset_inttype->get_con();
319 int d_offs = dest_offset_inttype->get_con();
320 int element_size = type2aelembytes(t);
321 aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
322 ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
323 if (s_offs >= d_offs) disjoint = true;
324 } else if (src_offset == dest_offset && src_offset != NULL) {
325 // This can occur if the offsets are identical non-constants.
326 disjoint = true;
327 }
328
329 return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
330 }
331
332 #define XTOP LP64_ONLY(COMMA top())
333
334 // Generate an optimized call to arraycopy.
335 // Caller must guard against non-arrays.
336 // Caller must determine a common array basic-type for both arrays.
337 // Caller must validate offsets against array bounds.
338 // The slow_region has already collected guard failure paths
339 // (such as out of bounds length or non-conformable array types).
340 // The generated code has this shape, in general:
341 //
342 // if (length == 0) return // via zero_path
343 // slowval = -1
344 // if (types unknown) {
345 // slowval = call generic copy loop
346 // if (slowval == 0) return // via checked_path
347 // } else if (indexes in bounds) {
348 // if ((is object array) && !(array type check)) {
349 // slowval = call checked copy loop
350 // if (slowval == 0) return // via checked_path
351 // } else {
352 // call bulk copy loop
353 // return // via fast_path
354 // }
355 // }
356 // // adjust params for remaining work:
357 // if (slowval != -1) {
358 // n = -1^slowval; src_offset += n; dest_offset += n; length -= n
359 // }
360 // slow_region:
361 // call slow arraycopy(src, src_offset, dest, dest_offset, length)
362 // return // via slow_call_path
363 //
364 // This routine is used from several intrinsics: System.arraycopy,
365 // Object.clone (the array subcase), and Arrays.copyOf[Range].
366 //
generate_arraycopy(ArrayCopyNode * ac,AllocateArrayNode * alloc,Node ** ctrl,MergeMemNode * mem,Node ** io,const TypePtr * adr_type,BasicType basic_elem_type,Node * src,Node * src_offset,Node * dest,Node * dest_offset,Node * copy_length,bool disjoint_bases,bool length_never_negative,RegionNode * slow_region)367 Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* alloc,
368 Node** ctrl, MergeMemNode* mem, Node** io,
369 const TypePtr* adr_type,
370 BasicType basic_elem_type,
371 Node* src, Node* src_offset,
372 Node* dest, Node* dest_offset,
373 Node* copy_length,
374 bool disjoint_bases,
375 bool length_never_negative,
376 RegionNode* slow_region) {
377 if (slow_region == NULL) {
378 slow_region = new RegionNode(1);
379 transform_later(slow_region);
380 }
381
382 Node* original_dest = dest;
383 bool dest_uninitialized = false;
384
385 // See if this is the initialization of a newly-allocated array.
386 // If so, we will take responsibility here for initializing it to zero.
387 // (Note: Because tightly_coupled_allocation performs checks on the
388 // out-edges of the dest, we need to avoid making derived pointers
389 // from it until we have checked its uses.)
390 if (ReduceBulkZeroing
391 && !(UseTLAB && ZeroTLAB) // pointless if already zeroed
392 && basic_elem_type != T_CONFLICT // avoid corner case
393 && !src->eqv_uncast(dest)
394 && alloc != NULL
395 && _igvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
396 && alloc->maybe_set_complete(&_igvn)) {
397 // "You break it, you buy it."
398 InitializeNode* init = alloc->initialization();
399 assert(init->is_complete(), "we just did this");
400 init->set_complete_with_arraycopy();
401 assert(dest->is_CheckCastPP(), "sanity");
402 assert(dest->in(0)->in(0) == init, "dest pinned");
403 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
404 // From this point on, every exit path is responsible for
405 // initializing any non-copied parts of the object to zero.
406 // Also, if this flag is set we make sure that arraycopy interacts properly
407 // with G1, eliding pre-barriers. See CR 6627983.
408 dest_uninitialized = true;
409 } else {
410 // No zeroing elimination here.
411 alloc = NULL;
412 //original_dest = dest;
413 //dest_uninitialized = false;
414 }
415
416 uint alias_idx = C->get_alias_index(adr_type);
417
418 // Results are placed here:
419 enum { fast_path = 1, // normal void-returning assembly stub
420 checked_path = 2, // special assembly stub with cleanup
421 slow_call_path = 3, // something went wrong; call the VM
422 zero_path = 4, // bypass when length of copy is zero
423 bcopy_path = 5, // copy primitive array by 64-bit blocks
424 PATH_LIMIT = 6
425 };
426 RegionNode* result_region = new RegionNode(PATH_LIMIT);
427 PhiNode* result_i_o = new PhiNode(result_region, Type::ABIO);
428 PhiNode* result_memory = new PhiNode(result_region, Type::MEMORY, adr_type);
429 assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
430 transform_later(result_region);
431 transform_later(result_i_o);
432 transform_later(result_memory);
433
434 // The slow_control path:
435 Node* slow_control;
436 Node* slow_i_o = *io;
437 Node* slow_mem = mem->memory_at(alias_idx);
438 DEBUG_ONLY(slow_control = (Node*) badAddress);
439
440 // Checked control path:
441 Node* checked_control = top();
442 Node* checked_mem = NULL;
443 Node* checked_i_o = NULL;
444 Node* checked_value = NULL;
445
446 if (basic_elem_type == T_CONFLICT) {
447 assert(!dest_uninitialized, "");
448 Node* cv = generate_generic_arraycopy(ctrl, &mem,
449 adr_type,
450 src, src_offset, dest, dest_offset,
451 copy_length, dest_uninitialized);
452 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
453 checked_control = *ctrl;
454 checked_i_o = *io;
455 checked_mem = mem->memory_at(alias_idx);
456 checked_value = cv;
457 *ctrl = top();
458 }
459
460 Node* not_pos = generate_nonpositive_guard(ctrl, copy_length, length_never_negative);
461 if (not_pos != NULL) {
462 Node* local_ctrl = not_pos, *local_io = *io;
463 MergeMemNode* local_mem = MergeMemNode::make(mem);
464 transform_later(local_mem);
465
466 // (6) length must not be negative.
467 if (!length_never_negative) {
468 generate_negative_guard(&local_ctrl, copy_length, slow_region);
469 }
470
471 // copy_length is 0.
472 if (dest_uninitialized) {
473 assert(!local_ctrl->is_top(), "no ctrl?");
474 Node* dest_length = alloc->in(AllocateNode::ALength);
475 if (copy_length->eqv_uncast(dest_length)
476 || _igvn.find_int_con(dest_length, 1) <= 0) {
477 // There is no zeroing to do. No need for a secondary raw memory barrier.
478 } else {
479 // Clear the whole thing since there are no source elements to copy.
480 generate_clear_array(local_ctrl, local_mem,
481 adr_type, dest, basic_elem_type,
482 intcon(0), NULL,
483 alloc->in(AllocateNode::AllocSize));
484 // Use a secondary InitializeNode as raw memory barrier.
485 // Currently it is needed only on this path since other
486 // paths have stub or runtime calls as raw memory barriers.
487 MemBarNode* mb = MemBarNode::make(C, Op_Initialize,
488 Compile::AliasIdxRaw,
489 top());
490 transform_later(mb);
491 mb->set_req(TypeFunc::Control,local_ctrl);
492 mb->set_req(TypeFunc::Memory, local_mem->memory_at(Compile::AliasIdxRaw));
493 local_ctrl = transform_later(new ProjNode(mb, TypeFunc::Control));
494 local_mem->set_memory_at(Compile::AliasIdxRaw, transform_later(new ProjNode(mb, TypeFunc::Memory)));
495
496 InitializeNode* init = mb->as_Initialize();
497 init->set_complete(&_igvn); // (there is no corresponding AllocateNode)
498 }
499 }
500
501 // Present the results of the fast call.
502 result_region->init_req(zero_path, local_ctrl);
503 result_i_o ->init_req(zero_path, local_io);
504 result_memory->init_req(zero_path, local_mem->memory_at(alias_idx));
505 }
506
507 if (!(*ctrl)->is_top() && dest_uninitialized) {
508 // We have to initialize the *uncopied* part of the array to zero.
509 // The copy destination is the slice dest[off..off+len]. The other slices
510 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
511 Node* dest_size = alloc->in(AllocateNode::AllocSize);
512 Node* dest_length = alloc->in(AllocateNode::ALength);
513 Node* dest_tail = transform_later( new AddINode(dest_offset, copy_length));
514
515 // If there is a head section that needs zeroing, do it now.
516 if (_igvn.find_int_con(dest_offset, -1) != 0) {
517 generate_clear_array(*ctrl, mem,
518 adr_type, dest, basic_elem_type,
519 intcon(0), dest_offset,
520 NULL);
521 }
522
523 // Next, perform a dynamic check on the tail length.
524 // It is often zero, and we can win big if we prove this.
525 // There are two wins: Avoid generating the ClearArray
526 // with its attendant messy index arithmetic, and upgrade
527 // the copy to a more hardware-friendly word size of 64 bits.
528 Node* tail_ctl = NULL;
529 if (!(*ctrl)->is_top() && !dest_tail->eqv_uncast(dest_length)) {
530 Node* cmp_lt = transform_later( new CmpINode(dest_tail, dest_length) );
531 Node* bol_lt = transform_later( new BoolNode(cmp_lt, BoolTest::lt) );
532 tail_ctl = generate_slow_guard(ctrl, bol_lt, NULL);
533 assert(tail_ctl != NULL || !(*ctrl)->is_top(), "must be an outcome");
534 }
535
536 // At this point, let's assume there is no tail.
537 if (!(*ctrl)->is_top() && alloc != NULL && basic_elem_type != T_OBJECT) {
538 // There is no tail. Try an upgrade to a 64-bit copy.
539 bool didit = false;
540 {
541 Node* local_ctrl = *ctrl, *local_io = *io;
542 MergeMemNode* local_mem = MergeMemNode::make(mem);
543 transform_later(local_mem);
544
545 didit = generate_block_arraycopy(&local_ctrl, &local_mem, local_io,
546 adr_type, basic_elem_type, alloc,
547 src, src_offset, dest, dest_offset,
548 dest_size, dest_uninitialized);
549 if (didit) {
550 // Present the results of the block-copying fast call.
551 result_region->init_req(bcopy_path, local_ctrl);
552 result_i_o ->init_req(bcopy_path, local_io);
553 result_memory->init_req(bcopy_path, local_mem->memory_at(alias_idx));
554 }
555 }
556 if (didit) {
557 *ctrl = top(); // no regular fast path
558 }
559 }
560
561 // Clear the tail, if any.
562 if (tail_ctl != NULL) {
563 Node* notail_ctl = (*ctrl)->is_top() ? NULL : *ctrl;
564 *ctrl = tail_ctl;
565 if (notail_ctl == NULL) {
566 generate_clear_array(*ctrl, mem,
567 adr_type, dest, basic_elem_type,
568 dest_tail, NULL,
569 dest_size);
570 } else {
571 // Make a local merge.
572 Node* done_ctl = transform_later(new RegionNode(3));
573 Node* done_mem = transform_later(new PhiNode(done_ctl, Type::MEMORY, adr_type));
574 done_ctl->init_req(1, notail_ctl);
575 done_mem->init_req(1, mem->memory_at(alias_idx));
576 generate_clear_array(*ctrl, mem,
577 adr_type, dest, basic_elem_type,
578 dest_tail, NULL,
579 dest_size);
580 done_ctl->init_req(2, *ctrl);
581 done_mem->init_req(2, mem->memory_at(alias_idx));
582 *ctrl = done_ctl;
583 mem->set_memory_at(alias_idx, done_mem);
584 }
585 }
586 }
587
588 BasicType copy_type = basic_elem_type;
589 assert(basic_elem_type != T_ARRAY, "caller must fix this");
590 if (!(*ctrl)->is_top() && copy_type == T_OBJECT) {
591 // If src and dest have compatible element types, we can copy bits.
592 // Types S[] and D[] are compatible if D is a supertype of S.
593 //
594 // If they are not, we will use checked_oop_disjoint_arraycopy,
595 // which performs a fast optimistic per-oop check, and backs off
596 // further to JVM_ArrayCopy on the first per-oop check that fails.
597 // (Actually, we don't move raw bits only; the GC requires card marks.)
598
599 // We don't need a subtype check for validated copies and Object[].clone()
600 bool skip_subtype_check = ac->is_arraycopy_validated() || ac->is_copyof_validated() ||
601 ac->is_copyofrange_validated() || ac->is_clone_oop_array();
602 if (!skip_subtype_check) {
603 // Get the klass* for both src and dest
604 Node* src_klass = ac->in(ArrayCopyNode::SrcKlass);
605 Node* dest_klass = ac->in(ArrayCopyNode::DestKlass);
606
607 assert(src_klass != NULL && dest_klass != NULL, "should have klasses");
608
609 // Generate the subtype check.
610 // This might fold up statically, or then again it might not.
611 //
612 // Non-static example: Copying List<String>.elements to a new String[].
613 // The backing store for a List<String> is always an Object[],
614 // but its elements are always type String, if the generic types
615 // are correct at the source level.
616 //
617 // Test S[] against D[], not S against D, because (probably)
618 // the secondary supertype cache is less busy for S[] than S.
619 // This usually only matters when D is an interface.
620 Node* not_subtype_ctrl = Phase::gen_subtype_check(src_klass, dest_klass, ctrl, mem, _igvn);
621 // Plug failing path into checked_oop_disjoint_arraycopy
622 if (not_subtype_ctrl != top()) {
623 Node* local_ctrl = not_subtype_ctrl;
624 MergeMemNode* local_mem = MergeMemNode::make(mem);
625 transform_later(local_mem);
626
627 // (At this point we can assume disjoint_bases, since types differ.)
628 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
629 Node* p1 = basic_plus_adr(dest_klass, ek_offset);
630 Node* n1 = LoadKlassNode::make(_igvn, NULL, C->immutable_memory(), p1, TypeRawPtr::BOTTOM);
631 Node* dest_elem_klass = transform_later(n1);
632 Node* cv = generate_checkcast_arraycopy(&local_ctrl, &local_mem,
633 adr_type,
634 dest_elem_klass,
635 src, src_offset, dest, dest_offset,
636 ConvI2X(copy_length), dest_uninitialized);
637 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
638 checked_control = local_ctrl;
639 checked_i_o = *io;
640 checked_mem = local_mem->memory_at(alias_idx);
641 checked_value = cv;
642 }
643 }
644 // At this point we know we do not need type checks on oop stores.
645
646 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
647 if (!bs->array_copy_requires_gc_barriers(alloc != NULL, copy_type, false, BarrierSetC2::Expansion)) {
648 // If we do not need gc barriers, copy using the jint or jlong stub.
649 copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
650 assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
651 "sizes agree");
652 }
653 }
654
655 bool is_partial_array_copy = false;
656 if (!(*ctrl)->is_top()) {
657 // Generate the fast path, if possible.
658 Node* local_ctrl = *ctrl;
659 MergeMemNode* local_mem = MergeMemNode::make(mem);
660 transform_later(local_mem);
661
662 is_partial_array_copy = generate_unchecked_arraycopy(&local_ctrl, &local_mem,
663 adr_type, copy_type, disjoint_bases,
664 src, src_offset, dest, dest_offset,
665 ConvI2X(copy_length), dest_uninitialized);
666
667 // Present the results of the fast call.
668 result_region->init_req(fast_path, local_ctrl);
669 result_i_o ->init_req(fast_path, *io);
670 result_memory->init_req(fast_path, local_mem->memory_at(alias_idx));
671 }
672
673 // Here are all the slow paths up to this point, in one bundle:
674 assert(slow_region != NULL, "allocated on entry");
675 slow_control = slow_region;
676 DEBUG_ONLY(slow_region = (RegionNode*)badAddress);
677
678 *ctrl = checked_control;
679 if (!(*ctrl)->is_top()) {
680 // Clean up after the checked call.
681 // The returned value is either 0 or -1^K,
682 // where K = number of partially transferred array elements.
683 Node* cmp = new CmpINode(checked_value, intcon(0));
684 transform_later(cmp);
685 Node* bol = new BoolNode(cmp, BoolTest::eq);
686 transform_later(bol);
687 IfNode* iff = new IfNode(*ctrl, bol, PROB_MAX, COUNT_UNKNOWN);
688 transform_later(iff);
689
690 // If it is 0, we are done, so transfer to the end.
691 Node* checks_done = new IfTrueNode(iff);
692 transform_later(checks_done);
693 result_region->init_req(checked_path, checks_done);
694 result_i_o ->init_req(checked_path, checked_i_o);
695 result_memory->init_req(checked_path, checked_mem);
696
697 // If it is not zero, merge into the slow call.
698 *ctrl = new IfFalseNode(iff);
699 transform_later(*ctrl);
700 RegionNode* slow_reg2 = new RegionNode(3);
701 PhiNode* slow_i_o2 = new PhiNode(slow_reg2, Type::ABIO);
702 PhiNode* slow_mem2 = new PhiNode(slow_reg2, Type::MEMORY, adr_type);
703 transform_later(slow_reg2);
704 transform_later(slow_i_o2);
705 transform_later(slow_mem2);
706 slow_reg2 ->init_req(1, slow_control);
707 slow_i_o2 ->init_req(1, slow_i_o);
708 slow_mem2 ->init_req(1, slow_mem);
709 slow_reg2 ->init_req(2, *ctrl);
710 slow_i_o2 ->init_req(2, checked_i_o);
711 slow_mem2 ->init_req(2, checked_mem);
712
713 slow_control = slow_reg2;
714 slow_i_o = slow_i_o2;
715 slow_mem = slow_mem2;
716
717 if (alloc != NULL) {
718 // We'll restart from the very beginning, after zeroing the whole thing.
719 // This can cause double writes, but that's OK since dest is brand new.
720 // So we ignore the low 31 bits of the value returned from the stub.
721 } else {
722 // We must continue the copy exactly where it failed, or else
723 // another thread might see the wrong number of writes to dest.
724 Node* checked_offset = new XorINode(checked_value, intcon(-1));
725 Node* slow_offset = new PhiNode(slow_reg2, TypeInt::INT);
726 transform_later(checked_offset);
727 transform_later(slow_offset);
728 slow_offset->init_req(1, intcon(0));
729 slow_offset->init_req(2, checked_offset);
730
731 // Adjust the arguments by the conditionally incoming offset.
732 Node* src_off_plus = new AddINode(src_offset, slow_offset);
733 transform_later(src_off_plus);
734 Node* dest_off_plus = new AddINode(dest_offset, slow_offset);
735 transform_later(dest_off_plus);
736 Node* length_minus = new SubINode(copy_length, slow_offset);
737 transform_later(length_minus);
738
739 // Tweak the node variables to adjust the code produced below:
740 src_offset = src_off_plus;
741 dest_offset = dest_off_plus;
742 copy_length = length_minus;
743 }
744 }
745 *ctrl = slow_control;
746 if (!(*ctrl)->is_top()) {
747 Node* local_ctrl = *ctrl, *local_io = slow_i_o;
748 MergeMemNode* local_mem = MergeMemNode::make(mem);
749 transform_later(local_mem);
750
751 // Generate the slow path, if needed.
752 local_mem->set_memory_at(alias_idx, slow_mem);
753
754 if (dest_uninitialized) {
755 generate_clear_array(local_ctrl, local_mem,
756 adr_type, dest, basic_elem_type,
757 intcon(0), NULL,
758 alloc->in(AllocateNode::AllocSize));
759 }
760
761 local_mem = generate_slow_arraycopy(ac,
762 &local_ctrl, local_mem, &local_io,
763 adr_type,
764 src, src_offset, dest, dest_offset,
765 copy_length, /*dest_uninitialized*/false);
766
767 result_region->init_req(slow_call_path, local_ctrl);
768 result_i_o ->init_req(slow_call_path, local_io);
769 result_memory->init_req(slow_call_path, local_mem->memory_at(alias_idx));
770 } else {
771 ShouldNotReachHere(); // no call to generate_slow_arraycopy:
772 // projections were not extracted
773 }
774
775 // Remove unused edges.
776 for (uint i = 1; i < result_region->req(); i++) {
777 if (result_region->in(i) == NULL) {
778 result_region->init_req(i, top());
779 }
780 }
781
782 // Finished; return the combined state.
783 *ctrl = result_region;
784 *io = result_i_o;
785 mem->set_memory_at(alias_idx, result_memory);
786
787 // mem no longer guaranteed to stay a MergeMemNode
788 Node* out_mem = mem;
789 DEBUG_ONLY(mem = NULL);
790
791 // The memory edges above are precise in order to model effects around
792 // array copies accurately to allow value numbering of field loads around
793 // arraycopy. Such field loads, both before and after, are common in Java
794 // collections and similar classes involving header/array data structures.
795 //
796 // But with low number of register or when some registers are used or killed
797 // by arraycopy calls it causes registers spilling on stack. See 6544710.
798 // The next memory barrier is added to avoid it. If the arraycopy can be
799 // optimized away (which it can, sometimes) then we can manually remove
800 // the membar also.
801 //
802 // Do not let reads from the cloned object float above the arraycopy.
803 if (alloc != NULL && !alloc->initialization()->does_not_escape()) {
804 // Do not let stores that initialize this object be reordered with
805 // a subsequent store that would make this object accessible by
806 // other threads.
807 insert_mem_bar(ctrl, &out_mem, Op_MemBarStoreStore);
808 } else {
809 insert_mem_bar(ctrl, &out_mem, Op_MemBarCPUOrder);
810 }
811
812 if (is_partial_array_copy) {
813 assert((*ctrl)->is_Proj(), "MemBar control projection");
814 assert((*ctrl)->in(0)->isa_MemBar(), "MemBar node");
815 (*ctrl)->in(0)->isa_MemBar()->set_trailing_partial_array_copy();
816 }
817
818 _igvn.replace_node(_memproj_fallthrough, out_mem);
819 _igvn.replace_node(_ioproj_fallthrough, *io);
820 _igvn.replace_node(_fallthroughcatchproj, *ctrl);
821
822 #ifdef ASSERT
823 const TypeOopPtr* dest_t = _igvn.type(dest)->is_oopptr();
824 if (dest_t->is_known_instance() && !is_partial_array_copy) {
825 ArrayCopyNode* ac = NULL;
826 assert(ArrayCopyNode::may_modify(dest_t, (*ctrl)->in(0)->as_MemBar(), &_igvn, ac), "dependency on arraycopy lost");
827 assert(ac == NULL, "no arraycopy anymore");
828 }
829 #endif
830
831 return out_mem;
832 }
833
834 // Helper for initialization of arrays, creating a ClearArray.
835 // It writes zero bits in [start..end), within the body of an array object.
836 // The memory effects are all chained onto the 'adr_type' alias category.
837 //
838 // Since the object is otherwise uninitialized, we are free
839 // to put a little "slop" around the edges of the cleared area,
840 // as long as it does not go back into the array's header,
841 // or beyond the array end within the heap.
842 //
843 // The lower edge can be rounded down to the nearest jint and the
844 // upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
845 //
846 // Arguments:
847 // adr_type memory slice where writes are generated
848 // dest oop of the destination array
849 // basic_elem_type element type of the destination
850 // slice_idx array index of first element to store
851 // slice_len number of elements to store (or NULL)
852 // dest_size total size in bytes of the array object
853 //
854 // Exactly one of slice_len or dest_size must be non-NULL.
855 // If dest_size is non-NULL, zeroing extends to the end of the object.
856 // If slice_len is non-NULL, the slice_idx value must be a constant.
generate_clear_array(Node * ctrl,MergeMemNode * merge_mem,const TypePtr * adr_type,Node * dest,BasicType basic_elem_type,Node * slice_idx,Node * slice_len,Node * dest_size)857 void PhaseMacroExpand::generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
858 const TypePtr* adr_type,
859 Node* dest,
860 BasicType basic_elem_type,
861 Node* slice_idx,
862 Node* slice_len,
863 Node* dest_size) {
864 // one or the other but not both of slice_len and dest_size:
865 assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
866 if (slice_len == NULL) slice_len = top();
867 if (dest_size == NULL) dest_size = top();
868
869 uint alias_idx = C->get_alias_index(adr_type);
870
871 // operate on this memory slice:
872 Node* mem = merge_mem->memory_at(alias_idx); // memory slice to operate on
873
874 // scaling and rounding of indexes:
875 int scale = exact_log2(type2aelembytes(basic_elem_type));
876 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
877 int clear_low = (-1 << scale) & (BytesPerInt - 1);
878 int bump_bit = (-1 << scale) & BytesPerInt;
879
880 // determine constant starts and ends
881 const intptr_t BIG_NEG = -128;
882 assert(BIG_NEG + 2*abase < 0, "neg enough");
883 intptr_t slice_idx_con = (intptr_t) _igvn.find_int_con(slice_idx, BIG_NEG);
884 intptr_t slice_len_con = (intptr_t) _igvn.find_int_con(slice_len, BIG_NEG);
885 if (slice_len_con == 0) {
886 return; // nothing to do here
887 }
888 intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
889 intptr_t end_con = _igvn.find_intptr_t_con(dest_size, -1);
890 if (slice_idx_con >= 0 && slice_len_con >= 0) {
891 assert(end_con < 0, "not two cons");
892 end_con = align_up(abase + ((slice_idx_con + slice_len_con) << scale),
893 BytesPerLong);
894 }
895
896 if (start_con >= 0 && end_con >= 0) {
897 // Constant start and end. Simple.
898 mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
899 start_con, end_con, &_igvn);
900 } else if (start_con >= 0 && dest_size != top()) {
901 // Constant start, pre-rounded end after the tail of the array.
902 Node* end = dest_size;
903 mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
904 start_con, end, &_igvn);
905 } else if (start_con >= 0 && slice_len != top()) {
906 // Constant start, non-constant end. End needs rounding up.
907 // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
908 intptr_t end_base = abase + (slice_idx_con << scale);
909 int end_round = (-1 << scale) & (BytesPerLong - 1);
910 Node* end = ConvI2X(slice_len);
911 if (scale != 0)
912 end = transform_later(new LShiftXNode(end, intcon(scale) ));
913 end_base += end_round;
914 end = transform_later(new AddXNode(end, MakeConX(end_base)) );
915 end = transform_later(new AndXNode(end, MakeConX(~end_round)) );
916 mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
917 start_con, end, &_igvn);
918 } else if (start_con < 0 && dest_size != top()) {
919 // Non-constant start, pre-rounded end after the tail of the array.
920 // This is almost certainly a "round-to-end" operation.
921 Node* start = slice_idx;
922 start = ConvI2X(start);
923 if (scale != 0)
924 start = transform_later(new LShiftXNode( start, intcon(scale) ));
925 start = transform_later(new AddXNode(start, MakeConX(abase)) );
926 if ((bump_bit | clear_low) != 0) {
927 int to_clear = (bump_bit | clear_low);
928 // Align up mod 8, then store a jint zero unconditionally
929 // just before the mod-8 boundary.
930 if (((abase + bump_bit) & ~to_clear) - bump_bit
931 < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
932 bump_bit = 0;
933 assert((abase & to_clear) == 0, "array base must be long-aligned");
934 } else {
935 // Bump 'start' up to (or past) the next jint boundary:
936 start = transform_later( new AddXNode(start, MakeConX(bump_bit)) );
937 assert((abase & clear_low) == 0, "array base must be int-aligned");
938 }
939 // Round bumped 'start' down to jlong boundary in body of array.
940 start = transform_later(new AndXNode(start, MakeConX(~to_clear)) );
941 if (bump_bit != 0) {
942 // Store a zero to the immediately preceding jint:
943 Node* x1 = transform_later(new AddXNode(start, MakeConX(-bump_bit)) );
944 Node* p1 = basic_plus_adr(dest, x1);
945 mem = StoreNode::make(_igvn, ctrl, mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered);
946 mem = transform_later(mem);
947 }
948 }
949 Node* end = dest_size; // pre-rounded
950 mem = ClearArrayNode::clear_memory(ctrl, mem, dest,
951 start, end, &_igvn);
952 } else {
953 // Non-constant start, unrounded non-constant end.
954 // (Nobody zeroes a random midsection of an array using this routine.)
955 ShouldNotReachHere(); // fix caller
956 }
957
958 // Done.
959 merge_mem->set_memory_at(alias_idx, mem);
960 }
961
generate_block_arraycopy(Node ** ctrl,MergeMemNode ** mem,Node * io,const TypePtr * adr_type,BasicType basic_elem_type,AllocateNode * alloc,Node * src,Node * src_offset,Node * dest,Node * dest_offset,Node * dest_size,bool dest_uninitialized)962 bool PhaseMacroExpand::generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
963 const TypePtr* adr_type,
964 BasicType basic_elem_type,
965 AllocateNode* alloc,
966 Node* src, Node* src_offset,
967 Node* dest, Node* dest_offset,
968 Node* dest_size, bool dest_uninitialized) {
969 // See if there is an advantage from block transfer.
970 int scale = exact_log2(type2aelembytes(basic_elem_type));
971 if (scale >= LogBytesPerLong)
972 return false; // it is already a block transfer
973
974 // Look at the alignment of the starting offsets.
975 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
976
977 intptr_t src_off_con = (intptr_t) _igvn.find_int_con(src_offset, -1);
978 intptr_t dest_off_con = (intptr_t) _igvn.find_int_con(dest_offset, -1);
979 if (src_off_con < 0 || dest_off_con < 0) {
980 // At present, we can only understand constants.
981 return false;
982 }
983
984 intptr_t src_off = abase + (src_off_con << scale);
985 intptr_t dest_off = abase + (dest_off_con << scale);
986
987 if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
988 // Non-aligned; too bad.
989 // One more chance: Pick off an initial 32-bit word.
990 // This is a common case, since abase can be odd mod 8.
991 if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
992 ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
993 Node* sptr = basic_plus_adr(src, src_off);
994 Node* dptr = basic_plus_adr(dest, dest_off);
995 const TypePtr* s_adr_type = _igvn.type(sptr)->is_ptr();
996 assert(s_adr_type->isa_aryptr(), "impossible slice");
997 uint s_alias_idx = C->get_alias_index(s_adr_type);
998 uint d_alias_idx = C->get_alias_index(adr_type);
999 bool is_mismatched = (basic_elem_type != T_INT);
1000 Node* sval = transform_later(
1001 LoadNode::make(_igvn, *ctrl, (*mem)->memory_at(s_alias_idx), sptr, s_adr_type,
1002 TypeInt::INT, T_INT, MemNode::unordered, LoadNode::DependsOnlyOnTest,
1003 false /*unaligned*/, is_mismatched));
1004 Node* st = transform_later(
1005 StoreNode::make(_igvn, *ctrl, (*mem)->memory_at(d_alias_idx), dptr, adr_type,
1006 sval, T_INT, MemNode::unordered));
1007 if (is_mismatched) {
1008 st->as_Store()->set_mismatched_access();
1009 }
1010 (*mem)->set_memory_at(d_alias_idx, st);
1011 src_off += BytesPerInt;
1012 dest_off += BytesPerInt;
1013 } else {
1014 return false;
1015 }
1016 }
1017 assert(src_off % BytesPerLong == 0, "");
1018 assert(dest_off % BytesPerLong == 0, "");
1019
1020 // Do this copy by giant steps.
1021 Node* sptr = basic_plus_adr(src, src_off);
1022 Node* dptr = basic_plus_adr(dest, dest_off);
1023 Node* countx = dest_size;
1024 countx = transform_later(new SubXNode(countx, MakeConX(dest_off)));
1025 countx = transform_later(new URShiftXNode(countx, intcon(LogBytesPerLong)));
1026
1027 bool disjoint_bases = true; // since alloc != NULL
1028 generate_unchecked_arraycopy(ctrl, mem,
1029 adr_type, T_LONG, disjoint_bases,
1030 sptr, NULL, dptr, NULL, countx, dest_uninitialized);
1031
1032 return true;
1033 }
1034
1035 // Helper function; generates code for the slow case.
1036 // We make a call to a runtime method which emulates the native method,
1037 // but without the native wrapper overhead.
generate_slow_arraycopy(ArrayCopyNode * ac,Node ** ctrl,Node * mem,Node ** io,const TypePtr * adr_type,Node * src,Node * src_offset,Node * dest,Node * dest_offset,Node * copy_length,bool dest_uninitialized)1038 MergeMemNode* PhaseMacroExpand::generate_slow_arraycopy(ArrayCopyNode *ac,
1039 Node** ctrl, Node* mem, Node** io,
1040 const TypePtr* adr_type,
1041 Node* src, Node* src_offset,
1042 Node* dest, Node* dest_offset,
1043 Node* copy_length, bool dest_uninitialized) {
1044 assert(!dest_uninitialized, "Invariant");
1045
1046 const TypeFunc* call_type = OptoRuntime::slow_arraycopy_Type();
1047 CallNode* call = new CallStaticJavaNode(call_type, OptoRuntime::slow_arraycopy_Java(),
1048 "slow_arraycopy",
1049 ac->jvms()->bci(), TypePtr::BOTTOM);
1050
1051 call->init_req(TypeFunc::Control, *ctrl);
1052 call->init_req(TypeFunc::I_O , *io);
1053 call->init_req(TypeFunc::Memory , mem);
1054 call->init_req(TypeFunc::ReturnAdr, top());
1055 call->init_req(TypeFunc::FramePtr, top());
1056 call->init_req(TypeFunc::Parms+0, src);
1057 call->init_req(TypeFunc::Parms+1, src_offset);
1058 call->init_req(TypeFunc::Parms+2, dest);
1059 call->init_req(TypeFunc::Parms+3, dest_offset);
1060 call->init_req(TypeFunc::Parms+4, copy_length);
1061 call->copy_call_debug_info(&_igvn, ac);
1062
1063 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
1064 _igvn.replace_node(ac, call);
1065 transform_later(call);
1066
1067 extract_call_projections(call);
1068 *ctrl = _fallthroughcatchproj->clone();
1069 transform_later(*ctrl);
1070
1071 Node* m = _memproj_fallthrough->clone();
1072 transform_later(m);
1073
1074 uint alias_idx = C->get_alias_index(adr_type);
1075 MergeMemNode* out_mem;
1076 if (alias_idx != Compile::AliasIdxBot) {
1077 out_mem = MergeMemNode::make(mem);
1078 out_mem->set_memory_at(alias_idx, m);
1079 } else {
1080 out_mem = MergeMemNode::make(m);
1081 }
1082 transform_later(out_mem);
1083
1084 *io = _ioproj_fallthrough->clone();
1085 transform_later(*io);
1086
1087 return out_mem;
1088 }
1089
1090 // Helper function; generates code for cases requiring runtime checks.
generate_checkcast_arraycopy(Node ** ctrl,MergeMemNode ** mem,const TypePtr * adr_type,Node * dest_elem_klass,Node * src,Node * src_offset,Node * dest,Node * dest_offset,Node * copy_length,bool dest_uninitialized)1091 Node* PhaseMacroExpand::generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
1092 const TypePtr* adr_type,
1093 Node* dest_elem_klass,
1094 Node* src, Node* src_offset,
1095 Node* dest, Node* dest_offset,
1096 Node* copy_length, bool dest_uninitialized) {
1097 if ((*ctrl)->is_top()) return NULL;
1098
1099 address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
1100 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
1101 return NULL;
1102 }
1103
1104 // Pick out the parameters required to perform a store-check
1105 // for the target array. This is an optimistic check. It will
1106 // look in each non-null element's class, at the desired klass's
1107 // super_check_offset, for the desired klass.
1108 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1109 Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
1110 Node* n3 = new LoadINode(NULL, *mem /*memory(p3)*/, p3, _igvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered);
1111 Node* check_offset = ConvI2X(transform_later(n3));
1112 Node* check_value = dest_elem_klass;
1113
1114 Node* src_start = array_element_address(src, src_offset, T_OBJECT);
1115 Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
1116
1117 const TypeFunc* call_type = OptoRuntime::checkcast_arraycopy_Type();
1118 Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "checkcast_arraycopy", adr_type,
1119 src_start, dest_start, copy_length XTOP, check_offset XTOP, check_value);
1120
1121 finish_arraycopy_call(call, ctrl, mem, adr_type);
1122
1123 Node* proj = new ProjNode(call, TypeFunc::Parms);
1124 transform_later(proj);
1125
1126 return proj;
1127 }
1128
1129 // Helper function; generates code for cases requiring runtime checks.
generate_generic_arraycopy(Node ** ctrl,MergeMemNode ** mem,const TypePtr * adr_type,Node * src,Node * src_offset,Node * dest,Node * dest_offset,Node * copy_length,bool dest_uninitialized)1130 Node* PhaseMacroExpand::generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
1131 const TypePtr* adr_type,
1132 Node* src, Node* src_offset,
1133 Node* dest, Node* dest_offset,
1134 Node* copy_length, bool dest_uninitialized) {
1135 if ((*ctrl)->is_top()) return NULL;
1136 assert(!dest_uninitialized, "Invariant");
1137
1138 address copyfunc_addr = StubRoutines::generic_arraycopy();
1139 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
1140 return NULL;
1141 }
1142
1143 const TypeFunc* call_type = OptoRuntime::generic_arraycopy_Type();
1144 Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "generic_arraycopy", adr_type,
1145 src, src_offset, dest, dest_offset, copy_length);
1146
1147 finish_arraycopy_call(call, ctrl, mem, adr_type);
1148
1149 Node* proj = new ProjNode(call, TypeFunc::Parms);
1150 transform_later(proj);
1151
1152 return proj;
1153 }
1154
1155 // Helper function; generates the fast out-of-line call to an arraycopy stub.
generate_unchecked_arraycopy(Node ** ctrl,MergeMemNode ** mem,const TypePtr * adr_type,BasicType basic_elem_type,bool disjoint_bases,Node * src,Node * src_offset,Node * dest,Node * dest_offset,Node * copy_length,bool dest_uninitialized)1156 bool PhaseMacroExpand::generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
1157 const TypePtr* adr_type,
1158 BasicType basic_elem_type,
1159 bool disjoint_bases,
1160 Node* src, Node* src_offset,
1161 Node* dest, Node* dest_offset,
1162 Node* copy_length, bool dest_uninitialized) {
1163 if ((*ctrl)->is_top()) return false;
1164
1165 Node* src_start = src;
1166 Node* dest_start = dest;
1167 if (src_offset != NULL || dest_offset != NULL) {
1168 src_start = array_element_address(src, src_offset, basic_elem_type);
1169 dest_start = array_element_address(dest, dest_offset, basic_elem_type);
1170 }
1171
1172 // Figure out which arraycopy runtime method to call.
1173 const char* copyfunc_name = "arraycopy";
1174 address copyfunc_addr =
1175 basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
1176 disjoint_bases, copyfunc_name, dest_uninitialized);
1177
1178 Node* result_memory = NULL;
1179 RegionNode* exit_block = NULL;
1180 if (ArrayCopyPartialInlineSize > 0 && is_subword_type(basic_elem_type) &&
1181 Matcher::vector_width_in_bytes(basic_elem_type) >= 16) {
1182 generate_partial_inlining_block(ctrl, mem, adr_type, &exit_block, &result_memory,
1183 copy_length, src_start, dest_start, basic_elem_type);
1184 }
1185
1186 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
1187 Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, copyfunc_name, adr_type,
1188 src_start, dest_start, copy_length XTOP);
1189
1190 finish_arraycopy_call(call, ctrl, mem, adr_type);
1191
1192 // Connecting remaining edges for exit_block coming from stub_block.
1193 if (exit_block) {
1194 exit_block->init_req(2, *ctrl);
1195
1196 // Memory edge corresponding to stub_region.
1197 result_memory->init_req(2, *mem);
1198
1199 uint alias_idx = C->get_alias_index(adr_type);
1200 if (alias_idx != Compile::AliasIdxBot) {
1201 *mem = MergeMemNode::make(*mem);
1202 (*mem)->set_memory_at(alias_idx, result_memory);
1203 } else {
1204 *mem = MergeMemNode::make(result_memory);
1205 }
1206 transform_later(*mem);
1207 *ctrl = exit_block;
1208 return true;
1209 }
1210 return false;
1211 }
1212
expand_arraycopy_node(ArrayCopyNode * ac)1213 void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {
1214 Node* ctrl = ac->in(TypeFunc::Control);
1215 Node* io = ac->in(TypeFunc::I_O);
1216 Node* src = ac->in(ArrayCopyNode::Src);
1217 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
1218 Node* dest = ac->in(ArrayCopyNode::Dest);
1219 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
1220 Node* length = ac->in(ArrayCopyNode::Length);
1221 MergeMemNode* merge_mem = NULL;
1222
1223 if (ac->is_clonebasic()) {
1224 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1225 bs->clone_at_expansion(this, ac);
1226 return;
1227 } else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_clone_oop_array()) {
1228 Node* mem = ac->in(TypeFunc::Memory);
1229 merge_mem = MergeMemNode::make(mem);
1230 transform_later(merge_mem);
1231
1232 AllocateArrayNode* alloc = NULL;
1233 if (ac->is_alloc_tightly_coupled()) {
1234 alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
1235 assert(alloc != NULL, "expect alloc");
1236 }
1237
1238 const TypePtr* adr_type = _igvn.type(dest)->is_oopptr()->add_offset(Type::OffsetBot);
1239 if (ac->_dest_type != TypeOopPtr::BOTTOM) {
1240 adr_type = ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr();
1241 }
1242 generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
1243 adr_type, T_OBJECT,
1244 src, src_offset, dest, dest_offset, length,
1245 true, !ac->is_copyofrange());
1246
1247 return;
1248 }
1249
1250 AllocateArrayNode* alloc = NULL;
1251 if (ac->is_alloc_tightly_coupled()) {
1252 alloc = AllocateArrayNode::Ideal_array_allocation(dest, &_igvn);
1253 assert(alloc != NULL, "expect alloc");
1254 }
1255
1256 assert(ac->is_arraycopy() || ac->is_arraycopy_validated(), "should be an arraycopy");
1257
1258 // Compile time checks. If any of these checks cannot be verified at compile time,
1259 // we do not make a fast path for this call. Instead, we let the call remain as it
1260 // is. The checks we choose to mandate at compile time are:
1261 //
1262 // (1) src and dest are arrays.
1263 const Type* src_type = src->Value(&_igvn);
1264 const Type* dest_type = dest->Value(&_igvn);
1265 const TypeAryPtr* top_src = src_type->isa_aryptr();
1266 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
1267
1268 BasicType src_elem = T_CONFLICT;
1269 BasicType dest_elem = T_CONFLICT;
1270
1271 if (top_dest != NULL && top_dest->klass() != NULL) {
1272 dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
1273 }
1274 if (top_src != NULL && top_src->klass() != NULL) {
1275 src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
1276 }
1277 if (is_reference_type(src_elem)) src_elem = T_OBJECT;
1278 if (is_reference_type(dest_elem)) dest_elem = T_OBJECT;
1279
1280 if (ac->is_arraycopy_validated() &&
1281 dest_elem != T_CONFLICT &&
1282 src_elem == T_CONFLICT) {
1283 src_elem = dest_elem;
1284 }
1285
1286 if (src_elem == T_CONFLICT || dest_elem == T_CONFLICT) {
1287 // Conservatively insert a memory barrier on all memory slices.
1288 // Do not let writes into the source float below the arraycopy.
1289 {
1290 Node* mem = ac->in(TypeFunc::Memory);
1291 insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder);
1292
1293 merge_mem = MergeMemNode::make(mem);
1294 transform_later(merge_mem);
1295 }
1296
1297 // Call StubRoutines::generic_arraycopy stub.
1298 Node* mem = generate_arraycopy(ac, NULL, &ctrl, merge_mem, &io,
1299 TypeRawPtr::BOTTOM, T_CONFLICT,
1300 src, src_offset, dest, dest_offset, length,
1301 // If a negative length guard was generated for the ArrayCopyNode,
1302 // the length of the array can never be negative.
1303 false, ac->has_negative_length_guard());
1304 return;
1305 }
1306
1307 assert(!ac->is_arraycopy_validated() || (src_elem == dest_elem && dest_elem != T_VOID), "validated but different basic types");
1308
1309 // (2) src and dest arrays must have elements of the same BasicType
1310 // Figure out the size and type of the elements we will be copying.
1311 if (src_elem != dest_elem || dest_elem == T_VOID) {
1312 // The component types are not the same or are not recognized. Punt.
1313 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
1314 {
1315 Node* mem = ac->in(TypeFunc::Memory);
1316 merge_mem = generate_slow_arraycopy(ac, &ctrl, mem, &io, TypePtr::BOTTOM, src, src_offset, dest, dest_offset, length, false);
1317 }
1318
1319 _igvn.replace_node(_memproj_fallthrough, merge_mem);
1320 _igvn.replace_node(_ioproj_fallthrough, io);
1321 _igvn.replace_node(_fallthroughcatchproj, ctrl);
1322 return;
1323 }
1324
1325 //---------------------------------------------------------------------------
1326 // We will make a fast path for this call to arraycopy.
1327
1328 // We have the following tests left to perform:
1329 //
1330 // (3) src and dest must not be null.
1331 // (4) src_offset must not be negative.
1332 // (5) dest_offset must not be negative.
1333 // (6) length must not be negative.
1334 // (7) src_offset + length must not exceed length of src.
1335 // (8) dest_offset + length must not exceed length of dest.
1336 // (9) each element of an oop array must be assignable
1337
1338 {
1339 Node* mem = ac->in(TypeFunc::Memory);
1340 merge_mem = MergeMemNode::make(mem);
1341 transform_later(merge_mem);
1342 }
1343
1344 RegionNode* slow_region = new RegionNode(1);
1345 transform_later(slow_region);
1346
1347 if (!ac->is_arraycopy_validated()) {
1348 // (3) operands must not be null
1349 // We currently perform our null checks with the null_check routine.
1350 // This means that the null exceptions will be reported in the caller
1351 // rather than (correctly) reported inside of the native arraycopy call.
1352 // This should be corrected, given time. We do our null check with the
1353 // stack pointer restored.
1354 // null checks done library_call.cpp
1355
1356 // (4) src_offset must not be negative.
1357 generate_negative_guard(&ctrl, src_offset, slow_region);
1358
1359 // (5) dest_offset must not be negative.
1360 generate_negative_guard(&ctrl, dest_offset, slow_region);
1361
1362 // (6) length must not be negative (moved to generate_arraycopy()).
1363 // generate_negative_guard(length, slow_region);
1364
1365 // (7) src_offset + length must not exceed length of src.
1366 Node* alen = ac->in(ArrayCopyNode::SrcLen);
1367 assert(alen != NULL, "need src len");
1368 generate_limit_guard(&ctrl,
1369 src_offset, length,
1370 alen,
1371 slow_region);
1372
1373 // (8) dest_offset + length must not exceed length of dest.
1374 alen = ac->in(ArrayCopyNode::DestLen);
1375 assert(alen != NULL, "need dest len");
1376 generate_limit_guard(&ctrl,
1377 dest_offset, length,
1378 alen,
1379 slow_region);
1380
1381 // (9) each element of an oop array must be assignable
1382 // The generate_arraycopy subroutine checks this.
1383 }
1384 // This is where the memory effects are placed:
1385 const TypePtr* adr_type = NULL;
1386 if (ac->_dest_type != TypeOopPtr::BOTTOM) {
1387 adr_type = ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr();
1388 } else {
1389 adr_type = TypeAryPtr::get_array_body_type(dest_elem);
1390 }
1391
1392 generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io,
1393 adr_type, dest_elem,
1394 src, src_offset, dest, dest_offset, length,
1395 // If a negative length guard was generated for the ArrayCopyNode,
1396 // the length of the array can never be negative.
1397 false, ac->has_negative_length_guard(), slow_region);
1398 }
1399