1 /*
2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "opto/library_call.hpp"
28 #include "opto/runtime.hpp"
29 #include "opto/vectornode.hpp"
30 #include "prims/vectorSupport.hpp"
31
32 #ifdef ASSERT
is_vector(ciKlass * klass)33 static bool is_vector(ciKlass* klass) {
34 return klass->is_subclass_of(ciEnv::current()->vector_VectorPayload_klass());
35 }
36
check_vbox(const TypeInstPtr * vbox_type)37 static bool check_vbox(const TypeInstPtr* vbox_type) {
38 assert(vbox_type->klass_is_exact(), "");
39
40 ciInstanceKlass* ik = vbox_type->klass()->as_instance_klass();
41 assert(is_vector(ik), "not a vector");
42
43 ciField* fd1 = ik->get_field_by_name(ciSymbol::ETYPE_name(), ciSymbol::class_signature(), /* is_static */ true);
44 assert(fd1 != NULL, "element type info is missing");
45
46 ciConstant val1 = fd1->constant_value();
47 BasicType elem_bt = val1.as_object()->as_instance()->java_mirror_type()->basic_type();
48 assert(is_java_primitive(elem_bt), "element type info is missing");
49
50 ciField* fd2 = ik->get_field_by_name(ciSymbol::VLENGTH_name(), ciSymbol::int_signature(), /* is_static */ true);
51 assert(fd2 != NULL, "vector length info is missing");
52
53 ciConstant val2 = fd2->constant_value();
54 assert(val2.as_int() > 0, "vector length info is missing");
55
56 return true;
57 }
58 #endif
59
box_vector(Node * vector,const TypeInstPtr * vbox_type,BasicType elem_bt,int num_elem,bool deoptimize_on_exception)60 Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception) {
61 assert(EnableVectorSupport, "");
62
63 PreserveReexecuteState preexecs(this);
64 jvms()->set_should_reexecute(true);
65
66 VectorBoxAllocateNode* alloc = new VectorBoxAllocateNode(C, vbox_type);
67 set_edges_for_java_call(alloc, /*must_throw=*/false, /*separate_io_proj=*/true);
68 make_slow_call_ex(alloc, env()->Throwable_klass(), /*separate_io_proj=*/true, deoptimize_on_exception);
69 set_i_o(gvn().transform( new ProjNode(alloc, TypeFunc::I_O) ));
70 set_all_memory(gvn().transform( new ProjNode(alloc, TypeFunc::Memory) ));
71 Node* ret = gvn().transform(new ProjNode(alloc, TypeFunc::Parms));
72
73 assert(check_vbox(vbox_type), "");
74 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
75 VectorBoxNode* vbox = new VectorBoxNode(C, ret, vector, vbox_type, vt);
76 return gvn().transform(vbox);
77 }
78
unbox_vector(Node * v,const TypeInstPtr * vbox_type,BasicType elem_bt,int num_elem,bool shuffle_to_vector)79 Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
80 assert(EnableVectorSupport, "");
81 const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
82 if (vbox_type->klass() != vbox_type_v->klass()) {
83 return NULL; // arguments don't agree on vector shapes
84 }
85 if (vbox_type_v->maybe_null()) {
86 return NULL; // no nulls are allowed
87 }
88 assert(check_vbox(vbox_type), "");
89 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
90 Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
91 return unbox;
92 }
93
vector_shift_count(Node * cnt,int shift_op,BasicType bt,int num_elem)94 Node* GraphKit::vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem) {
95 assert(bt == T_INT || bt == T_LONG || bt == T_SHORT || bt == T_BYTE, "byte, short, long and int are supported");
96 juint mask = (type2aelembytes(bt) * BitsPerByte - 1);
97 Node* nmask = gvn().transform(ConNode::make(TypeInt::make(mask)));
98 Node* mcnt = gvn().transform(new AndINode(cnt, nmask));
99 return gvn().transform(VectorNode::shift_count(shift_op, mcnt, num_elem, bt));
100 }
101
arch_supports_vector(int sopc,int num_elem,BasicType type,VectorMaskUseType mask_use_type,bool has_scalar_args)102 bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type, VectorMaskUseType mask_use_type, bool has_scalar_args) {
103 // Check that the operation is valid.
104 if (sopc <= 0) {
105 #ifndef PRODUCT
106 if (C->print_intrinsics()) {
107 tty->print_cr(" ** Rejected intrinsification because no valid vector op could be extracted");
108 }
109 #endif
110 return false;
111 }
112
113 // Check that architecture supports this op-size-type combination.
114 if (!Matcher::match_rule_supported_vector(sopc, num_elem, type)) {
115 #ifndef PRODUCT
116 if (C->print_intrinsics()) {
117 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support it",
118 NodeClassNames[sopc], type2name(type), num_elem);
119 }
120 #endif
121 return false;
122 } else {
123 assert(Matcher::match_rule_supported(sopc), "must be supported");
124 }
125
126 if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
127 Matcher::supports_vector_variable_shifts() == false) {
128 if (C->print_intrinsics()) {
129 tty->print_cr(" ** Rejected vector op (%s,%s,%d) because architecture does not support variable vector shifts",
130 NodeClassNames[sopc], type2name(type), num_elem);
131 }
132 return false;
133 }
134
135 // Check whether mask unboxing is supported.
136 if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseLoad) {
137 if (!Matcher::match_rule_supported_vector(Op_VectorLoadMask, num_elem, type)) {
138 #ifndef PRODUCT
139 if (C->print_intrinsics()) {
140 tty->print_cr(" ** Rejected vector mask loading (%s,%s,%d) because architecture does not support it",
141 NodeClassNames[Op_VectorLoadMask], type2name(type), num_elem);
142 }
143 #endif
144 return false;
145 }
146 }
147
148 // Check whether mask boxing is supported.
149 if (mask_use_type == VecMaskUseAll || mask_use_type == VecMaskUseStore) {
150 if (!Matcher::match_rule_supported_vector(Op_VectorStoreMask, num_elem, type)) {
151 #ifndef PRODUCT
152 if (C->print_intrinsics()) {
153 tty->print_cr("Rejected vector mask storing (%s,%s,%d) because architecture does not support it",
154 NodeClassNames[Op_VectorStoreMask], type2name(type), num_elem);
155 }
156 #endif
157 return false;
158 }
159 }
160
161 return true;
162 }
163
is_vector_mask(ciKlass * klass)164 static bool is_vector_mask(ciKlass* klass) {
165 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
166 }
167
is_vector_shuffle(ciKlass * klass)168 static bool is_vector_shuffle(ciKlass* klass) {
169 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
170 }
171
is_klass_initialized(const TypeInstPtr * vec_klass)172 static bool is_klass_initialized(const TypeInstPtr* vec_klass) {
173 if (vec_klass->const_oop() == NULL) {
174 return false; // uninitialized or some kind of unsafe access
175 }
176 assert(vec_klass->const_oop()->as_instance()->java_lang_Class_klass() != NULL, "klass instance expected");
177 ciInstanceKlass* klass = vec_klass->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
178 return klass->is_initialized();
179 }
180
181 // public static
182 // <VM>
183 // VM unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
184 // VM vm,
185 // Function<VM, VM> defaultImpl) {
186 //
187 // public static
188 // <VM>
189 // VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
190 // VM vm1, VM vm2,
191 // BiFunction<VM, VM, VM> defaultImpl) {
192 //
193 // public static
194 // <VM>
195 // VM ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length,
196 // VM vm1, VM vm2, VM vm3,
197 // TernaryOperation<VM> defaultImpl) {
198 //
inline_vector_nary_operation(int n)199 bool LibraryCallKit::inline_vector_nary_operation(int n) {
200 const TypeInt* opr = gvn().type(argument(0))->isa_int();
201 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
202 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
203 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
204
205 if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
206 !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
207 if (C->print_intrinsics()) {
208 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
209 NodeClassNames[argument(0)->Opcode()],
210 NodeClassNames[argument(1)->Opcode()],
211 NodeClassNames[argument(2)->Opcode()],
212 NodeClassNames[argument(3)->Opcode()]);
213 }
214 return false; // not enough info for intrinsification
215 }
216 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
217 if (!elem_type->is_primitive_type()) {
218 if (C->print_intrinsics()) {
219 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
220 }
221 return false; // should be primitive type
222 }
223 if (!is_klass_initialized(vector_klass)) {
224 if (C->print_intrinsics()) {
225 tty->print_cr(" ** klass argument not initialized");
226 }
227 return false;
228 }
229 BasicType elem_bt = elem_type->basic_type();
230 int num_elem = vlen->get_con();
231 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
232 int sopc = VectorNode::opcode(opc, elem_bt);
233 if (sopc == 0) {
234 if (C->print_intrinsics()) {
235 tty->print_cr(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
236 }
237 return false; // operation not supported
238 }
239 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
240 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
241
242 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
243 if (!arch_supports_vector(sopc, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseAll : VecMaskNotUsed)) {
244 if (C->print_intrinsics()) {
245 tty->print_cr(" ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d",
246 n, sopc, num_elem, type2name(elem_bt),
247 is_vector_mask(vbox_klass) ? 1 : 0);
248 }
249 return false; // not supported
250 }
251
252 Node* opd1 = NULL; Node* opd2 = NULL; Node* opd3 = NULL;
253 switch (n) {
254 case 3: {
255 opd3 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
256 if (opd3 == NULL) {
257 if (C->print_intrinsics()) {
258 tty->print_cr(" ** unbox failed v3=%s",
259 NodeClassNames[argument(6)->Opcode()]);
260 }
261 return false;
262 }
263 // fall-through
264 }
265 case 2: {
266 opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
267 if (opd2 == NULL) {
268 if (C->print_intrinsics()) {
269 tty->print_cr(" ** unbox failed v2=%s",
270 NodeClassNames[argument(5)->Opcode()]);
271 }
272 return false;
273 }
274 // fall-through
275 }
276 case 1: {
277 opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
278 if (opd1 == NULL) {
279 if (C->print_intrinsics()) {
280 tty->print_cr(" ** unbox failed v1=%s",
281 NodeClassNames[argument(4)->Opcode()]);
282 }
283 return false;
284 }
285 break;
286 }
287 default: fatal("unsupported arity: %d", n);
288 }
289
290 Node* operation = NULL;
291 const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
292 switch (n) {
293 case 1:
294 case 2: {
295 operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, vt));
296 break;
297 }
298 case 3: {
299 operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, opd3, vt));
300 break;
301 }
302 default: fatal("unsupported arity: %d", n);
303 }
304 // Wrap it up in VectorBox to keep object type information.
305 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
306 set_result(vbox);
307 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
308 return true;
309 }
310
311 // <Sh extends VectorShuffle<E>, E>
312 // Sh ShuffleIota(Class<?> E, Class<?> ShuffleClass, Vector.Species<E> s, int length,
313 // int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
inline_vector_shuffle_iota()314 bool LibraryCallKit::inline_vector_shuffle_iota() {
315 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
316 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
317 const TypeInt* start_val = gvn().type(argument(4))->isa_int();
318 const TypeInt* step_val = gvn().type(argument(5))->isa_int();
319 const TypeInt* wrap = gvn().type(argument(6))->isa_int();
320
321 Node* start = argument(4);
322 Node* step = argument(5);
323
324 if (shuffle_klass == NULL || vlen == NULL || start_val == NULL || step_val == NULL || wrap == NULL) {
325 return false; // dead code
326 }
327 if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
328 shuffle_klass->const_oop() == NULL || !wrap->is_con()) {
329 return false; // not enough info for intrinsification
330 }
331 if (!is_klass_initialized(shuffle_klass)) {
332 if (C->print_intrinsics()) {
333 tty->print_cr(" ** klass argument not initialized");
334 }
335 return false;
336 }
337
338 int do_wrap = wrap->get_con();
339 int num_elem = vlen->get_con();
340 BasicType elem_bt = T_BYTE;
341
342 if (num_elem < 4)
343 return false;
344
345 if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
346 return false;
347 }
348 if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
349 return false;
350 }
351 if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
352 return false;
353 }
354 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
355 return false;
356 }
357 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
358 return false;
359 }
360
361 const Type * type_bt = Type::get_const_basic_type(elem_bt);
362 const TypeVect * vt = TypeVect::make(type_bt, num_elem);
363
364 Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
365
366 if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
367 Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
368 res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
369 } else if (step_val->get_con() > 1) {
370 Node* cnt = gvn().makecon(TypeInt::make(log2_int(step_val->get_con())));
371 res = gvn().transform(VectorNode::make(Op_LShiftVB, res, cnt, vt));
372 }
373
374 if (!start_val->is_con() || start_val->get_con() != 0) {
375 Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
376 res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
377 }
378
379 Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
380 Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
381 if(do_wrap) {
382 // Wrap the indices greater than lane count.
383 res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
384 } else {
385 ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(1));
386 Node * lane_cnt = gvn().makecon(TypeInt::make(num_elem));
387 Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
388 Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vt));
389
390 // Make the indices greater than lane count as -ve values. This matches the java side implementation.
391 res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
392 Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
393 res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
394 }
395
396 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
397 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
398
399 // Wrap it up in VectorBox to keep object type information.
400 res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
401 set_result(res);
402 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
403 return true;
404 }
405
406 // <VM ,Sh extends VectorShuffle<E>, E>
407 // VM shuffleToVector(Class<VM> VecClass, Class<?>E , Class<?> ShuffleClass, Sh s, int length,
408 // ShuffleToVectorOperation<VM,Sh,E> defaultImpl)
inline_vector_shuffle_to_vector()409 bool LibraryCallKit::inline_vector_shuffle_to_vector() {
410 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
411 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
412 const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
413 Node* shuffle = argument(3);
414 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
415
416 if (vector_klass == NULL || elem_klass == NULL || shuffle_klass == NULL || shuffle->is_top() || vlen == NULL) {
417 return false; // dead code
418 }
419 if (!vlen->is_con() || vector_klass->const_oop() == NULL || shuffle_klass->const_oop() == NULL) {
420 return false; // not enough info for intrinsification
421 }
422 if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
423 if (C->print_intrinsics()) {
424 tty->print_cr(" ** klass argument not initialized");
425 }
426 return false;
427 }
428
429 int num_elem = vlen->get_con();
430 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
431 BasicType elem_bt = elem_type->basic_type();
432
433 if (num_elem < 4) {
434 return false;
435 }
436
437 int cast_vopc = VectorCastNode::opcode(T_BYTE); // from shuffle of type T_BYTE
438 // Make sure that cast is implemented to particular type/size combination.
439 if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
440 if (C->print_intrinsics()) {
441 tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
442 cast_vopc, num_elem, type2name(elem_bt));
443 }
444 return false;
445 }
446
447 ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
448 const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
449
450 // Unbox shuffle with true flag to indicate its load shuffle to vector
451 Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, elem_bt, num_elem, true);
452
453 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
454 const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
455
456 // Box vector
457 Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
458 set_result(res);
459 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
460 return true;
461 }
462
463 // <V extends Vector<?,?>>
464 // V broadcastCoerced(Class<?> vectorClass, Class<?> elementType, int vlen,
465 // long bits,
466 // LongFunction<V> defaultImpl)
inline_vector_broadcast_coerced()467 bool LibraryCallKit::inline_vector_broadcast_coerced() {
468 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
469 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
470 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
471
472 if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
473 vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
474 if (C->print_intrinsics()) {
475 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s",
476 NodeClassNames[argument(0)->Opcode()],
477 NodeClassNames[argument(1)->Opcode()],
478 NodeClassNames[argument(2)->Opcode()]);
479 }
480 return false; // not enough info for intrinsification
481 }
482
483 if (!is_klass_initialized(vector_klass)) {
484 if (C->print_intrinsics()) {
485 tty->print_cr(" ** klass argument not initialized");
486 }
487 return false;
488 }
489 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
490 if (!elem_type->is_primitive_type()) {
491 if (C->print_intrinsics()) {
492 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
493 }
494 return false; // should be primitive type
495 }
496 BasicType elem_bt = elem_type->basic_type();
497 int num_elem = vlen->get_con();
498 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
499 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
500
501 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
502 if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt,
503 (is_vector_mask(vbox_klass) ? VecMaskUseStore : VecMaskNotUsed), true /*has_scalar_args*/)) {
504 if (C->print_intrinsics()) {
505 tty->print_cr(" ** not supported: arity=0 op=broadcast vlen=%d etype=%s ismask=%d",
506 num_elem, type2name(elem_bt),
507 is_vector_mask(vbox_klass) ? 1 : 0);
508 }
509 return false; // not supported
510 }
511
512 Node* bits = argument(3); // long
513
514 Node* elem = NULL;
515 switch (elem_bt) {
516 case T_BOOLEAN: // fall-through
517 case T_BYTE: // fall-through
518 case T_SHORT: // fall-through
519 case T_CHAR: // fall-through
520 case T_INT: {
521 elem = gvn().transform(new ConvL2INode(bits));
522 break;
523 }
524 case T_DOUBLE: {
525 elem = gvn().transform(new MoveL2DNode(bits));
526 break;
527 }
528 case T_FLOAT: {
529 bits = gvn().transform(new ConvL2INode(bits));
530 elem = gvn().transform(new MoveI2FNode(bits));
531 break;
532 }
533 case T_LONG: {
534 elem = bits; // no conversion needed
535 break;
536 }
537 default: fatal("%s", type2name(elem_bt));
538 }
539
540 Node* broadcast = VectorNode::scalar2vector(elem, num_elem, Type::get_const_basic_type(elem_bt));
541 broadcast = gvn().transform(broadcast);
542
543 Node* box = box_vector(broadcast, vbox_type, elem_bt, num_elem);
544 set_result(box);
545 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
546 return true;
547 }
548
549 // <C, V extends Vector<?,?>>
550 // V load(Class<?> vectorClass, Class<?> elementType, int vlen,
551 // Object base, long offset,
552 // /* Vector.Mask<E,S> m*/
553 // Object container, int index,
554 // LoadOperation<C, VM> defaultImpl) {
555 //
556 // <C, V extends Vector<?,?>>
557 // void store(Class<?> vectorClass, Class<?> elementType, int vlen,
558 // Object base, long offset,
559 // V v, /*Vector.Mask<E,S> m*/
560 // Object container, int index,
561 // StoreVectorOperation<C, V> defaultImpl) {
562
inline_vector_mem_operation(bool is_store)563 bool LibraryCallKit::inline_vector_mem_operation(bool is_store) {
564 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
565 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
566 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
567
568 if (vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
569 vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
570 if (C->print_intrinsics()) {
571 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s",
572 NodeClassNames[argument(0)->Opcode()],
573 NodeClassNames[argument(1)->Opcode()],
574 NodeClassNames[argument(2)->Opcode()]);
575 }
576 return false; // not enough info for intrinsification
577 }
578 if (!is_klass_initialized(vector_klass)) {
579 if (C->print_intrinsics()) {
580 tty->print_cr(" ** klass argument not initialized");
581 }
582 return false;
583 }
584
585 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
586 if (!elem_type->is_primitive_type()) {
587 if (C->print_intrinsics()) {
588 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
589 }
590 return false; // should be primitive type
591 }
592 BasicType elem_bt = elem_type->basic_type();
593 int num_elem = vlen->get_con();
594
595 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
596 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, num_elem, elem_bt, VecMaskNotUsed)) {
597 if (C->print_intrinsics()) {
598 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
599 is_store, is_store ? "store" : "load",
600 num_elem, type2name(elem_bt));
601 }
602 return false; // not supported
603 }
604
605 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
606 bool is_mask = is_vector_mask(vbox_klass);
607
608 Node* base = argument(3);
609 Node* offset = ConvL2X(argument(4));
610 DecoratorSet decorators = C2_UNSAFE_ACCESS;
611 Node* addr = make_unsafe_address(base, offset, decorators, (is_mask ? T_BOOLEAN : elem_bt), true);
612
613 // Can base be NULL? Otherwise, always on-heap access.
614 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(gvn().type(base));
615
616 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
617 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
618
619 // Now handle special case where load/store happens from/to byte array but element type is not byte.
620 bool using_byte_array = arr_type != NULL && arr_type->elem()->array_element_basic_type() == T_BYTE && elem_bt != T_BYTE;
621 // Handle loading masks.
622 // If there is no consistency between array and vector element types, it must be special byte array case or loading masks
623 if (arr_type != NULL && !using_byte_array && elem_bt != arr_type->elem()->array_element_basic_type() && !is_mask) {
624 return false;
625 }
626 // Since we are using byte array, we need to double check that the byte operations are supported by backend.
627 if (using_byte_array) {
628 int byte_num_elem = num_elem * type2aelembytes(elem_bt);
629 if (!arch_supports_vector(is_store ? Op_StoreVector : Op_LoadVector, byte_num_elem, T_BYTE, VecMaskNotUsed)
630 || !arch_supports_vector(Op_VectorReinterpret, byte_num_elem, T_BYTE, VecMaskNotUsed)) {
631 if (C->print_intrinsics()) {
632 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d*8 etype=%s/8 ismask=no",
633 is_store, is_store ? "store" : "load",
634 byte_num_elem, type2name(elem_bt));
635 }
636 return false; // not supported
637 }
638 }
639 if (is_mask) {
640 if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) {
641 if (C->print_intrinsics()) {
642 tty->print_cr(" ** not supported: arity=%d op=%s/mask vlen=%d etype=bit ismask=no",
643 is_store, is_store ? "store" : "load",
644 num_elem);
645 }
646 return false; // not supported
647 }
648 if (!is_store) {
649 if (!arch_supports_vector(Op_LoadVector, num_elem, elem_bt, VecMaskUseLoad)) {
650 return false; // not supported
651 }
652 } else {
653 if (!arch_supports_vector(Op_StoreVector, num_elem, elem_bt, VecMaskUseStore)) {
654 return false; // not supported
655 }
656 }
657 }
658
659 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
660
661 if (can_access_non_heap) {
662 insert_mem_bar(Op_MemBarCPUOrder);
663 }
664
665 if (is_store) {
666 Node* val = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
667 if (val == NULL) {
668 return false; // operand unboxing failed
669 }
670 set_all_memory(reset_memory());
671
672 // In case the store needs to happen to byte array, reinterpret the incoming vector to byte vector.
673 int store_num_elem = num_elem;
674 if (using_byte_array) {
675 store_num_elem = num_elem * type2aelembytes(elem_bt);
676 const TypeVect* to_vect_type = TypeVect::make(T_BYTE, store_num_elem);
677 val = gvn().transform(new VectorReinterpretNode(val, val->bottom_type()->is_vect(), to_vect_type));
678 }
679
680 Node* vstore = gvn().transform(StoreVectorNode::make(0, control(), memory(addr), addr, addr_type, val, store_num_elem));
681 set_memory(vstore, addr_type);
682 } else {
683 // When using byte array, we need to load as byte then reinterpret the value. Otherwise, do a simple vector load.
684 Node* vload = NULL;
685 if (using_byte_array) {
686 int load_num_elem = num_elem * type2aelembytes(elem_bt);
687 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, load_num_elem, T_BYTE));
688 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
689 vload = gvn().transform(new VectorReinterpretNode(vload, vload->bottom_type()->is_vect(), to_vect_type));
690 } else {
691 // Special handle for masks
692 if (is_mask) {
693 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, T_BOOLEAN));
694 const TypeVect* to_vect_type = TypeVect::make(elem_bt, num_elem);
695 vload = gvn().transform(new VectorLoadMaskNode(vload, to_vect_type));
696 } else {
697 vload = gvn().transform(LoadVectorNode::make(0, control(), memory(addr), addr, addr_type, num_elem, elem_bt));
698 }
699 }
700 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
701 set_result(box);
702 }
703
704 if (can_access_non_heap) {
705 insert_mem_bar(Op_MemBarCPUOrder);
706 }
707
708 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
709 return true;
710 }
711
712 // <C, V extends Vector<?>, W extends IntVector, E, S extends VectorSpecies<E>>
713 // void loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass,
714 // Object base, long offset, // Unsafe addressing
715 // W index_vector,
716 // C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation
717 // LoadVectorOperationWithMap<C, V, E, S> defaultImpl)
718 //
719 // <C, V extends Vector<?>, W extends IntVector>
720 // void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass,
721 // Object base, long offset, // Unsafe addressing
722 // W index_vector, V v,
723 // C container, int index, int[] indexMap, int indexM, // Arguments for default implementation
724 // StoreVectorOperationWithMap<C, V> defaultImpl) {
725 //
inline_vector_gather_scatter(bool is_scatter)726 bool LibraryCallKit::inline_vector_gather_scatter(bool is_scatter) {
727 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
728 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
729 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
730 const TypeInstPtr* vector_idx_klass = gvn().type(argument(3))->isa_instptr();
731
732 if (vector_klass == NULL || elem_klass == NULL || vector_idx_klass == NULL || vlen == NULL ||
733 vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || vector_idx_klass->const_oop() == NULL || !vlen->is_con()) {
734 if (C->print_intrinsics()) {
735 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s viclass=%s",
736 NodeClassNames[argument(0)->Opcode()],
737 NodeClassNames[argument(1)->Opcode()],
738 NodeClassNames[argument(2)->Opcode()],
739 NodeClassNames[argument(3)->Opcode()]);
740 }
741 return false; // not enough info for intrinsification
742 }
743
744 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(vector_idx_klass)) {
745 if (C->print_intrinsics()) {
746 tty->print_cr(" ** klass argument not initialized");
747 }
748 return false;
749 }
750 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
751 if (!elem_type->is_primitive_type()) {
752 if (C->print_intrinsics()) {
753 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
754 }
755 return false; // should be primitive type
756 }
757 BasicType elem_bt = elem_type->basic_type();
758 int num_elem = vlen->get_con();
759
760 if (!arch_supports_vector(is_scatter ? Op_StoreVectorScatter : Op_LoadVectorGather, num_elem, elem_bt, VecMaskNotUsed)) {
761 if (C->print_intrinsics()) {
762 tty->print_cr(" ** not supported: arity=%d op=%s vlen=%d etype=%s ismask=no",
763 is_scatter, is_scatter ? "scatter" : "gather",
764 num_elem, type2name(elem_bt));
765 }
766 return false; // not supported
767 }
768
769 // Check that the vector holding indices is supported by architecture
770 if (!arch_supports_vector(Op_LoadVector, num_elem, T_INT, VecMaskNotUsed)) {
771 if (C->print_intrinsics()) {
772 tty->print_cr(" ** not supported: arity=%d op=%s/loadindex vlen=%d etype=int ismask=no",
773 is_scatter, is_scatter ? "scatter" : "gather",
774 num_elem);
775 }
776 return false; // not supported
777 }
778
779 Node* base = argument(4);
780 Node* offset = ConvL2X(argument(5));
781 Node* addr = make_unsafe_address(base, offset, C2_UNSAFE_ACCESS, elem_bt, true);
782
783 const TypePtr *addr_type = gvn().type(addr)->isa_ptr();
784 const TypeAryPtr* arr_type = addr_type->isa_aryptr();
785
786 // The array must be consistent with vector type
787 if (arr_type == NULL || (arr_type != NULL && elem_bt != arr_type->elem()->array_element_basic_type())) {
788 return false;
789 }
790 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
791 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
792
793 ciKlass* vbox_idx_klass = vector_idx_klass->const_oop()->as_instance()->java_lang_Class_klass();
794
795 if (vbox_idx_klass == NULL) {
796 return false;
797 }
798
799 const TypeInstPtr* vbox_idx_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_idx_klass);
800
801 Node* index_vect = unbox_vector(argument(7), vbox_idx_type, T_INT, num_elem);
802 if (index_vect == NULL) {
803 return false;
804 }
805 const TypeVect* vector_type = TypeVect::make(elem_bt, num_elem);
806 if (is_scatter) {
807 Node* val = unbox_vector(argument(8), vbox_type, elem_bt, num_elem);
808 if (val == NULL) {
809 return false; // operand unboxing failed
810 }
811 set_all_memory(reset_memory());
812
813 Node* vstore = gvn().transform(new StoreVectorScatterNode(control(), memory(addr), addr, addr_type, val, index_vect));
814 set_memory(vstore, addr_type);
815 } else {
816 Node* vload = gvn().transform(new LoadVectorGatherNode(control(), memory(addr), addr, addr_type, vector_type, index_vect));
817
818 Node* box = box_vector(vload, vbox_type, elem_bt, num_elem);
819 set_result(box);
820 }
821
822 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
823 return true;
824 }
825
826 // <V extends Vector<?,?>>
827 // long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int vlen,
828 // V v,
829 // Function<V,Long> defaultImpl)
830
inline_vector_reduction()831 bool LibraryCallKit::inline_vector_reduction() {
832 const TypeInt* opr = gvn().type(argument(0))->isa_int();
833 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
834 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
835 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
836
837 if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
838 !opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
839 if (C->print_intrinsics()) {
840 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
841 NodeClassNames[argument(0)->Opcode()],
842 NodeClassNames[argument(1)->Opcode()],
843 NodeClassNames[argument(2)->Opcode()],
844 NodeClassNames[argument(3)->Opcode()]);
845 }
846 return false; // not enough info for intrinsification
847 }
848 if (!is_klass_initialized(vector_klass)) {
849 if (C->print_intrinsics()) {
850 tty->print_cr(" ** klass argument not initialized");
851 }
852 return false;
853 }
854 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
855 if (!elem_type->is_primitive_type()) {
856 if (C->print_intrinsics()) {
857 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
858 }
859 return false; // should be primitive type
860 }
861 BasicType elem_bt = elem_type->basic_type();
862 int num_elem = vlen->get_con();
863
864 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
865 int sopc = ReductionNode::opcode(opc, elem_bt);
866
867 // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
868 if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed)) {
869 if (C->print_intrinsics()) {
870 tty->print_cr(" ** not supported: arity=1 op=%d/reduce vlen=%d etype=%s ismask=no",
871 sopc, num_elem, type2name(elem_bt));
872 }
873 return false;
874 }
875
876 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
877 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
878
879 Node* opd = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
880 if (opd == NULL) {
881 return false; // operand unboxing failed
882 }
883
884 Node* init = ReductionNode::make_reduction_input(gvn(), opc, elem_bt);
885 Node* rn = gvn().transform(ReductionNode::make(opc, NULL, init, opd, elem_bt));
886
887 Node* bits = NULL;
888 switch (elem_bt) {
889 case T_BYTE:
890 case T_SHORT:
891 case T_INT: {
892 bits = gvn().transform(new ConvI2LNode(rn));
893 break;
894 }
895 case T_FLOAT: {
896 rn = gvn().transform(new MoveF2INode(rn));
897 bits = gvn().transform(new ConvI2LNode(rn));
898 break;
899 }
900 case T_DOUBLE: {
901 bits = gvn().transform(new MoveD2LNode(rn));
902 break;
903 }
904 case T_LONG: {
905 bits = rn; // no conversion needed
906 break;
907 }
908 default: fatal("%s", type2name(elem_bt));
909 }
910 set_result(bits);
911 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
912 return true;
913 }
914
915 // public static <V> boolean test(int cond, Class<?> vectorClass, Class<?> elementType, int vlen,
916 // V v1, V v2,
917 // BiFunction<V, V, Boolean> defaultImpl) {
918 //
inline_vector_test()919 bool LibraryCallKit::inline_vector_test() {
920 const TypeInt* cond = gvn().type(argument(0))->isa_int();
921 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
922 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
923 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
924
925 if (cond == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL ||
926 !cond->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
927 if (C->print_intrinsics()) {
928 tty->print_cr(" ** missing constant: cond=%s vclass=%s etype=%s vlen=%s",
929 NodeClassNames[argument(0)->Opcode()],
930 NodeClassNames[argument(1)->Opcode()],
931 NodeClassNames[argument(2)->Opcode()],
932 NodeClassNames[argument(3)->Opcode()]);
933 }
934 return false; // not enough info for intrinsification
935 }
936 if (!is_klass_initialized(vector_klass)) {
937 if (C->print_intrinsics()) {
938 tty->print_cr(" ** klass argument not initialized");
939 }
940 return false;
941 }
942 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
943 if (!elem_type->is_primitive_type()) {
944 if (C->print_intrinsics()) {
945 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
946 }
947 return false; // should be primitive type
948 }
949 BasicType elem_bt = elem_type->basic_type();
950 int num_elem = vlen->get_con();
951 BoolTest::mask booltest = (BoolTest::mask)cond->get_con();
952 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
953 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
954
955 if (!arch_supports_vector(Op_VectorTest, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseLoad : VecMaskNotUsed)) {
956 if (C->print_intrinsics()) {
957 tty->print_cr(" ** not supported: arity=2 op=test/%d vlen=%d etype=%s ismask=%d",
958 cond->get_con(), num_elem, type2name(elem_bt),
959 is_vector_mask(vbox_klass));
960 }
961 return false;
962 }
963
964 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
965 Node* opd2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
966 if (opd1 == NULL || opd2 == NULL) {
967 return false; // operand unboxing failed
968 }
969 Node* test = new VectorTestNode(opd1, opd2, booltest);
970 test = gvn().transform(test);
971
972 set_result(test);
973 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
974 return true;
975 }
976
977 // public static
978 // <V extends Vector, M extends Mask>
979 // V blend(Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen,
980 // V v1, V v2, M m,
981 // VectorBlendOp<V,M> defaultImpl) { ...
982 //
inline_vector_blend()983 bool LibraryCallKit::inline_vector_blend() {
984 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
985 const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
986 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
987 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
988
989 if (mask_klass == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
990 return false; // dead code
991 }
992 if (mask_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
993 elem_klass->const_oop() == NULL || !vlen->is_con()) {
994 if (C->print_intrinsics()) {
995 tty->print_cr(" ** missing constant: vclass=%s mclass=%s etype=%s vlen=%s",
996 NodeClassNames[argument(0)->Opcode()],
997 NodeClassNames[argument(1)->Opcode()],
998 NodeClassNames[argument(2)->Opcode()],
999 NodeClassNames[argument(3)->Opcode()]);
1000 }
1001 return false; // not enough info for intrinsification
1002 }
1003 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1004 if (C->print_intrinsics()) {
1005 tty->print_cr(" ** klass argument not initialized");
1006 }
1007 return false;
1008 }
1009 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1010 if (!elem_type->is_primitive_type()) {
1011 if (C->print_intrinsics()) {
1012 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1013 }
1014 return false; // should be primitive type
1015 }
1016 BasicType elem_bt = elem_type->basic_type();
1017 BasicType mask_bt = elem_bt;
1018 int num_elem = vlen->get_con();
1019
1020 if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
1021 if (C->print_intrinsics()) {
1022 tty->print_cr(" ** not supported: arity=2 op=blend vlen=%d etype=%s ismask=useload",
1023 num_elem, type2name(elem_bt));
1024 }
1025 return false; // not supported
1026 }
1027 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1028 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1029
1030 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1031 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1032
1033 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1034 Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1035 Node* mask = unbox_vector(argument(6), mbox_type, mask_bt, num_elem);
1036
1037 if (v1 == NULL || v2 == NULL || mask == NULL) {
1038 return false; // operand unboxing failed
1039 }
1040
1041 Node* blend = gvn().transform(new VectorBlendNode(v1, v2, mask));
1042
1043 Node* box = box_vector(blend, vbox_type, elem_bt, num_elem);
1044 set_result(box);
1045 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1046 return true;
1047 }
1048
1049 // public static <V extends Vector<E,S>,
1050 // M extends Vector.Mask<E,S>,
1051 // S extends Vector.Shape, E>
1052 // M compare(int cond, Class<V> vectorClass, Class<M> maskClass, Class<?> elementType, int vlen,
1053 // V v1, V v2,
1054 // VectorCompareOp<V,M> defaultImpl) { ...
1055 //
inline_vector_compare()1056 bool LibraryCallKit::inline_vector_compare() {
1057 const TypeInt* cond = gvn().type(argument(0))->isa_int();
1058 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1059 const TypeInstPtr* mask_klass = gvn().type(argument(2))->isa_instptr();
1060 const TypeInstPtr* elem_klass = gvn().type(argument(3))->isa_instptr();
1061 const TypeInt* vlen = gvn().type(argument(4))->isa_int();
1062
1063 if (cond == NULL || vector_klass == NULL || mask_klass == NULL || elem_klass == NULL || vlen == NULL) {
1064 return false; // dead code
1065 }
1066 if (!cond->is_con() || vector_klass->const_oop() == NULL || mask_klass->const_oop() == NULL ||
1067 elem_klass->const_oop() == NULL || !vlen->is_con()) {
1068 if (C->print_intrinsics()) {
1069 tty->print_cr(" ** missing constant: cond=%s vclass=%s mclass=%s etype=%s vlen=%s",
1070 NodeClassNames[argument(0)->Opcode()],
1071 NodeClassNames[argument(1)->Opcode()],
1072 NodeClassNames[argument(2)->Opcode()],
1073 NodeClassNames[argument(3)->Opcode()],
1074 NodeClassNames[argument(4)->Opcode()]);
1075 }
1076 return false; // not enough info for intrinsification
1077 }
1078 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(mask_klass)) {
1079 if (C->print_intrinsics()) {
1080 tty->print_cr(" ** klass argument not initialized");
1081 }
1082 return false;
1083 }
1084 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1085 if (!elem_type->is_primitive_type()) {
1086 if (C->print_intrinsics()) {
1087 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1088 }
1089 return false; // should be primitive type
1090 }
1091
1092 int num_elem = vlen->get_con();
1093 BasicType elem_bt = elem_type->basic_type();
1094 BasicType mask_bt = elem_bt;
1095
1096 if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
1097 if (C->print_intrinsics()) {
1098 tty->print_cr(" ** not supported: arity=2 op=comp/%d vlen=%d etype=%s ismask=usestore",
1099 cond->get_con(), num_elem, type2name(elem_bt));
1100 }
1101 return false;
1102 }
1103
1104 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1105 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1106
1107 ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
1108 const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
1109
1110 Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
1111 Node* v2 = unbox_vector(argument(6), vbox_type, elem_bt, num_elem);
1112
1113 if (v1 == NULL || v2 == NULL) {
1114 return false; // operand unboxing failed
1115 }
1116 BoolTest::mask pred = (BoolTest::mask)cond->get_con();
1117 ConINode* pred_node = (ConINode*)gvn().makecon(cond);
1118
1119 const TypeVect* vt = TypeVect::make(mask_bt, num_elem);
1120 Node* operation = gvn().transform(new VectorMaskCmpNode(pred, v1, v2, pred_node, vt));
1121
1122 Node* box = box_vector(operation, mbox_type, mask_bt, num_elem);
1123 set_result(box);
1124 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1125 return true;
1126 }
1127
1128 // public static
1129 // <V extends Vector, Sh extends Shuffle>
1130 // V rearrangeOp(Class<V> vectorClass, Class<Sh> shuffleClass, Class< ? > elementType, int vlen,
1131 // V v1, Sh sh,
1132 // VectorSwizzleOp<V, Sh, S, E> defaultImpl) { ...
1133
inline_vector_rearrange()1134 bool LibraryCallKit::inline_vector_rearrange() {
1135 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1136 const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
1137 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1138 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1139
1140 if (vector_klass == NULL || shuffle_klass == NULL || elem_klass == NULL || vlen == NULL) {
1141 return false; // dead code
1142 }
1143 if (shuffle_klass->const_oop() == NULL || vector_klass->const_oop() == NULL ||
1144 elem_klass->const_oop() == NULL || !vlen->is_con()) {
1145 if (C->print_intrinsics()) {
1146 tty->print_cr(" ** missing constant: vclass=%s sclass=%s etype=%s vlen=%s",
1147 NodeClassNames[argument(0)->Opcode()],
1148 NodeClassNames[argument(1)->Opcode()],
1149 NodeClassNames[argument(2)->Opcode()],
1150 NodeClassNames[argument(3)->Opcode()]);
1151 }
1152 return false; // not enough info for intrinsification
1153 }
1154 if (!is_klass_initialized(vector_klass) || !is_klass_initialized(shuffle_klass)) {
1155 if (C->print_intrinsics()) {
1156 tty->print_cr(" ** klass argument not initialized");
1157 }
1158 return false;
1159 }
1160 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1161 if (!elem_type->is_primitive_type()) {
1162 if (C->print_intrinsics()) {
1163 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1164 }
1165 return false; // should be primitive type
1166 }
1167 BasicType elem_bt = elem_type->basic_type();
1168 BasicType shuffle_bt = elem_bt;
1169 int num_elem = vlen->get_con();
1170
1171 if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
1172 if (C->print_intrinsics()) {
1173 tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
1174 num_elem, type2name(elem_bt));
1175 }
1176 return false; // not supported
1177 }
1178 if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed)) {
1179 if (C->print_intrinsics()) {
1180 tty->print_cr(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
1181 num_elem, type2name(elem_bt));
1182 }
1183 return false; // not supported
1184 }
1185 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1186 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1187
1188 ciKlass* shbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
1189 const TypeInstPtr* shbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, shbox_klass);
1190
1191 Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1192 Node* shuffle = unbox_vector(argument(5), shbox_type, shuffle_bt, num_elem);
1193
1194 if (v1 == NULL || shuffle == NULL) {
1195 return false; // operand unboxing failed
1196 }
1197
1198 Node* rearrange = gvn().transform(new VectorRearrangeNode(v1, shuffle));
1199
1200 Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
1201 set_result(box);
1202 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1203 return true;
1204 }
1205
1206 // public static
1207 // <V extends Vector<?,?>>
1208 // V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int vlen,
1209 // V v, int i,
1210 // VectorBroadcastIntOp<V> defaultImpl) {
1211 //
inline_vector_broadcast_int()1212 bool LibraryCallKit::inline_vector_broadcast_int() {
1213 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1214 const TypeInstPtr* vector_klass = gvn().type(argument(1))->isa_instptr();
1215 const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
1216 const TypeInt* vlen = gvn().type(argument(3))->isa_int();
1217
1218 if (opr == NULL || vector_klass == NULL || elem_klass == NULL || vlen == NULL) {
1219 return false; // dead code
1220 }
1221 if (!opr->is_con() || vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con()) {
1222 if (C->print_intrinsics()) {
1223 tty->print_cr(" ** missing constant: opr=%s vclass=%s etype=%s vlen=%s",
1224 NodeClassNames[argument(0)->Opcode()],
1225 NodeClassNames[argument(1)->Opcode()],
1226 NodeClassNames[argument(2)->Opcode()],
1227 NodeClassNames[argument(3)->Opcode()]);
1228 }
1229 return false; // not enough info for intrinsification
1230 }
1231 if (!is_klass_initialized(vector_klass)) {
1232 if (C->print_intrinsics()) {
1233 tty->print_cr(" ** klass argument not initialized");
1234 }
1235 return false;
1236 }
1237 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1238 if (!elem_type->is_primitive_type()) {
1239 if (C->print_intrinsics()) {
1240 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1241 }
1242 return false; // should be primitive type
1243 }
1244 BasicType elem_bt = elem_type->basic_type();
1245 int num_elem = vlen->get_con();
1246 int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
1247 if (opc == 0 || !VectorNode::is_shift_opcode(opc)) {
1248 if (C->print_intrinsics()) {
1249 tty->print_cr(" ** operation not supported: op=%d bt=%s", opr->get_con(), type2name(elem_bt));
1250 }
1251 return false; // operation not supported
1252 }
1253 int sopc = VectorNode::opcode(opc, elem_bt);
1254 if (sopc == 0) {
1255 if (C->print_intrinsics()) {
1256 tty->print_cr(" ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
1257 }
1258 return false; // operation not supported
1259 }
1260 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1261 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1262
1263 if (!arch_supports_vector(sopc, num_elem, elem_bt, VecMaskNotUsed, true /*has_scalar_args*/)) {
1264 if (C->print_intrinsics()) {
1265 tty->print_cr(" ** not supported: arity=0 op=int/%d vlen=%d etype=%s ismask=no",
1266 sopc, num_elem, type2name(elem_bt));
1267 }
1268 return false; // not supported
1269 }
1270 Node* opd1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
1271 Node* opd2 = vector_shift_count(argument(5), opc, elem_bt, num_elem);
1272 if (opd1 == NULL || opd2 == NULL) {
1273 return false;
1274 }
1275 Node* operation = gvn().transform(VectorNode::make(opc, opd1, opd2, num_elem, elem_bt));
1276
1277 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
1278 set_result(vbox);
1279 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1280 return true;
1281 }
1282
1283 // public static <VOUT extends VectorPayload,
1284 // VIN extends VectorPayload,
1285 // S extends VectorSpecies>
1286 // VOUT convert(int oprId,
1287 // Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen,
1288 // Class<?> toVectorClass, Class<?> toElementType, int toVLen,
1289 // VIN v, S s,
1290 // VectorConvertOp<VOUT, VIN, S> defaultImpl) {
1291 //
inline_vector_convert()1292 bool LibraryCallKit::inline_vector_convert() {
1293 const TypeInt* opr = gvn().type(argument(0))->isa_int();
1294
1295 const TypeInstPtr* vector_klass_from = gvn().type(argument(1))->isa_instptr();
1296 const TypeInstPtr* elem_klass_from = gvn().type(argument(2))->isa_instptr();
1297 const TypeInt* vlen_from = gvn().type(argument(3))->isa_int();
1298
1299 const TypeInstPtr* vector_klass_to = gvn().type(argument(4))->isa_instptr();
1300 const TypeInstPtr* elem_klass_to = gvn().type(argument(5))->isa_instptr();
1301 const TypeInt* vlen_to = gvn().type(argument(6))->isa_int();
1302
1303 if (opr == NULL ||
1304 vector_klass_from == NULL || elem_klass_from == NULL || vlen_from == NULL ||
1305 vector_klass_to == NULL || elem_klass_to == NULL || vlen_to == NULL) {
1306 return false; // dead code
1307 }
1308 if (!opr->is_con() ||
1309 vector_klass_from->const_oop() == NULL || elem_klass_from->const_oop() == NULL || !vlen_from->is_con() ||
1310 vector_klass_to->const_oop() == NULL || elem_klass_to->const_oop() == NULL || !vlen_to->is_con()) {
1311 if (C->print_intrinsics()) {
1312 tty->print_cr(" ** missing constant: opr=%s vclass_from=%s etype_from=%s vlen_from=%s vclass_to=%s etype_to=%s vlen_to=%s",
1313 NodeClassNames[argument(0)->Opcode()],
1314 NodeClassNames[argument(1)->Opcode()],
1315 NodeClassNames[argument(2)->Opcode()],
1316 NodeClassNames[argument(3)->Opcode()],
1317 NodeClassNames[argument(4)->Opcode()],
1318 NodeClassNames[argument(5)->Opcode()],
1319 NodeClassNames[argument(6)->Opcode()]);
1320 }
1321 return false; // not enough info for intrinsification
1322 }
1323 if (!is_klass_initialized(vector_klass_from) || !is_klass_initialized(vector_klass_to)) {
1324 if (C->print_intrinsics()) {
1325 tty->print_cr(" ** klass argument not initialized");
1326 }
1327 return false;
1328 }
1329
1330 assert(opr->get_con() == VectorSupport::VECTOR_OP_CAST ||
1331 opr->get_con() == VectorSupport::VECTOR_OP_REINTERPRET, "wrong opcode");
1332 bool is_cast = (opr->get_con() == VectorSupport::VECTOR_OP_CAST);
1333
1334 ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
1335 ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
1336 if (is_vector_shuffle(vbox_klass_from) || is_vector_shuffle(vbox_klass_to)) {
1337 return false; // vector shuffles aren't supported
1338 }
1339 bool is_mask = is_vector_mask(vbox_klass_from);
1340
1341 ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();
1342 if (!elem_type_from->is_primitive_type()) {
1343 return false; // should be primitive type
1344 }
1345 BasicType elem_bt_from = elem_type_from->basic_type();
1346 ciType* elem_type_to = elem_klass_to->const_oop()->as_instance()->java_mirror_type();
1347 if (!elem_type_to->is_primitive_type()) {
1348 return false; // should be primitive type
1349 }
1350 BasicType elem_bt_to = elem_type_to->basic_type();
1351 if (is_mask && elem_bt_from != elem_bt_to) {
1352 return false; // type mismatch
1353 }
1354 int num_elem_from = vlen_from->get_con();
1355 int num_elem_to = vlen_to->get_con();
1356
1357 // Check whether we can unbox to appropriate size. Even with casting, checking for reinterpret is needed
1358 // since we may need to change size.
1359 if (!arch_supports_vector(Op_VectorReinterpret,
1360 num_elem_from,
1361 elem_bt_from,
1362 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
1363 if (C->print_intrinsics()) {
1364 tty->print_cr(" ** not supported: arity=1 op=%s/1 vlen1=%d etype1=%s ismask=%d",
1365 is_cast ? "cast" : "reinterpret",
1366 num_elem_from, type2name(elem_bt_from), is_mask);
1367 }
1368 return false;
1369 }
1370
1371 // Check whether we can support resizing/reinterpreting to the new size.
1372 if (!arch_supports_vector(Op_VectorReinterpret,
1373 num_elem_to,
1374 elem_bt_to,
1375 is_mask ? VecMaskUseAll : VecMaskNotUsed)) {
1376 if (C->print_intrinsics()) {
1377 tty->print_cr(" ** not supported: arity=1 op=%s/2 vlen2=%d etype2=%s ismask=%d",
1378 is_cast ? "cast" : "reinterpret",
1379 num_elem_to, type2name(elem_bt_to), is_mask);
1380 }
1381 return false;
1382 }
1383
1384 // At this point, we know that both input and output vector registers are supported
1385 // by the architecture. Next check if the casted type is simply to same type - which means
1386 // that it is actually a resize and not a cast.
1387 if (is_cast && elem_bt_from == elem_bt_to) {
1388 is_cast = false;
1389 }
1390
1391 const TypeInstPtr* vbox_type_from = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_from);
1392
1393 Node* opd1 = unbox_vector(argument(7), vbox_type_from, elem_bt_from, num_elem_from);
1394 if (opd1 == NULL) {
1395 return false;
1396 }
1397
1398 const TypeVect* src_type = TypeVect::make(elem_bt_from, num_elem_from);
1399 const TypeVect* dst_type = TypeVect::make(elem_bt_to, num_elem_to);
1400
1401 Node* op = opd1;
1402 if (is_cast) {
1403 assert(!is_mask, "masks cannot be casted");
1404 int cast_vopc = VectorCastNode::opcode(elem_bt_from);
1405 // Make sure that cast is implemented to particular type/size combination.
1406 if (!arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) {
1407 if (C->print_intrinsics()) {
1408 tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d",
1409 cast_vopc,
1410 num_elem_to, type2name(elem_bt_to), is_mask);
1411 }
1412 return false;
1413 }
1414
1415 if (num_elem_from < num_elem_to) {
1416 // Since input and output number of elements are not consistent, we need to make sure we
1417 // properly size. Thus, first make a cast that retains the number of elements from source.
1418 // In case the size exceeds the arch size, we do the minimum.
1419 int num_elem_for_cast = MIN2(num_elem_from, Matcher::max_vector_size(elem_bt_to));
1420
1421 // It is possible that arch does not support this intermediate vector size
1422 // TODO More complex logic required here to handle this corner case for the sizes.
1423 if (!arch_supports_vector(cast_vopc, num_elem_for_cast, elem_bt_to, VecMaskNotUsed)) {
1424 if (C->print_intrinsics()) {
1425 tty->print_cr(" ** not supported: arity=1 op=cast#%d/4 vlen1=%d etype2=%s ismask=%d",
1426 cast_vopc,
1427 num_elem_for_cast, type2name(elem_bt_to), is_mask);
1428 }
1429 return false;
1430 }
1431
1432 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_for_cast));
1433 // Now ensure that the destination gets properly resized to needed size.
1434 op = gvn().transform(new VectorReinterpretNode(op, op->bottom_type()->is_vect(), dst_type));
1435 } else if (num_elem_from > num_elem_to) {
1436 // Since number elements from input is larger than output, simply reduce size of input (we are supposed to
1437 // drop top elements anyway).
1438 int num_elem_for_resize = MAX2(num_elem_to, Matcher::min_vector_size(elem_bt_from));
1439
1440 // It is possible that arch does not support this intermediate vector size
1441 // TODO More complex logic required here to handle this corner case for the sizes.
1442 if (!arch_supports_vector(Op_VectorReinterpret,
1443 num_elem_for_resize,
1444 elem_bt_from,
1445 VecMaskNotUsed)) {
1446 if (C->print_intrinsics()) {
1447 tty->print_cr(" ** not supported: arity=1 op=cast/5 vlen2=%d etype1=%s ismask=%d",
1448 num_elem_for_resize, type2name(elem_bt_from), is_mask);
1449 }
1450 return false;
1451 }
1452
1453 op = gvn().transform(new VectorReinterpretNode(op,
1454 src_type,
1455 TypeVect::make(elem_bt_from,
1456 num_elem_for_resize)));
1457 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
1458 } else {
1459 // Since input and output number of elements match, and since we know this vector size is
1460 // supported, simply do a cast with no resize needed.
1461 op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to));
1462 }
1463 } else if (Type::cmp(src_type, dst_type) != 0) {
1464 assert(!is_cast, "must be reinterpret");
1465 op = gvn().transform(new VectorReinterpretNode(op, src_type, dst_type));
1466 }
1467
1468 const TypeInstPtr* vbox_type_to = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass_to);
1469 Node* vbox = box_vector(op, vbox_type_to, elem_bt_to, num_elem_to);
1470 set_result(vbox);
1471 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem_to * type2aelembytes(elem_bt_to))));
1472 return true;
1473 }
1474
1475 // public static
1476 // <V extends Vector<?>>
1477 // V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen,
1478 // V vec, int ix, long val,
1479 // VecInsertOp<V> defaultImpl) {
1480 //
inline_vector_insert()1481 bool LibraryCallKit::inline_vector_insert() {
1482 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1483 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
1484 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
1485 const TypeInt* idx = gvn().type(argument(4))->isa_int();
1486
1487 if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
1488 return false; // dead code
1489 }
1490 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1491 if (C->print_intrinsics()) {
1492 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1493 NodeClassNames[argument(0)->Opcode()],
1494 NodeClassNames[argument(1)->Opcode()],
1495 NodeClassNames[argument(2)->Opcode()],
1496 NodeClassNames[argument(4)->Opcode()]);
1497 }
1498 return false; // not enough info for intrinsification
1499 }
1500 if (!is_klass_initialized(vector_klass)) {
1501 if (C->print_intrinsics()) {
1502 tty->print_cr(" ** klass argument not initialized");
1503 }
1504 return false;
1505 }
1506 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1507 if (!elem_type->is_primitive_type()) {
1508 if (C->print_intrinsics()) {
1509 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1510 }
1511 return false; // should be primitive type
1512 }
1513 BasicType elem_bt = elem_type->basic_type();
1514 int num_elem = vlen->get_con();
1515 if (!arch_supports_vector(Op_VectorInsert, num_elem, elem_bt, VecMaskNotUsed)) {
1516 if (C->print_intrinsics()) {
1517 tty->print_cr(" ** not supported: arity=1 op=insert vlen=%d etype=%s ismask=no",
1518 num_elem, type2name(elem_bt));
1519 }
1520 return false; // not supported
1521 }
1522
1523 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1524 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1525
1526 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
1527 if (opd == NULL) {
1528 return false;
1529 }
1530
1531 Node* insert_val = argument(5);
1532 assert(gvn().type(insert_val)->isa_long() != NULL, "expected to be long");
1533
1534 // Convert insert value back to its appropriate type.
1535 switch (elem_bt) {
1536 case T_BYTE:
1537 insert_val = gvn().transform(new ConvL2INode(insert_val));
1538 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::BYTE));
1539 break;
1540 case T_SHORT:
1541 insert_val = gvn().transform(new ConvL2INode(insert_val));
1542 insert_val = gvn().transform(new CastIINode(insert_val, TypeInt::SHORT));
1543 break;
1544 case T_INT:
1545 insert_val = gvn().transform(new ConvL2INode(insert_val));
1546 break;
1547 case T_FLOAT:
1548 insert_val = gvn().transform(new ConvL2INode(insert_val));
1549 insert_val = gvn().transform(new MoveI2FNode(insert_val));
1550 break;
1551 case T_DOUBLE:
1552 insert_val = gvn().transform(new MoveL2DNode(insert_val));
1553 break;
1554 case T_LONG:
1555 // no conversion needed
1556 break;
1557 default: fatal("%s", type2name(elem_bt)); break;
1558 }
1559
1560 Node* operation = gvn().transform(VectorInsertNode::make(opd, insert_val, idx->get_con()));
1561
1562 Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
1563 set_result(vbox);
1564 C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
1565 return true;
1566 }
1567
1568 // public static
1569 // <V extends Vector<?>>
1570 // long extract(Class<?> vectorClass, Class<?> elementType, int vlen,
1571 // V vec, int ix,
1572 // VecExtractOp<V> defaultImpl) {
1573 //
inline_vector_extract()1574 bool LibraryCallKit::inline_vector_extract() {
1575 const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
1576 const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
1577 const TypeInt* vlen = gvn().type(argument(2))->isa_int();
1578 const TypeInt* idx = gvn().type(argument(4))->isa_int();
1579
1580 if (vector_klass == NULL || elem_klass == NULL || vlen == NULL || idx == NULL) {
1581 return false; // dead code
1582 }
1583 if (vector_klass->const_oop() == NULL || elem_klass->const_oop() == NULL || !vlen->is_con() || !idx->is_con()) {
1584 if (C->print_intrinsics()) {
1585 tty->print_cr(" ** missing constant: vclass=%s etype=%s vlen=%s idx=%s",
1586 NodeClassNames[argument(0)->Opcode()],
1587 NodeClassNames[argument(1)->Opcode()],
1588 NodeClassNames[argument(2)->Opcode()],
1589 NodeClassNames[argument(4)->Opcode()]);
1590 }
1591 return false; // not enough info for intrinsification
1592 }
1593 if (!is_klass_initialized(vector_klass)) {
1594 if (C->print_intrinsics()) {
1595 tty->print_cr(" ** klass argument not initialized");
1596 }
1597 return false;
1598 }
1599 ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
1600 if (!elem_type->is_primitive_type()) {
1601 if (C->print_intrinsics()) {
1602 tty->print_cr(" ** not a primitive bt=%d", elem_type->basic_type());
1603 }
1604 return false; // should be primitive type
1605 }
1606 BasicType elem_bt = elem_type->basic_type();
1607 int num_elem = vlen->get_con();
1608 int vopc = ExtractNode::opcode(elem_bt);
1609 if (!arch_supports_vector(vopc, num_elem, elem_bt, VecMaskNotUsed)) {
1610 if (C->print_intrinsics()) {
1611 tty->print_cr(" ** not supported: arity=1 op=extract vlen=%d etype=%s ismask=no",
1612 num_elem, type2name(elem_bt));
1613 }
1614 return false; // not supported
1615 }
1616
1617 ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
1618 const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
1619
1620 Node* opd = unbox_vector(argument(3), vbox_type, elem_bt, num_elem);
1621 if (opd == NULL) {
1622 return false;
1623 }
1624
1625 Node* operation = gvn().transform(ExtractNode::make(opd, idx->get_con(), elem_bt));
1626
1627 Node* bits = NULL;
1628 switch (elem_bt) {
1629 case T_BYTE:
1630 case T_SHORT:
1631 case T_INT: {
1632 bits = gvn().transform(new ConvI2LNode(operation));
1633 break;
1634 }
1635 case T_FLOAT: {
1636 bits = gvn().transform(new MoveF2INode(operation));
1637 bits = gvn().transform(new ConvI2LNode(bits));
1638 break;
1639 }
1640 case T_DOUBLE: {
1641 bits = gvn().transform(new MoveD2LNode(operation));
1642 break;
1643 }
1644 case T_LONG: {
1645 bits = operation; // no conversion needed
1646 break;
1647 }
1648 default: fatal("%s", type2name(elem_bt));
1649 }
1650
1651 set_result(bits);
1652 return true;
1653 }
1654
1655