1 /*
2 * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.inline.hpp"
26 #include "opto/connode.hpp"
27 #include "opto/mulnode.hpp"
28 #include "opto/subnode.hpp"
29 #include "opto/vectornode.hpp"
30 #include "utilities/powerOfTwo.hpp"
31 #include "utilities/globalDefinitions.hpp"
32
33 //------------------------------VectorNode--------------------------------------
34
35 // Return the vector operator for the specified scalar operation
36 // and vector length.
opcode(int sopc,BasicType bt)37 int VectorNode::opcode(int sopc, BasicType bt) {
38 switch (sopc) {
39 case Op_AddI:
40 switch (bt) {
41 case T_BOOLEAN:
42 case T_BYTE: return Op_AddVB;
43 case T_CHAR:
44 case T_SHORT: return Op_AddVS;
45 case T_INT: return Op_AddVI;
46 default: return 0;
47 }
48 case Op_AddL: return (bt == T_LONG ? Op_AddVL : 0);
49 case Op_AddF: return (bt == T_FLOAT ? Op_AddVF : 0);
50 case Op_AddD: return (bt == T_DOUBLE ? Op_AddVD : 0);
51
52 case Op_SubI:
53 switch (bt) {
54 case T_BOOLEAN:
55 case T_BYTE: return Op_SubVB;
56 case T_CHAR:
57 case T_SHORT: return Op_SubVS;
58 case T_INT: return Op_SubVI;
59 default: return 0;
60 }
61 case Op_SubL: return (bt == T_LONG ? Op_SubVL : 0);
62 case Op_SubF: return (bt == T_FLOAT ? Op_SubVF : 0);
63 case Op_SubD: return (bt == T_DOUBLE ? Op_SubVD : 0);
64
65 case Op_MulI:
66 switch (bt) {
67 case T_BOOLEAN:return 0;
68 case T_BYTE: return Op_MulVB;
69 case T_CHAR:
70 case T_SHORT: return Op_MulVS;
71 case T_INT: return Op_MulVI;
72 default: return 0;
73 }
74 case Op_MulL: return (bt == T_LONG ? Op_MulVL : 0);
75 case Op_MulF:
76 return (bt == T_FLOAT ? Op_MulVF : 0);
77 case Op_MulD:
78 return (bt == T_DOUBLE ? Op_MulVD : 0);
79 case Op_FmaD:
80 return (bt == T_DOUBLE ? Op_FmaVD : 0);
81 case Op_FmaF:
82 return (bt == T_FLOAT ? Op_FmaVF : 0);
83 case Op_CMoveF:
84 return (bt == T_FLOAT ? Op_CMoveVF : 0);
85 case Op_CMoveD:
86 return (bt == T_DOUBLE ? Op_CMoveVD : 0);
87 case Op_DivF:
88 return (bt == T_FLOAT ? Op_DivVF : 0);
89 case Op_DivD:
90 return (bt == T_DOUBLE ? Op_DivVD : 0);
91 case Op_AbsI:
92 switch (bt) {
93 case T_BOOLEAN:
94 case T_CHAR: return 0; // abs does not make sense for unsigned
95 case T_BYTE: return Op_AbsVB;
96 case T_SHORT: return Op_AbsVS;
97 case T_INT: return Op_AbsVI;
98 default: return 0;
99 }
100 case Op_AbsL:
101 return (bt == T_LONG ? Op_AbsVL : 0);
102 case Op_MinI:
103 switch (bt) {
104 case T_BOOLEAN:
105 case T_CHAR: return 0;
106 case T_BYTE:
107 case T_SHORT:
108 case T_INT: return Op_MinV;
109 default: return 0;
110 }
111 case Op_MinL:
112 return (bt == T_LONG ? Op_MinV : 0);
113 case Op_MinF:
114 return (bt == T_FLOAT ? Op_MinV : 0);
115 case Op_MinD:
116 return (bt == T_DOUBLE ? Op_MinV : 0);
117 case Op_MaxI:
118 switch (bt) {
119 case T_BOOLEAN:
120 case T_CHAR: return 0;
121 case T_BYTE:
122 case T_SHORT:
123 case T_INT: return Op_MaxV;
124 default: return 0;
125 }
126 case Op_MaxL:
127 return (bt == T_LONG ? Op_MaxV : 0);
128 case Op_MaxF:
129 return (bt == T_FLOAT ? Op_MaxV : 0);
130 case Op_MaxD:
131 return (bt == T_DOUBLE ? Op_MaxV : 0);
132 case Op_AbsF:
133 return (bt == T_FLOAT ? Op_AbsVF : 0);
134 case Op_AbsD:
135 return (bt == T_DOUBLE ? Op_AbsVD : 0);
136 case Op_NegI:
137 return (bt == T_INT ? Op_NegVI : 0);
138 case Op_NegF:
139 return (bt == T_FLOAT ? Op_NegVF : 0);
140 case Op_NegD:
141 return (bt == T_DOUBLE ? Op_NegVD : 0);
142 case Op_RoundDoubleMode:
143 return (bt == T_DOUBLE ? Op_RoundDoubleModeV : 0);
144 case Op_RotateLeft:
145 return (bt == T_LONG || bt == T_INT ? Op_RotateLeftV : 0);
146 case Op_RotateRight:
147 return (bt == T_LONG || bt == T_INT ? Op_RotateRightV : 0);
148 case Op_SqrtF:
149 return (bt == T_FLOAT ? Op_SqrtVF : 0);
150 case Op_SqrtD:
151 return (bt == T_DOUBLE ? Op_SqrtVD : 0);
152 case Op_PopCountI:
153 // Unimplemented for subword types since bit count changes
154 // depending on size of lane (and sign bit).
155 return (bt == T_INT ? Op_PopCountVI : 0);
156 case Op_LShiftI:
157 switch (bt) {
158 case T_BOOLEAN:
159 case T_BYTE: return Op_LShiftVB;
160 case T_CHAR:
161 case T_SHORT: return Op_LShiftVS;
162 case T_INT: return Op_LShiftVI;
163 default: return 0;
164 }
165 case Op_LShiftL:
166 return (bt == T_LONG ? Op_LShiftVL : 0);
167 case Op_RShiftI:
168 switch (bt) {
169 case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value
170 case T_CHAR: return Op_URShiftVS; // char is unsigned value
171 case T_BYTE: return Op_RShiftVB;
172 case T_SHORT: return Op_RShiftVS;
173 case T_INT: return Op_RShiftVI;
174 default: return 0;
175 }
176 case Op_RShiftL:
177 return (bt == T_LONG ? Op_RShiftVL : 0);
178 case Op_URShiftB:
179 return (bt == T_BYTE ? Op_URShiftVB : 0);
180 case Op_URShiftS:
181 return (bt == T_SHORT ? Op_URShiftVS : 0);
182 case Op_URShiftI:
183 switch (bt) {
184 case T_BOOLEAN:return Op_URShiftVB;
185 case T_CHAR: return Op_URShiftVS;
186 case T_BYTE:
187 case T_SHORT: return 0; // Vector logical right shift for signed short
188 // values produces incorrect Java result for
189 // negative data because java code should convert
190 // a short value into int value with sign
191 // extension before a shift.
192 case T_INT: return Op_URShiftVI;
193 default: return 0;
194 }
195 case Op_URShiftL:
196 return (bt == T_LONG ? Op_URShiftVL : 0);
197 case Op_AndI:
198 case Op_AndL:
199 return Op_AndV;
200 case Op_OrI:
201 case Op_OrL:
202 return Op_OrV;
203 case Op_XorI:
204 case Op_XorL:
205 return Op_XorV;
206
207 case Op_LoadB:
208 case Op_LoadUB:
209 case Op_LoadUS:
210 case Op_LoadS:
211 case Op_LoadI:
212 case Op_LoadL:
213 case Op_LoadF:
214 case Op_LoadD:
215 return Op_LoadVector;
216
217 case Op_StoreB:
218 case Op_StoreC:
219 case Op_StoreI:
220 case Op_StoreL:
221 case Op_StoreF:
222 case Op_StoreD:
223 return Op_StoreVector;
224 case Op_MulAddS2I:
225 return Op_MulAddVS2VI;
226
227 default:
228 return 0; // Unimplemented
229 }
230 }
231
replicate_opcode(BasicType bt)232 int VectorNode::replicate_opcode(BasicType bt) {
233 switch(bt) {
234 case T_BOOLEAN:
235 case T_BYTE:
236 return Op_ReplicateB;
237 case T_SHORT:
238 case T_CHAR:
239 return Op_ReplicateS;
240 case T_INT:
241 return Op_ReplicateI;
242 case T_LONG:
243 return Op_ReplicateL;
244 case T_FLOAT:
245 return Op_ReplicateF;
246 case T_DOUBLE:
247 return Op_ReplicateD;
248 default:
249 assert(false, "wrong type: %s", type2name(bt));
250 return 0;
251 }
252 }
253
254 // Also used to check if the code generator
255 // supports the vector operation.
implemented(int opc,uint vlen,BasicType bt)256 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) {
257 if (is_java_primitive(bt) &&
258 (vlen > 1) && is_power_of_2(vlen) &&
259 Matcher::vector_size_supported(bt, vlen)) {
260 int vopc = VectorNode::opcode(opc, bt);
261 // For rotate operation we will do a lazy de-generation into
262 // OrV/LShiftV/URShiftV pattern if the target does not support
263 // vector rotation instruction.
264 if (vopc == Op_RotateLeftV || vopc == Op_RotateRightV) {
265 return is_vector_rotate_supported(vopc, vlen, bt);
266 }
267 return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt);
268 }
269 return false;
270 }
271
is_type_transition_short_to_int(Node * n)272 bool VectorNode::is_type_transition_short_to_int(Node* n) {
273 switch (n->Opcode()) {
274 case Op_MulAddS2I:
275 return true;
276 }
277 return false;
278 }
279
is_type_transition_to_int(Node * n)280 bool VectorNode::is_type_transition_to_int(Node* n) {
281 return is_type_transition_short_to_int(n);
282 }
283
is_muladds2i(Node * n)284 bool VectorNode::is_muladds2i(Node* n) {
285 if (n->Opcode() == Op_MulAddS2I) {
286 return true;
287 }
288 return false;
289 }
290
is_roundopD(Node * n)291 bool VectorNode::is_roundopD(Node* n) {
292 if (n->Opcode() == Op_RoundDoubleMode) {
293 return true;
294 }
295 return false;
296 }
297
is_scalar_rotate(Node * n)298 bool VectorNode::is_scalar_rotate(Node* n) {
299 if (n->Opcode() == Op_RotateLeft || n->Opcode() == Op_RotateRight) {
300 return true;
301 }
302 return false;
303 }
304
is_vector_rotate_supported(int vopc,uint vlen,BasicType bt)305 bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) {
306 assert(vopc == Op_RotateLeftV || vopc == Op_RotateRightV, "wrong opcode");
307
308 // If target defines vector rotation patterns then no
309 // need for degeneration.
310 if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) {
311 return true;
312 }
313
314 // Validate existence of nodes created in case of rotate degeneration.
315 switch (bt) {
316 case T_INT:
317 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) &&
318 Matcher::match_rule_supported_vector(Op_LShiftVI, vlen, bt) &&
319 Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt);
320 case T_LONG:
321 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) &&
322 Matcher::match_rule_supported_vector(Op_LShiftVL, vlen, bt) &&
323 Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt);
324 default:
325 assert(false, "not supported: %s", type2name(bt));
326 return false;
327 }
328 }
329
is_shift_opcode(int opc)330 bool VectorNode::is_shift_opcode(int opc) {
331 switch (opc) {
332 case Op_LShiftI:
333 case Op_LShiftL:
334 case Op_RShiftI:
335 case Op_RShiftL:
336 case Op_URShiftB:
337 case Op_URShiftS:
338 case Op_URShiftI:
339 case Op_URShiftL:
340 return true;
341 default:
342 return false;
343 }
344 }
345
is_shift(Node * n)346 bool VectorNode::is_shift(Node* n) {
347 return is_shift_opcode(n->Opcode());
348 }
349
is_vshift_cnt(Node * n)350 bool VectorNode::is_vshift_cnt(Node* n) {
351 switch (n->Opcode()) {
352 case Op_LShiftCntV:
353 case Op_RShiftCntV:
354 return true;
355 default:
356 return false;
357 }
358 }
359
360 // Check if input is loop invariant vector.
is_invariant_vector(Node * n)361 bool VectorNode::is_invariant_vector(Node* n) {
362 // Only Replicate vector nodes are loop invariant for now.
363 switch (n->Opcode()) {
364 case Op_ReplicateB:
365 case Op_ReplicateS:
366 case Op_ReplicateI:
367 case Op_ReplicateL:
368 case Op_ReplicateF:
369 case Op_ReplicateD:
370 return true;
371 default:
372 return false;
373 }
374 }
375
376 // [Start, end) half-open range defining which operands are vectors
vector_operands(Node * n,uint * start,uint * end)377 void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
378 switch (n->Opcode()) {
379 case Op_LoadB: case Op_LoadUB:
380 case Op_LoadS: case Op_LoadUS:
381 case Op_LoadI: case Op_LoadL:
382 case Op_LoadF: case Op_LoadD:
383 case Op_LoadP: case Op_LoadN:
384 *start = 0;
385 *end = 0; // no vector operands
386 break;
387 case Op_StoreB: case Op_StoreC:
388 case Op_StoreI: case Op_StoreL:
389 case Op_StoreF: case Op_StoreD:
390 case Op_StoreP: case Op_StoreN:
391 *start = MemNode::ValueIn;
392 *end = MemNode::ValueIn + 1; // 1 vector operand
393 break;
394 case Op_LShiftI: case Op_LShiftL:
395 case Op_RShiftI: case Op_RShiftL:
396 case Op_URShiftI: case Op_URShiftL:
397 *start = 1;
398 *end = 2; // 1 vector operand
399 break;
400 case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD:
401 case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD:
402 case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD:
403 case Op_DivF: case Op_DivD:
404 case Op_AndI: case Op_AndL:
405 case Op_OrI: case Op_OrL:
406 case Op_XorI: case Op_XorL:
407 case Op_MulAddS2I:
408 *start = 1;
409 *end = 3; // 2 vector operands
410 break;
411 case Op_CMoveI: case Op_CMoveL: case Op_CMoveF: case Op_CMoveD:
412 *start = 2;
413 *end = n->req();
414 break;
415 case Op_FmaD:
416 case Op_FmaF:
417 *start = 1;
418 *end = 4; // 3 vector operands
419 break;
420 default:
421 *start = 1;
422 *end = n->req(); // default is all operands
423 }
424 }
425
426 // Make a vector node for binary operation
make(int vopc,Node * n1,Node * n2,const TypeVect * vt)427 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt) {
428 // This method should not be called for unimplemented vectors.
429 guarantee(vopc > 0, "vopc must be > 0");
430 switch (vopc) {
431 case Op_AddVB: return new AddVBNode(n1, n2, vt);
432 case Op_AddVS: return new AddVSNode(n1, n2, vt);
433 case Op_AddVI: return new AddVINode(n1, n2, vt);
434 case Op_AddVL: return new AddVLNode(n1, n2, vt);
435 case Op_AddVF: return new AddVFNode(n1, n2, vt);
436 case Op_AddVD: return new AddVDNode(n1, n2, vt);
437
438 case Op_SubVB: return new SubVBNode(n1, n2, vt);
439 case Op_SubVS: return new SubVSNode(n1, n2, vt);
440 case Op_SubVI: return new SubVINode(n1, n2, vt);
441 case Op_SubVL: return new SubVLNode(n1, n2, vt);
442 case Op_SubVF: return new SubVFNode(n1, n2, vt);
443 case Op_SubVD: return new SubVDNode(n1, n2, vt);
444
445 case Op_MulVB: return new MulVBNode(n1, n2, vt);
446 case Op_MulVS: return new MulVSNode(n1, n2, vt);
447 case Op_MulVI: return new MulVINode(n1, n2, vt);
448 case Op_MulVL: return new MulVLNode(n1, n2, vt);
449 case Op_MulVF: return new MulVFNode(n1, n2, vt);
450 case Op_MulVD: return new MulVDNode(n1, n2, vt);
451
452 case Op_DivVF: return new DivVFNode(n1, n2, vt);
453 case Op_DivVD: return new DivVDNode(n1, n2, vt);
454
455 case Op_MinV: return new MinVNode(n1, n2, vt);
456 case Op_MaxV: return new MaxVNode(n1, n2, vt);
457
458 case Op_AbsVF: return new AbsVFNode(n1, vt);
459 case Op_AbsVD: return new AbsVDNode(n1, vt);
460 case Op_AbsVB: return new AbsVBNode(n1, vt);
461 case Op_AbsVS: return new AbsVSNode(n1, vt);
462 case Op_AbsVI: return new AbsVINode(n1, vt);
463 case Op_AbsVL: return new AbsVLNode(n1, vt);
464
465 case Op_NegVI: return new NegVINode(n1, vt);
466 case Op_NegVF: return new NegVFNode(n1, vt);
467 case Op_NegVD: return new NegVDNode(n1, vt);
468
469 case Op_SqrtVF: return new SqrtVFNode(n1, vt);
470 case Op_SqrtVD: return new SqrtVDNode(n1, vt);
471
472 case Op_PopCountVI: return new PopCountVINode(n1, vt);
473 case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt);
474 case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt);
475
476 case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt);
477 case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt);
478 case Op_LShiftVI: return new LShiftVINode(n1, n2, vt);
479 case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt);
480
481 case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt);
482 case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt);
483 case Op_RShiftVI: return new RShiftVINode(n1, n2, vt);
484 case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt);
485
486 case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt);
487 case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt);
488 case Op_URShiftVI: return new URShiftVINode(n1, n2, vt);
489 case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt);
490
491 case Op_AndV: return new AndVNode(n1, n2, vt);
492 case Op_OrV: return new OrVNode (n1, n2, vt);
493 case Op_XorV: return new XorVNode(n1, n2, vt);
494
495 case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt);
496
497 case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt);
498 default:
499 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
500 return NULL;
501 }
502 }
503
504 // Return the vector version of a scalar binary operation node.
make(int opc,Node * n1,Node * n2,uint vlen,BasicType bt)505 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt) {
506 const TypeVect* vt = TypeVect::make(bt, vlen);
507 int vopc = VectorNode::opcode(opc, bt);
508 // This method should not be called for unimplemented vectors.
509 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
510 return make(vopc, n1, n2, vt);
511 }
512
513 // Make a vector node for ternary operation
make(int vopc,Node * n1,Node * n2,Node * n3,const TypeVect * vt)514 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt) {
515 // This method should not be called for unimplemented vectors.
516 guarantee(vopc > 0, "vopc must be > 0");
517 switch (vopc) {
518 case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt);
519 case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt);
520 default:
521 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
522 return NULL;
523 }
524 }
525
526 // Return the vector version of a scalar ternary operation node.
make(int opc,Node * n1,Node * n2,Node * n3,uint vlen,BasicType bt)527 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) {
528 const TypeVect* vt = TypeVect::make(bt, vlen);
529 int vopc = VectorNode::opcode(opc, bt);
530 // This method should not be called for unimplemented vectors.
531 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]);
532 return make(vopc, n1, n2, n3, vt);
533 }
534
535 // Scalar promotion
scalar2vector(Node * s,uint vlen,const Type * opd_t)536 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) {
537 BasicType bt = opd_t->array_element_basic_type();
538 const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen)
539 : TypeVect::make(bt, vlen);
540 switch (bt) {
541 case T_BOOLEAN:
542 case T_BYTE:
543 return new ReplicateBNode(s, vt);
544 case T_CHAR:
545 case T_SHORT:
546 return new ReplicateSNode(s, vt);
547 case T_INT:
548 return new ReplicateINode(s, vt);
549 case T_LONG:
550 return new ReplicateLNode(s, vt);
551 case T_FLOAT:
552 return new ReplicateFNode(s, vt);
553 case T_DOUBLE:
554 return new ReplicateDNode(s, vt);
555 default:
556 fatal("Type '%s' is not supported for vectors", type2name(bt));
557 return NULL;
558 }
559 }
560
shift_count(int opc,Node * cnt,uint vlen,BasicType bt)561 VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) {
562 // Match shift count type with shift vector type.
563 const TypeVect* vt = TypeVect::make(bt, vlen);
564 switch (opc) {
565 case Op_LShiftI:
566 case Op_LShiftL:
567 return new LShiftCntVNode(cnt, vt);
568 case Op_RShiftI:
569 case Op_RShiftL:
570 case Op_URShiftB:
571 case Op_URShiftS:
572 case Op_URShiftI:
573 case Op_URShiftL:
574 return new RShiftCntVNode(cnt, vt);
575 default:
576 fatal("Missed vector creation for '%s'", NodeClassNames[opc]);
577 return NULL;
578 }
579 }
580
is_vector_shift(int opc)581 bool VectorNode::is_vector_shift(int opc) {
582 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
583 switch (opc) {
584 case Op_LShiftVB:
585 case Op_LShiftVS:
586 case Op_LShiftVI:
587 case Op_LShiftVL:
588 case Op_RShiftVB:
589 case Op_RShiftVS:
590 case Op_RShiftVI:
591 case Op_RShiftVL:
592 case Op_URShiftVB:
593 case Op_URShiftVS:
594 case Op_URShiftVI:
595 case Op_URShiftVL:
596 return true;
597 default:
598 return false;
599 }
600 }
601
is_vector_shift_count(int opc)602 bool VectorNode::is_vector_shift_count(int opc) {
603 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
604 switch (opc) {
605 case Op_RShiftCntV:
606 case Op_LShiftCntV:
607 return true;
608 default:
609 return false;
610 }
611 }
612
is_con_M1(Node * n)613 static bool is_con_M1(Node* n) {
614 if (n->is_Con()) {
615 const Type* t = n->bottom_type();
616 if (t->isa_int() && t->is_int()->get_con() == -1) {
617 return true;
618 }
619 if (t->isa_long() && t->is_long()->get_con() == -1) {
620 return true;
621 }
622 }
623 return false;
624 }
625
is_all_ones_vector(Node * n)626 bool VectorNode::is_all_ones_vector(Node* n) {
627 switch (n->Opcode()) {
628 case Op_ReplicateB:
629 case Op_ReplicateS:
630 case Op_ReplicateI:
631 case Op_ReplicateL:
632 return is_con_M1(n->in(1));
633 default:
634 return false;
635 }
636 }
637
is_vector_bitwise_not_pattern(Node * n)638 bool VectorNode::is_vector_bitwise_not_pattern(Node* n) {
639 if (n->Opcode() == Op_XorV) {
640 return is_all_ones_vector(n->in(1)) ||
641 is_all_ones_vector(n->in(2));
642 }
643 return false;
644 }
645
646 // Return initial Pack node. Additional operands added with add_opd() calls.
make(Node * s,uint vlen,BasicType bt)647 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) {
648 const TypeVect* vt = TypeVect::make(bt, vlen);
649 switch (bt) {
650 case T_BOOLEAN:
651 case T_BYTE:
652 return new PackBNode(s, vt);
653 case T_CHAR:
654 case T_SHORT:
655 return new PackSNode(s, vt);
656 case T_INT:
657 return new PackINode(s, vt);
658 case T_LONG:
659 return new PackLNode(s, vt);
660 case T_FLOAT:
661 return new PackFNode(s, vt);
662 case T_DOUBLE:
663 return new PackDNode(s, vt);
664 default:
665 fatal("Type '%s' is not supported for vectors", type2name(bt));
666 return NULL;
667 }
668 }
669
670 // Create a binary tree form for Packs. [lo, hi) (half-open) range
binary_tree_pack(int lo,int hi)671 PackNode* PackNode::binary_tree_pack(int lo, int hi) {
672 int ct = hi - lo;
673 assert(is_power_of_2(ct), "power of 2");
674 if (ct == 2) {
675 PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type());
676 pk->add_opd(in(lo+1));
677 return pk;
678 } else {
679 int mid = lo + ct/2;
680 PackNode* n1 = binary_tree_pack(lo, mid);
681 PackNode* n2 = binary_tree_pack(mid, hi );
682
683 BasicType bt = n1->vect_type()->element_basic_type();
684 assert(bt == n2->vect_type()->element_basic_type(), "should be the same");
685 switch (bt) {
686 case T_BOOLEAN:
687 case T_BYTE:
688 return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2));
689 case T_CHAR:
690 case T_SHORT:
691 return new PackINode(n1, n2, TypeVect::make(T_INT, 2));
692 case T_INT:
693 return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2));
694 case T_LONG:
695 return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2));
696 case T_FLOAT:
697 return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
698 case T_DOUBLE:
699 return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2));
700 default:
701 fatal("Type '%s' is not supported for vectors", type2name(bt));
702 return NULL;
703 }
704 }
705 }
706
707 // Return the vector version of a scalar load node.
make(int opc,Node * ctl,Node * mem,Node * adr,const TypePtr * atyp,uint vlen,BasicType bt,ControlDependency control_dependency)708 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem,
709 Node* adr, const TypePtr* atyp,
710 uint vlen, BasicType bt,
711 ControlDependency control_dependency) {
712 const TypeVect* vt = TypeVect::make(bt, vlen);
713 return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency);
714 }
715
716 // Return the vector version of a scalar store node.
make(int opc,Node * ctl,Node * mem,Node * adr,const TypePtr * atyp,Node * val,uint vlen)717 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem,
718 Node* adr, const TypePtr* atyp, Node* val,
719 uint vlen) {
720 return new StoreVectorNode(ctl, mem, adr, atyp, val);
721 }
722
Ideal(PhaseGVN * phase,bool can_reshape)723 Node* LoadVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
724 Node* mask_len = in(3)->in(1);
725 const TypeLong* ty = phase->type(mask_len)->isa_long();
726 if (ty && ty->is_con()) {
727 BasicType mask_bt = ((VectorMaskGenNode*)in(3))->get_elem_type()->array_element_basic_type();
728 uint load_sz = type2aelembytes(mask_bt) * ty->get_con();
729 if ( load_sz == 32 || load_sz == 64) {
730 assert(load_sz == 32 || MaxVectorSize > 32, "Unexpected load size");
731 Node* ctr = in(MemNode::Control);
732 Node* mem = in(MemNode::Memory);
733 Node* adr = in(MemNode::Address);
734 return phase->transform(new LoadVectorNode(ctr, mem, adr, adr_type(), vect_type()));
735 }
736 }
737 return NULL;
738 }
739
Ideal(PhaseGVN * phase,bool can_reshape)740 Node* StoreVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) {
741 Node* mask_len = in(4)->in(1);
742 const TypeLong* ty = phase->type(mask_len)->isa_long();
743 if (ty && ty->is_con()) {
744 BasicType mask_bt = ((VectorMaskGenNode*)in(4))->get_elem_type()->array_element_basic_type();
745 uint load_sz = type2aelembytes(mask_bt) * ty->get_con();
746 if ( load_sz == 32 || load_sz == 64) {
747 assert(load_sz == 32 || MaxVectorSize > 32, "Unexpected store size");
748 Node* ctr = in(MemNode::Control);
749 Node* mem = in(MemNode::Memory);
750 Node* adr = in(MemNode::Address);
751 Node* val = in(MemNode::ValueIn);
752 return phase->transform(new StoreVectorNode(ctr, mem, adr, adr_type(), val));
753 }
754 }
755 return NULL;
756 }
757
opcode(BasicType bt)758 int ExtractNode::opcode(BasicType bt) {
759 switch (bt) {
760 case T_BOOLEAN: return Op_ExtractUB;
761 case T_BYTE: return Op_ExtractB;
762 case T_CHAR: return Op_ExtractC;
763 case T_SHORT: return Op_ExtractS;
764 case T_INT: return Op_ExtractI;
765 case T_LONG: return Op_ExtractL;
766 case T_FLOAT: return Op_ExtractF;
767 case T_DOUBLE: return Op_ExtractD;
768 default:
769 assert(false, "wrong type: %s", type2name(bt));
770 return 0;
771 }
772 }
773
774 // Extract a scalar element of vector.
make(Node * v,uint position,BasicType bt)775 Node* ExtractNode::make(Node* v, uint position, BasicType bt) {
776 assert((int)position < Matcher::max_vector_size(bt), "pos in range");
777 ConINode* pos = ConINode::make((int)position);
778 switch (bt) {
779 case T_BOOLEAN: return new ExtractUBNode(v, pos);
780 case T_BYTE: return new ExtractBNode(v, pos);
781 case T_CHAR: return new ExtractCNode(v, pos);
782 case T_SHORT: return new ExtractSNode(v, pos);
783 case T_INT: return new ExtractINode(v, pos);
784 case T_LONG: return new ExtractLNode(v, pos);
785 case T_FLOAT: return new ExtractFNode(v, pos);
786 case T_DOUBLE: return new ExtractDNode(v, pos);
787 default:
788 assert(false, "wrong type: %s", type2name(bt));
789 return NULL;
790 }
791 }
792
opcode(int opc,BasicType bt)793 int ReductionNode::opcode(int opc, BasicType bt) {
794 int vopc = opc;
795 switch (opc) {
796 case Op_AddI:
797 switch (bt) {
798 case T_BOOLEAN:
799 case T_CHAR: return 0;
800 case T_BYTE:
801 case T_SHORT:
802 case T_INT:
803 vopc = Op_AddReductionVI;
804 break;
805 default: ShouldNotReachHere(); return 0;
806 }
807 break;
808 case Op_AddL:
809 assert(bt == T_LONG, "must be");
810 vopc = Op_AddReductionVL;
811 break;
812 case Op_AddF:
813 assert(bt == T_FLOAT, "must be");
814 vopc = Op_AddReductionVF;
815 break;
816 case Op_AddD:
817 assert(bt == T_DOUBLE, "must be");
818 vopc = Op_AddReductionVD;
819 break;
820 case Op_MulI:
821 switch (bt) {
822 case T_BOOLEAN:
823 case T_CHAR: return 0;
824 case T_BYTE:
825 case T_SHORT:
826 case T_INT:
827 vopc = Op_MulReductionVI;
828 break;
829 default: ShouldNotReachHere(); return 0;
830 }
831 break;
832 case Op_MulL:
833 assert(bt == T_LONG, "must be");
834 vopc = Op_MulReductionVL;
835 break;
836 case Op_MulF:
837 assert(bt == T_FLOAT, "must be");
838 vopc = Op_MulReductionVF;
839 break;
840 case Op_MulD:
841 assert(bt == T_DOUBLE, "must be");
842 vopc = Op_MulReductionVD;
843 break;
844 case Op_MinI:
845 switch (bt) {
846 case T_BOOLEAN:
847 case T_CHAR: return 0;
848 case T_BYTE:
849 case T_SHORT:
850 case T_INT:
851 vopc = Op_MinReductionV;
852 break;
853 default: ShouldNotReachHere(); return 0;
854 }
855 break;
856 case Op_MinL:
857 assert(bt == T_LONG, "must be");
858 vopc = Op_MinReductionV;
859 break;
860 case Op_MinF:
861 assert(bt == T_FLOAT, "must be");
862 vopc = Op_MinReductionV;
863 break;
864 case Op_MinD:
865 assert(bt == T_DOUBLE, "must be");
866 vopc = Op_MinReductionV;
867 break;
868 case Op_MaxI:
869 switch (bt) {
870 case T_BOOLEAN:
871 case T_CHAR: return 0;
872 case T_BYTE:
873 case T_SHORT:
874 case T_INT:
875 vopc = Op_MaxReductionV;
876 break;
877 default: ShouldNotReachHere(); return 0;
878 }
879 break;
880 case Op_MaxL:
881 assert(bt == T_LONG, "must be");
882 vopc = Op_MaxReductionV;
883 break;
884 case Op_MaxF:
885 assert(bt == T_FLOAT, "must be");
886 vopc = Op_MaxReductionV;
887 break;
888 case Op_MaxD:
889 assert(bt == T_DOUBLE, "must be");
890 vopc = Op_MaxReductionV;
891 break;
892 case Op_AndI:
893 switch (bt) {
894 case T_BOOLEAN:
895 case T_CHAR: return 0;
896 case T_BYTE:
897 case T_SHORT:
898 case T_INT:
899 vopc = Op_AndReductionV;
900 break;
901 default: ShouldNotReachHere(); return 0;
902 }
903 break;
904 case Op_AndL:
905 assert(bt == T_LONG, "must be");
906 vopc = Op_AndReductionV;
907 break;
908 case Op_OrI:
909 switch(bt) {
910 case T_BOOLEAN:
911 case T_CHAR: return 0;
912 case T_BYTE:
913 case T_SHORT:
914 case T_INT:
915 vopc = Op_OrReductionV;
916 break;
917 default: ShouldNotReachHere(); return 0;
918 }
919 break;
920 case Op_OrL:
921 assert(bt == T_LONG, "must be");
922 vopc = Op_OrReductionV;
923 break;
924 case Op_XorI:
925 switch(bt) {
926 case T_BOOLEAN:
927 case T_CHAR: return 0;
928 case T_BYTE:
929 case T_SHORT:
930 case T_INT:
931 vopc = Op_XorReductionV;
932 break;
933 default: ShouldNotReachHere(); return 0;
934 }
935 break;
936 case Op_XorL:
937 assert(bt == T_LONG, "must be");
938 vopc = Op_XorReductionV;
939 break;
940 default:
941 break;
942 }
943 return vopc;
944 }
945
946 // Return the appropriate reduction node.
make(int opc,Node * ctrl,Node * n1,Node * n2,BasicType bt)947 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) {
948
949 int vopc = opcode(opc, bt);
950
951 // This method should not be called for unimplemented vectors.
952 guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]);
953
954 switch (vopc) {
955 case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2);
956 case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2);
957 case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2);
958 case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2);
959 case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2);
960 case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2);
961 case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2);
962 case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2);
963 case Op_MinReductionV: return new MinReductionVNode(ctrl, n1, n2);
964 case Op_MaxReductionV: return new MaxReductionVNode(ctrl, n1, n2);
965 case Op_AndReductionV: return new AndReductionVNode(ctrl, n1, n2);
966 case Op_OrReductionV: return new OrReductionVNode(ctrl, n1, n2);
967 case Op_XorReductionV: return new XorReductionVNode(ctrl, n1, n2);
968 default:
969 assert(false, "unknown node: %s", NodeClassNames[vopc]);
970 return NULL;
971 }
972 }
973
Identity(PhaseGVN * phase)974 Node* VectorLoadMaskNode::Identity(PhaseGVN* phase) {
975 BasicType out_bt = type()->is_vect()->element_basic_type();
976 if (out_bt == T_BOOLEAN) {
977 return in(1); // redundant conversion
978 }
979 return this;
980 }
981
Identity(PhaseGVN * phase)982 Node* VectorStoreMaskNode::Identity(PhaseGVN* phase) {
983 // Identity transformation on boolean vectors.
984 // VectorStoreMask (VectorLoadMask bv) elem_size ==> bv
985 // vector[n]{bool} => vector[n]{t} => vector[n]{bool}
986 if (in(1)->Opcode() == Op_VectorLoadMask) {
987 return in(1)->in(1);
988 }
989 return this;
990 }
991
make(PhaseGVN & gvn,Node * in,BasicType in_type,uint num_elem)992 VectorStoreMaskNode* VectorStoreMaskNode::make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem) {
993 assert(in->bottom_type()->isa_vect(), "sanity");
994 const TypeVect* vt = TypeVect::make(T_BOOLEAN, num_elem);
995 int elem_size = type2aelembytes(in_type);
996 return new VectorStoreMaskNode(in, gvn.intcon(elem_size), vt);
997 }
998
make(int vopc,Node * n1,BasicType bt,uint vlen)999 VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) {
1000 const TypeVect* vt = TypeVect::make(bt, vlen);
1001 switch (vopc) {
1002 case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt);
1003 case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt);
1004 case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt);
1005 case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt);
1006 case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt);
1007 case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt);
1008 default:
1009 assert(false, "unknown node: %s", NodeClassNames[vopc]);
1010 return NULL;
1011 }
1012 }
1013
opcode(BasicType bt)1014 int VectorCastNode::opcode(BasicType bt) {
1015 switch (bt) {
1016 case T_BYTE: return Op_VectorCastB2X;
1017 case T_SHORT: return Op_VectorCastS2X;
1018 case T_INT: return Op_VectorCastI2X;
1019 case T_LONG: return Op_VectorCastL2X;
1020 case T_FLOAT: return Op_VectorCastF2X;
1021 case T_DOUBLE: return Op_VectorCastD2X;
1022 default:
1023 assert(false, "unknown type: %s", type2name(bt));
1024 return 0;
1025 }
1026 }
1027
Identity(PhaseGVN * phase)1028 Node* VectorCastNode::Identity(PhaseGVN* phase) {
1029 if (!in(1)->is_top()) {
1030 BasicType in_bt = in(1)->bottom_type()->is_vect()->element_basic_type();
1031 BasicType out_bt = vect_type()->element_basic_type();
1032 if (in_bt == out_bt) {
1033 return in(1); // redundant cast
1034 }
1035 }
1036 return this;
1037 }
1038
make_reduction_input(PhaseGVN & gvn,int opc,BasicType bt)1039 Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) {
1040 int vopc = opcode(opc, bt);
1041 guarantee(vopc != opc, "Vector reduction for '%s' is not implemented", NodeClassNames[opc]);
1042
1043 switch (vopc) {
1044 case Op_AndReductionV:
1045 switch (bt) {
1046 case T_BYTE:
1047 case T_SHORT:
1048 case T_INT:
1049 return gvn.makecon(TypeInt::MINUS_1);
1050 case T_LONG:
1051 return gvn.makecon(TypeLong::MINUS_1);
1052 default:
1053 fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]);
1054 return NULL;
1055 }
1056 break;
1057 case Op_AddReductionVI: // fallthrough
1058 case Op_AddReductionVL: // fallthrough
1059 case Op_AddReductionVF: // fallthrough
1060 case Op_AddReductionVD:
1061 case Op_OrReductionV:
1062 case Op_XorReductionV:
1063 return gvn.zerocon(bt);
1064 case Op_MulReductionVI:
1065 return gvn.makecon(TypeInt::ONE);
1066 case Op_MulReductionVL:
1067 return gvn.makecon(TypeLong::ONE);
1068 case Op_MulReductionVF:
1069 return gvn.makecon(TypeF::ONE);
1070 case Op_MulReductionVD:
1071 return gvn.makecon(TypeD::ONE);
1072 case Op_MinReductionV:
1073 switch (bt) {
1074 case T_BYTE:
1075 case T_SHORT:
1076 case T_INT:
1077 return gvn.makecon(TypeInt::MAX);
1078 case T_LONG:
1079 return gvn.makecon(TypeLong::MAX);
1080 case T_FLOAT:
1081 return gvn.makecon(TypeF::POS_INF);
1082 case T_DOUBLE:
1083 return gvn.makecon(TypeD::POS_INF);
1084 default: Unimplemented(); return NULL;
1085 }
1086 break;
1087 case Op_MaxReductionV:
1088 switch (bt) {
1089 case T_BYTE:
1090 case T_SHORT:
1091 case T_INT:
1092 return gvn.makecon(TypeInt::MIN);
1093 case T_LONG:
1094 return gvn.makecon(TypeLong::MIN);
1095 case T_FLOAT:
1096 return gvn.makecon(TypeF::NEG_INF);
1097 case T_DOUBLE:
1098 return gvn.makecon(TypeD::NEG_INF);
1099 default: Unimplemented(); return NULL;
1100 }
1101 break;
1102 default:
1103 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]);
1104 return NULL;
1105 }
1106 }
1107
implemented(int opc,uint vlen,BasicType bt)1108 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) {
1109 if (is_java_primitive(bt) &&
1110 (vlen > 1) && is_power_of_2(vlen) &&
1111 Matcher::vector_size_supported(bt, vlen)) {
1112 int vopc = ReductionNode::opcode(opc, bt);
1113 return vopc != opc && Matcher::match_rule_supported_vector(vopc, vlen, bt);
1114 }
1115 return false;
1116 }
1117
make(PhaseGVN & gvn,Node * in1,Node * in2,Node * in3,uint truth_table,const TypeVect * vt)1118 MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3,
1119 uint truth_table, const TypeVect* vt) {
1120 assert(truth_table <= 0xFF, "invalid");
1121 assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1122 assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1123 assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch");
1124 Node* fn = gvn.intcon(truth_table);
1125 return new MacroLogicVNode(in1, in2, in3, fn, vt);
1126 }
1127
degenerate_vector_rotate(Node * src,Node * cnt,bool is_rotate_left,int vlen,BasicType bt,PhaseGVN * phase)1128 Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left,
1129 int vlen, BasicType bt, PhaseGVN* phase) {
1130 assert(bt == T_INT || bt == T_LONG, "sanity");
1131 const TypeVect* vt = TypeVect::make(bt, vlen);
1132
1133 int shift_mask = (bt == T_INT) ? 0x1F : 0x3F;
1134 int shiftLOpc = (bt == T_INT) ? Op_LShiftI : Op_LShiftL;
1135 int shiftROpc = (bt == T_INT) ? Op_URShiftI: Op_URShiftL;
1136
1137 // Compute shift values for right rotation and
1138 // later swap them in case of left rotation.
1139 Node* shiftRCnt = NULL;
1140 Node* shiftLCnt = NULL;
1141 if (cnt->is_Con() && cnt->bottom_type()->isa_int()) {
1142 // Constant shift case.
1143 int shift = cnt->get_int() & shift_mask;
1144 shiftRCnt = phase->intcon(shift);
1145 shiftLCnt = phase->intcon(shift_mask + 1 - shift);
1146 } else {
1147 // Variable shift case.
1148 assert(VectorNode::is_invariant_vector(cnt), "Broadcast expected");
1149 cnt = cnt->in(1);
1150 if (bt == T_LONG) {
1151 // Shift count vector for Rotate vector has long elements too.
1152 assert(cnt->Opcode() == Op_ConvI2L, "ConvI2L expected");
1153 cnt = cnt->in(1);
1154 }
1155 shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(shift_mask)));
1156 shiftLCnt = phase->transform(new SubINode(phase->intcon(shift_mask + 1), shiftRCnt));
1157 }
1158
1159 // Swap the computed left and right shift counts.
1160 if (is_rotate_left) {
1161 swap(shiftRCnt,shiftLCnt);
1162 }
1163
1164 shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt));
1165 shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt));
1166
1167 return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt)),
1168 phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt)),
1169 vt);
1170 }
1171
Ideal(PhaseGVN * phase,bool can_reshape)1172 Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1173 int vlen = length();
1174 BasicType bt = vect_type()->element_basic_type();
1175 if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1176 !Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) {
1177 return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase);
1178 }
1179 return NULL;
1180 }
1181
Ideal(PhaseGVN * phase,bool can_reshape)1182 Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1183 int vlen = length();
1184 BasicType bt = vect_type()->element_basic_type();
1185 if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) ||
1186 !Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) {
1187 return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase);
1188 }
1189 return NULL;
1190 }
1191
1192 #ifndef PRODUCT
dump_spec(outputStream * st) const1193 void VectorMaskCmpNode::dump_spec(outputStream *st) const {
1194 st->print(" %d #", _predicate); _type->dump_on(st);
1195 }
1196 #endif // PRODUCT
1197
Identity(PhaseGVN * phase)1198 Node* VectorReinterpretNode::Identity(PhaseGVN *phase) {
1199 Node* n = in(1);
1200 if (n->Opcode() == Op_VectorReinterpret) {
1201 // "VectorReinterpret (VectorReinterpret node) ==> node" if:
1202 // 1) Types of 'node' and 'this' are identical
1203 // 2) Truncations are not introduced by the first VectorReinterpret
1204 if (Type::cmp(bottom_type(), n->in(1)->bottom_type()) == 0 &&
1205 length_in_bytes() <= n->bottom_type()->is_vect()->length_in_bytes()) {
1206 return n->in(1);
1207 }
1208 }
1209 return this;
1210 }
1211
make(Node * vec,Node * new_val,int position)1212 Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) {
1213 assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range");
1214 ConINode* pos = ConINode::make(position);
1215 return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect());
1216 }
1217
Ideal(PhaseGVN * phase,bool can_reshape)1218 Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) {
1219 Node* n = obj()->uncast();
1220 if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1221 if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1222 // Handled by VectorUnboxNode::Identity()
1223 } else {
1224 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
1225 ciKlass* vbox_klass = vbox->box_type()->klass();
1226 const TypeVect* in_vt = vbox->vec_type();
1227 const TypeVect* out_vt = type()->is_vect();
1228 assert(in_vt->length() == out_vt->length(), "mismatch on number of elements");
1229 Node* value = vbox->in(VectorBoxNode::Value);
1230
1231 bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
1232 bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
1233 if (is_vector_mask) {
1234 // VectorUnbox (VectorBox vmask) ==> VectorLoadMask (VectorStoreMask vmask)
1235 value = phase->transform(VectorStoreMaskNode::make(*phase, value, in_vt->element_basic_type(), in_vt->length()));
1236 return new VectorLoadMaskNode(value, out_vt);
1237 } else if (is_vector_shuffle) {
1238 if (is_shuffle_to_vector()) {
1239 // VectorUnbox (VectorBox vshuffle) ==> VectorCastB2X vshuffle
1240 return new VectorCastB2XNode(value, out_vt);
1241 } else {
1242 // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle
1243 return new VectorLoadShuffleNode(value, out_vt);
1244 }
1245 } else {
1246 assert(false, "type mismatch on vector: %s", vbox_klass->name()->as_utf8());
1247 }
1248 }
1249 }
1250 return NULL;
1251 }
1252
Identity(PhaseGVN * phase)1253 Node* VectorUnboxNode::Identity(PhaseGVN* phase) {
1254 Node* n = obj()->uncast();
1255 if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) {
1256 if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) {
1257 return n->in(VectorBoxNode::Value); // VectorUnbox (VectorBox v) ==> v
1258 } else {
1259 // Handled by VectorUnboxNode::Ideal().
1260 }
1261 }
1262 return this;
1263 }
1264
vec_box_type(const TypeInstPtr * box_type)1265 const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) {
1266 const Type** fields = TypeTuple::fields(0);
1267 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
1268
1269 fields = TypeTuple::fields(1);
1270 fields[TypeFunc::Parms+0] = box_type;
1271 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
1272
1273 return TypeFunc::make(domain, range);
1274 }
1275
1276 #ifndef PRODUCT
dump_spec(outputStream * st) const1277 void VectorBoxAllocateNode::dump_spec(outputStream *st) const {
1278 CallStaticJavaNode::dump_spec(st);
1279 }
1280 #endif // !PRODUCT
1281