1 /* 2 * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 package jdk.incubator.vector; 26 27 import java.util.function.IntFunction; 28 import java.util.HashMap; 29 import java.util.ArrayList; 30 31 import jdk.internal.vm.annotation.ForceInline; 32 import jdk.internal.vm.annotation.Stable; 33 34 import jdk.internal.vm.vector.VectorSupport; 35 36 /** 37 * This class consists solely of static constants 38 * that describe lane-wise vector operations, plus nested interfaces 39 * which classify them. 40 * The static constants serve as tokens denoting specifically 41 * requested lane operations in vector expressions, such 42 * as the token {@code ADD} in 43 * {@code w = v0.}{@link 44 * Vector#lanewise(VectorOperators.Binary,Vector) 45 * lanewise}{@code (ADD, v1)}. 46 * 47 * <p> 48 * 49 * The documentation for each individual operator token is very brief, 50 * giving a symbolic Java expression for the operation that the token 51 * requests. Those symbolic expressions use the following conventional 52 * elements: 53 * <ul> 54 * <li>{@code a}, {@code b}, {@code c} — names of lane values 55 * 56 * <li>Java operators like {@code +}, {@code ?:}, etc. — 57 * expression operators 58 * 59 * <li>Java method names like {@code max}, {@code sin}, etc. — 60 * methods in standard classes like {@code Math}, {@code Double}, etc. 61 * Unqualified method names should be read as if in the context of a 62 * static import, and with resolution of overloading. 63 * 64 * <li>{@code bits(x)} — a function call which produces the 65 * underlying bits of the value {@code x}. If {@code x} is a floating 66 * point value, this is either {@code doubleToLongBits(x)} or 67 * {@code floatToIntBits(x)}. Otherwise, the value is just {@code x}. 68 * 69 * <li>{@code ESIZE} — the size in bytes of the operand type 70 * 71 * <li>{@code intVal}, {@code byteVal}, etc. — the operand of a 72 * conversion, with the indicated type 73 * </ul> 74 * 75 * <h2>Operations on floating point vectors</h2> 76 * <ul> 77 * <li>Lane-wise vector operations that apply to floating point vectors 78 * follow the accuracy and monotonicity specifications of the equivalent 79 * Java operation or method mentioned in its documentation unless specified otherwise. 80 * If the vector element type is {@code float} and the Java operation or 81 * method only accepts and returns {@code double} values, then the scalar operation 82 * on each lane is adapted to cast operands and the result, specifically widening 83 * {@code float} operands to {@code double} operands and narrowing the {@code double} 84 * result to a {@code float}. 85 * 86 * <li id="fp_assoc">Certain associative operations that apply to floating point 87 * vectors are not truly associative on the floating point lane values. 88 * Specifically, {@link #ADD} and {@link #MUL} used with cross-lane reduction 89 * operations, such as {@link FloatVector#reduceLanes(Associative)}. 90 * The result of such an operation is a function both of the input 91 * values (vector and mask) as well as the order of the scalar operations 92 * applied to combine lane values. 93 * In such cases the order is intentionally not defined. 94 * This allows the JVM to generate optimal machine code for the underlying 95 * platform at runtime. If the platform supports a vector instruction 96 * to add or multiply all values in the vector, or if there is some 97 * other efficient machine code sequence, then the JVM has the option of 98 * generating this machine code. Otherwise, the default 99 * implementation is applied, which adds vector elements 100 * sequentially from beginning to end. For this reason, the 101 * result of such an operation may vary for the same input values. 102 * </ul> 103 * 104 * <p> Note that a particular operator token may apply to several 105 * different lane types. Thus, these tokens behave like overloaded 106 * operators or methods, not like type-specific method handles or 107 * lambdas. Also unlike method handles or lambdas, these operators do 108 * not possess operational semantics; they have no {@code apply} or 109 * {@code invoke} method. They are used only to request lane 110 * operations from vector objects, and cannot (by themselves) perform 111 * operations on individual lane values. 112 * 113 */ 114 public abstract class VectorOperators { VectorOperators()115 private VectorOperators() { } 116 117 /** 118 * Root type for all operator tokens, providing queries for common 119 * properties such as arity, argument and return types, symbolic 120 * name, and operator name. 121 * 122 * @see VectorOperators.Unary Unary 123 * @see VectorOperators.Binary Binary 124 * @see VectorOperators.Ternary Ternary 125 * @see VectorOperators.Associative Associative 126 * @see VectorOperators.Comparison Comparison 127 * @see VectorOperators.Test Test 128 * @see VectorOperators.Conversion Conversion 129 * 130 * @apiNote 131 * User code should not implement this interface. A future release of 132 * this type may restrict implementations to be members of the same 133 * package. 134 */ 135 public interface Operator { 136 /** 137 * Returns the symbolic name of this operator, 138 * as a constant in {@link VectorOperators}. 139 * 140 * The operator symbol, Java method name, 141 * or example expression, 142 * such as {@code "+"}, {@code "max"} or {@code "-a"}, 143 * is also available as {@link #operatorName()}. 144 * 145 * @return the symbolic name of this operator, 146 * such as {@code "ADD"} 147 */ name()148 String name(); 149 150 /** 151 * Returns the Java operator symbol or method 152 * name corresponding to this operator. 153 * If there is no symbol or method, return a 154 * string containing a representative expression 155 * for the operator, using operand names 156 * {@code a}, {@code b} (for non-unary operators), 157 * and {@code c} (for ternary operators). 158 * 159 * The symbolic name of the constant, 160 * such as {@code "ADD"}, 161 * is also available as {@link #name()}. 162 * 163 * @return an operator token, such as {@code "+"}, 164 * or a method name, such as {@code "max"}, 165 * or a representative expression, such as {@code "-a"} 166 */ operatorName()167 String operatorName(); 168 169 /** 170 * Returns the arity of this operator (1, 2, or 3). 171 * @return the arity of this operator (1, 2, or 3) 172 */ arity()173 int arity(); 174 175 /** 176 * Reports whether this operator returns a boolean (a mask). 177 * A boolean operator also reports {@code boolean} as the 178 * {@code rangeType}. 179 * @return whether this operator returns a boolean 180 */ isBoolean()181 boolean isBoolean(); 182 183 /** 184 * Reports the special return type of this operator. 185 * If this operator is a boolean, returns {@code boolean.class}. 186 * If this operator is a {@code Conversion}, 187 * returns its {@linkplain Conversion#rangeType range type}. 188 * 189 * Otherwise, the operator's return value always has 190 * whatever type was given as an input, and this method 191 * returns {@code Object.class} to denote that fact. 192 * @return the special return type, or {@code Object.class} if none 193 */ rangeType()194 Class<?> rangeType(); 195 196 /** 197 * Returns the associativity of this operator. 198 * Only binary operators can be associative. 199 * @return the associativity of this operator 200 */ isAssociative()201 boolean isAssociative(); 202 203 /** 204 * Reports whether this operator is compatible with 205 * the proposed element type. 206 * 207 * First, unrestricted operators are compatible with all element 208 * types. 209 * 210 * Next, if the element type is {@code double} or {@code float} 211 * and the operator is restricted to floating point types, it is 212 * compatible. 213 * 214 * Otherwise, if the element type is neither {@code double} nor 215 * {@code float} and the operator is restricted to integral 216 * types, it is compatible. Otherwise, the operator is not 217 * compatible. 218 * 219 * @param elementType the proposed operand type for the operator 220 * @return whether the proposed type is compatible with this operator 221 */ compatibleWith(Class<?> elementType)222 boolean compatibleWith(Class<?> elementType); 223 224 // FIXME: Maybe add a query about architectural support. 225 } 226 227 /** 228 * Type for all 229 * <a href="Vector.html#lane-wise">lane-wise</a> 230 * unary (one-argument) operators, 231 * usable in expressions like {@code w = v0.}{@link 232 * Vector#lanewise(VectorOperators.Unary) 233 * lanewise}{@code (NEG)}. 234 * 235 * @apiNote 236 * User code should not implement this interface. A future release of 237 * this type may restrict implementations to be members of the same 238 * package. 239 */ 240 public interface Unary extends Operator { 241 } 242 243 /** 244 * Type for all 245 * <a href="Vector.html#lane-wise">lane-wise</a> 246 * binary (two-argument) operators, 247 * usable in expressions like {@code w = v0.}{@link 248 * Vector#lanewise(VectorOperators.Binary,Vector) 249 * lanewise}{@code (ADD, v1)}. 250 * 251 * @apiNote 252 * User code should not implement this interface. A future release of 253 * this type may restrict implementations to be members of the same 254 * package. 255 */ 256 public interface Binary extends Operator { 257 } 258 259 /** 260 * Type for all 261 * <a href="Vector.html#lane-wise">lane-wise</a> 262 * ternary (three-argument) operators, 263 * usable in expressions like {@code w = v0.}{@link 264 * Vector#lanewise(VectorOperators.Ternary,Vector,Vector) 265 * lanewise}{@code (FMA, v1, v2)}. 266 * 267 * @apiNote 268 * User code should not implement this interface. A future release of 269 * this type may restrict implementations to be members of the same 270 * package. 271 */ 272 public interface Ternary extends Operator { 273 } 274 275 /** 276 * Type for all reassociating 277 * <a href="Vector.html#lane-wise">lane-wise</a> 278 * binary operators, 279 * usable in expressions like {@code e = v0.}{@link 280 * IntVector#reduceLanes(VectorOperators.Associative) 281 * reduceLanes}{@code (ADD)}. 282 * 283 * @apiNote 284 * User code should not implement this interface. A future release of 285 * this type may restrict implementations to be members of the same 286 * package. 287 */ 288 public interface Associative extends Binary { 289 } 290 291 /** 292 * Type for all unary 293 * <a href="Vector.html#lane-wise">lane-wise</a> 294 * boolean tests on lane values, 295 * usable in expressions like {@code m = v0.}{@link 296 * FloatVector#test(VectorOperators.Test) 297 * test}{@code (IS_FINITE)}. 298 * 299 * @apiNote 300 * User code should not implement this interface. A future release of 301 * this type may restrict implementations to be members of the same 302 * package. 303 */ 304 public interface Test extends Operator { 305 } 306 307 /** 308 * Type for all binary 309 * <a href="Vector.html#lane-wise">lane-wise</a> 310 * boolean comparisons on lane values, 311 * usable in expressions like {@code m = v0.}{@link 312 * Vector#compare(VectorOperators.Comparison,Vector) 313 * compare}{@code (LT, v1)}. 314 * 315 * @apiNote 316 * User code should not implement this interface. A future release of 317 * this type may restrict implementations to be members of the same 318 * package. 319 */ 320 public interface Comparison extends Operator { 321 } 322 323 /** 324 * Type for all 325 * <a href="Vector.html#lane-wise">lane-wise</a> 326 * conversions on lane values, 327 * usable in expressions like {@code w1 = v0.}{@link 328 * Vector#convert(VectorOperators.Conversion,int) 329 * convert}{@code (I2D, 1)}. 330 * 331 * @param <E> the boxed element type for the conversion 332 * domain type (the input lane type) 333 * @param <F> the boxed element type for the conversion 334 * range type (the output lane type) 335 * 336 * @apiNote 337 * User code should not implement this interface. A future release of 338 * this type may restrict implementations to be members of the same 339 * package. 340 */ 341 public interface Conversion<E,F> extends Operator { 342 /** 343 * The domain of this conversion, a primitive type. 344 * @return the domain of this conversion 345 */ domainType()346 Class<E> domainType(); 347 348 /** 349 * The range of this conversion, a primitive type. 350 * @return the range of this conversion 351 */ 352 @Override rangeType()353 Class<F> rangeType(); 354 355 /** 356 * Ensures that this conversion has the 357 * desired domain and range types. 358 * @param from the desired domain type 359 * @param to the desired range type 360 * @param <E> the desired domain type 361 * @param <F> the desired range type 362 * @return this conversion object, with validated domain and range 363 */ check(Class<E> from, Class<F> to)364 <E,F> Conversion<E,F> check(Class<E> from, Class<F> to); 365 366 /// Factories. 367 368 /** 369 * The Java language assignment or casting conversion between two types. 370 * @param <E> the domain type (boxed version of a lane type) 371 * @param <F> the range type (boxed version of a lane type) 372 * @param from the type of the value to convert 373 * @param to the desired type after conversion 374 * @return a Java assignment or casting conversion 375 */ 376 @ForceInline ofCast(Class<E> from, Class<F> to)377 static <E,F> Conversion<E,F> ofCast(Class<E> from, Class<F> to) { 378 LaneType dom = LaneType.of(from); 379 LaneType ran = LaneType.of(to); 380 return ConversionImpl.ofCast(dom, ran).check(from, to); 381 } 382 383 /** 384 * The bitwise reinterpretation between two types. 385 * @param <E> the domain type (boxed version of a lane type) 386 * @param <F> the range type (boxed version of a lane type) 387 * @param from the type of the value to reinterpret 388 * @param to the desired type after reinterpretation 389 * @return a bitwise reinterpretation conversion 390 */ 391 @ForceInline ofReinterpret(Class<E> from, Class<F> to)392 static <E,F> Conversion<E,F> ofReinterpret(Class<E> from, Class<F> to) { 393 LaneType dom = LaneType.of(from); 394 LaneType ran = LaneType.of(to); 395 return ConversionImpl.ofReinterpret(dom, ran).check(from, to); 396 } 397 398 } 399 400 /*package-private*/ 401 @ForceInline opCode(Operator op, int requireKind, int forbidKind)402 static int opCode(Operator op, int requireKind, int forbidKind) { 403 return ((OperatorImpl)op).opCode(requireKind, forbidKind); 404 } 405 406 /*package-private*/ 407 @ForceInline opKind(Operator op, int bit)408 static boolean opKind(Operator op, int bit) { 409 return ((OperatorImpl)op).opKind(bit); 410 } 411 412 /*package-private*/ 413 static final int 414 VO_ALL = 0, 415 VO_UNARY = 0x001, 416 VO_BINARY = 0x002, 417 VO_TERNARY = 0x003, 418 VO_ARITY_MASK = (VO_UNARY|VO_BINARY|VO_TERNARY), 419 VO_ASSOC = 0x004, 420 VO_SHIFT = 0x008, 421 VO_BOOL = 0x010, 422 VO_CONV = 0x020, 423 VO_PRIVATE = 0x040, // not public, invisible 424 VO_SPECIAL = 0x080, // random special handling 425 VO_NOFP = 0x100, 426 VO_ONLYFP = 0x200, 427 VO_OPCODE_VALID = 0x800, 428 VO_OPCODE_SHIFT = 12, 429 VO_OPCODE_LIMIT = 0x400, 430 VO_RAN_SHIFT = 0, 431 VO_DOM_SHIFT = 4, 432 VO_DOM_RAN_MASK = 0x0FF, 433 VO_KIND_CAST = 0x000, 434 VO_KIND_BITWISE = 0x100; 435 436 private static final HashMap<Integer, String> OPC_NAME 437 = new HashMap<>(); 438 private static final HashMap<Integer, String> CMP_OPC_NAME 439 = new HashMap<>(); 440 private static final HashMap<Integer, String> CONV_OPC_NAME 441 = new HashMap<>(); 442 443 // Unary operators 444 445 /** Produce {@code ~a}. Integral only. */ 446 public static final /*bitwise*/ Unary NOT = unary("NOT", "~", -1 /*VectorSupport.VECTOR_OP_NOT*/, VO_NOFP | VO_SPECIAL); 447 /** Produce {@code a==0?0:-1} (zero or minus one). Integral only. */ 448 public static final /*bitwise*/ Unary ZOMO = unary("ZOMO", "a==0?0:-1", -1 /*VectorSupport.VECTOR_OP_ZOMO*/, VO_NOFP); 449 /** Produce {@code abs(a)}. */ 450 public static final Unary ABS = unary("ABS", "abs", VectorSupport.VECTOR_OP_ABS, VO_ALL); 451 /** Produce {@code -a}. */ 452 public static final Unary NEG = unary("NEG", "-a", VectorSupport.VECTOR_OP_NEG, VO_ALL|VO_SPECIAL); 453 454 /** Produce {@code sin(a)}. Floating only. 455 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 456 */ 457 public static final /*float*/ Unary SIN = unary("SIN", "sin", VectorSupport.VECTOR_OP_SIN, VO_ONLYFP); 458 /** Produce {@code cos(a)}. Floating only. 459 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 460 */ 461 public static final /*float*/ Unary COS = unary("COS", "cos", VectorSupport.VECTOR_OP_COS, VO_ONLYFP); 462 /** Produce {@code tan(a)}. Floating only. 463 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 464 */ 465 public static final /*float*/ Unary TAN = unary("TAN", "tan", VectorSupport.VECTOR_OP_TAN, VO_ONLYFP); 466 /** Produce {@code asin(a)}. Floating only. 467 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 468 */ 469 public static final /*float*/ Unary ASIN = unary("ASIN", "asin", VectorSupport.VECTOR_OP_ASIN, VO_ONLYFP); 470 /** Produce {@code acos(a)}. Floating only. 471 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 472 */ 473 public static final /*float*/ Unary ACOS = unary("ACOS", "acos", VectorSupport.VECTOR_OP_ACOS, VO_ONLYFP); 474 /** Produce {@code atan(a)}. Floating only. 475 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 476 */ 477 public static final /*float*/ Unary ATAN = unary("ATAN", "atan", VectorSupport.VECTOR_OP_ATAN, VO_ONLYFP); 478 479 /** Produce {@code exp(a)}. Floating only. 480 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 481 */ 482 public static final /*float*/ Unary EXP = unary("EXP", "exp", VectorSupport.VECTOR_OP_EXP, VO_ONLYFP); 483 /** Produce {@code log(a)}. Floating only. 484 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 485 */ 486 public static final /*float*/ Unary LOG = unary("LOG", "log", VectorSupport.VECTOR_OP_LOG, VO_ONLYFP); 487 /** Produce {@code log10(a)}. Floating only. 488 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 489 */ 490 public static final /*float*/ Unary LOG10 = unary("LOG10", "log10", VectorSupport.VECTOR_OP_LOG10, VO_ONLYFP); 491 /** Produce {@code sqrt(a)}. Floating only. See section "Operations on floating point vectors" above */ 492 public static final /*float*/ Unary SQRT = unary("SQRT", "sqrt", VectorSupport.VECTOR_OP_SQRT, VO_ONLYFP); 493 /** Produce {@code cbrt(a)}. Floating only. 494 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 495 */ 496 public static final /*float*/ Unary CBRT = unary("CBRT", "cbrt", VectorSupport.VECTOR_OP_CBRT, VO_ONLYFP); 497 498 /** Produce {@code sinh(a)}. Floating only. 499 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 500 */ 501 public static final /*float*/ Unary SINH = unary("SINH", "sinh", VectorSupport.VECTOR_OP_SINH, VO_ONLYFP); 502 /** Produce {@code cosh(a)}. Floating only. 503 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 504 */ 505 public static final /*float*/ Unary COSH = unary("COSH", "cosh", VectorSupport.VECTOR_OP_COSH, VO_ONLYFP); 506 /** Produce {@code tanh(a)}. Floating only. 507 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 508 */ 509 public static final /*float*/ Unary TANH = unary("TANH", "tanh", VectorSupport.VECTOR_OP_TANH, VO_ONLYFP); 510 /** Produce {@code expm1(a)}. Floating only. 511 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 512 */ 513 public static final /*float*/ Unary EXPM1 = unary("EXPM1", "expm1", VectorSupport.VECTOR_OP_EXPM1, VO_ONLYFP); 514 /** Produce {@code log1p(a)}. Floating only. 515 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 516 */ 517 public static final /*float*/ Unary LOG1P = unary("LOG1P", "log1p", VectorSupport.VECTOR_OP_LOG1P, VO_ONLYFP); 518 519 // Binary operators 520 521 /** Produce {@code a+b}. */ 522 public static final Associative ADD = assoc("ADD", "+", VectorSupport.VECTOR_OP_ADD, VO_ALL+VO_ASSOC); 523 /** Produce {@code a-b}. */ 524 public static final Binary SUB = binary("SUB", "-", VectorSupport.VECTOR_OP_SUB, VO_ALL); 525 /** Produce {@code a*b}. */ 526 public static final Associative MUL = assoc("MUL", "*", VectorSupport.VECTOR_OP_MUL, VO_ALL+VO_ASSOC); 527 /** Produce {@code a/b}. Floating only. */ 528 public static final Binary DIV = binary("DIV", "/", VectorSupport.VECTOR_OP_DIV, VO_ALL| VO_SPECIAL); 529 /** Produce {@code min(a,b)}. */ 530 public static final Associative MIN = assoc("MIN", "min", VectorSupport.VECTOR_OP_MIN, VO_ALL+VO_ASSOC); 531 /** Produce {@code max(a,b)}. */ 532 public static final Associative MAX = assoc("MAX", "max", VectorSupport.VECTOR_OP_MAX, VO_ALL+VO_ASSOC); 533 /** Produce {@code bits(a)!=0?a:b}. */ 534 public static final Associative FIRST_NONZERO = assoc("FIRST_NONZERO", "a!=0?a:b", -1 /*VectorSupport.VECTOR_OP_FIRST_NONZERO*/, VO_ALL+VO_ASSOC); 535 536 /** Produce {@code a&b}. Integral only. */ 537 public static final Associative AND = assoc("AND", "&", VectorSupport.VECTOR_OP_AND, VO_NOFP+VO_ASSOC); 538 /** Produce {@code a&~b}. Integral only. */ 539 public static final /*bitwise*/ Binary AND_NOT = binary("AND_NOT", "&~", -1 /*VectorSupport.VECTOR_OP_AND_NOT*/, VO_NOFP); // FIXME 540 /** Produce {@code a|b}. Integral only. */ 541 public static final /*bitwise*/ Associative OR = assoc("OR", "|", VectorSupport.VECTOR_OP_OR, VO_NOFP+VO_ASSOC); 542 /*package-private*/ /** Version of OR which works on float and double too. */ 543 static final Associative OR_UNCHECKED = assoc("OR_UNCHECKED", "|", VectorSupport.VECTOR_OP_OR, VO_ASSOC+VO_PRIVATE); 544 /** Produce {@code a^b}. Integral only. */ 545 public static final /*bitwise*/ Associative XOR = assoc("XOR", "^", VectorSupport.VECTOR_OP_XOR, VO_NOFP+VO_ASSOC); 546 547 /** Produce {@code a<<(n&(ESIZE*8-1))}. Integral only. */ 548 public static final /*bitwise*/ Binary LSHL = binary("LSHL", "<<", VectorSupport.VECTOR_OP_LSHIFT, VO_SHIFT); 549 /** Produce {@code a>>(n&(ESIZE*8-1))}. Integral only. */ 550 public static final /*bitwise*/ Binary ASHR = binary("ASHR", ">>", VectorSupport.VECTOR_OP_RSHIFT, VO_SHIFT); 551 /** Produce {@code a>>>(n&(ESIZE*8-1))}. Integral only. */ 552 public static final /*bitwise*/ Binary LSHR = binary("LSHR", ">>>", VectorSupport.VECTOR_OP_URSHIFT, VO_SHIFT); 553 /** Produce {@code rotateLeft(a,n)}. Integral only. */ 554 public static final /*bitwise*/ Binary ROL = binary("ROL", "rotateLeft", -1 /*VectorSupport.VECTOR_OP_LROTATE*/, VO_SHIFT | VO_SPECIAL); 555 /** Produce {@code rotateRight(a,n)}. Integral only. */ 556 public static final /*bitwise*/ Binary ROR = binary("ROR", "rotateRight", -1 /*VectorSupport.VECTOR_OP_RROTATE*/, VO_SHIFT | VO_SPECIAL); 557 558 /** Produce {@code atan2(a,b)}. See Floating only. 559 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 560 */ 561 public static final /*float*/ Binary ATAN2 = binary("ATAN2", "atan2", VectorSupport.VECTOR_OP_ATAN2, VO_ONLYFP); 562 /** Produce {@code pow(a,b)}. Floating only. 563 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 564 */ 565 public static final /*float*/ Binary POW = binary("POW", "pow", VectorSupport.VECTOR_OP_POW, VO_ONLYFP); 566 /** Produce {@code hypot(a,b)}. Floating only. 567 * Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above 568 */ 569 public static final /*float*/ Binary HYPOT = binary("HYPOT", "hypot", VectorSupport.VECTOR_OP_HYPOT, VO_ONLYFP); 570 571 // Ternary operators 572 573 /** Produce {@code a^((a^b)&c)}. (Bitwise {@code (c(i)?b(i):a(i))}.) Integral only. */ 574 public static final /*float*/ Ternary BITWISE_BLEND = ternary("BITWISE_BLEND", "a^((a^b)&c)", -1 /*VectorSupport.VECTOR_OP_BITWISE_BLEND*/, VO_NOFP); 575 /** Produce {@code fma(a,b,c)}. Floating only. */ 576 public static final /*float*/ Ternary FMA = ternary("FMA", "fma", VectorSupport.VECTOR_OP_FMA, VO_ONLYFP); 577 578 // Unary boolean operators 579 /** Test {@code bits(a)==0}. (Not true of {@code -0.0}.) */ 580 public static final Test IS_DEFAULT = predicate("IS_DEFAULT", "bits(a)==0", -1 /*VectorSupport.VECTOR_OP_TEST_DEFAULT*/, VO_ALL); 581 /** Test {@code bits(a)<0}. (True of {@code -0.0}.) */ 582 public static final Test IS_NEGATIVE = predicate("IS_NEGATIVE", "bits(a)<0", -1 /*VectorSupport.VECTOR_OP_TEST_NEGATIVE*/, VO_ALL); 583 /** Test {@code isFinite(a)}. Floating only. */ 584 public static final Test IS_FINITE = predicate("IS_FINITE", "isFinite", -1 /*VectorSupport.VECTOR_OP_TEST_FINITE*/, VO_ONLYFP); 585 /** Test {@code isNaN(a)}. Floating only. */ 586 public static final Test IS_NAN = predicate("IS_NAN", "isNaN", -1 /*VectorSupport.VECTOR_OP_TEST_NAN*/, VO_ONLYFP); 587 /** Test {@code isInfinite(a)}. Floating only. */ 588 public static final Test IS_INFINITE = predicate("IS_INFINITE", "isInfinite", -1 /*VectorSupport.VECTOR_OP_TEST_INFINITE*/, VO_ONLYFP); 589 590 // Binary boolean operators 591 592 /** Compare {@code a==b}. */ 593 public static final Comparison EQ = compare("EQ", "==", VectorSupport.BT_eq, VO_ALL); 594 /** Compare {@code a!=b}. */ 595 public static final Comparison NE = compare("NE", "!=", VectorSupport.BT_ne, VO_ALL); 596 /** Compare {@code a<b}. */ 597 public static final Comparison LT = compare("LT", "<", VectorSupport.BT_lt, VO_ALL); 598 /** Compare {@code a<=b}. */ 599 public static final Comparison LE = compare("LE", "<=", VectorSupport.BT_le, VO_ALL); 600 /** Compare {@code a>b}. */ 601 public static final Comparison GT = compare("GT", ">", VectorSupport.BT_gt, VO_ALL); 602 /** Compare {@code a>=b}. */ 603 public static final Comparison GE = compare("GE", ">=", VectorSupport.BT_ge, VO_ALL); 604 /** Unsigned compare {@code a<b}. Integral only. 605 * @see java.lang.Integer#compareUnsigned 606 * @see java.lang.Long#compareUnsigned 607 */ 608 public static final Comparison UNSIGNED_LT = compare("UNSIGNED_LT", "<", VectorSupport.BT_ult, VO_NOFP); 609 /** Unsigned compare {@code a<=b}. Integral only. 610 * @see java.lang.Integer#compareUnsigned 611 * @see java.lang.Long#compareUnsigned 612 */ 613 public static final Comparison UNSIGNED_LE = compare("UNSIGNED_LE", "<=", VectorSupport.BT_ule, VO_NOFP); 614 /** Unsigned compare {@code a>b}. Integral only. 615 * @see java.lang.Integer#compareUnsigned 616 * @see java.lang.Long#compareUnsigned 617 */ 618 public static final Comparison UNSIGNED_GT = compare("UNSIGNED_GT", ">", VectorSupport.BT_ugt, VO_NOFP); 619 /** Unsigned compare {@code a>=b}. Integral only. 620 * @see java.lang.Integer#compareUnsigned 621 * @see java.lang.Long#compareUnsigned 622 */ 623 public static final Comparison UNSIGNED_GE = compare("UNSIGNED_GE", ">=", VectorSupport.BT_uge, VO_NOFP); 624 625 // Conversion operators 626 627 /** Convert {@code byteVal} to {@code (double)byteVal}. */ 628 public static final Conversion<Byte,Double> B2D = convert("B2D", 'C', byte.class, double.class, VO_KIND_CAST, VO_ALL); 629 /** Convert {@code byteVal} to {@code (float)byteVal}. */ 630 public static final Conversion<Byte,Float> B2F = convert("B2F", 'C', byte.class, float.class, VO_KIND_CAST, VO_ALL); 631 /** Convert {@code byteVal} to {@code (int)byteVal}. */ 632 public static final Conversion<Byte,Integer> B2I = convert("B2I", 'C', byte.class, int.class, VO_KIND_CAST, VO_ALL); 633 /** Convert {@code byteVal} to {@code (long)byteVal}. */ 634 public static final Conversion<Byte,Long> B2L = convert("B2L", 'C', byte.class, long.class, VO_KIND_CAST, VO_ALL); 635 /** Convert {@code byteVal} to {@code (short)byteVal}. */ 636 public static final Conversion<Byte,Short> B2S = convert("B2S", 'C', byte.class, short.class, VO_KIND_CAST, VO_ALL); 637 /** Convert {@code doubleVal} to {@code (byte)doubleVal}. */ 638 public static final Conversion<Double,Byte> D2B = convert("D2B", 'C', double.class, byte.class, VO_KIND_CAST, VO_ALL); 639 /** Convert {@code doubleVal} to {@code (float)doubleVal}. */ 640 public static final Conversion<Double,Float> D2F = convert("D2F", 'C', double.class, float.class, VO_KIND_CAST, VO_ALL); 641 /** Convert {@code doubleVal} to {@code (int)doubleVal}. */ 642 public static final Conversion<Double,Integer> D2I = convert("D2I", 'C', double.class, int.class, VO_KIND_CAST, VO_ALL); 643 /** Convert {@code doubleVal} to {@code (long)doubleVal}. */ 644 public static final Conversion<Double,Long> D2L = convert("D2L", 'C', double.class, long.class, VO_KIND_CAST, VO_ALL); 645 /** Convert {@code doubleVal} to {@code (short)doubleVal}. */ 646 public static final Conversion<Double,Short> D2S = convert("D2S", 'C', double.class, short.class, VO_KIND_CAST, VO_ALL); 647 /** Convert {@code floatVal} to {@code (byte)floatVal}. */ 648 public static final Conversion<Float,Byte> F2B = convert("F2B", 'C', float.class, byte.class, VO_KIND_CAST, VO_ALL); 649 /** Convert {@code floatVal} to {@code (double)floatVal}. */ 650 public static final Conversion<Float,Double> F2D = convert("F2D", 'C', float.class, double.class, VO_KIND_CAST, VO_ALL); 651 /** Convert {@code floatVal} to {@code (int)floatVal}. */ 652 public static final Conversion<Float,Integer> F2I = convert("F2I", 'C', float.class, int.class, VO_KIND_CAST, VO_ALL); 653 /** Convert {@code floatVal} to {@code (long)floatVal}. */ 654 public static final Conversion<Float,Long> F2L = convert("F2L", 'C', float.class, long.class, VO_KIND_CAST, VO_ALL); 655 /** Convert {@code floatVal} to {@code (short)floatVal}. */ 656 public static final Conversion<Float,Short> F2S = convert("F2S", 'C', float.class, short.class, VO_KIND_CAST, VO_ALL); 657 /** Convert {@code intVal} to {@code (byte)intVal}. */ 658 public static final Conversion<Integer,Byte> I2B = convert("I2B", 'C', int.class, byte.class, VO_KIND_CAST, VO_ALL); 659 /** Convert {@code intVal} to {@code (double)intVal}. */ 660 public static final Conversion<Integer,Double> I2D = convert("I2D", 'C', int.class, double.class, VO_KIND_CAST, VO_ALL); 661 /** Convert {@code intVal} to {@code (float)intVal}. */ 662 public static final Conversion<Integer,Float> I2F = convert("I2F", 'C', int.class, float.class, VO_KIND_CAST, VO_ALL); 663 /** Convert {@code intVal} to {@code (long)intVal}. */ 664 public static final Conversion<Integer,Long> I2L = convert("I2L", 'C', int.class, long.class, VO_KIND_CAST, VO_ALL); 665 /** Convert {@code intVal} to {@code (short)intVal}. */ 666 public static final Conversion<Integer,Short> I2S = convert("I2S", 'C', int.class, short.class, VO_KIND_CAST, VO_ALL); 667 /** Convert {@code longVal} to {@code (byte)longVal}. */ 668 public static final Conversion<Long,Byte> L2B = convert("L2B", 'C', long.class, byte.class, VO_KIND_CAST, VO_ALL); 669 /** Convert {@code longVal} to {@code (double)longVal}. */ 670 public static final Conversion<Long,Double> L2D = convert("L2D", 'C', long.class, double.class, VO_KIND_CAST, VO_ALL); 671 /** Convert {@code longVal} to {@code (float)longVal}. */ 672 public static final Conversion<Long,Float> L2F = convert("L2F", 'C', long.class, float.class, VO_KIND_CAST, VO_ALL); 673 /** Convert {@code longVal} to {@code (int)longVal}. */ 674 public static final Conversion<Long,Integer> L2I = convert("L2I", 'C', long.class, int.class, VO_KIND_CAST, VO_ALL); 675 /** Convert {@code longVal} to {@code (short)longVal}. */ 676 public static final Conversion<Long,Short> L2S = convert("L2S", 'C', long.class, short.class, VO_KIND_CAST, VO_ALL); 677 /** Convert {@code shortVal} to {@code (byte)shortVal}. */ 678 public static final Conversion<Short,Byte> S2B = convert("S2B", 'C', short.class, byte.class, VO_KIND_CAST, VO_ALL); 679 /** Convert {@code shortVal} to {@code (double)shortVal}. */ 680 public static final Conversion<Short,Double> S2D = convert("S2D", 'C', short.class, double.class, VO_KIND_CAST, VO_ALL); 681 /** Convert {@code shortVal} to {@code (float)shortVal}. */ 682 public static final Conversion<Short,Float> S2F = convert("S2F", 'C', short.class, float.class, VO_KIND_CAST, VO_ALL); 683 /** Convert {@code shortVal} to {@code (int)shortVal}. */ 684 public static final Conversion<Short,Integer> S2I = convert("S2I", 'C', short.class, int.class, VO_KIND_CAST, VO_ALL); 685 /** Convert {@code shortVal} to {@code (long)shortVal}. */ 686 public static final Conversion<Short,Long> S2L = convert("S2L", 'C', short.class, long.class, VO_KIND_CAST, VO_ALL); 687 /** Reinterpret bits of {@code doubleVal} as {@code long}. As if by {@link Double#doubleToRawLongBits(double)} */ 688 public static final Conversion<Double,Long> REINTERPRET_D2L = convert("REINTERPRET_D2L", 'R', double.class, long.class, VO_KIND_BITWISE, VO_ALL); 689 /** Reinterpret bits of {@code floatVal} as {@code int}. As if by {@link Float#floatToRawIntBits(float)} */ 690 public static final Conversion<Float,Integer> REINTERPRET_F2I = convert("REINTERPRET_F2I", 'R', float.class, int.class, VO_KIND_BITWISE, VO_ALL); 691 /** Reinterpret bits of {@code intVal} as {@code float}. As if by {@link Float#intBitsToFloat(int)} */ 692 public static final Conversion<Integer,Float> REINTERPRET_I2F = convert("REINTERPRET_I2F", 'R', int.class, float.class, VO_KIND_BITWISE, VO_ALL); 693 /** Reinterpret bits of {@code longVal} as {@code double}. As if by {@link Double#longBitsToDouble(long)} */ 694 public static final Conversion<Long,Double> REINTERPRET_L2D = convert("REINTERPRET_L2D", 'R', long.class, double.class, VO_KIND_BITWISE, VO_ALL); 695 /** Zero-extend {@code byteVal} to {@code int}. */ 696 public static final Conversion<Byte,Integer> ZERO_EXTEND_B2I = convert("ZERO_EXTEND_B2I", 'Z', byte.class, int.class, VO_KIND_BITWISE, VO_ALL); 697 /** Zero-extend {@code byteVal} to {@code long}. */ 698 public static final Conversion<Byte,Long> ZERO_EXTEND_B2L = convert("ZERO_EXTEND_B2L", 'Z', byte.class, long.class, VO_KIND_BITWISE, VO_ALL); 699 /** Zero-extend {@code byteVal} to {@code short}. */ 700 public static final Conversion<Byte,Short> ZERO_EXTEND_B2S = convert("ZERO_EXTEND_B2S", 'Z', byte.class, short.class, VO_KIND_BITWISE, VO_ALL); 701 /** Zero-extend {@code intVal} to {@code long}. */ 702 public static final Conversion<Integer,Long> ZERO_EXTEND_I2L = convert("ZERO_EXTEND_I2L", 'Z', int.class, long.class, VO_KIND_BITWISE, VO_ALL); 703 /** Zero-extend {@code shortVal} to {@code int}. */ 704 public static final Conversion<Short,Integer> ZERO_EXTEND_S2I = convert("ZERO_EXTEND_S2I", 'Z', short.class, int.class, VO_KIND_BITWISE, VO_ALL); 705 /** Zero-extend {@code shortVal} to {@code long}. */ 706 public static final Conversion<Short,Long> ZERO_EXTEND_S2L = convert("ZERO_EXTEND_S2L", 'Z', short.class, long.class, VO_KIND_BITWISE, VO_ALL); 707 708 // (End of conversion operators) 709 opInfo(int opCode, int bits)710 private static int opInfo(int opCode, int bits) { 711 if (opCode >= 0) { 712 bits |= VO_OPCODE_VALID; 713 } else { 714 opCode &= (VO_OPCODE_LIMIT - 1); // not a valid op 715 bits |= VO_SPECIAL; // mark for special handling 716 } 717 assert((bits >> VO_OPCODE_SHIFT) == 0); 718 assert(opCode >= 0 && opCode < VO_OPCODE_LIMIT); 719 return (opCode << VO_OPCODE_SHIFT) + bits; 720 } 721 unary(String name, String opName, int opCode, int flags)722 private static Unary unary(String name, String opName, int opCode, int flags) { 723 if (opCode >= 0 && (flags & VO_PRIVATE) == 0) 724 OPC_NAME.put(opCode, name); 725 return new UnaryImpl(name, opName, opInfo(opCode, flags | VO_UNARY)); 726 } 727 binary(String name, String opName, int opCode, int flags)728 private static Binary binary(String name, String opName, int opCode, int flags) { 729 if (opCode >= 0 && (flags & VO_PRIVATE) == 0) 730 OPC_NAME.put(opCode, name); 731 return new BinaryImpl(name, opName, opInfo(opCode, flags | VO_BINARY)); 732 } 733 ternary(String name, String opName, int opCode, int flags)734 private static Ternary ternary(String name, String opName, int opCode, int flags) { 735 if (opCode >= 0 && (flags & VO_PRIVATE) == 0) 736 OPC_NAME.put(opCode, name); 737 return new TernaryImpl(name, opName, opInfo(opCode, flags | VO_TERNARY)); 738 } 739 assoc(String name, String opName, int opCode, int flags)740 private static Associative assoc(String name, String opName, int opCode, int flags) { 741 if (opCode >= 0 && (flags & VO_PRIVATE) == 0) 742 OPC_NAME.put(opCode, name); 743 return new AssociativeImpl(name, opName, opInfo(opCode, flags | VO_BINARY | VO_ASSOC)); 744 } 745 predicate(String name, String opName, int opCode, int flags)746 private static Test predicate(String name, String opName, int opCode, int flags) { 747 if (opCode >= 0 && (flags & VO_PRIVATE) == 0) 748 CMP_OPC_NAME.put(opCode, name); 749 return new TestImpl(name, opName, opInfo(opCode, flags | VO_UNARY | VO_BOOL)); 750 } 751 compare(String name, String opName, int opCode, int flags)752 private static Comparison compare(String name, String opName, int opCode, int flags) { 753 if (opCode >= 0 && (flags & VO_PRIVATE) == 0) 754 CMP_OPC_NAME.put(opCode, name); 755 return new ComparisonImpl(name, opName, opInfo(opCode, flags | VO_BINARY | VO_BOOL)); 756 } 757 758 private static <E,F> ConversionImpl<E,F> convert(String name, char kind, Class<E> dom, Class<F> ran, int opCode, int flags)759 convert(String name, char kind, Class<E> dom, Class<F> ran, int opCode, int flags) { 760 int domran = ((LaneType.of(dom).basicType << VO_DOM_SHIFT) + 761 (LaneType.of(ran).basicType << VO_RAN_SHIFT)); 762 if (opCode >= 0) { 763 if ((opCode & VO_DOM_RAN_MASK) == 0) { 764 opCode += domran; 765 } 766 if ((flags & VO_PRIVATE) == 0) 767 CONV_OPC_NAME.put(opCode, name); 768 } 769 String opName = dom+"-"+kind+"-"+ran; //?? 770 return new ConversionImpl<>(name, opName, opInfo(opCode, flags | VO_UNARY | VO_CONV), 771 kind, dom, ran); 772 } 773 774 private static abstract class OperatorImpl implements Operator { 775 private final String symName; 776 private final String opName; 777 private final int opInfo; 778 OperatorImpl(String symName, String opName, int opInfo)779 OperatorImpl(String symName, String opName, int opInfo) { 780 this.symName = symName; 781 this.opName = opName; 782 this.opInfo = opInfo; 783 assert(opInfo != 0); 784 } 785 786 @Override name()787 public final String name() { 788 return symName; 789 } 790 @Override operatorName()791 public final String operatorName() { 792 return opName; 793 } 794 @Override toString()795 public final String toString() { 796 return name(); 797 } 798 @Override arity()799 public final int arity() { 800 return opInfo & VO_ARITY_MASK; 801 } 802 @Override isBoolean()803 public final boolean isBoolean() { 804 return opKind(VO_BOOL); 805 } 806 @Override rangeType()807 public Class<?> rangeType() { 808 return Object.class; 809 } 810 @Override isAssociative()811 public final boolean isAssociative() { 812 return opKind(VO_ASSOC); 813 } 814 815 @ForceInline compatibleWith(Class<?> elementType)816 public boolean compatibleWith(Class<?> elementType) { 817 return compatibleWith(LaneType.of(elementType)); 818 } 819 820 /*package-private*/ 821 @ForceInline opInfo()822 int opInfo() { 823 return opInfo; 824 } 825 826 /*package-private*/ 827 @ForceInline opCode(int requireKind, int forbidKind)828 int opCode(int requireKind, int forbidKind) { 829 int opc = opCodeRaw(); 830 if ((opInfo & requireKind) != requireKind || 831 (forbidKind != 0 && 832 (opInfo & forbidKind) == forbidKind)) { 833 throw illegalOperation(requireKind, forbidKind); 834 } 835 return opc; 836 } 837 838 /*package-private*/ 839 @ForceInline opCodeRaw()840 int opCodeRaw() { 841 return (opInfo >> VO_OPCODE_SHIFT); 842 } 843 844 /*package-private*/ 845 UnsupportedOperationException illegalOperation(int requireKind, int forbidKind)846 illegalOperation(int requireKind, int forbidKind) { 847 String msg1 = ""; 848 requireKind &=~ VO_OPCODE_VALID; 849 switch (requireKind) { 850 case VO_ONLYFP: msg1 = "floating point operator required here"; break; 851 case VO_NOFP: msg1 = "integral/bitwise operator required here"; break; 852 case VO_ASSOC: msg1 = "associative operator required here"; break; 853 } 854 String msg2 = ""; 855 switch (forbidKind) { 856 case VO_ONLYFP: msg2 = "inapplicable floating point operator"; break; 857 case VO_NOFP: msg2 = "inapplicable integral/bitwise operator"; break; 858 } 859 if ((opInfo & VO_OPCODE_VALID) == 0) { 860 msg2 = "operator is not implemented"; 861 } 862 return illegalOperation(msg1, msg2); 863 } 864 865 /*package-private*/ 866 UnsupportedOperationException illegalOperation(String msg1, String msg2)867 illegalOperation(String msg1, String msg2) { 868 String dot = ""; 869 if (!msg1.isEmpty() && !msg2.isEmpty()) { 870 dot = "; "; 871 } else if (msg1.isEmpty() && msg2.isEmpty()) { 872 // Couldn't decode the *kind bits. 873 msg1 = "illegal operator"; 874 } 875 String msg = String.format("%s: %s%s%s", this, msg1, dot, msg2); 876 return new UnsupportedOperationException(msg); 877 } 878 879 880 /*package-private*/ 881 @ForceInline opKind(int kindBit)882 boolean opKind(int kindBit) { 883 return (opInfo & kindBit) != 0; 884 } 885 886 @ForceInline 887 /*package-private*/ compatibleWith(LaneType laneType)888 boolean compatibleWith(LaneType laneType) { 889 if (laneType.elementKind == 'F') { 890 return !opKind(VO_NOFP); 891 } else if (laneType.elementKind == 'I') { 892 return !opKind(VO_ONLYFP); 893 } else { 894 throw new AssertionError(); 895 } 896 } 897 } 898 899 private static class UnaryImpl extends OperatorImpl implements Unary { UnaryImpl(String symName, String opName, int opInfo)900 private UnaryImpl(String symName, String opName, int opInfo) { 901 super(symName, opName, opInfo); 902 assert((opInfo & VO_ARITY_MASK) == VO_UNARY); 903 } 904 } 905 906 private static class BinaryImpl extends OperatorImpl implements Binary { BinaryImpl(String symName, String opName, int opInfo)907 private BinaryImpl(String symName, String opName, int opInfo) { 908 super(symName, opName, opInfo); 909 assert((opInfo & VO_ARITY_MASK) == VO_BINARY); 910 } 911 } 912 913 private static class TernaryImpl extends OperatorImpl implements Ternary { TernaryImpl(String symName, String opName, int opInfo)914 private TernaryImpl(String symName, String opName, int opInfo) { 915 super(symName, opName, opInfo); 916 assert((opInfo & VO_ARITY_MASK) == VO_TERNARY); 917 } 918 } 919 920 private static class AssociativeImpl extends BinaryImpl implements Associative { AssociativeImpl(String symName, String opName, int opInfo)921 private AssociativeImpl(String symName, String opName, int opInfo) { 922 super(symName, opName, opInfo); 923 } 924 } 925 926 /*package-private*/ 927 static 928 class ConversionImpl<E,F> extends OperatorImpl 929 implements Conversion<E,F> { ConversionImpl(String symName, String opName, int opInfo, char kind, Class<E> dom, Class<F> ran)930 private ConversionImpl(String symName, String opName, int opInfo, 931 char kind, Class<E> dom, Class<F> ran) { 932 super(symName, opName, opInfo); 933 assert((opInfo & VO_ARITY_MASK) == VO_UNARY); 934 this.kind = kind; 935 this.dom = LaneType.of(dom); 936 this.ran = LaneType.of(ran); 937 check(dom, ran); // make sure it all lines up 938 } 939 private final char kind; 940 private final LaneType dom; 941 private final LaneType ran; 942 943 // non-overrides are all package-private 944 kind()945 char kind() { return kind; } domain()946 LaneType domain() { return dom; } range()947 LaneType range() { return ran; } 948 sizeChangeLog2()949 int sizeChangeLog2() { 950 return ran.elementSizeLog2 - dom.elementSizeLog2; 951 } 952 953 @SuppressWarnings("unchecked") 954 @Override domainType()955 public Class<E> domainType() { 956 return (Class<E>) dom.elementType; 957 } 958 @SuppressWarnings("unchecked") 959 @Override rangeType()960 public Class<F> rangeType() { 961 return (Class<F>) ran.elementType; 962 } 963 964 @SuppressWarnings("unchecked") 965 @ForceInline 966 public 967 <E,F> Conversion<E,F> check(Class<E> dom, Class<F> ran)968 check(Class<E> dom, Class<F> ran) { 969 if (this.dom.elementType != dom || 970 this.ran.elementType != ran) 971 throw checkFailed(dom, ran); 972 return (Conversion<E,F>) this; 973 } checkFailed(Class<?> dom, Class<?> ran)974 private RuntimeException checkFailed(Class<?> dom, Class<?> ran) { 975 return new ClassCastException(toString()+": not "+dom+" -> "+ran); 976 } 977 ofCopy(LaneType dom)978 static ConversionImpl<?,?> ofCopy(LaneType dom) { 979 return findConv('I', dom, dom); 980 } ofCast(LaneType dom, LaneType ran)981 static ConversionImpl<?,?> ofCast(LaneType dom, LaneType ran) { 982 if (dom == ran) return ofCopy(dom); 983 return findConv('C', dom, ran); 984 } ofReinterpret(LaneType dom, LaneType ran)985 static ConversionImpl<?,?> ofReinterpret(LaneType dom, LaneType ran) { 986 if (dom == ran) return ofCopy(dom); 987 if (dom.elementKind == 'I' && 988 ran.elementKind == 'I' && 989 dom.elementSize < ran.elementSize) { 990 // Zero extension of field (unsigned semantics). 991 return findConv('Z', dom, ran); 992 } 993 // if (dom.elementSize != ran.elementSize) { 994 // throw new IllegalArgumentException("bad reinterpret"); 995 // } 996 return findConv('R', dom, ran); 997 } 998 999 @ForceInline 1000 private static ConversionImpl<?,?> findConv(char kind, LaneType dom, LaneType ran)1001 findConv(char kind, LaneType dom, LaneType ran) { 1002 ConversionImpl<?,?>[] cache = cacheOf(kind, dom); 1003 int ranKey = ran.switchKey; 1004 ConversionImpl<?,?> conv = cache[ranKey]; 1005 if (conv != null) { 1006 return conv; 1007 } 1008 return makeConv(kind, dom, ran); 1009 } 1010 a2b(LaneType dom, LaneType ran)1011 static String a2b(LaneType dom, LaneType ran) { 1012 return dom.typeChar + "2" + ran.typeChar; 1013 } 1014 1015 static ConversionImpl<?,?> makeConv(char kind, LaneType dom, LaneType ran)1016 makeConv(char kind, LaneType dom, LaneType ran) { 1017 String name; 1018 Class<?> domType = dom.elementType; 1019 Class<?> ranType = ran.elementType; 1020 int domCode = (dom.basicType << VO_DOM_SHIFT); 1021 int ranCode = (ran.basicType << VO_RAN_SHIFT); 1022 int opCode = domCode + ranCode; 1023 switch (kind) { 1024 case 'I': 1025 assert(dom == ran); 1026 name = "COPY_"+a2b(dom, ran); 1027 opCode = VO_KIND_CAST; 1028 break; 1029 case 'C': 1030 name = ""+a2b(dom, ran); 1031 opCode = VO_KIND_CAST; 1032 break; 1033 case 'R': 1034 name = "REINTERPRET_"+a2b(dom, ran); 1035 opCode = VO_KIND_BITWISE; 1036 break; 1037 case 'Z': 1038 name = "ZERO_EXTEND_"+a2b(dom, ran); 1039 opCode = VO_KIND_BITWISE; 1040 break; 1041 default: throw new AssertionError(); 1042 } 1043 ConversionImpl<?,?> conv = convert(name, kind, domType, ranType, opCode, VO_ALL); 1044 // Put into the cache for next time. 1045 // The JIT can see into this cache 1046 // when kind/dom/ran are all constant. 1047 ConversionImpl<?,?>[] cache = cacheOf(kind, dom); 1048 int ranKey = ran.switchKey; 1049 // The extra "check" calls help validate that 1050 // we aren't cross-wiring the cache. 1051 conv.check(domType, ranType); 1052 synchronized (ConversionImpl.class) { 1053 if (cache[ranKey] == null) { 1054 cache[ranKey] = conv; 1055 } else { 1056 conv = cache[ranKey]; 1057 conv.check(domType, ranType); 1058 } 1059 } 1060 return conv; 1061 } check(char kind, LaneType dom, LaneType ran)1062 private final void check(char kind, LaneType dom, LaneType ran) { 1063 if (this.kind != kind || this.dom != dom || this.ran != ran) { 1064 throw new AssertionError(this + " != " + dom + kind + ran); 1065 } 1066 } 1067 1068 /** Helper for cache probes. */ 1069 @ForceInline 1070 private static ConversionImpl<?,?>[] cacheOf(char kind, LaneType dom)1071 cacheOf(char kind, LaneType dom) { 1072 assert("CIRZWN".indexOf(kind) >= 0); 1073 int k = (kind <= 'I' ? KIND_CI : 1074 (kind == 'R' || kind == 'Z') ? KIND_RZ : 1075 KIND_WN); 1076 return CACHES[k][dom.switchKey]; 1077 } 1078 private static final int 1079 LINE_LIMIT = LaneType.SK_LIMIT, 1080 KIND_CI = 0, KIND_RZ = 1, KIND_WN = 2, KIND_LIMIT = 3; 1081 private static final @Stable ConversionImpl<?,?>[][][] 1082 CACHES = new ConversionImpl<?,?>[KIND_LIMIT][LINE_LIMIT][LINE_LIMIT]; 1083 initCaches()1084 private synchronized static void initCaches() { 1085 for (var f : VectorOperators.class.getFields()) { 1086 if (f.getType() != Conversion.class) continue; 1087 ConversionImpl<?,?> conv; 1088 try { 1089 conv = (ConversionImpl) f.get(null); 1090 } catch (ReflectiveOperationException ex) { 1091 throw new AssertionError(ex); 1092 } 1093 LaneType dom = conv.dom; 1094 LaneType ran = conv.ran; 1095 int opc = conv.opCodeRaw(); 1096 switch (conv.kind) { 1097 case 'W': 1098 int domCode = (opc >> VO_DOM_SHIFT) & 0xF; 1099 dom = LaneType.ofBasicType(domCode); 1100 break; 1101 case 'N': 1102 int ranCode = (opc >> VO_RAN_SHIFT) & 0xF; 1103 ran = LaneType.ofBasicType(ranCode); 1104 break; 1105 } 1106 assert((opc & VO_DOM_RAN_MASK) == 1107 ((dom.basicType << VO_DOM_SHIFT) + 1108 (ran.basicType << VO_RAN_SHIFT))); 1109 ConversionImpl<?,?>[] cache = cacheOf(conv.kind, dom); 1110 int ranKey = ran.switchKey; 1111 if (cache[ranKey] != conv) { 1112 assert(cache[ranKey] == null || 1113 cache[ranKey].name().equals(conv.name())) 1114 : conv + " vs. " + cache[ranKey]; 1115 cache[ranKey] = conv; 1116 } 1117 } 1118 } 1119 1120 // hack for generating static field defs genCode()1121 static { assert(genCode()); } genCode()1122 private static boolean genCode() { 1123 if (true) return true; // remove to enable code 1124 ArrayList<String> defs = new ArrayList<>(); 1125 for (LaneType l1 : LaneType.values()) { 1126 for (LaneType l2 : LaneType.values()) { 1127 for (int i = 0; i <= 1; i++) { 1128 ConversionImpl<?,?> c; 1129 try { 1130 c = ((i == 0) ? ofCast(l1, l2) : ofReinterpret(l1, l2)); 1131 } catch (IllegalArgumentException ex) { 1132 assert((i == 1 && l1.elementSize != l2.elementSize) || 1133 (i == 2 && l1.elementSize == l2.elementSize)); 1134 continue; // ignore this combo 1135 } 1136 if (c.kind == 'C' || 1137 c.kind == 'Z' || 1138 (c.kind == 'R' && 1139 l1.elementKind+l2.elementKind == 'F'+'I' && 1140 l1.elementSize == l2.elementSize) || 1141 (c.kind == 'N' || c.kind == 'W')) { 1142 int opc = c.opCodeRaw(); 1143 String opcs; 1144 switch (opc & ~VO_DOM_RAN_MASK) { 1145 case VO_KIND_CAST: opcs = "VO_KIND_CAST"; break; 1146 case VO_KIND_BITWISE: opcs = "VO_KIND_BITWISE"; break; 1147 default: opcs = Integer.toHexString(opc); 1148 } 1149 String code = c.genCode(opcs); 1150 if (!defs.contains(code)) defs.add(code); 1151 } 1152 } 1153 } 1154 } 1155 java.util.Collections.sort(defs); 1156 for (String def : defs) System.out.println(def); 1157 return true; 1158 } genCode(String opcs)1159 private String genCode(String opcs) { 1160 if (true) return null; // remove to enable code 1161 int domran = opCodeRaw() & VO_DOM_RAN_MASK; 1162 switch (kind()) { 1163 case 'W': case 'N': 1164 opcs += " + 0x" + Integer.toHexString(domran); 1165 } 1166 String doc; 1167 switch (kind()) { 1168 case 'R': 1169 doc = "Reinterpret bits of {@code _domVal} as {@code _ran}"; 1170 break; 1171 case 'Z': 1172 doc = "Zero-extend {@code _domVal} to {@code _ran}"; 1173 break; 1174 case 'W': 1175 doc = "In-place widen {@code _domVal} inside _ran to {@code (_ran)_domVal}"; 1176 LaneType logdom = LaneType.ofBasicType(domran >> VO_DOM_SHIFT & 0xF); 1177 doc = doc.replace("_dom", logdom.elementType.getSimpleName()); 1178 break; 1179 case 'N': 1180 doc = "In-place narrow {@code _domVal} to {@code (_ran)_domVal} inside _dom"; 1181 LaneType logran = LaneType.ofBasicType(domran >> VO_RAN_SHIFT & 0xF); 1182 doc = doc.replace("_ran", logran.elementType.getSimpleName()); 1183 break; 1184 default: 1185 doc = "Convert {@code _domVal} to {@code (_ran)_domVal}"; 1186 } 1187 String code = ( 1188 " /** _Doc. */" + "\n" + 1189 " public static final Conversion<_Dom,_Ran> _N" + 1190 " = convert(\"_N\", '_K', _dom.class, _ran.class, _opc, VO_ALL);"); 1191 return code 1192 .replace("_Doc", doc) 1193 .replace("_dom", dom.elementType.getSimpleName()) 1194 .replace("_ran", ran.elementType.getSimpleName()) 1195 .replace("_Dom", dom.genericElementType.getSimpleName()) 1196 .replace("_Ran", ran.genericElementType.getSimpleName()) 1197 .replace("_N", name()) 1198 .replace("_K", ""+kind()) 1199 .replace("_opc", ""+opcs); 1200 } 1201 } 1202 1203 private static class TestImpl extends OperatorImpl implements Test { TestImpl(String symName, String opName, int opInfo)1204 private TestImpl(String symName, String opName, int opInfo) { 1205 super(symName, opName, opInfo); 1206 assert((opInfo & VO_ARITY_MASK) == VO_UNARY); 1207 assert((opInfo & VO_BOOL) == VO_BOOL); 1208 } 1209 @Override rangeType()1210 public Class<?> rangeType() { 1211 return boolean.class; 1212 } 1213 } 1214 1215 private static class ComparisonImpl extends OperatorImpl implements Comparison { ComparisonImpl(String symName, String opName, int opInfo)1216 private ComparisonImpl(String symName, String opName, int opInfo) { 1217 super(symName, opName, opInfo); 1218 assert((opInfo & VO_ARITY_MASK) == VO_BINARY); 1219 assert((opInfo & VO_BOOL) == VO_BOOL); 1220 } 1221 @Override rangeType()1222 public Class<?> rangeType() { 1223 return boolean.class; 1224 } 1225 /* --- * 1226 boolean test(long a, long b) { 1227 switch (opInfo() >> VO_OPCODE_SHIFT) { 1228 case VectorSupport.BT_eq: return a == b; 1229 case VectorSupport.BT_ne: return a != b; 1230 case VectorSupport.BT_lt: return a < b; 1231 case VectorSupport.BT_le: return a <= b; 1232 case VectorSupport.BT_gt: return a > b; 1233 case VectorSupport.BT_ge: return a >= b; 1234 } 1235 throw new AssertionError(); 1236 } 1237 * --- */ 1238 } 1239 1240 static { ConversionImpl.initCaches()1241 ConversionImpl.initCaches(); checkConstants()1242 assert(checkConstants()); 1243 } 1244 checkConstants()1245 private static boolean checkConstants() { 1246 // Check uniqueness of opcodes, to prevent dumb aliasing errors. 1247 OperatorImpl[] ops = new OperatorImpl[VO_OPCODE_LIMIT << VO_OPCODE_SHIFT]; 1248 for (var f : VectorOperators.class.getFields()) { 1249 Class<?> ft = f.getType(); 1250 OperatorImpl op; 1251 try { 1252 op = (OperatorImpl) f.get(null); 1253 } catch (ReflectiveOperationException ex) { 1254 throw new AssertionError(ex); 1255 } 1256 assert(op.name().equals(f.getName())) : op; 1257 assert(op.isAssociative() == (ft == Associative.class)) : op; 1258 if (op.isBoolean()) { 1259 assert(ft == (op.arity() == 2 ? Comparison.class : Test.class)) : op; 1260 } 1261 if (ft == Unary.class || ft == Conversion.class || ft == Test.class) { 1262 assert(op.arity() == 1) : op; 1263 } else if (ft == Ternary.class) { 1264 assert(op.arity() == 3) : op; 1265 } else { 1266 assert(op.arity() == 2) : op; 1267 if (ft != Associative.class && 1268 ft != Comparison.class) { 1269 assert(ft == Binary.class) : op; 1270 } 1271 } 1272 if (op.opKind(VO_OPCODE_VALID)) { 1273 int opsMask = (((VO_OPCODE_LIMIT-1) << VO_OPCODE_SHIFT) 1274 | VO_BOOL | VO_CONV 1275 | VO_ARITY_MASK); 1276 int opsIndex = op.opInfo & opsMask; 1277 OperatorImpl op0 = ops[opsIndex]; 1278 assert(op0 == null) 1279 : java.util.Arrays.asList(op0, Integer.toHexString(op0.opInfo), op, Integer.toHexString(op.opInfo)); 1280 ops[opsIndex] = op; 1281 } else { 1282 // These are all the "-1" opcode guys we know about: 1283 assert(op == ZOMO || 1284 op == FIRST_NONZERO || 1285 op == AND_NOT || op == NOT || 1286 op == ROL || 1287 op == ROR || 1288 op == IS_DEFAULT || op == IS_NEGATIVE || 1289 op == IS_FINITE || op == IS_NAN || op == IS_INFINITE || 1290 op == BITWISE_BLEND) : op; 1291 } 1292 } 1293 return true; 1294 } 1295 1296 // Managing behavioral information on slow paths: 1297 /*package-private*/ 1298 static class ImplCache<OP extends Operator,T> { ImplCache(Class<OP> whatKind, Class<? extends Vector<?>> whatVec)1299 public ImplCache(Class<OP> whatKind, 1300 Class<? extends Vector<?>> whatVec) { 1301 this.whatKind = whatKind; 1302 this.whatVec = whatVec; 1303 } 1304 1305 // These are used only for forming diagnostics: 1306 private final Class<OP> whatKind; 1307 private final Class<? extends Vector<?>> whatVec; 1308 1309 private final @Stable 1310 Object[] cache = new Object[VO_OPCODE_LIMIT]; 1311 1312 @ForceInline find(OP op, int opc, IntFunction<T> supplier)1313 public T find(OP op, int opc, IntFunction<T> supplier) { 1314 @SuppressWarnings("unchecked") 1315 T fn = (T) cache[opc]; 1316 if (fn != null) return fn; 1317 fn = supplier.apply(opc); 1318 if (fn == null) throw badOp(op); 1319 assert(VectorSupport.isNonCapturingLambda(fn)) : fn; 1320 // The JIT can see into this cache: 1321 cache[opc] = fn; 1322 return fn; 1323 } 1324 badOp(Operator op)1325 private UnsupportedOperationException badOp(Operator op) { 1326 String msg = String.format("%s: illegal %s in %s", 1327 op, 1328 whatKind.getSimpleName().toLowerCase(), 1329 whatVec.getSimpleName()); 1330 return new UnsupportedOperationException(msg); 1331 } 1332 toString()1333 @Override public String toString() { 1334 ArrayList<String> entries = new ArrayList<>(); 1335 for (int i = 0; i < cache.length; i++) { 1336 Object fn = cache[i]; 1337 if (fn != null) entries.add(i+": "+fn); 1338 } 1339 return String.format("ImplCache<%s,%s>[%s]", 1340 whatKind.getSimpleName(), 1341 whatVec.getSimpleName(), 1342 String.join(", ", entries)); 1343 } 1344 } 1345 } 1346