1 /* 2 * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26 package jdk.internal.vm.vector; 27 28 import jdk.internal.vm.annotation.IntrinsicCandidate; 29 import jdk.internal.misc.Unsafe; 30 31 import java.util.function.*; 32 33 public class VectorSupport { 34 static { registerNatives()35 registerNatives(); 36 } 37 38 private static final Unsafe U = Unsafe.getUnsafe(); 39 40 // Unary 41 public static final int VECTOR_OP_ABS = 0; 42 public static final int VECTOR_OP_NEG = 1; 43 public static final int VECTOR_OP_SQRT = 2; 44 45 // Binary 46 public static final int VECTOR_OP_ADD = 4; 47 public static final int VECTOR_OP_SUB = 5; 48 public static final int VECTOR_OP_MUL = 6; 49 public static final int VECTOR_OP_DIV = 7; 50 public static final int VECTOR_OP_MIN = 8; 51 public static final int VECTOR_OP_MAX = 9; 52 53 public static final int VECTOR_OP_AND = 10; 54 public static final int VECTOR_OP_OR = 11; 55 public static final int VECTOR_OP_XOR = 12; 56 57 // Ternary 58 public static final int VECTOR_OP_FMA = 13; 59 60 // Broadcast int 61 public static final int VECTOR_OP_LSHIFT = 14; 62 public static final int VECTOR_OP_RSHIFT = 15; 63 public static final int VECTOR_OP_URSHIFT = 16; 64 65 public static final int VECTOR_OP_CAST = 17; 66 public static final int VECTOR_OP_REINTERPRET = 18; 67 68 // Mask manipulation operations 69 public static final int VECTOR_OP_MASK_TRUECOUNT = 19; 70 public static final int VECTOR_OP_MASK_FIRSTTRUE = 20; 71 public static final int VECTOR_OP_MASK_LASTTRUE = 21; 72 73 // Math routines 74 public static final int VECTOR_OP_TAN = 101; 75 public static final int VECTOR_OP_TANH = 102; 76 public static final int VECTOR_OP_SIN = 103; 77 public static final int VECTOR_OP_SINH = 104; 78 public static final int VECTOR_OP_COS = 105; 79 public static final int VECTOR_OP_COSH = 106; 80 public static final int VECTOR_OP_ASIN = 107; 81 public static final int VECTOR_OP_ACOS = 108; 82 public static final int VECTOR_OP_ATAN = 109; 83 public static final int VECTOR_OP_ATAN2 = 110; 84 public static final int VECTOR_OP_CBRT = 111; 85 public static final int VECTOR_OP_LOG = 112; 86 public static final int VECTOR_OP_LOG10 = 113; 87 public static final int VECTOR_OP_LOG1P = 114; 88 public static final int VECTOR_OP_POW = 115; 89 public static final int VECTOR_OP_EXP = 116; 90 public static final int VECTOR_OP_EXPM1 = 117; 91 public static final int VECTOR_OP_HYPOT = 118; 92 93 // See src/hotspot/share/opto/subnode.hpp 94 // struct BoolTest, and enclosed enum mask 95 public static final int BT_eq = 0; // 0000 96 public static final int BT_ne = 4; // 0100 97 public static final int BT_le = 5; // 0101 98 public static final int BT_ge = 7; // 0111 99 public static final int BT_lt = 3; // 0011 100 public static final int BT_gt = 1; // 0001 101 public static final int BT_overflow = 2; // 0010 102 public static final int BT_no_overflow = 6; // 0110 103 // never = 8 1000 104 // illegal = 9 1001 105 // Unsigned comparisons apply to BT_le, BT_ge, BT_lt, BT_gt for integral types 106 public static final int BT_unsigned_compare = 0b10000; 107 public static final int BT_ule = BT_le | BT_unsigned_compare; 108 public static final int BT_uge = BT_ge | BT_unsigned_compare; 109 public static final int BT_ult = BT_lt | BT_unsigned_compare; 110 public static final int BT_ugt = BT_gt | BT_unsigned_compare; 111 112 // BasicType codes, for primitives only: 113 public static final int 114 T_FLOAT = 6, 115 T_DOUBLE = 7, 116 T_BYTE = 8, 117 T_SHORT = 9, 118 T_INT = 10, 119 T_LONG = 11; 120 121 /* ============================================================================ */ 122 123 public static class VectorSpecies<E> {} 124 125 public static class VectorPayload { 126 private final Object payload; // array of primitives 127 VectorPayload(Object payload)128 public VectorPayload(Object payload) { 129 this.payload = payload; 130 } 131 getPayload()132 protected final Object getPayload() { 133 return VectorSupport.maybeRebox(this).payload; 134 } 135 } 136 137 public static class Vector<E> extends VectorPayload { Vector(Object payload)138 public Vector(Object payload) { 139 super(payload); 140 } 141 } 142 143 public static class VectorShuffle<E> extends VectorPayload { VectorShuffle(Object payload)144 public VectorShuffle(Object payload) { 145 super(payload); 146 } 147 } 148 public static class VectorMask<E> extends VectorPayload { VectorMask(Object payload)149 public VectorMask(Object payload) { 150 super(payload); 151 } 152 } 153 154 /* ============================================================================ */ 155 public interface BroadcastOperation<VM, E, S extends VectorSpecies<E>> { broadcast(long l, S s)156 VM broadcast(long l, S s); 157 } 158 159 @IntrinsicCandidate 160 public static 161 <VM, E, S extends VectorSpecies<E>> broadcastCoerced(Class<? extends VM> vmClass, Class<E> E, int length, long bits, S s, BroadcastOperation<VM, E, S> defaultImpl)162 VM broadcastCoerced(Class<? extends VM> vmClass, Class<E> E, int length, 163 long bits, S s, 164 BroadcastOperation<VM, E, S> defaultImpl) { 165 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 166 return defaultImpl.broadcast(bits, s); 167 } 168 169 /* ============================================================================ */ 170 public interface ShuffleIotaOperation<E, S extends VectorSpecies<E>> { apply(int length, int start, int step, S s)171 VectorShuffle<E> apply(int length, int start, int step, S s); 172 } 173 174 @IntrinsicCandidate 175 public static 176 <E, S extends VectorSpecies<E>> shuffleIota(Class<?> E, Class<?> ShuffleClass, S s, int length, int start, int step, int wrap, ShuffleIotaOperation<E, S> defaultImpl)177 VectorShuffle<E> shuffleIota(Class<?> E, Class<?> ShuffleClass, S s, int length, 178 int start, int step, int wrap, ShuffleIotaOperation<E, S> defaultImpl) { 179 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 180 return defaultImpl.apply(length, start, step, s); 181 } 182 183 public interface ShuffleToVectorOperation<VM, Sh, E> { apply(Sh s)184 VM apply(Sh s); 185 } 186 187 @IntrinsicCandidate 188 public static 189 <VM ,Sh extends VectorShuffle<E>, E> shuffleToVector(Class<?> VM, Class<?>E , Class<?> ShuffleClass, Sh s, int length, ShuffleToVectorOperation<VM,Sh,E> defaultImpl)190 VM shuffleToVector(Class<?> VM, Class<?>E , Class<?> ShuffleClass, Sh s, int length, 191 ShuffleToVectorOperation<VM,Sh,E> defaultImpl) { 192 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 193 return defaultImpl.apply(s); 194 } 195 196 /* ============================================================================ */ 197 public interface IndexOperation<V extends Vector<E>, E, S extends VectorSpecies<E>> { index(V v, int step, S s)198 V index(V v, int step, S s); 199 } 200 201 //FIXME @IntrinsicCandidate 202 public static 203 <V extends Vector<E>, E, S extends VectorSpecies<E>> indexVector(Class<? extends V> vClass, Class<E> E, int length, V v, int step, S s, IndexOperation<V, E, S> defaultImpl)204 V indexVector(Class<? extends V> vClass, Class<E> E, int length, 205 V v, int step, S s, 206 IndexOperation<V, E, S> defaultImpl) { 207 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 208 return defaultImpl.index(v, step, s); 209 } 210 211 /* ============================================================================ */ 212 213 @IntrinsicCandidate 214 public static 215 <V extends Vector<?>> reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length, V v, Function<V,Long> defaultImpl)216 long reductionCoerced(int oprId, Class<?> vectorClass, Class<?> elementType, int length, 217 V v, 218 Function<V,Long> defaultImpl) { 219 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 220 return defaultImpl.apply(v); 221 } 222 223 /* ============================================================================ */ 224 225 public interface VecExtractOp<V> { apply(V v1, int idx)226 long apply(V v1, int idx); 227 } 228 229 @IntrinsicCandidate 230 public static 231 <V extends Vector<?>> extract(Class<?> vectorClass, Class<?> elementType, int vlen, V vec, int ix, VecExtractOp<V> defaultImpl)232 long extract(Class<?> vectorClass, Class<?> elementType, int vlen, 233 V vec, int ix, 234 VecExtractOp<V> defaultImpl) { 235 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 236 return defaultImpl.apply(vec, ix); 237 } 238 239 /* ============================================================================ */ 240 241 public interface VecInsertOp<V> { apply(V v1, int idx, long val)242 V apply(V v1, int idx, long val); 243 } 244 245 @IntrinsicCandidate 246 public static 247 <V extends Vector<?>> insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen, V vec, int ix, long val, VecInsertOp<V> defaultImpl)248 V insert(Class<? extends V> vectorClass, Class<?> elementType, int vlen, 249 V vec, int ix, long val, 250 VecInsertOp<V> defaultImpl) { 251 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 252 return defaultImpl.apply(vec, ix, val); 253 } 254 255 /* ============================================================================ */ 256 257 @IntrinsicCandidate 258 public static 259 <VM> unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length, VM vm, Function<VM, VM> defaultImpl)260 VM unaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length, 261 VM vm, 262 Function<VM, VM> defaultImpl) { 263 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 264 return defaultImpl.apply(vm); 265 } 266 267 /* ============================================================================ */ 268 269 @IntrinsicCandidate 270 public static 271 <VM> binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length, VM vm1, VM vm2, BiFunction<VM, VM, VM> defaultImpl)272 VM binaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length, 273 VM vm1, VM vm2, 274 BiFunction<VM, VM, VM> defaultImpl) { 275 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 276 return defaultImpl.apply(vm1, vm2); 277 } 278 279 /* ============================================================================ */ 280 281 public interface TernaryOperation<V> { apply(V v1, V v2, V v3)282 V apply(V v1, V v2, V v3); 283 } 284 285 @IntrinsicCandidate 286 public static 287 <VM> ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length, VM vm1, VM vm2, VM vm3, TernaryOperation<VM> defaultImpl)288 VM ternaryOp(int oprId, Class<? extends VM> vmClass, Class<?> elementType, int length, 289 VM vm1, VM vm2, VM vm3, 290 TernaryOperation<VM> defaultImpl) { 291 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 292 return defaultImpl.apply(vm1, vm2, vm3); 293 } 294 295 /* ============================================================================ */ 296 297 // Memory operations 298 299 public interface LoadOperation<C, V, E, S extends VectorSpecies<E>> { load(C container, int index, S s)300 V load(C container, int index, S s); 301 } 302 303 @IntrinsicCandidate 304 public static 305 <C, VM, E, S extends VectorSpecies<E>> load(Class<? extends VM> vmClass, Class<E> E, int length, Object base, long offset, C container, int index, S s, LoadOperation<C, VM, E, S> defaultImpl)306 VM load(Class<? extends VM> vmClass, Class<E> E, int length, 307 Object base, long offset, // Unsafe addressing 308 C container, int index, S s, // Arguments for default implementation 309 LoadOperation<C, VM, E, S> defaultImpl) { 310 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 311 return defaultImpl.load(container, index, s); 312 } 313 314 /* ============================================================================ */ 315 316 public interface LoadVectorOperationWithMap<C, V extends Vector<?>, E, S extends VectorSpecies<E>> { loadWithMap(C container, int index, int[] indexMap, int indexM, S s)317 V loadWithMap(C container, int index, int[] indexMap, int indexM, S s); 318 } 319 320 @IntrinsicCandidate 321 public static 322 <C, V extends Vector<?>, W extends Vector<Integer>, E, S extends VectorSpecies<E>> loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass, Object base, long offset, W index_vector, C container, int index, int[] indexMap, int indexM, S s, LoadVectorOperationWithMap<C, V, E, S> defaultImpl)323 V loadWithMap(Class<?> vectorClass, Class<E> E, int length, Class<?> vectorIndexClass, 324 Object base, long offset, // Unsafe addressing 325 W index_vector, 326 C container, int index, int[] indexMap, int indexM, S s, // Arguments for default implementation 327 LoadVectorOperationWithMap<C, V, E, S> defaultImpl) { 328 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 329 return defaultImpl.loadWithMap(container, index, indexMap, indexM, s); 330 } 331 332 /* ============================================================================ */ 333 334 public interface StoreVectorOperation<C, V extends Vector<?>> { store(C container, int index, V v)335 void store(C container, int index, V v); 336 } 337 338 @IntrinsicCandidate 339 public static 340 <C, V extends Vector<?>> store(Class<?> vectorClass, Class<?> elementType, int length, Object base, long offset, V v, C container, int index, StoreVectorOperation<C, V> defaultImpl)341 void store(Class<?> vectorClass, Class<?> elementType, int length, 342 Object base, long offset, // Unsafe addressing 343 V v, 344 C container, int index, // Arguments for default implementation 345 StoreVectorOperation<C, V> defaultImpl) { 346 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 347 defaultImpl.store(container, index, v); 348 } 349 350 /* ============================================================================ */ 351 352 public interface StoreVectorOperationWithMap<C, V extends Vector<?>> { storeWithMap(C container, int index, V v, int[] indexMap, int indexM)353 void storeWithMap(C container, int index, V v, int[] indexMap, int indexM); 354 } 355 356 @IntrinsicCandidate 357 public static 358 <C, V extends Vector<?>, W extends Vector<Integer>> storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass, Object base, long offset, W index_vector, V v, C container, int index, int[] indexMap, int indexM, StoreVectorOperationWithMap<C, V> defaultImpl)359 void storeWithMap(Class<?> vectorClass, Class<?> elementType, int length, Class<?> vectorIndexClass, 360 Object base, long offset, // Unsafe addressing 361 W index_vector, V v, 362 C container, int index, int[] indexMap, int indexM, // Arguments for default implementation 363 StoreVectorOperationWithMap<C, V> defaultImpl) { 364 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 365 defaultImpl.storeWithMap(container, index, v, indexMap, indexM); 366 } 367 368 /* ============================================================================ */ 369 370 @IntrinsicCandidate 371 public static 372 <VM> test(int cond, Class<?> vmClass, Class<?> elementType, int length, VM vm1, VM vm2, BiFunction<VM, VM, Boolean> defaultImpl)373 boolean test(int cond, Class<?> vmClass, Class<?> elementType, int length, 374 VM vm1, VM vm2, 375 BiFunction<VM, VM, Boolean> defaultImpl) { 376 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 377 return defaultImpl.apply(vm1, vm2); 378 } 379 380 /* ============================================================================ */ 381 382 public interface VectorCompareOp<V,M> { apply(int cond, V v1, V v2)383 M apply(int cond, V v1, V v2); 384 } 385 386 @IntrinsicCandidate 387 public static <V extends Vector<E>, 388 M extends VectorMask<E>, 389 E> compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length, V v1, V v2, VectorCompareOp<V,M> defaultImpl)390 M compare(int cond, Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length, 391 V v1, V v2, 392 VectorCompareOp<V,M> defaultImpl) { 393 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 394 return defaultImpl.apply(cond, v1, v2); 395 } 396 397 /* ============================================================================ */ 398 399 public interface VectorRearrangeOp<V extends Vector<E>, 400 Sh extends VectorShuffle<E>, 401 E> { apply(V v1, Sh shuffle)402 V apply(V v1, Sh shuffle); 403 } 404 405 @IntrinsicCandidate 406 public static 407 <V extends Vector<E>, 408 Sh extends VectorShuffle<E>, 409 E> rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<?> elementType, int vlen, V v1, Sh sh, VectorRearrangeOp<V,Sh, E> defaultImpl)410 V rearrangeOp(Class<? extends V> vectorClass, Class<Sh> shuffleClass, Class<?> elementType, int vlen, 411 V v1, Sh sh, 412 VectorRearrangeOp<V,Sh, E> defaultImpl) { 413 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 414 return defaultImpl.apply(v1, sh); 415 } 416 417 /* ============================================================================ */ 418 419 public interface VectorBlendOp<V extends Vector<E>, 420 M extends VectorMask<E>, 421 E> { apply(V v1, V v2, M mask)422 V apply(V v1, V v2, M mask); 423 } 424 425 @IntrinsicCandidate 426 public static 427 <V extends Vector<E>, 428 M extends VectorMask<E>, 429 E> blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length, V v1, V v2, M m, VectorBlendOp<V,M, E> defaultImpl)430 V blend(Class<? extends V> vectorClass, Class<M> maskClass, Class<?> elementType, int length, 431 V v1, V v2, M m, 432 VectorBlendOp<V,M, E> defaultImpl) { 433 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 434 return defaultImpl.apply(v1, v2, m); 435 } 436 437 /* ============================================================================ */ 438 439 public interface VectorBroadcastIntOp<V extends Vector<?>> { apply(V v, int n)440 V apply(V v, int n); 441 } 442 443 @IntrinsicCandidate 444 public static 445 <V extends Vector<?>> broadcastInt(int opr, Class<? extends V> vectorClass, Class<?> elementType, int length, V v, int n, VectorBroadcastIntOp<V> defaultImpl)446 V broadcastInt(int opr, Class<? extends V> vectorClass, Class<?> elementType, int length, 447 V v, int n, 448 VectorBroadcastIntOp<V> defaultImpl) { 449 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 450 return defaultImpl.apply(v, n); 451 } 452 453 /* ============================================================================ */ 454 455 public interface VectorConvertOp<VOUT, VIN, S> { apply(VIN v, S species)456 VOUT apply(VIN v, S species); 457 } 458 459 // Users of this intrinsic assume that it respects 460 // REGISTER_ENDIAN, which is currently ByteOrder.LITTLE_ENDIAN. 461 // See javadoc for REGISTER_ENDIAN. 462 463 @IntrinsicCandidate 464 public static <VOUT extends VectorPayload, 465 VIN extends VectorPayload, 466 S extends VectorSpecies<?>> convert(int oprId, Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen, Class<?> toVectorClass, Class<?> toElementType, int toVLen, VIN v, S s, VectorConvertOp<VOUT, VIN, S> defaultImpl)467 VOUT convert(int oprId, 468 Class<?> fromVectorClass, Class<?> fromElementType, int fromVLen, 469 Class<?> toVectorClass, Class<?> toElementType, int toVLen, 470 VIN v, S s, 471 VectorConvertOp<VOUT, VIN, S> defaultImpl) { 472 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 473 return defaultImpl.apply(v, s); 474 } 475 476 /* ============================================================================ */ 477 478 @IntrinsicCandidate maybeRebox(V v)479 public static <V> V maybeRebox(V v) { 480 // The fence is added here to avoid memory aliasing problems in C2 between scalar & vector accesses. 481 // TODO: move the fence generation into C2. Generate only when reboxing is taking place. 482 U.loadFence(); 483 return v; 484 } 485 486 /* ============================================================================ */ 487 public interface VectorMaskOp<M> { apply(M m)488 int apply(M m); 489 } 490 491 @IntrinsicCandidate 492 public static 493 <E, M> maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass, int length, M m, VectorMaskOp<M> defaultImpl)494 int maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass, int length, M m, 495 VectorMaskOp<M> defaultImpl) { 496 assert isNonCapturingLambda(defaultImpl) : defaultImpl; 497 return defaultImpl.apply(m); 498 } 499 500 /* ============================================================================ */ 501 502 // query the JVM's supported vector sizes and types getMaxLaneCount(Class<?> etype)503 public static native int getMaxLaneCount(Class<?> etype); 504 505 /* ============================================================================ */ 506 isNonCapturingLambda(Object o)507 public static boolean isNonCapturingLambda(Object o) { 508 return o.getClass().getDeclaredFields().length == 0; 509 } 510 511 /* ============================================================================ */ 512 registerNatives()513 private static native int registerNatives(); 514 } 515