1 //===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This tablegen backend is responsible for emitting arm_sve.h, which includes 10 // a declaration and definition of each function specified by the ARM C/C++ 11 // Language Extensions (ACLE). 12 // 13 // For details, visit: 14 // https://developer.arm.com/architectures/system-architectures/software-standards/acle 15 // 16 // Each SVE instruction is implemented in terms of 1 or more functions which 17 // are suffixed with the element type of the input vectors. Functions may be 18 // implemented in terms of generic vector operations such as +, *, -, etc. or 19 // by calling a __builtin_-prefixed function which will be handled by clang's 20 // CodeGen library. 21 // 22 // See also the documentation in include/clang/Basic/arm_sve.td. 23 // 24 //===----------------------------------------------------------------------===// 25 26 #include "llvm/ADT/ArrayRef.h" 27 #include "llvm/ADT/STLExtras.h" 28 #include "llvm/ADT/StringExtras.h" 29 #include "llvm/ADT/StringMap.h" 30 #include "llvm/TableGen/Error.h" 31 #include "llvm/TableGen/Record.h" 32 #include <array> 33 #include <cctype> 34 #include <set> 35 #include <sstream> 36 #include <string> 37 #include <tuple> 38 39 using namespace llvm; 40 41 enum ClassKind { 42 ClassNone, 43 ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix 44 ClassG, // Overloaded name without type suffix 45 }; 46 47 enum class ACLEKind { SVE, SME }; 48 49 using TypeSpec = std::string; 50 51 namespace { 52 53 class ImmCheck { 54 unsigned Arg; 55 unsigned Kind; 56 unsigned ElementSizeInBits; 57 58 public: 59 ImmCheck(unsigned Arg, unsigned Kind, unsigned ElementSizeInBits = 0) 60 : Arg(Arg), Kind(Kind), ElementSizeInBits(ElementSizeInBits) {} 61 ImmCheck(const ImmCheck &Other) = default; 62 ~ImmCheck() = default; 63 64 unsigned getArg() const { return Arg; } 65 unsigned getKind() const { return Kind; } 66 unsigned getElementSizeInBits() const { return ElementSizeInBits; } 67 }; 68 69 class SVEType { 70 bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat; 71 bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp, 72 Svcount; 73 unsigned Bitwidth, ElementBitwidth, NumVectors; 74 75 public: 76 SVEType() : SVEType("", 'v') {} 77 78 SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1) 79 : Float(false), Signed(true), Immediate(false), Void(false), 80 Constant(false), Pointer(false), BFloat(false), DefaultType(false), 81 IsScalable(true), Predicate(false), PredicatePattern(false), 82 PrefetchOp(false), Svcount(false), Bitwidth(128), ElementBitwidth(~0U), 83 NumVectors(NumVectors) { 84 if (!TS.empty()) 85 applyTypespec(TS); 86 applyModifier(CharMod); 87 } 88 89 SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) { 90 NumVectors = NumV; 91 } 92 93 bool isPointer() const { return Pointer; } 94 bool isVoidPointer() const { return Pointer && Void; } 95 bool isSigned() const { return Signed; } 96 bool isImmediate() const { return Immediate; } 97 bool isScalar() const { return NumVectors == 0; } 98 bool isVector() const { return NumVectors > 0; } 99 bool isScalableVector() const { return isVector() && IsScalable; } 100 bool isFixedLengthVector() const { return isVector() && !IsScalable; } 101 bool isChar() const { return ElementBitwidth == 8; } 102 bool isVoid() const { return Void & !Pointer; } 103 bool isDefault() const { return DefaultType; } 104 bool isFloat() const { return Float && !BFloat; } 105 bool isBFloat() const { return BFloat && !Float; } 106 bool isFloatingPoint() const { return Float || BFloat; } 107 bool isInteger() const { 108 return !isFloatingPoint() && !Predicate && !Svcount; 109 } 110 bool isScalarPredicate() const { 111 return !isFloatingPoint() && Predicate && NumVectors == 0; 112 } 113 bool isPredicateVector() const { return Predicate; } 114 bool isPredicatePattern() const { return PredicatePattern; } 115 bool isPrefetchOp() const { return PrefetchOp; } 116 bool isSvcount() const { return Svcount; } 117 bool isConstant() const { return Constant; } 118 unsigned getElementSizeInBits() const { return ElementBitwidth; } 119 unsigned getNumVectors() const { return NumVectors; } 120 121 unsigned getNumElements() const { 122 assert(ElementBitwidth != ~0U); 123 return Bitwidth / ElementBitwidth; 124 } 125 unsigned getSizeInBits() const { 126 return Bitwidth; 127 } 128 129 /// Return the string representation of a type, which is an encoded 130 /// string for passing to the BUILTIN() macro in Builtins.def. 131 std::string builtin_str() const; 132 133 /// Return the C/C++ string representation of a type for use in the 134 /// arm_sve.h header file. 135 std::string str() const; 136 137 private: 138 /// Creates the type based on the typespec string in TS. 139 void applyTypespec(StringRef TS); 140 141 /// Applies a prototype modifier to the type. 142 void applyModifier(char Mod); 143 }; 144 145 class SVEEmitter; 146 147 /// The main grunt class. This represents an instantiation of an intrinsic with 148 /// a particular typespec and prototype. 149 class Intrinsic { 150 /// The unmangled name. 151 std::string Name; 152 153 /// The name of the corresponding LLVM IR intrinsic. 154 std::string LLVMName; 155 156 /// Intrinsic prototype. 157 std::string Proto; 158 159 /// The base type spec for this intrinsic. 160 TypeSpec BaseTypeSpec; 161 162 /// The base class kind. Most intrinsics use ClassS, which has full type 163 /// info for integers (_s32/_u32), or ClassG which is used for overloaded 164 /// intrinsics. 165 ClassKind Class; 166 167 /// The architectural #ifdef guard. 168 std::string Guard; 169 170 // The merge suffix such as _m, _x or _z. 171 std::string MergeSuffix; 172 173 /// The types of return value [0] and parameters [1..]. 174 std::vector<SVEType> Types; 175 176 /// The "base type", which is VarType('d', BaseTypeSpec). 177 SVEType BaseType; 178 179 uint64_t Flags; 180 181 SmallVector<ImmCheck, 2> ImmChecks; 182 183 public: 184 Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy, 185 StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName, 186 uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT, 187 ClassKind Class, SVEEmitter &Emitter, StringRef Guard); 188 189 ~Intrinsic()=default; 190 191 std::string getName() const { return Name; } 192 std::string getLLVMName() const { return LLVMName; } 193 std::string getProto() const { return Proto; } 194 TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; } 195 SVEType getBaseType() const { return BaseType; } 196 197 StringRef getGuard() const { return Guard; } 198 ClassKind getClassKind() const { return Class; } 199 200 SVEType getReturnType() const { return Types[0]; } 201 ArrayRef<SVEType> getTypes() const { return Types; } 202 SVEType getParamType(unsigned I) const { return Types[I + 1]; } 203 unsigned getNumParams() const { 204 return Proto.size() - (2 * llvm::count(Proto, '.')) - 1; 205 } 206 207 uint64_t getFlags() const { return Flags; } 208 bool isFlagSet(uint64_t Flag) const { return Flags & Flag;} 209 210 ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; } 211 212 /// Return the type string for a BUILTIN() macro in Builtins.def. 213 std::string getBuiltinTypeStr(); 214 215 /// Return the name, mangled with type information. The name is mangled for 216 /// ClassS, so will add type suffixes such as _u32/_s32. 217 std::string getMangledName() const { return mangleName(ClassS); } 218 219 /// As above, but mangles the LLVM name instead. 220 std::string getMangledLLVMName() const { return mangleLLVMName(); } 221 222 /// Returns true if the intrinsic is overloaded, in that it should also generate 223 /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of 224 /// 'svld1_u32(..)'. 225 static bool isOverloadedIntrinsic(StringRef Name) { 226 auto BrOpen = Name.find('['); 227 auto BrClose = Name.find(']'); 228 return BrOpen != std::string::npos && BrClose != std::string::npos; 229 } 230 231 /// Return true if the intrinsic takes a splat operand. 232 bool hasSplat() const { 233 // These prototype modifiers are described in arm_sve.td. 234 return Proto.find_first_of("ajfrKLR@") != std::string::npos; 235 } 236 237 /// Return the parameter index of the splat operand. 238 unsigned getSplatIdx() const { 239 unsigned I = 1, Param = 0; 240 for (; I < Proto.size(); ++I, ++Param) { 241 if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' || 242 Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' || 243 Proto[I] == 'R' || Proto[I] == '@') 244 break; 245 246 // Multivector modifier can be skipped 247 if (Proto[I] == '.') 248 I += 2; 249 } 250 assert(I != Proto.size() && "Prototype has no splat operand"); 251 return Param; 252 } 253 254 /// Emits the intrinsic declaration to the ostream. 255 void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const; 256 257 private: 258 std::string getMergeSuffix() const { return MergeSuffix; } 259 std::string mangleName(ClassKind LocalCK) const; 260 std::string mangleLLVMName() const; 261 std::string replaceTemplatedArgs(std::string Name, TypeSpec TS, 262 std::string Proto) const; 263 }; 264 265 class SVEEmitter { 266 private: 267 // The reinterpret builtins are generated separately because they 268 // need the cross product of all types (121 functions in total), 269 // which is inconvenient to specify in the arm_sve.td file or 270 // generate in CGBuiltin.cpp. 271 struct ReinterpretTypeInfo { 272 SVEType BaseType; 273 const char *Suffix; 274 }; 275 276 static const std::array<ReinterpretTypeInfo, 12> Reinterprets; 277 278 RecordKeeper &Records; 279 llvm::StringMap<uint64_t> EltTypes; 280 llvm::StringMap<uint64_t> MemEltTypes; 281 llvm::StringMap<uint64_t> FlagTypes; 282 llvm::StringMap<uint64_t> MergeTypes; 283 llvm::StringMap<uint64_t> ImmCheckTypes; 284 285 public: 286 SVEEmitter(RecordKeeper &R) : Records(R) { 287 for (auto *RV : Records.getAllDerivedDefinitions("EltType")) 288 EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); 289 for (auto *RV : Records.getAllDerivedDefinitions("MemEltType")) 290 MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); 291 for (auto *RV : Records.getAllDerivedDefinitions("FlagType")) 292 FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); 293 for (auto *RV : Records.getAllDerivedDefinitions("MergeType")) 294 MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); 295 for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType")) 296 ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); 297 } 298 299 /// Returns the enum value for the immcheck type 300 unsigned getEnumValueForImmCheck(StringRef C) const { 301 auto It = ImmCheckTypes.find(C); 302 if (It != ImmCheckTypes.end()) 303 return It->getValue(); 304 llvm_unreachable("Unsupported imm check"); 305 } 306 307 /// Returns the enum value for the flag type 308 uint64_t getEnumValueForFlag(StringRef C) const { 309 auto Res = FlagTypes.find(C); 310 if (Res != FlagTypes.end()) 311 return Res->getValue(); 312 llvm_unreachable("Unsupported flag"); 313 } 314 315 // Returns the SVETypeFlags for a given value and mask. 316 uint64_t encodeFlag(uint64_t V, StringRef MaskName) const { 317 auto It = FlagTypes.find(MaskName); 318 if (It != FlagTypes.end()) { 319 uint64_t Mask = It->getValue(); 320 unsigned Shift = llvm::countr_zero(Mask); 321 assert(Shift < 64 && "Mask value produced an invalid shift value"); 322 return (V << Shift) & Mask; 323 } 324 llvm_unreachable("Unsupported flag"); 325 } 326 327 // Returns the SVETypeFlags for the given element type. 328 uint64_t encodeEltType(StringRef EltName) { 329 auto It = EltTypes.find(EltName); 330 if (It != EltTypes.end()) 331 return encodeFlag(It->getValue(), "EltTypeMask"); 332 llvm_unreachable("Unsupported EltType"); 333 } 334 335 // Returns the SVETypeFlags for the given memory element type. 336 uint64_t encodeMemoryElementType(uint64_t MT) { 337 return encodeFlag(MT, "MemEltTypeMask"); 338 } 339 340 // Returns the SVETypeFlags for the given merge type. 341 uint64_t encodeMergeType(uint64_t MT) { 342 return encodeFlag(MT, "MergeTypeMask"); 343 } 344 345 // Returns the SVETypeFlags for the given splat operand. 346 unsigned encodeSplatOperand(unsigned SplatIdx) { 347 assert(SplatIdx < 7 && "SplatIdx out of encodable range"); 348 return encodeFlag(SplatIdx + 1, "SplatOperandMask"); 349 } 350 351 // Returns the SVETypeFlags value for the given SVEType. 352 uint64_t encodeTypeFlags(const SVEType &T); 353 354 /// Emit arm_sve.h. 355 void createHeader(raw_ostream &o); 356 357 // Emits core intrinsics in both arm_sme.h and arm_sve.h 358 void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter, 359 ACLEKind Kind); 360 361 /// Emit all the __builtin prototypes and code needed by Sema. 362 void createBuiltins(raw_ostream &o); 363 364 /// Emit all the information needed to map builtin -> LLVM IR intrinsic. 365 void createCodeGenMap(raw_ostream &o); 366 367 /// Emit all the range checks for the immediates. 368 void createRangeChecks(raw_ostream &o); 369 370 /// Create the SVETypeFlags used in CGBuiltins 371 void createTypeFlags(raw_ostream &o); 372 373 /// Emit arm_sme.h. 374 void createSMEHeader(raw_ostream &o); 375 376 /// Emit all the SME __builtin prototypes and code needed by Sema. 377 void createSMEBuiltins(raw_ostream &o); 378 379 /// Emit all the information needed to map builtin -> LLVM IR intrinsic. 380 void createSMECodeGenMap(raw_ostream &o); 381 382 /// Emit all the range checks for the immediates. 383 void createSMERangeChecks(raw_ostream &o); 384 385 /// Create intrinsic and add it to \p Out 386 void createIntrinsic(Record *R, 387 SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out); 388 }; 389 390 const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets = 391 {{{SVEType("c", 'd'), "s8"}, 392 {SVEType("Uc", 'd'), "u8"}, 393 {SVEType("s", 'd'), "s16"}, 394 {SVEType("Us", 'd'), "u16"}, 395 {SVEType("i", 'd'), "s32"}, 396 {SVEType("Ui", 'd'), "u32"}, 397 {SVEType("l", 'd'), "s64"}, 398 {SVEType("Ul", 'd'), "u64"}, 399 {SVEType("h", 'd'), "f16"}, 400 {SVEType("b", 'd'), "bf16"}, 401 {SVEType("f", 'd'), "f32"}, 402 {SVEType("d", 'd'), "f64"}}}; 403 404 } // end anonymous namespace 405 406 407 //===----------------------------------------------------------------------===// 408 // Type implementation 409 //===----------------------------------------------------------------------===// 410 411 std::string SVEType::builtin_str() const { 412 std::string S; 413 if (isVoid()) 414 return "v"; 415 416 if (isScalarPredicate()) 417 return "b"; 418 419 if (isSvcount()) 420 return "Qa"; 421 422 if (isVoidPointer()) 423 S += "v"; 424 else if (!isFloatingPoint()) 425 switch (ElementBitwidth) { 426 case 1: S += "b"; break; 427 case 8: S += "c"; break; 428 case 16: S += "s"; break; 429 case 32: S += "i"; break; 430 case 64: S += "Wi"; break; 431 case 128: S += "LLLi"; break; 432 default: llvm_unreachable("Unhandled case!"); 433 } 434 else if (isFloat()) 435 switch (ElementBitwidth) { 436 case 16: S += "h"; break; 437 case 32: S += "f"; break; 438 case 64: S += "d"; break; 439 default: llvm_unreachable("Unhandled case!"); 440 } 441 else if (isBFloat()) { 442 assert(ElementBitwidth == 16 && "Not a valid BFloat."); 443 S += "y"; 444 } 445 446 if (!isFloatingPoint()) { 447 if ((isChar() || isPointer()) && !isVoidPointer()) { 448 // Make chars and typed pointers explicitly signed. 449 if (Signed) 450 S = "S" + S; 451 else if (!Signed) 452 S = "U" + S; 453 } else if (!isVoidPointer() && !Signed) { 454 S = "U" + S; 455 } 456 } 457 458 // Constant indices are "int", but have the "constant expression" modifier. 459 if (isImmediate()) { 460 assert(!isFloat() && "fp immediates are not supported"); 461 S = "I" + S; 462 } 463 464 if (isScalar()) { 465 if (Constant) S += "C"; 466 if (Pointer) S += "*"; 467 return S; 468 } 469 470 if (isFixedLengthVector()) 471 return "V" + utostr(getNumElements() * NumVectors) + S; 472 return "q" + utostr(getNumElements() * NumVectors) + S; 473 } 474 475 std::string SVEType::str() const { 476 if (isPredicatePattern()) 477 return "enum svpattern"; 478 479 if (isPrefetchOp()) 480 return "enum svprfop"; 481 482 std::string S; 483 if (Void) 484 S += "void"; 485 else { 486 if (isScalableVector() || isSvcount()) 487 S += "sv"; 488 if (!Signed && !isFloatingPoint()) 489 S += "u"; 490 491 if (Float) 492 S += "float"; 493 else if (isSvcount()) 494 S += "count"; 495 else if (isScalarPredicate() || isPredicateVector()) 496 S += "bool"; 497 else if (isBFloat()) 498 S += "bfloat"; 499 else 500 S += "int"; 501 502 if (!isScalarPredicate() && !isPredicateVector() && !isSvcount()) 503 S += utostr(ElementBitwidth); 504 if (isFixedLengthVector()) 505 S += "x" + utostr(getNumElements()); 506 if (NumVectors > 1) 507 S += "x" + utostr(NumVectors); 508 if (!isScalarPredicate()) 509 S += "_t"; 510 } 511 512 if (Constant) 513 S += " const"; 514 if (Pointer) 515 S += " *"; 516 517 return S; 518 } 519 520 void SVEType::applyTypespec(StringRef TS) { 521 for (char I : TS) { 522 switch (I) { 523 case 'Q': 524 Svcount = true; 525 break; 526 case 'P': 527 Predicate = true; 528 break; 529 case 'U': 530 Signed = false; 531 break; 532 case 'c': 533 ElementBitwidth = 8; 534 break; 535 case 's': 536 ElementBitwidth = 16; 537 break; 538 case 'i': 539 ElementBitwidth = 32; 540 break; 541 case 'l': 542 ElementBitwidth = 64; 543 break; 544 case 'q': 545 ElementBitwidth = 128; 546 break; 547 case 'h': 548 Float = true; 549 ElementBitwidth = 16; 550 break; 551 case 'f': 552 Float = true; 553 ElementBitwidth = 32; 554 break; 555 case 'd': 556 Float = true; 557 ElementBitwidth = 64; 558 break; 559 case 'b': 560 BFloat = true; 561 Float = false; 562 ElementBitwidth = 16; 563 break; 564 default: 565 llvm_unreachable("Unhandled type code!"); 566 } 567 } 568 assert(ElementBitwidth != ~0U && "Bad element bitwidth!"); 569 } 570 571 void SVEType::applyModifier(char Mod) { 572 switch (Mod) { 573 case 'v': 574 Void = true; 575 break; 576 case 'd': 577 DefaultType = true; 578 break; 579 case 'c': 580 Constant = true; 581 [[fallthrough]]; 582 case 'p': 583 Pointer = true; 584 Bitwidth = ElementBitwidth; 585 NumVectors = 0; 586 break; 587 case 'e': 588 Signed = false; 589 ElementBitwidth /= 2; 590 break; 591 case 'h': 592 ElementBitwidth /= 2; 593 break; 594 case 'q': 595 ElementBitwidth /= 4; 596 break; 597 case 'b': 598 Signed = false; 599 Float = false; 600 BFloat = false; 601 ElementBitwidth /= 4; 602 break; 603 case 'o': 604 ElementBitwidth *= 4; 605 break; 606 case 'P': 607 Signed = true; 608 Float = false; 609 BFloat = false; 610 Predicate = true; 611 Svcount = false; 612 Bitwidth = 16; 613 ElementBitwidth = 1; 614 break; 615 case '{': 616 IsScalable = false; 617 Bitwidth = 128; 618 NumVectors = 1; 619 break; 620 case 's': 621 case 'a': 622 Bitwidth = ElementBitwidth; 623 NumVectors = 0; 624 break; 625 case 'R': 626 ElementBitwidth /= 2; 627 NumVectors = 0; 628 break; 629 case 'r': 630 ElementBitwidth /= 4; 631 NumVectors = 0; 632 break; 633 case '@': 634 Signed = false; 635 Float = false; 636 BFloat = false; 637 ElementBitwidth /= 4; 638 NumVectors = 0; 639 break; 640 case 'K': 641 Signed = true; 642 Float = false; 643 BFloat = false; 644 Bitwidth = ElementBitwidth; 645 NumVectors = 0; 646 break; 647 case 'L': 648 Signed = false; 649 Float = false; 650 BFloat = false; 651 Bitwidth = ElementBitwidth; 652 NumVectors = 0; 653 break; 654 case 'u': 655 Predicate = false; 656 Svcount = false; 657 Signed = false; 658 Float = false; 659 BFloat = false; 660 break; 661 case 'x': 662 Predicate = false; 663 Svcount = false; 664 Signed = true; 665 Float = false; 666 BFloat = false; 667 break; 668 case 'i': 669 Predicate = false; 670 Svcount = false; 671 Float = false; 672 BFloat = false; 673 ElementBitwidth = Bitwidth = 64; 674 NumVectors = 0; 675 Signed = false; 676 Immediate = true; 677 break; 678 case 'I': 679 Predicate = false; 680 Svcount = false; 681 Float = false; 682 BFloat = false; 683 ElementBitwidth = Bitwidth = 32; 684 NumVectors = 0; 685 Signed = true; 686 Immediate = true; 687 PredicatePattern = true; 688 break; 689 case 'J': 690 Predicate = false; 691 Svcount = false; 692 Float = false; 693 BFloat = false; 694 ElementBitwidth = Bitwidth = 32; 695 NumVectors = 0; 696 Signed = true; 697 Immediate = true; 698 PrefetchOp = true; 699 break; 700 case 'k': 701 Predicate = false; 702 Svcount = false; 703 Signed = true; 704 Float = false; 705 BFloat = false; 706 ElementBitwidth = Bitwidth = 32; 707 NumVectors = 0; 708 break; 709 case 'l': 710 Predicate = false; 711 Svcount = false; 712 Signed = true; 713 Float = false; 714 BFloat = false; 715 ElementBitwidth = Bitwidth = 64; 716 NumVectors = 0; 717 break; 718 case 'm': 719 Predicate = false; 720 Svcount = false; 721 Signed = false; 722 Float = false; 723 BFloat = false; 724 ElementBitwidth = Bitwidth = 32; 725 NumVectors = 0; 726 break; 727 case 'n': 728 Predicate = false; 729 Svcount = false; 730 Signed = false; 731 Float = false; 732 BFloat = false; 733 ElementBitwidth = Bitwidth = 64; 734 NumVectors = 0; 735 break; 736 case 'w': 737 ElementBitwidth = 64; 738 break; 739 case 'j': 740 ElementBitwidth = Bitwidth = 64; 741 NumVectors = 0; 742 break; 743 case 'f': 744 Signed = false; 745 ElementBitwidth = Bitwidth = 64; 746 NumVectors = 0; 747 break; 748 case 'g': 749 Signed = false; 750 Float = false; 751 BFloat = false; 752 ElementBitwidth = 64; 753 break; 754 case '[': 755 Signed = false; 756 Float = false; 757 BFloat = false; 758 ElementBitwidth = 8; 759 break; 760 case 't': 761 Signed = true; 762 Float = false; 763 BFloat = false; 764 ElementBitwidth = 32; 765 break; 766 case 'z': 767 Signed = false; 768 Float = false; 769 BFloat = false; 770 ElementBitwidth = 32; 771 break; 772 case 'O': 773 Predicate = false; 774 Svcount = false; 775 Float = true; 776 ElementBitwidth = 16; 777 break; 778 case 'M': 779 Predicate = false; 780 Svcount = false; 781 Float = true; 782 BFloat = false; 783 ElementBitwidth = 32; 784 break; 785 case 'N': 786 Predicate = false; 787 Svcount = false; 788 Float = true; 789 ElementBitwidth = 64; 790 break; 791 case 'Q': 792 Constant = true; 793 Pointer = true; 794 Void = true; 795 NumVectors = 0; 796 break; 797 case 'S': 798 Constant = true; 799 Pointer = true; 800 ElementBitwidth = Bitwidth = 8; 801 NumVectors = 0; 802 Signed = true; 803 break; 804 case 'W': 805 Constant = true; 806 Pointer = true; 807 ElementBitwidth = Bitwidth = 8; 808 NumVectors = 0; 809 Signed = false; 810 break; 811 case 'T': 812 Constant = true; 813 Pointer = true; 814 ElementBitwidth = Bitwidth = 16; 815 NumVectors = 0; 816 Signed = true; 817 break; 818 case 'X': 819 Constant = true; 820 Pointer = true; 821 ElementBitwidth = Bitwidth = 16; 822 NumVectors = 0; 823 Signed = false; 824 break; 825 case 'Y': 826 Constant = true; 827 Pointer = true; 828 ElementBitwidth = Bitwidth = 32; 829 NumVectors = 0; 830 Signed = false; 831 break; 832 case 'U': 833 Constant = true; 834 Pointer = true; 835 ElementBitwidth = Bitwidth = 32; 836 NumVectors = 0; 837 Signed = true; 838 break; 839 case '%': 840 Pointer = true; 841 Void = true; 842 NumVectors = 0; 843 break; 844 case 'A': 845 Pointer = true; 846 ElementBitwidth = Bitwidth = 8; 847 NumVectors = 0; 848 Signed = true; 849 break; 850 case 'B': 851 Pointer = true; 852 ElementBitwidth = Bitwidth = 16; 853 NumVectors = 0; 854 Signed = true; 855 break; 856 case 'C': 857 Pointer = true; 858 ElementBitwidth = Bitwidth = 32; 859 NumVectors = 0; 860 Signed = true; 861 break; 862 case 'D': 863 Pointer = true; 864 ElementBitwidth = Bitwidth = 64; 865 NumVectors = 0; 866 Signed = true; 867 break; 868 case 'E': 869 Pointer = true; 870 ElementBitwidth = Bitwidth = 8; 871 NumVectors = 0; 872 Signed = false; 873 break; 874 case 'F': 875 Pointer = true; 876 ElementBitwidth = Bitwidth = 16; 877 NumVectors = 0; 878 Signed = false; 879 break; 880 case 'G': 881 Pointer = true; 882 ElementBitwidth = Bitwidth = 32; 883 NumVectors = 0; 884 Signed = false; 885 break; 886 case '$': 887 Predicate = false; 888 Svcount = false; 889 Float = false; 890 BFloat = true; 891 ElementBitwidth = 16; 892 break; 893 case '}': 894 Predicate = false; 895 Signed = true; 896 Svcount = true; 897 NumVectors = 0; 898 Float = false; 899 BFloat = false; 900 break; 901 case '.': 902 llvm_unreachable(". is never a type in itself"); 903 break; 904 default: 905 llvm_unreachable("Unhandled character!"); 906 } 907 } 908 909 /// Returns the modifier and number of vectors for the given operand \p Op. 910 std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) { 911 for (unsigned P = 0; !Proto.empty(); ++P) { 912 unsigned NumVectors = 1; 913 unsigned CharsToSkip = 1; 914 char Mod = Proto[0]; 915 if (Mod == '2' || Mod == '3' || Mod == '4') { 916 NumVectors = Mod - '0'; 917 Mod = 'd'; 918 if (Proto.size() > 1 && Proto[1] == '.') { 919 Mod = Proto[2]; 920 CharsToSkip = 3; 921 } 922 } 923 924 if (P == Op) 925 return {Mod, NumVectors}; 926 927 Proto = Proto.drop_front(CharsToSkip); 928 } 929 llvm_unreachable("Unexpected Op"); 930 } 931 932 //===----------------------------------------------------------------------===// 933 // Intrinsic implementation 934 //===----------------------------------------------------------------------===// 935 936 Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy, 937 StringRef MergeSuffix, uint64_t MemoryElementTy, 938 StringRef LLVMName, uint64_t Flags, 939 ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class, 940 SVEEmitter &Emitter, StringRef Guard) 941 : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()), 942 BaseTypeSpec(BT), Class(Class), Guard(Guard.str()), 943 MergeSuffix(MergeSuffix.str()), BaseType(BT, 'd'), Flags(Flags), 944 ImmChecks(Checks.begin(), Checks.end()) { 945 // Types[0] is the return value. 946 for (unsigned I = 0; I < (getNumParams() + 1); ++I) { 947 char Mod; 948 unsigned NumVectors; 949 std::tie(Mod, NumVectors) = getProtoModifier(Proto, I); 950 SVEType T(BaseTypeSpec, Mod, NumVectors); 951 Types.push_back(T); 952 953 // Add range checks for immediates 954 if (I > 0) { 955 if (T.isPredicatePattern()) 956 ImmChecks.emplace_back( 957 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31")); 958 else if (T.isPrefetchOp()) 959 ImmChecks.emplace_back( 960 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13")); 961 } 962 } 963 964 // Set flags based on properties 965 this->Flags |= Emitter.encodeTypeFlags(BaseType); 966 this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy); 967 this->Flags |= Emitter.encodeMergeType(MergeTy); 968 if (hasSplat()) 969 this->Flags |= Emitter.encodeSplatOperand(getSplatIdx()); 970 } 971 972 std::string Intrinsic::getBuiltinTypeStr() { 973 std::string S = getReturnType().builtin_str(); 974 for (unsigned I = 0; I < getNumParams(); ++I) 975 S += getParamType(I).builtin_str(); 976 977 return S; 978 } 979 980 std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS, 981 std::string Proto) const { 982 std::string Ret = Name; 983 while (Ret.find('{') != std::string::npos) { 984 size_t Pos = Ret.find('{'); 985 size_t End = Ret.find('}'); 986 unsigned NumChars = End - Pos + 1; 987 assert(NumChars == 3 && "Unexpected template argument"); 988 989 SVEType T; 990 char C = Ret[Pos+1]; 991 switch(C) { 992 default: 993 llvm_unreachable("Unknown predication specifier"); 994 case 'd': 995 T = SVEType(TS, 'd'); 996 break; 997 case '0': 998 case '1': 999 case '2': 1000 case '3': 1001 T = SVEType(TS, Proto[C - '0']); 1002 break; 1003 } 1004 1005 // Replace templated arg with the right suffix (e.g. u32) 1006 std::string TypeCode; 1007 if (T.isInteger()) 1008 TypeCode = T.isSigned() ? 's' : 'u'; 1009 else if (T.isSvcount()) 1010 TypeCode = 'c'; 1011 else if (T.isPredicateVector()) 1012 TypeCode = 'b'; 1013 else if (T.isBFloat()) 1014 TypeCode = "bf"; 1015 else 1016 TypeCode = 'f'; 1017 Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits())); 1018 } 1019 1020 return Ret; 1021 } 1022 1023 std::string Intrinsic::mangleLLVMName() const { 1024 std::string S = getLLVMName(); 1025 1026 // Replace all {d} like expressions with e.g. 'u32' 1027 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()); 1028 } 1029 1030 std::string Intrinsic::mangleName(ClassKind LocalCK) const { 1031 std::string S = getName(); 1032 1033 if (LocalCK == ClassG) { 1034 // Remove the square brackets and everything in between. 1035 while (S.find('[') != std::string::npos) { 1036 auto Start = S.find('['); 1037 auto End = S.find(']'); 1038 S.erase(Start, (End-Start)+1); 1039 } 1040 } else { 1041 // Remove the square brackets. 1042 while (S.find('[') != std::string::npos) { 1043 auto BrPos = S.find('['); 1044 if (BrPos != std::string::npos) 1045 S.erase(BrPos, 1); 1046 BrPos = S.find(']'); 1047 if (BrPos != std::string::npos) 1048 S.erase(BrPos, 1); 1049 } 1050 } 1051 1052 // Replace all {d} like expressions with e.g. 'u32' 1053 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) + 1054 getMergeSuffix(); 1055 } 1056 1057 void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, 1058 ACLEKind Kind) const { 1059 bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1; 1060 1061 std::string FullName = mangleName(ClassS); 1062 std::string ProtoName = mangleName(getClassKind()); 1063 std::string SMEAttrs = ""; 1064 1065 if (Flags & Emitter.getEnumValueForFlag("IsStreaming")) 1066 SMEAttrs += ", arm_streaming"; 1067 if (Flags & Emitter.getEnumValueForFlag("IsStreamingCompatible")) 1068 SMEAttrs += ", arm_streaming_compatible"; 1069 if (Flags & Emitter.getEnumValueForFlag("IsSharedZA")) 1070 SMEAttrs += ", arm_shared_za"; 1071 if (Flags & Emitter.getEnumValueForFlag("IsPreservesZA")) 1072 SMEAttrs += ", arm_preserves_za"; 1073 1074 OS << (IsOverloaded ? "__aio " : "__ai ") 1075 << "__attribute__((__clang_arm_builtin_alias("; 1076 1077 switch (Kind) { 1078 case ACLEKind::SME: 1079 OS << "__builtin_sme_" << FullName << ")"; 1080 break; 1081 case ACLEKind::SVE: 1082 OS << "__builtin_sve_" << FullName << ")"; 1083 break; 1084 } 1085 1086 if (!SMEAttrs.empty()) 1087 OS << SMEAttrs; 1088 OS << "))\n"; 1089 1090 OS << getTypes()[0].str() << " " << ProtoName << "("; 1091 for (unsigned I = 0; I < getTypes().size() - 1; ++I) { 1092 if (I != 0) 1093 OS << ", "; 1094 OS << getTypes()[I + 1].str(); 1095 } 1096 OS << ");\n"; 1097 } 1098 1099 //===----------------------------------------------------------------------===// 1100 // SVEEmitter implementation 1101 //===----------------------------------------------------------------------===// 1102 uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) { 1103 if (T.isFloat()) { 1104 switch (T.getElementSizeInBits()) { 1105 case 16: 1106 return encodeEltType("EltTyFloat16"); 1107 case 32: 1108 return encodeEltType("EltTyFloat32"); 1109 case 64: 1110 return encodeEltType("EltTyFloat64"); 1111 default: 1112 llvm_unreachable("Unhandled float element bitwidth!"); 1113 } 1114 } 1115 1116 if (T.isBFloat()) { 1117 assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat."); 1118 return encodeEltType("EltTyBFloat16"); 1119 } 1120 1121 if (T.isPredicateVector() || T.isSvcount()) { 1122 switch (T.getElementSizeInBits()) { 1123 case 8: 1124 return encodeEltType("EltTyBool8"); 1125 case 16: 1126 return encodeEltType("EltTyBool16"); 1127 case 32: 1128 return encodeEltType("EltTyBool32"); 1129 case 64: 1130 return encodeEltType("EltTyBool64"); 1131 default: 1132 llvm_unreachable("Unhandled predicate element bitwidth!"); 1133 } 1134 } 1135 1136 switch (T.getElementSizeInBits()) { 1137 case 8: 1138 return encodeEltType("EltTyInt8"); 1139 case 16: 1140 return encodeEltType("EltTyInt16"); 1141 case 32: 1142 return encodeEltType("EltTyInt32"); 1143 case 64: 1144 return encodeEltType("EltTyInt64"); 1145 case 128: 1146 return encodeEltType("EltTyInt128"); 1147 default: 1148 llvm_unreachable("Unhandled integer element bitwidth!"); 1149 } 1150 } 1151 1152 void SVEEmitter::createIntrinsic( 1153 Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) { 1154 StringRef Name = R->getValueAsString("Name"); 1155 StringRef Proto = R->getValueAsString("Prototype"); 1156 StringRef Types = R->getValueAsString("Types"); 1157 StringRef Guard = R->getValueAsString("TargetGuard"); 1158 StringRef LLVMName = R->getValueAsString("LLVMIntrinsic"); 1159 uint64_t Merge = R->getValueAsInt("Merge"); 1160 StringRef MergeSuffix = R->getValueAsString("MergeSuffix"); 1161 uint64_t MemEltType = R->getValueAsInt("MemEltType"); 1162 std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags"); 1163 std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks"); 1164 1165 int64_t Flags = 0; 1166 for (auto FlagRec : FlagsList) 1167 Flags |= FlagRec->getValueAsInt("Value"); 1168 1169 // Create a dummy TypeSpec for non-overloaded builtins. 1170 if (Types.empty()) { 1171 assert((Flags & getEnumValueForFlag("IsOverloadNone")) && 1172 "Expect TypeSpec for overloaded builtin!"); 1173 Types = "i"; 1174 } 1175 1176 // Extract type specs from string 1177 SmallVector<TypeSpec, 8> TypeSpecs; 1178 TypeSpec Acc; 1179 for (char I : Types) { 1180 Acc.push_back(I); 1181 if (islower(I)) { 1182 TypeSpecs.push_back(TypeSpec(Acc)); 1183 Acc.clear(); 1184 } 1185 } 1186 1187 // Remove duplicate type specs. 1188 llvm::sort(TypeSpecs); 1189 TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()), 1190 TypeSpecs.end()); 1191 1192 // Create an Intrinsic for each type spec. 1193 for (auto TS : TypeSpecs) { 1194 // Collate a list of range/option checks for the immediates. 1195 SmallVector<ImmCheck, 2> ImmChecks; 1196 for (auto *R : ImmCheckList) { 1197 int64_t Arg = R->getValueAsInt("Arg"); 1198 int64_t EltSizeArg = R->getValueAsInt("EltSizeArg"); 1199 int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value"); 1200 assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative"); 1201 1202 unsigned ElementSizeInBits = 0; 1203 char Mod; 1204 unsigned NumVectors; 1205 std::tie(Mod, NumVectors) = getProtoModifier(Proto, EltSizeArg + 1); 1206 if (EltSizeArg >= 0) 1207 ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits(); 1208 ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits)); 1209 } 1210 1211 Out.push_back(std::make_unique<Intrinsic>( 1212 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks, 1213 TS, ClassS, *this, Guard)); 1214 1215 // Also generate the short-form (e.g. svadd_m) for the given type-spec. 1216 if (Intrinsic::isOverloadedIntrinsic(Name)) 1217 Out.push_back(std::make_unique<Intrinsic>( 1218 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, 1219 ImmChecks, TS, ClassG, *this, Guard)); 1220 } 1221 } 1222 1223 void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS, 1224 SVEEmitter &Emitter, 1225 ACLEKind Kind) { 1226 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1227 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1228 for (auto *R : RV) 1229 createIntrinsic(R, Defs); 1230 1231 // Sort intrinsics in header file by following order/priority: 1232 // - Architectural guard (i.e. does it require SVE2 or SVE2_AES) 1233 // - Class (is intrinsic overloaded or not) 1234 // - Intrinsic name 1235 std::stable_sort(Defs.begin(), Defs.end(), 1236 [](const std::unique_ptr<Intrinsic> &A, 1237 const std::unique_ptr<Intrinsic> &B) { 1238 auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) { 1239 return std::make_tuple(I->getGuard(), 1240 (unsigned)I->getClassKind(), 1241 I->getName()); 1242 }; 1243 return ToTuple(A) < ToTuple(B); 1244 }); 1245 1246 // Actually emit the intrinsic declarations. 1247 for (auto &I : Defs) 1248 I->emitIntrinsic(OS, Emitter, Kind); 1249 } 1250 1251 void SVEEmitter::createHeader(raw_ostream &OS) { 1252 OS << "/*===---- arm_sve.h - ARM SVE intrinsics " 1253 "-----------------------------------===\n" 1254 " *\n" 1255 " *\n" 1256 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM " 1257 "Exceptions.\n" 1258 " * See https://llvm.org/LICENSE.txt for license information.\n" 1259 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n" 1260 " *\n" 1261 " *===-----------------------------------------------------------------" 1262 "------===\n" 1263 " */\n\n"; 1264 1265 OS << "#ifndef __ARM_SVE_H\n"; 1266 OS << "#define __ARM_SVE_H\n\n"; 1267 1268 OS << "#if !defined(__LITTLE_ENDIAN__)\n"; 1269 OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n"; 1270 OS << "#endif\n"; 1271 1272 OS << "#include <stdint.h>\n\n"; 1273 OS << "#ifdef __cplusplus\n"; 1274 OS << "extern \"C\" {\n"; 1275 OS << "#else\n"; 1276 OS << "#include <stdbool.h>\n"; 1277 OS << "#endif\n\n"; 1278 1279 OS << "typedef __fp16 float16_t;\n"; 1280 OS << "typedef float float32_t;\n"; 1281 OS << "typedef double float64_t;\n"; 1282 1283 OS << "typedef __SVInt8_t svint8_t;\n"; 1284 OS << "typedef __SVInt16_t svint16_t;\n"; 1285 OS << "typedef __SVInt32_t svint32_t;\n"; 1286 OS << "typedef __SVInt64_t svint64_t;\n"; 1287 OS << "typedef __SVUint8_t svuint8_t;\n"; 1288 OS << "typedef __SVUint16_t svuint16_t;\n"; 1289 OS << "typedef __SVUint32_t svuint32_t;\n"; 1290 OS << "typedef __SVUint64_t svuint64_t;\n"; 1291 OS << "typedef __SVFloat16_t svfloat16_t;\n\n"; 1292 1293 OS << "typedef __SVBfloat16_t svbfloat16_t;\n"; 1294 1295 OS << "#include <arm_bf16.h>\n"; 1296 OS << "#include <arm_vector_types.h>\n"; 1297 1298 OS << "typedef __SVFloat32_t svfloat32_t;\n"; 1299 OS << "typedef __SVFloat64_t svfloat64_t;\n"; 1300 OS << "typedef __clang_svint8x2_t svint8x2_t;\n"; 1301 OS << "typedef __clang_svint16x2_t svint16x2_t;\n"; 1302 OS << "typedef __clang_svint32x2_t svint32x2_t;\n"; 1303 OS << "typedef __clang_svint64x2_t svint64x2_t;\n"; 1304 OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n"; 1305 OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n"; 1306 OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n"; 1307 OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n"; 1308 OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n"; 1309 OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n"; 1310 OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n"; 1311 OS << "typedef __clang_svint8x3_t svint8x3_t;\n"; 1312 OS << "typedef __clang_svint16x3_t svint16x3_t;\n"; 1313 OS << "typedef __clang_svint32x3_t svint32x3_t;\n"; 1314 OS << "typedef __clang_svint64x3_t svint64x3_t;\n"; 1315 OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n"; 1316 OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n"; 1317 OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n"; 1318 OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n"; 1319 OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n"; 1320 OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n"; 1321 OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n"; 1322 OS << "typedef __clang_svint8x4_t svint8x4_t;\n"; 1323 OS << "typedef __clang_svint16x4_t svint16x4_t;\n"; 1324 OS << "typedef __clang_svint32x4_t svint32x4_t;\n"; 1325 OS << "typedef __clang_svint64x4_t svint64x4_t;\n"; 1326 OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n"; 1327 OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n"; 1328 OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n"; 1329 OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n"; 1330 OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n"; 1331 OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n"; 1332 OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n"; 1333 OS << "typedef __SVBool_t svbool_t;\n"; 1334 OS << "typedef __clang_svboolx2_t svboolx2_t;\n"; 1335 OS << "typedef __clang_svboolx4_t svboolx4_t;\n\n"; 1336 1337 OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n"; 1338 OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n"; 1339 OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n"; 1340 1341 OS << "typedef __SVCount_t svcount_t;\n\n"; 1342 1343 OS << "enum svpattern\n"; 1344 OS << "{\n"; 1345 OS << " SV_POW2 = 0,\n"; 1346 OS << " SV_VL1 = 1,\n"; 1347 OS << " SV_VL2 = 2,\n"; 1348 OS << " SV_VL3 = 3,\n"; 1349 OS << " SV_VL4 = 4,\n"; 1350 OS << " SV_VL5 = 5,\n"; 1351 OS << " SV_VL6 = 6,\n"; 1352 OS << " SV_VL7 = 7,\n"; 1353 OS << " SV_VL8 = 8,\n"; 1354 OS << " SV_VL16 = 9,\n"; 1355 OS << " SV_VL32 = 10,\n"; 1356 OS << " SV_VL64 = 11,\n"; 1357 OS << " SV_VL128 = 12,\n"; 1358 OS << " SV_VL256 = 13,\n"; 1359 OS << " SV_MUL4 = 29,\n"; 1360 OS << " SV_MUL3 = 30,\n"; 1361 OS << " SV_ALL = 31\n"; 1362 OS << "};\n\n"; 1363 1364 OS << "enum svprfop\n"; 1365 OS << "{\n"; 1366 OS << " SV_PLDL1KEEP = 0,\n"; 1367 OS << " SV_PLDL1STRM = 1,\n"; 1368 OS << " SV_PLDL2KEEP = 2,\n"; 1369 OS << " SV_PLDL2STRM = 3,\n"; 1370 OS << " SV_PLDL3KEEP = 4,\n"; 1371 OS << " SV_PLDL3STRM = 5,\n"; 1372 OS << " SV_PSTL1KEEP = 8,\n"; 1373 OS << " SV_PSTL1STRM = 9,\n"; 1374 OS << " SV_PSTL2KEEP = 10,\n"; 1375 OS << " SV_PSTL2STRM = 11,\n"; 1376 OS << " SV_PSTL3KEEP = 12,\n"; 1377 OS << " SV_PSTL3STRM = 13\n"; 1378 OS << "};\n\n"; 1379 1380 OS << "/* Function attributes */\n"; 1381 OS << "#define __ai static __inline__ __attribute__((__always_inline__, " 1382 "__nodebug__))\n\n"; 1383 OS << "#define __aio static __inline__ __attribute__((__always_inline__, " 1384 "__nodebug__, __overloadable__))\n\n"; 1385 1386 // Add reinterpret functions. 1387 for (auto [N, Suffix] : 1388 std::initializer_list<std::pair<unsigned, const char *>>{ 1389 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) { 1390 for (auto ShortForm : {false, true}) 1391 for (const ReinterpretTypeInfo &To : Reinterprets) { 1392 SVEType ToV(To.BaseType, N); 1393 for (const ReinterpretTypeInfo &From : Reinterprets) { 1394 SVEType FromV(From.BaseType, N); 1395 if (ShortForm) { 1396 OS << "__aio __attribute__((target(\"sve\"))) " << ToV.str() 1397 << " svreinterpret_" << To.Suffix; 1398 OS << "(" << FromV.str() << " op) __arm_streaming_compatible {\n"; 1399 OS << " return __builtin_sve_reinterpret_" << To.Suffix << "_" 1400 << From.Suffix << Suffix << "(op);\n"; 1401 OS << "}\n\n"; 1402 } else 1403 OS << "#define svreinterpret_" << To.Suffix << "_" << From.Suffix 1404 << Suffix << "(...) __builtin_sve_reinterpret_" << To.Suffix 1405 << "_" << From.Suffix << Suffix << "(__VA_ARGS__)\n"; 1406 } 1407 } 1408 } 1409 1410 createCoreHeaderIntrinsics(OS, *this, ACLEKind::SVE); 1411 1412 OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n"; 1413 OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n"; 1414 1415 OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n"; 1416 OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n"; 1417 OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n"; 1418 OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n"; 1419 1420 OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n"; 1421 OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n"; 1422 1423 OS << "#ifdef __cplusplus\n"; 1424 OS << "} // extern \"C\"\n"; 1425 OS << "#endif\n\n"; 1426 OS << "#undef __ai\n\n"; 1427 OS << "#undef __aio\n\n"; 1428 OS << "#endif /* __ARM_SVE_H */\n"; 1429 } 1430 1431 void SVEEmitter::createBuiltins(raw_ostream &OS) { 1432 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1433 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1434 for (auto *R : RV) 1435 createIntrinsic(R, Defs); 1436 1437 // The mappings must be sorted based on BuiltinID. 1438 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, 1439 const std::unique_ptr<Intrinsic> &B) { 1440 return A->getMangledName() < B->getMangledName(); 1441 }); 1442 1443 OS << "#ifdef GET_SVE_BUILTINS\n"; 1444 for (auto &Def : Defs) { 1445 // Only create BUILTINs for non-overloaded intrinsics, as overloaded 1446 // declarations only live in the header file. 1447 if (Def->getClassKind() != ClassG) 1448 OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \"" 1449 << Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard() 1450 << "\")\n"; 1451 } 1452 1453 // Add reinterpret functions. 1454 for (auto [N, Suffix] : 1455 std::initializer_list<std::pair<unsigned, const char *>>{ 1456 {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) { 1457 for (const ReinterpretTypeInfo &To : Reinterprets) { 1458 SVEType ToV(To.BaseType, N); 1459 for (const ReinterpretTypeInfo &From : Reinterprets) { 1460 SVEType FromV(From.BaseType, N); 1461 OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << To.Suffix << "_" 1462 << From.Suffix << Suffix << +", \"" << ToV.builtin_str() 1463 << FromV.builtin_str() << "\", \"n\", \"sve\")\n"; 1464 } 1465 } 1466 } 1467 1468 OS << "#endif\n\n"; 1469 } 1470 1471 void SVEEmitter::createCodeGenMap(raw_ostream &OS) { 1472 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1473 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1474 for (auto *R : RV) 1475 createIntrinsic(R, Defs); 1476 1477 // The mappings must be sorted based on BuiltinID. 1478 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, 1479 const std::unique_ptr<Intrinsic> &B) { 1480 return A->getMangledName() < B->getMangledName(); 1481 }); 1482 1483 OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n"; 1484 for (auto &Def : Defs) { 1485 // Builtins only exist for non-overloaded intrinsics, overloaded 1486 // declarations only live in the header file. 1487 if (Def->getClassKind() == ClassG) 1488 continue; 1489 1490 uint64_t Flags = Def->getFlags(); 1491 auto FlagString = std::to_string(Flags); 1492 1493 std::string LLVMName = Def->getMangledLLVMName(); 1494 std::string Builtin = Def->getMangledName(); 1495 if (!LLVMName.empty()) 1496 OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString 1497 << "),\n"; 1498 else 1499 OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n"; 1500 } 1501 OS << "#endif\n\n"; 1502 } 1503 1504 void SVEEmitter::createRangeChecks(raw_ostream &OS) { 1505 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1506 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1507 for (auto *R : RV) 1508 createIntrinsic(R, Defs); 1509 1510 // The mappings must be sorted based on BuiltinID. 1511 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, 1512 const std::unique_ptr<Intrinsic> &B) { 1513 return A->getMangledName() < B->getMangledName(); 1514 }); 1515 1516 1517 OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n"; 1518 1519 // Ensure these are only emitted once. 1520 std::set<std::string> Emitted; 1521 1522 for (auto &Def : Defs) { 1523 if (Emitted.find(Def->getMangledName()) != Emitted.end() || 1524 Def->getImmChecks().empty()) 1525 continue; 1526 1527 OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n"; 1528 for (auto &Check : Def->getImmChecks()) 1529 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", " 1530 << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n"; 1531 OS << " break;\n"; 1532 1533 Emitted.insert(Def->getMangledName()); 1534 } 1535 1536 OS << "#endif\n\n"; 1537 } 1538 1539 /// Create the SVETypeFlags used in CGBuiltins 1540 void SVEEmitter::createTypeFlags(raw_ostream &OS) { 1541 OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n"; 1542 for (auto &KV : FlagTypes) 1543 OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n"; 1544 OS << "#endif\n\n"; 1545 1546 OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n"; 1547 for (auto &KV : EltTypes) 1548 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; 1549 OS << "#endif\n\n"; 1550 1551 OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n"; 1552 for (auto &KV : MemEltTypes) 1553 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; 1554 OS << "#endif\n\n"; 1555 1556 OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n"; 1557 for (auto &KV : MergeTypes) 1558 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; 1559 OS << "#endif\n\n"; 1560 1561 OS << "#ifdef LLVM_GET_SVE_IMMCHECKTYPES\n"; 1562 for (auto &KV : ImmCheckTypes) 1563 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; 1564 OS << "#endif\n\n"; 1565 } 1566 1567 void SVEEmitter::createSMEHeader(raw_ostream &OS) { 1568 OS << "/*===---- arm_sme_draft_spec_subject_to_change.h - ARM SME intrinsics " 1569 "------===\n" 1570 " *\n" 1571 " *\n" 1572 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM " 1573 "Exceptions.\n" 1574 " * See https://llvm.org/LICENSE.txt for license information.\n" 1575 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n" 1576 " *\n" 1577 " *===-----------------------------------------------------------------" 1578 "------===\n" 1579 " */\n\n"; 1580 1581 OS << "#ifndef __ARM_SME_H\n"; 1582 OS << "#define __ARM_SME_H\n\n"; 1583 1584 OS << "#if !defined(__LITTLE_ENDIAN__)\n"; 1585 OS << "#error \"Big endian is currently not supported for arm_sme_draft_spec_subject_to_change.h\"\n"; 1586 OS << "#endif\n"; 1587 1588 OS << "#include <arm_sve.h>\n\n"; 1589 1590 OS << "/* Function attributes */\n"; 1591 OS << "#define __ai static __inline__ __attribute__((__always_inline__, " 1592 "__nodebug__))\n\n"; 1593 OS << "#define __aio static __inline__ __attribute__((__always_inline__, " 1594 "__nodebug__, __overloadable__))\n\n"; 1595 1596 OS << "#ifdef __cplusplus\n"; 1597 OS << "extern \"C\" {\n"; 1598 OS << "#endif\n\n"; 1599 1600 createCoreHeaderIntrinsics(OS, *this, ACLEKind::SME); 1601 1602 OS << "#ifdef __cplusplus\n"; 1603 OS << "} // extern \"C\"\n"; 1604 OS << "#endif\n\n"; 1605 OS << "#undef __ai\n\n"; 1606 OS << "#endif /* __ARM_SME_H */\n"; 1607 } 1608 1609 void SVEEmitter::createSMEBuiltins(raw_ostream &OS) { 1610 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1611 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1612 for (auto *R : RV) { 1613 createIntrinsic(R, Defs); 1614 } 1615 1616 // The mappings must be sorted based on BuiltinID. 1617 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, 1618 const std::unique_ptr<Intrinsic> &B) { 1619 return A->getMangledName() < B->getMangledName(); 1620 }); 1621 1622 OS << "#ifdef GET_SME_BUILTINS\n"; 1623 for (auto &Def : Defs) { 1624 // Only create BUILTINs for non-overloaded intrinsics, as overloaded 1625 // declarations only live in the header file. 1626 if (Def->getClassKind() != ClassG) 1627 OS << "TARGET_BUILTIN(__builtin_sme_" << Def->getMangledName() << ", \"" 1628 << Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard() 1629 << "\")\n"; 1630 } 1631 1632 OS << "#endif\n\n"; 1633 } 1634 1635 void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) { 1636 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1637 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1638 for (auto *R : RV) { 1639 createIntrinsic(R, Defs); 1640 } 1641 1642 // The mappings must be sorted based on BuiltinID. 1643 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, 1644 const std::unique_ptr<Intrinsic> &B) { 1645 return A->getMangledName() < B->getMangledName(); 1646 }); 1647 1648 OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n"; 1649 for (auto &Def : Defs) { 1650 // Builtins only exist for non-overloaded intrinsics, overloaded 1651 // declarations only live in the header file. 1652 if (Def->getClassKind() == ClassG) 1653 continue; 1654 1655 uint64_t Flags = Def->getFlags(); 1656 auto FlagString = std::to_string(Flags); 1657 1658 std::string LLVMName = Def->getLLVMName(); 1659 std::string Builtin = Def->getMangledName(); 1660 if (!LLVMName.empty()) 1661 OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString 1662 << "),\n"; 1663 else 1664 OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n"; 1665 } 1666 OS << "#endif\n\n"; 1667 } 1668 1669 void SVEEmitter::createSMERangeChecks(raw_ostream &OS) { 1670 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1671 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1672 for (auto *R : RV) { 1673 createIntrinsic(R, Defs); 1674 } 1675 1676 // The mappings must be sorted based on BuiltinID. 1677 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, 1678 const std::unique_ptr<Intrinsic> &B) { 1679 return A->getMangledName() < B->getMangledName(); 1680 }); 1681 1682 1683 OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n"; 1684 1685 // Ensure these are only emitted once. 1686 std::set<std::string> Emitted; 1687 1688 for (auto &Def : Defs) { 1689 if (Emitted.find(Def->getMangledName()) != Emitted.end() || 1690 Def->getImmChecks().empty()) 1691 continue; 1692 1693 OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n"; 1694 for (auto &Check : Def->getImmChecks()) 1695 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", " 1696 << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n"; 1697 OS << " break;\n"; 1698 1699 Emitted.insert(Def->getMangledName()); 1700 } 1701 1702 OS << "#endif\n\n"; 1703 } 1704 1705 namespace clang { 1706 void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) { 1707 SVEEmitter(Records).createHeader(OS); 1708 } 1709 1710 void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) { 1711 SVEEmitter(Records).createBuiltins(OS); 1712 } 1713 1714 void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) { 1715 SVEEmitter(Records).createCodeGenMap(OS); 1716 } 1717 1718 void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) { 1719 SVEEmitter(Records).createRangeChecks(OS); 1720 } 1721 1722 void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) { 1723 SVEEmitter(Records).createTypeFlags(OS); 1724 } 1725 1726 void EmitSmeHeader(RecordKeeper &Records, raw_ostream &OS) { 1727 SVEEmitter(Records).createSMEHeader(OS); 1728 } 1729 1730 void EmitSmeBuiltins(RecordKeeper &Records, raw_ostream &OS) { 1731 SVEEmitter(Records).createSMEBuiltins(OS); 1732 } 1733 1734 void EmitSmeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) { 1735 SVEEmitter(Records).createSMECodeGenMap(OS); 1736 } 1737 1738 void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) { 1739 SVEEmitter(Records).createSMERangeChecks(OS); 1740 } 1741 1742 } // End namespace clang 1743