1 //===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This tablegen backend is responsible for emitting arm_sve.h, which includes 10 // a declaration and definition of each function specified by the ARM C/C++ 11 // Language Extensions (ACLE). 12 // 13 // For details, visit: 14 // https://developer.arm.com/architectures/system-architectures/software-standards/acle 15 // 16 // Each SVE instruction is implemented in terms of 1 or more functions which 17 // are suffixed with the element type of the input vectors. Functions may be 18 // implemented in terms of generic vector operations such as +, *, -, etc. or 19 // by calling a __builtin_-prefixed function which will be handled by clang's 20 // CodeGen library. 21 // 22 // See also the documentation in include/clang/Basic/arm_sve.td. 23 // 24 //===----------------------------------------------------------------------===// 25 26 #include "llvm/ADT/STLExtras.h" 27 #include "llvm/ADT/StringMap.h" 28 #include "llvm/ADT/ArrayRef.h" 29 #include "llvm/ADT/StringExtras.h" 30 #include "llvm/TableGen/Record.h" 31 #include "llvm/TableGen/Error.h" 32 #include <string> 33 #include <sstream> 34 #include <set> 35 #include <cctype> 36 #include <tuple> 37 38 using namespace llvm; 39 40 enum ClassKind { 41 ClassNone, 42 ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix 43 ClassG, // Overloaded name without type suffix 44 }; 45 46 using TypeSpec = std::string; 47 48 namespace { 49 50 class ImmCheck { 51 unsigned Arg; 52 unsigned Kind; 53 unsigned ElementSizeInBits; 54 55 public: 56 ImmCheck(unsigned Arg, unsigned Kind, unsigned ElementSizeInBits = 0) 57 : Arg(Arg), Kind(Kind), ElementSizeInBits(ElementSizeInBits) {} 58 ImmCheck(const ImmCheck &Other) = default; 59 ~ImmCheck() = default; 60 61 unsigned getArg() const { return Arg; } 62 unsigned getKind() const { return Kind; } 63 unsigned getElementSizeInBits() const { return ElementSizeInBits; } 64 }; 65 66 class SVEType { 67 TypeSpec TS; 68 bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat; 69 bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp; 70 unsigned Bitwidth, ElementBitwidth, NumVectors; 71 72 public: 73 SVEType() : SVEType(TypeSpec(), 'v') {} 74 75 SVEType(TypeSpec TS, char CharMod) 76 : TS(TS), Float(false), Signed(true), Immediate(false), Void(false), 77 Constant(false), Pointer(false), BFloat(false), DefaultType(false), 78 IsScalable(true), Predicate(false), PredicatePattern(false), 79 PrefetchOp(false), Bitwidth(128), ElementBitwidth(~0U), NumVectors(1) { 80 if (!TS.empty()) 81 applyTypespec(); 82 applyModifier(CharMod); 83 } 84 85 bool isPointer() const { return Pointer; } 86 bool isVoidPointer() const { return Pointer && Void; } 87 bool isSigned() const { return Signed; } 88 bool isImmediate() const { return Immediate; } 89 bool isScalar() const { return NumVectors == 0; } 90 bool isVector() const { return NumVectors > 0; } 91 bool isScalableVector() const { return isVector() && IsScalable; } 92 bool isChar() const { return ElementBitwidth == 8; } 93 bool isVoid() const { return Void & !Pointer; } 94 bool isDefault() const { return DefaultType; } 95 bool isFloat() const { return Float && !BFloat; } 96 bool isBFloat() const { return BFloat && !Float; } 97 bool isFloatingPoint() const { return Float || BFloat; } 98 bool isInteger() const { return !isFloatingPoint() && !Predicate; } 99 bool isScalarPredicate() const { 100 return !isFloatingPoint() && Predicate && NumVectors == 0; 101 } 102 bool isPredicateVector() const { return Predicate; } 103 bool isPredicatePattern() const { return PredicatePattern; } 104 bool isPrefetchOp() const { return PrefetchOp; } 105 bool isConstant() const { return Constant; } 106 unsigned getElementSizeInBits() const { return ElementBitwidth; } 107 unsigned getNumVectors() const { return NumVectors; } 108 109 unsigned getNumElements() const { 110 assert(ElementBitwidth != ~0U); 111 return Bitwidth / ElementBitwidth; 112 } 113 unsigned getSizeInBits() const { 114 return Bitwidth; 115 } 116 117 /// Return the string representation of a type, which is an encoded 118 /// string for passing to the BUILTIN() macro in Builtins.def. 119 std::string builtin_str() const; 120 121 /// Return the C/C++ string representation of a type for use in the 122 /// arm_sve.h header file. 123 std::string str() const; 124 125 private: 126 /// Creates the type based on the typespec string in TS. 127 void applyTypespec(); 128 129 /// Applies a prototype modifier to the type. 130 void applyModifier(char Mod); 131 }; 132 133 134 class SVEEmitter; 135 136 /// The main grunt class. This represents an instantiation of an intrinsic with 137 /// a particular typespec and prototype. 138 class Intrinsic { 139 /// The unmangled name. 140 std::string Name; 141 142 /// The name of the corresponding LLVM IR intrinsic. 143 std::string LLVMName; 144 145 /// Intrinsic prototype. 146 std::string Proto; 147 148 /// The base type spec for this intrinsic. 149 TypeSpec BaseTypeSpec; 150 151 /// The base class kind. Most intrinsics use ClassS, which has full type 152 /// info for integers (_s32/_u32), or ClassG which is used for overloaded 153 /// intrinsics. 154 ClassKind Class; 155 156 /// The architectural #ifdef guard. 157 std::string Guard; 158 159 // The merge suffix such as _m, _x or _z. 160 std::string MergeSuffix; 161 162 /// The types of return value [0] and parameters [1..]. 163 std::vector<SVEType> Types; 164 165 /// The "base type", which is VarType('d', BaseTypeSpec). 166 SVEType BaseType; 167 168 uint64_t Flags; 169 170 SmallVector<ImmCheck, 2> ImmChecks; 171 172 public: 173 Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy, 174 StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName, 175 uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT, 176 ClassKind Class, SVEEmitter &Emitter, StringRef Guard); 177 178 ~Intrinsic()=default; 179 180 std::string getName() const { return Name; } 181 std::string getLLVMName() const { return LLVMName; } 182 std::string getProto() const { return Proto; } 183 TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; } 184 SVEType getBaseType() const { return BaseType; } 185 186 StringRef getGuard() const { return Guard; } 187 ClassKind getClassKind() const { return Class; } 188 189 SVEType getReturnType() const { return Types[0]; } 190 ArrayRef<SVEType> getTypes() const { return Types; } 191 SVEType getParamType(unsigned I) const { return Types[I + 1]; } 192 unsigned getNumParams() const { return Proto.size() - 1; } 193 194 uint64_t getFlags() const { return Flags; } 195 bool isFlagSet(uint64_t Flag) const { return Flags & Flag;} 196 197 ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; } 198 199 /// Return the type string for a BUILTIN() macro in Builtins.def. 200 std::string getBuiltinTypeStr(); 201 202 /// Return the name, mangled with type information. The name is mangled for 203 /// ClassS, so will add type suffixes such as _u32/_s32. 204 std::string getMangledName() const { return mangleName(ClassS); } 205 206 /// Returns true if the intrinsic is overloaded, in that it should also generate 207 /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of 208 /// 'svld1_u32(..)'. 209 static bool isOverloadedIntrinsic(StringRef Name) { 210 auto BrOpen = Name.find('['); 211 auto BrClose = Name.find(']'); 212 return BrOpen != std::string::npos && BrClose != std::string::npos; 213 } 214 215 /// Return true if the intrinsic takes a splat operand. 216 bool hasSplat() const { 217 // These prototype modifiers are described in arm_sve.td. 218 return Proto.find_first_of("ajfrKLR@") != std::string::npos; 219 } 220 221 /// Return the parameter index of the splat operand. 222 unsigned getSplatIdx() const { 223 // These prototype modifiers are described in arm_sve.td. 224 auto Idx = Proto.find_first_of("ajfrKLR@"); 225 assert(Idx != std::string::npos && Idx > 0 && 226 "Prototype has no splat operand"); 227 return Idx - 1; 228 } 229 230 /// Emits the intrinsic declaration to the ostream. 231 void emitIntrinsic(raw_ostream &OS) const; 232 233 private: 234 std::string getMergeSuffix() const { return MergeSuffix; } 235 std::string mangleName(ClassKind LocalCK) const; 236 std::string replaceTemplatedArgs(std::string Name, TypeSpec TS, 237 std::string Proto) const; 238 }; 239 240 class SVEEmitter { 241 private: 242 // The reinterpret builtins are generated separately because they 243 // need the cross product of all types (121 functions in total), 244 // which is inconvenient to specify in the arm_sve.td file or 245 // generate in CGBuiltin.cpp. 246 struct ReinterpretTypeInfo { 247 const char *Suffix; 248 const char *Type; 249 const char *BuiltinType; 250 }; 251 SmallVector<ReinterpretTypeInfo, 12> Reinterprets = { 252 {"s8", "svint8_t", "q16Sc"}, {"s16", "svint16_t", "q8Ss"}, 253 {"s32", "svint32_t", "q4Si"}, {"s64", "svint64_t", "q2SWi"}, 254 {"u8", "svuint8_t", "q16Uc"}, {"u16", "svuint16_t", "q8Us"}, 255 {"u32", "svuint32_t", "q4Ui"}, {"u64", "svuint64_t", "q2UWi"}, 256 {"f16", "svfloat16_t", "q8h"}, {"bf16", "svbfloat16_t", "q8y"}, 257 {"f32", "svfloat32_t", "q4f"}, {"f64", "svfloat64_t", "q2d"}}; 258 259 RecordKeeper &Records; 260 llvm::StringMap<uint64_t> EltTypes; 261 llvm::StringMap<uint64_t> MemEltTypes; 262 llvm::StringMap<uint64_t> FlagTypes; 263 llvm::StringMap<uint64_t> MergeTypes; 264 llvm::StringMap<uint64_t> ImmCheckTypes; 265 266 public: 267 SVEEmitter(RecordKeeper &R) : Records(R) { 268 for (auto *RV : Records.getAllDerivedDefinitions("EltType")) 269 EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); 270 for (auto *RV : Records.getAllDerivedDefinitions("MemEltType")) 271 MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); 272 for (auto *RV : Records.getAllDerivedDefinitions("FlagType")) 273 FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); 274 for (auto *RV : Records.getAllDerivedDefinitions("MergeType")) 275 MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); 276 for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType")) 277 ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); 278 } 279 280 /// Returns the enum value for the immcheck type 281 unsigned getEnumValueForImmCheck(StringRef C) const { 282 auto It = ImmCheckTypes.find(C); 283 if (It != ImmCheckTypes.end()) 284 return It->getValue(); 285 llvm_unreachable("Unsupported imm check"); 286 } 287 288 /// Returns the enum value for the flag type 289 uint64_t getEnumValueForFlag(StringRef C) const { 290 auto Res = FlagTypes.find(C); 291 if (Res != FlagTypes.end()) 292 return Res->getValue(); 293 llvm_unreachable("Unsupported flag"); 294 } 295 296 // Returns the SVETypeFlags for a given value and mask. 297 uint64_t encodeFlag(uint64_t V, StringRef MaskName) const { 298 auto It = FlagTypes.find(MaskName); 299 if (It != FlagTypes.end()) { 300 uint64_t Mask = It->getValue(); 301 unsigned Shift = llvm::countTrailingZeros(Mask); 302 return (V << Shift) & Mask; 303 } 304 llvm_unreachable("Unsupported flag"); 305 } 306 307 // Returns the SVETypeFlags for the given element type. 308 uint64_t encodeEltType(StringRef EltName) { 309 auto It = EltTypes.find(EltName); 310 if (It != EltTypes.end()) 311 return encodeFlag(It->getValue(), "EltTypeMask"); 312 llvm_unreachable("Unsupported EltType"); 313 } 314 315 // Returns the SVETypeFlags for the given memory element type. 316 uint64_t encodeMemoryElementType(uint64_t MT) { 317 return encodeFlag(MT, "MemEltTypeMask"); 318 } 319 320 // Returns the SVETypeFlags for the given merge type. 321 uint64_t encodeMergeType(uint64_t MT) { 322 return encodeFlag(MT, "MergeTypeMask"); 323 } 324 325 // Returns the SVETypeFlags for the given splat operand. 326 unsigned encodeSplatOperand(unsigned SplatIdx) { 327 assert(SplatIdx < 7 && "SplatIdx out of encodable range"); 328 return encodeFlag(SplatIdx + 1, "SplatOperandMask"); 329 } 330 331 // Returns the SVETypeFlags value for the given SVEType. 332 uint64_t encodeTypeFlags(const SVEType &T); 333 334 /// Emit arm_sve.h. 335 void createHeader(raw_ostream &o); 336 337 /// Emit all the __builtin prototypes and code needed by Sema. 338 void createBuiltins(raw_ostream &o); 339 340 /// Emit all the information needed to map builtin -> LLVM IR intrinsic. 341 void createCodeGenMap(raw_ostream &o); 342 343 /// Emit all the range checks for the immediates. 344 void createRangeChecks(raw_ostream &o); 345 346 /// Create the SVETypeFlags used in CGBuiltins 347 void createTypeFlags(raw_ostream &o); 348 349 /// Create intrinsic and add it to \p Out 350 void createIntrinsic(Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out); 351 }; 352 353 } // end anonymous namespace 354 355 356 //===----------------------------------------------------------------------===// 357 // Type implementation 358 //===----------------------------------------------------------------------===// 359 360 std::string SVEType::builtin_str() const { 361 std::string S; 362 if (isVoid()) 363 return "v"; 364 365 if (isScalarPredicate()) 366 return "b"; 367 368 if (isVoidPointer()) 369 S += "v"; 370 else if (!isFloatingPoint()) 371 switch (ElementBitwidth) { 372 case 1: S += "b"; break; 373 case 8: S += "c"; break; 374 case 16: S += "s"; break; 375 case 32: S += "i"; break; 376 case 64: S += "Wi"; break; 377 case 128: S += "LLLi"; break; 378 default: llvm_unreachable("Unhandled case!"); 379 } 380 else if (isFloat()) 381 switch (ElementBitwidth) { 382 case 16: S += "h"; break; 383 case 32: S += "f"; break; 384 case 64: S += "d"; break; 385 default: llvm_unreachable("Unhandled case!"); 386 } 387 else if (isBFloat()) { 388 assert(ElementBitwidth == 16 && "Not a valid BFloat."); 389 S += "y"; 390 } 391 392 if (!isFloatingPoint()) { 393 if ((isChar() || isPointer()) && !isVoidPointer()) { 394 // Make chars and typed pointers explicitly signed. 395 if (Signed) 396 S = "S" + S; 397 else if (!Signed) 398 S = "U" + S; 399 } else if (!isVoidPointer() && !Signed) { 400 S = "U" + S; 401 } 402 } 403 404 // Constant indices are "int", but have the "constant expression" modifier. 405 if (isImmediate()) { 406 assert(!isFloat() && "fp immediates are not supported"); 407 S = "I" + S; 408 } 409 410 if (isScalar()) { 411 if (Constant) S += "C"; 412 if (Pointer) S += "*"; 413 return S; 414 } 415 416 assert(isScalableVector() && "Unsupported type"); 417 return "q" + utostr(getNumElements() * NumVectors) + S; 418 } 419 420 std::string SVEType::str() const { 421 if (isPredicatePattern()) 422 return "enum svpattern"; 423 424 if (isPrefetchOp()) 425 return "enum svprfop"; 426 427 std::string S; 428 if (Void) 429 S += "void"; 430 else { 431 if (isScalableVector()) 432 S += "sv"; 433 if (!Signed && !isFloatingPoint()) 434 S += "u"; 435 436 if (Float) 437 S += "float"; 438 else if (isScalarPredicate() || isPredicateVector()) 439 S += "bool"; 440 else if (isBFloat()) 441 S += "bfloat"; 442 else 443 S += "int"; 444 445 if (!isScalarPredicate() && !isPredicateVector()) 446 S += utostr(ElementBitwidth); 447 if (!isScalableVector() && isVector()) 448 S += "x" + utostr(getNumElements()); 449 if (NumVectors > 1) 450 S += "x" + utostr(NumVectors); 451 if (!isScalarPredicate()) 452 S += "_t"; 453 } 454 455 if (Constant) 456 S += " const"; 457 if (Pointer) 458 S += " *"; 459 460 return S; 461 } 462 void SVEType::applyTypespec() { 463 for (char I : TS) { 464 switch (I) { 465 case 'P': 466 Predicate = true; 467 break; 468 case 'U': 469 Signed = false; 470 break; 471 case 'c': 472 ElementBitwidth = 8; 473 break; 474 case 's': 475 ElementBitwidth = 16; 476 break; 477 case 'i': 478 ElementBitwidth = 32; 479 break; 480 case 'l': 481 ElementBitwidth = 64; 482 break; 483 case 'h': 484 Float = true; 485 ElementBitwidth = 16; 486 break; 487 case 'f': 488 Float = true; 489 ElementBitwidth = 32; 490 break; 491 case 'd': 492 Float = true; 493 ElementBitwidth = 64; 494 break; 495 case 'b': 496 BFloat = true; 497 Float = false; 498 ElementBitwidth = 16; 499 break; 500 default: 501 llvm_unreachable("Unhandled type code!"); 502 } 503 } 504 assert(ElementBitwidth != ~0U && "Bad element bitwidth!"); 505 } 506 507 void SVEType::applyModifier(char Mod) { 508 switch (Mod) { 509 case '2': 510 NumVectors = 2; 511 break; 512 case '3': 513 NumVectors = 3; 514 break; 515 case '4': 516 NumVectors = 4; 517 break; 518 case 'v': 519 Void = true; 520 break; 521 case 'd': 522 DefaultType = true; 523 break; 524 case 'c': 525 Constant = true; 526 LLVM_FALLTHROUGH; 527 case 'p': 528 Pointer = true; 529 Bitwidth = ElementBitwidth; 530 NumVectors = 0; 531 break; 532 case 'e': 533 Signed = false; 534 ElementBitwidth /= 2; 535 break; 536 case 'h': 537 ElementBitwidth /= 2; 538 break; 539 case 'q': 540 ElementBitwidth /= 4; 541 break; 542 case 'b': 543 Signed = false; 544 Float = false; 545 BFloat = false; 546 ElementBitwidth /= 4; 547 break; 548 case 'o': 549 ElementBitwidth *= 4; 550 break; 551 case 'P': 552 Signed = true; 553 Float = false; 554 BFloat = false; 555 Predicate = true; 556 Bitwidth = 16; 557 ElementBitwidth = 1; 558 break; 559 case 's': 560 case 'a': 561 Bitwidth = ElementBitwidth; 562 NumVectors = 0; 563 break; 564 case 'R': 565 ElementBitwidth /= 2; 566 NumVectors = 0; 567 break; 568 case 'r': 569 ElementBitwidth /= 4; 570 NumVectors = 0; 571 break; 572 case '@': 573 Signed = false; 574 Float = false; 575 BFloat = false; 576 ElementBitwidth /= 4; 577 NumVectors = 0; 578 break; 579 case 'K': 580 Signed = true; 581 Float = false; 582 BFloat = false; 583 Bitwidth = ElementBitwidth; 584 NumVectors = 0; 585 break; 586 case 'L': 587 Signed = false; 588 Float = false; 589 BFloat = false; 590 Bitwidth = ElementBitwidth; 591 NumVectors = 0; 592 break; 593 case 'u': 594 Predicate = false; 595 Signed = false; 596 Float = false; 597 BFloat = false; 598 break; 599 case 'x': 600 Predicate = false; 601 Signed = true; 602 Float = false; 603 BFloat = false; 604 break; 605 case 'i': 606 Predicate = false; 607 Float = false; 608 BFloat = false; 609 ElementBitwidth = Bitwidth = 64; 610 NumVectors = 0; 611 Signed = false; 612 Immediate = true; 613 break; 614 case 'I': 615 Predicate = false; 616 Float = false; 617 BFloat = false; 618 ElementBitwidth = Bitwidth = 32; 619 NumVectors = 0; 620 Signed = true; 621 Immediate = true; 622 PredicatePattern = true; 623 break; 624 case 'J': 625 Predicate = false; 626 Float = false; 627 BFloat = false; 628 ElementBitwidth = Bitwidth = 32; 629 NumVectors = 0; 630 Signed = true; 631 Immediate = true; 632 PrefetchOp = true; 633 break; 634 case 'k': 635 Predicate = false; 636 Signed = true; 637 Float = false; 638 BFloat = false; 639 ElementBitwidth = Bitwidth = 32; 640 NumVectors = 0; 641 break; 642 case 'l': 643 Predicate = false; 644 Signed = true; 645 Float = false; 646 BFloat = false; 647 ElementBitwidth = Bitwidth = 64; 648 NumVectors = 0; 649 break; 650 case 'm': 651 Predicate = false; 652 Signed = false; 653 Float = false; 654 BFloat = false; 655 ElementBitwidth = Bitwidth = 32; 656 NumVectors = 0; 657 break; 658 case 'n': 659 Predicate = false; 660 Signed = false; 661 Float = false; 662 BFloat = false; 663 ElementBitwidth = Bitwidth = 64; 664 NumVectors = 0; 665 break; 666 case 'w': 667 ElementBitwidth = 64; 668 break; 669 case 'j': 670 ElementBitwidth = Bitwidth = 64; 671 NumVectors = 0; 672 break; 673 case 'f': 674 Signed = false; 675 ElementBitwidth = Bitwidth = 64; 676 NumVectors = 0; 677 break; 678 case 'g': 679 Signed = false; 680 Float = false; 681 BFloat = false; 682 ElementBitwidth = 64; 683 break; 684 case 't': 685 Signed = true; 686 Float = false; 687 BFloat = false; 688 ElementBitwidth = 32; 689 break; 690 case 'z': 691 Signed = false; 692 Float = false; 693 BFloat = false; 694 ElementBitwidth = 32; 695 break; 696 case 'O': 697 Predicate = false; 698 Float = true; 699 ElementBitwidth = 16; 700 break; 701 case 'M': 702 Predicate = false; 703 Float = true; 704 BFloat = false; 705 ElementBitwidth = 32; 706 break; 707 case 'N': 708 Predicate = false; 709 Float = true; 710 ElementBitwidth = 64; 711 break; 712 case 'Q': 713 Constant = true; 714 Pointer = true; 715 Void = true; 716 NumVectors = 0; 717 break; 718 case 'S': 719 Constant = true; 720 Pointer = true; 721 ElementBitwidth = Bitwidth = 8; 722 NumVectors = 0; 723 Signed = true; 724 break; 725 case 'W': 726 Constant = true; 727 Pointer = true; 728 ElementBitwidth = Bitwidth = 8; 729 NumVectors = 0; 730 Signed = false; 731 break; 732 case 'T': 733 Constant = true; 734 Pointer = true; 735 ElementBitwidth = Bitwidth = 16; 736 NumVectors = 0; 737 Signed = true; 738 break; 739 case 'X': 740 Constant = true; 741 Pointer = true; 742 ElementBitwidth = Bitwidth = 16; 743 NumVectors = 0; 744 Signed = false; 745 break; 746 case 'Y': 747 Constant = true; 748 Pointer = true; 749 ElementBitwidth = Bitwidth = 32; 750 NumVectors = 0; 751 Signed = false; 752 break; 753 case 'U': 754 Constant = true; 755 Pointer = true; 756 ElementBitwidth = Bitwidth = 32; 757 NumVectors = 0; 758 Signed = true; 759 break; 760 case 'A': 761 Pointer = true; 762 ElementBitwidth = Bitwidth = 8; 763 NumVectors = 0; 764 Signed = true; 765 break; 766 case 'B': 767 Pointer = true; 768 ElementBitwidth = Bitwidth = 16; 769 NumVectors = 0; 770 Signed = true; 771 break; 772 case 'C': 773 Pointer = true; 774 ElementBitwidth = Bitwidth = 32; 775 NumVectors = 0; 776 Signed = true; 777 break; 778 case 'D': 779 Pointer = true; 780 ElementBitwidth = Bitwidth = 64; 781 NumVectors = 0; 782 Signed = true; 783 break; 784 case 'E': 785 Pointer = true; 786 ElementBitwidth = Bitwidth = 8; 787 NumVectors = 0; 788 Signed = false; 789 break; 790 case 'F': 791 Pointer = true; 792 ElementBitwidth = Bitwidth = 16; 793 NumVectors = 0; 794 Signed = false; 795 break; 796 case 'G': 797 Pointer = true; 798 ElementBitwidth = Bitwidth = 32; 799 NumVectors = 0; 800 Signed = false; 801 break; 802 default: 803 llvm_unreachable("Unhandled character!"); 804 } 805 } 806 807 808 //===----------------------------------------------------------------------===// 809 // Intrinsic implementation 810 //===----------------------------------------------------------------------===// 811 812 Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy, 813 StringRef MergeSuffix, uint64_t MemoryElementTy, 814 StringRef LLVMName, uint64_t Flags, 815 ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class, 816 SVEEmitter &Emitter, StringRef Guard) 817 : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()), 818 BaseTypeSpec(BT), Class(Class), Guard(Guard.str()), 819 MergeSuffix(MergeSuffix.str()), BaseType(BT, 'd'), Flags(Flags), 820 ImmChecks(Checks.begin(), Checks.end()) { 821 // Types[0] is the return value. 822 for (unsigned I = 0; I < Proto.size(); ++I) { 823 SVEType T(BaseTypeSpec, Proto[I]); 824 Types.push_back(T); 825 826 // Add range checks for immediates 827 if (I > 0) { 828 if (T.isPredicatePattern()) 829 ImmChecks.emplace_back( 830 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31")); 831 else if (T.isPrefetchOp()) 832 ImmChecks.emplace_back( 833 I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13")); 834 } 835 } 836 837 // Set flags based on properties 838 this->Flags |= Emitter.encodeTypeFlags(BaseType); 839 this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy); 840 this->Flags |= Emitter.encodeMergeType(MergeTy); 841 if (hasSplat()) 842 this->Flags |= Emitter.encodeSplatOperand(getSplatIdx()); 843 } 844 845 std::string Intrinsic::getBuiltinTypeStr() { 846 std::string S = getReturnType().builtin_str(); 847 for (unsigned I = 0; I < getNumParams(); ++I) 848 S += getParamType(I).builtin_str(); 849 850 return S; 851 } 852 853 std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS, 854 std::string Proto) const { 855 std::string Ret = Name; 856 while (Ret.find('{') != std::string::npos) { 857 size_t Pos = Ret.find('{'); 858 size_t End = Ret.find('}'); 859 unsigned NumChars = End - Pos + 1; 860 assert(NumChars == 3 && "Unexpected template argument"); 861 862 SVEType T; 863 char C = Ret[Pos+1]; 864 switch(C) { 865 default: 866 llvm_unreachable("Unknown predication specifier"); 867 case 'd': 868 T = SVEType(TS, 'd'); 869 break; 870 case '0': 871 case '1': 872 case '2': 873 case '3': 874 T = SVEType(TS, Proto[C - '0']); 875 break; 876 } 877 878 // Replace templated arg with the right suffix (e.g. u32) 879 std::string TypeCode; 880 if (T.isInteger()) 881 TypeCode = T.isSigned() ? 's' : 'u'; 882 else if (T.isPredicateVector()) 883 TypeCode = 'b'; 884 else if (T.isBFloat()) 885 TypeCode = "bf"; 886 else 887 TypeCode = 'f'; 888 Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits())); 889 } 890 891 return Ret; 892 } 893 894 std::string Intrinsic::mangleName(ClassKind LocalCK) const { 895 std::string S = getName(); 896 897 if (LocalCK == ClassG) { 898 // Remove the square brackets and everything in between. 899 while (S.find('[') != std::string::npos) { 900 auto Start = S.find('['); 901 auto End = S.find(']'); 902 S.erase(Start, (End-Start)+1); 903 } 904 } else { 905 // Remove the square brackets. 906 while (S.find('[') != std::string::npos) { 907 auto BrPos = S.find('['); 908 if (BrPos != std::string::npos) 909 S.erase(BrPos, 1); 910 BrPos = S.find(']'); 911 if (BrPos != std::string::npos) 912 S.erase(BrPos, 1); 913 } 914 } 915 916 // Replace all {d} like expressions with e.g. 'u32' 917 return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) + 918 getMergeSuffix(); 919 } 920 921 void Intrinsic::emitIntrinsic(raw_ostream &OS) const { 922 bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1; 923 924 std::string FullName = mangleName(ClassS); 925 std::string ProtoName = mangleName(getClassKind()); 926 927 OS << (IsOverloaded ? "__aio " : "__ai ") 928 << "__attribute__((__clang_arm_builtin_alias(" 929 << "__builtin_sve_" << FullName << ")))\n"; 930 931 OS << getTypes()[0].str() << " " << ProtoName << "("; 932 for (unsigned I = 0; I < getTypes().size() - 1; ++I) { 933 if (I != 0) 934 OS << ", "; 935 OS << getTypes()[I + 1].str(); 936 } 937 OS << ");\n"; 938 } 939 940 //===----------------------------------------------------------------------===// 941 // SVEEmitter implementation 942 //===----------------------------------------------------------------------===// 943 uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) { 944 if (T.isFloat()) { 945 switch (T.getElementSizeInBits()) { 946 case 16: 947 return encodeEltType("EltTyFloat16"); 948 case 32: 949 return encodeEltType("EltTyFloat32"); 950 case 64: 951 return encodeEltType("EltTyFloat64"); 952 default: 953 llvm_unreachable("Unhandled float element bitwidth!"); 954 } 955 } 956 957 if (T.isBFloat()) { 958 assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat."); 959 return encodeEltType("EltTyBFloat16"); 960 } 961 962 if (T.isPredicateVector()) { 963 switch (T.getElementSizeInBits()) { 964 case 8: 965 return encodeEltType("EltTyBool8"); 966 case 16: 967 return encodeEltType("EltTyBool16"); 968 case 32: 969 return encodeEltType("EltTyBool32"); 970 case 64: 971 return encodeEltType("EltTyBool64"); 972 default: 973 llvm_unreachable("Unhandled predicate element bitwidth!"); 974 } 975 } 976 977 switch (T.getElementSizeInBits()) { 978 case 8: 979 return encodeEltType("EltTyInt8"); 980 case 16: 981 return encodeEltType("EltTyInt16"); 982 case 32: 983 return encodeEltType("EltTyInt32"); 984 case 64: 985 return encodeEltType("EltTyInt64"); 986 default: 987 llvm_unreachable("Unhandled integer element bitwidth!"); 988 } 989 } 990 991 void SVEEmitter::createIntrinsic( 992 Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) { 993 StringRef Name = R->getValueAsString("Name"); 994 StringRef Proto = R->getValueAsString("Prototype"); 995 StringRef Types = R->getValueAsString("Types"); 996 StringRef Guard = R->getValueAsString("ArchGuard"); 997 StringRef LLVMName = R->getValueAsString("LLVMIntrinsic"); 998 uint64_t Merge = R->getValueAsInt("Merge"); 999 StringRef MergeSuffix = R->getValueAsString("MergeSuffix"); 1000 uint64_t MemEltType = R->getValueAsInt("MemEltType"); 1001 std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags"); 1002 std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks"); 1003 1004 int64_t Flags = 0; 1005 for (auto FlagRec : FlagsList) 1006 Flags |= FlagRec->getValueAsInt("Value"); 1007 1008 // Create a dummy TypeSpec for non-overloaded builtins. 1009 if (Types.empty()) { 1010 assert((Flags & getEnumValueForFlag("IsOverloadNone")) && 1011 "Expect TypeSpec for overloaded builtin!"); 1012 Types = "i"; 1013 } 1014 1015 // Extract type specs from string 1016 SmallVector<TypeSpec, 8> TypeSpecs; 1017 TypeSpec Acc; 1018 for (char I : Types) { 1019 Acc.push_back(I); 1020 if (islower(I)) { 1021 TypeSpecs.push_back(TypeSpec(Acc)); 1022 Acc.clear(); 1023 } 1024 } 1025 1026 // Remove duplicate type specs. 1027 llvm::sort(TypeSpecs); 1028 TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()), 1029 TypeSpecs.end()); 1030 1031 // Create an Intrinsic for each type spec. 1032 for (auto TS : TypeSpecs) { 1033 // Collate a list of range/option checks for the immediates. 1034 SmallVector<ImmCheck, 2> ImmChecks; 1035 for (auto *R : ImmCheckList) { 1036 int64_t Arg = R->getValueAsInt("Arg"); 1037 int64_t EltSizeArg = R->getValueAsInt("EltSizeArg"); 1038 int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value"); 1039 assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative"); 1040 1041 unsigned ElementSizeInBits = 0; 1042 if (EltSizeArg >= 0) 1043 ElementSizeInBits = 1044 SVEType(TS, Proto[EltSizeArg + /* offset by return arg */ 1]) 1045 .getElementSizeInBits(); 1046 ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits)); 1047 } 1048 1049 Out.push_back(std::make_unique<Intrinsic>( 1050 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks, 1051 TS, ClassS, *this, Guard)); 1052 1053 // Also generate the short-form (e.g. svadd_m) for the given type-spec. 1054 if (Intrinsic::isOverloadedIntrinsic(Name)) 1055 Out.push_back(std::make_unique<Intrinsic>( 1056 Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, 1057 ImmChecks, TS, ClassG, *this, Guard)); 1058 } 1059 } 1060 1061 void SVEEmitter::createHeader(raw_ostream &OS) { 1062 OS << "/*===---- arm_sve.h - ARM SVE intrinsics " 1063 "-----------------------------------===\n" 1064 " *\n" 1065 " *\n" 1066 " * Part of the LLVM Project, under the Apache License v2.0 with LLVM " 1067 "Exceptions.\n" 1068 " * See https://llvm.org/LICENSE.txt for license information.\n" 1069 " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n" 1070 " *\n" 1071 " *===-----------------------------------------------------------------" 1072 "------===\n" 1073 " */\n\n"; 1074 1075 OS << "#ifndef __ARM_SVE_H\n"; 1076 OS << "#define __ARM_SVE_H\n\n"; 1077 1078 OS << "#if !defined(__ARM_FEATURE_SVE)\n"; 1079 OS << "#error \"SVE support not enabled\"\n"; 1080 OS << "#else\n\n"; 1081 1082 OS << "#if !defined(__LITTLE_ENDIAN__)\n"; 1083 OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n"; 1084 OS << "#endif\n"; 1085 1086 OS << "#include <stdint.h>\n\n"; 1087 OS << "#ifdef __cplusplus\n"; 1088 OS << "extern \"C\" {\n"; 1089 OS << "#else\n"; 1090 OS << "#include <stdbool.h>\n"; 1091 OS << "#endif\n\n"; 1092 1093 OS << "typedef __fp16 float16_t;\n"; 1094 OS << "typedef float float32_t;\n"; 1095 OS << "typedef double float64_t;\n"; 1096 1097 OS << "typedef __SVInt8_t svint8_t;\n"; 1098 OS << "typedef __SVInt16_t svint16_t;\n"; 1099 OS << "typedef __SVInt32_t svint32_t;\n"; 1100 OS << "typedef __SVInt64_t svint64_t;\n"; 1101 OS << "typedef __SVUint8_t svuint8_t;\n"; 1102 OS << "typedef __SVUint16_t svuint16_t;\n"; 1103 OS << "typedef __SVUint32_t svuint32_t;\n"; 1104 OS << "typedef __SVUint64_t svuint64_t;\n"; 1105 OS << "typedef __SVFloat16_t svfloat16_t;\n\n"; 1106 1107 OS << "#if defined(__ARM_FEATURE_SVE_BF16) && " 1108 "!defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)\n"; 1109 OS << "#error \"__ARM_FEATURE_BF16_SCALAR_ARITHMETIC must be defined when " 1110 "__ARM_FEATURE_SVE_BF16 is defined\"\n"; 1111 OS << "#endif\n\n"; 1112 1113 OS << "#if defined(__ARM_FEATURE_SVE_BF16)\n"; 1114 OS << "typedef __SVBFloat16_t svbfloat16_t;\n"; 1115 OS << "#endif\n\n"; 1116 1117 OS << "#if defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC)\n"; 1118 OS << "#include <arm_bf16.h>\n"; 1119 OS << "typedef __bf16 bfloat16_t;\n"; 1120 OS << "#endif\n\n"; 1121 1122 OS << "typedef __SVFloat32_t svfloat32_t;\n"; 1123 OS << "typedef __SVFloat64_t svfloat64_t;\n"; 1124 OS << "typedef __clang_svint8x2_t svint8x2_t;\n"; 1125 OS << "typedef __clang_svint16x2_t svint16x2_t;\n"; 1126 OS << "typedef __clang_svint32x2_t svint32x2_t;\n"; 1127 OS << "typedef __clang_svint64x2_t svint64x2_t;\n"; 1128 OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n"; 1129 OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n"; 1130 OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n"; 1131 OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n"; 1132 OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n"; 1133 OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n"; 1134 OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n"; 1135 OS << "typedef __clang_svint8x3_t svint8x3_t;\n"; 1136 OS << "typedef __clang_svint16x3_t svint16x3_t;\n"; 1137 OS << "typedef __clang_svint32x3_t svint32x3_t;\n"; 1138 OS << "typedef __clang_svint64x3_t svint64x3_t;\n"; 1139 OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n"; 1140 OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n"; 1141 OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n"; 1142 OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n"; 1143 OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n"; 1144 OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n"; 1145 OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n"; 1146 OS << "typedef __clang_svint8x4_t svint8x4_t;\n"; 1147 OS << "typedef __clang_svint16x4_t svint16x4_t;\n"; 1148 OS << "typedef __clang_svint32x4_t svint32x4_t;\n"; 1149 OS << "typedef __clang_svint64x4_t svint64x4_t;\n"; 1150 OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n"; 1151 OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n"; 1152 OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n"; 1153 OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n"; 1154 OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n"; 1155 OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n"; 1156 OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n"; 1157 OS << "typedef __SVBool_t svbool_t;\n\n"; 1158 1159 OS << "#ifdef __ARM_FEATURE_SVE_BF16\n"; 1160 OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n"; 1161 OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n"; 1162 OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n"; 1163 OS << "#endif\n"; 1164 1165 OS << "enum svpattern\n"; 1166 OS << "{\n"; 1167 OS << " SV_POW2 = 0,\n"; 1168 OS << " SV_VL1 = 1,\n"; 1169 OS << " SV_VL2 = 2,\n"; 1170 OS << " SV_VL3 = 3,\n"; 1171 OS << " SV_VL4 = 4,\n"; 1172 OS << " SV_VL5 = 5,\n"; 1173 OS << " SV_VL6 = 6,\n"; 1174 OS << " SV_VL7 = 7,\n"; 1175 OS << " SV_VL8 = 8,\n"; 1176 OS << " SV_VL16 = 9,\n"; 1177 OS << " SV_VL32 = 10,\n"; 1178 OS << " SV_VL64 = 11,\n"; 1179 OS << " SV_VL128 = 12,\n"; 1180 OS << " SV_VL256 = 13,\n"; 1181 OS << " SV_MUL4 = 29,\n"; 1182 OS << " SV_MUL3 = 30,\n"; 1183 OS << " SV_ALL = 31\n"; 1184 OS << "};\n\n"; 1185 1186 OS << "enum svprfop\n"; 1187 OS << "{\n"; 1188 OS << " SV_PLDL1KEEP = 0,\n"; 1189 OS << " SV_PLDL1STRM = 1,\n"; 1190 OS << " SV_PLDL2KEEP = 2,\n"; 1191 OS << " SV_PLDL2STRM = 3,\n"; 1192 OS << " SV_PLDL3KEEP = 4,\n"; 1193 OS << " SV_PLDL3STRM = 5,\n"; 1194 OS << " SV_PSTL1KEEP = 8,\n"; 1195 OS << " SV_PSTL1STRM = 9,\n"; 1196 OS << " SV_PSTL2KEEP = 10,\n"; 1197 OS << " SV_PSTL2STRM = 11,\n"; 1198 OS << " SV_PSTL3KEEP = 12,\n"; 1199 OS << " SV_PSTL3STRM = 13\n"; 1200 OS << "};\n\n"; 1201 1202 OS << "/* Function attributes */\n"; 1203 OS << "#define __ai static __inline__ __attribute__((__always_inline__, " 1204 "__nodebug__))\n\n"; 1205 OS << "#define __aio static __inline__ __attribute__((__always_inline__, " 1206 "__nodebug__, __overloadable__))\n\n"; 1207 1208 // Add reinterpret functions. 1209 for (auto ShortForm : { false, true } ) 1210 for (const ReinterpretTypeInfo &From : Reinterprets) 1211 for (const ReinterpretTypeInfo &To : Reinterprets) { 1212 const bool IsBFloat = StringRef(From.Suffix).equals("bf16") || 1213 StringRef(To.Suffix).equals("bf16"); 1214 if (IsBFloat) 1215 OS << "#if defined(__ARM_FEATURE_SVE_BF16)\n"; 1216 if (ShortForm) { 1217 OS << "__aio " << From.Type << " svreinterpret_" << From.Suffix; 1218 OS << "(" << To.Type << " op) {\n"; 1219 OS << " return __builtin_sve_reinterpret_" << From.Suffix << "_" 1220 << To.Suffix << "(op);\n"; 1221 OS << "}\n\n"; 1222 } else 1223 OS << "#define svreinterpret_" << From.Suffix << "_" << To.Suffix 1224 << "(...) __builtin_sve_reinterpret_" << From.Suffix << "_" 1225 << To.Suffix << "(__VA_ARGS__)\n"; 1226 if (IsBFloat) 1227 OS << "#endif /* #if defined(__ARM_FEATURE_SVE_BF16) */\n"; 1228 } 1229 1230 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1231 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1232 for (auto *R : RV) 1233 createIntrinsic(R, Defs); 1234 1235 // Sort intrinsics in header file by following order/priority: 1236 // - Architectural guard (i.e. does it require SVE2 or SVE2_AES) 1237 // - Class (is intrinsic overloaded or not) 1238 // - Intrinsic name 1239 std::stable_sort( 1240 Defs.begin(), Defs.end(), [](const std::unique_ptr<Intrinsic> &A, 1241 const std::unique_ptr<Intrinsic> &B) { 1242 auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) { 1243 return std::make_tuple(I->getGuard(), (unsigned)I->getClassKind(), I->getName()); 1244 }; 1245 return ToTuple(A) < ToTuple(B); 1246 }); 1247 1248 StringRef InGuard = ""; 1249 for (auto &I : Defs) { 1250 // Emit #endif/#if pair if needed. 1251 if (I->getGuard() != InGuard) { 1252 if (!InGuard.empty()) 1253 OS << "#endif //" << InGuard << "\n"; 1254 InGuard = I->getGuard(); 1255 if (!InGuard.empty()) 1256 OS << "\n#if " << InGuard << "\n"; 1257 } 1258 1259 // Actually emit the intrinsic declaration. 1260 I->emitIntrinsic(OS); 1261 } 1262 1263 if (!InGuard.empty()) 1264 OS << "#endif //" << InGuard << "\n"; 1265 1266 OS << "#if defined(__ARM_FEATURE_SVE_BF16)\n"; 1267 OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n"; 1268 OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n"; 1269 OS << "#endif /*__ARM_FEATURE_SVE_BF16 */\n\n"; 1270 1271 OS << "#if defined(__ARM_FEATURE_SVE2)\n"; 1272 OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n"; 1273 OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n"; 1274 OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n"; 1275 OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n"; 1276 1277 OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n"; 1278 OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n"; 1279 1280 OS << "#endif /*__ARM_FEATURE_SVE2 */\n\n"; 1281 1282 OS << "#ifdef __cplusplus\n"; 1283 OS << "} // extern \"C\"\n"; 1284 OS << "#endif\n\n"; 1285 OS << "#undef __ai\n\n"; 1286 OS << "#undef __aio\n\n"; 1287 OS << "#endif /*__ARM_FEATURE_SVE */\n\n"; 1288 OS << "#endif /* __ARM_SVE_H */\n"; 1289 } 1290 1291 void SVEEmitter::createBuiltins(raw_ostream &OS) { 1292 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1293 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1294 for (auto *R : RV) 1295 createIntrinsic(R, Defs); 1296 1297 // The mappings must be sorted based on BuiltinID. 1298 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, 1299 const std::unique_ptr<Intrinsic> &B) { 1300 return A->getMangledName() < B->getMangledName(); 1301 }); 1302 1303 OS << "#ifdef GET_SVE_BUILTINS\n"; 1304 for (auto &Def : Defs) { 1305 // Only create BUILTINs for non-overloaded intrinsics, as overloaded 1306 // declarations only live in the header file. 1307 if (Def->getClassKind() != ClassG) 1308 OS << "BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \"" 1309 << Def->getBuiltinTypeStr() << "\", \"n\")\n"; 1310 } 1311 1312 // Add reinterpret builtins 1313 for (const ReinterpretTypeInfo &From : Reinterprets) 1314 for (const ReinterpretTypeInfo &To : Reinterprets) 1315 OS << "BUILTIN(__builtin_sve_reinterpret_" << From.Suffix << "_" 1316 << To.Suffix << +", \"" << From.BuiltinType << To.BuiltinType 1317 << "\", \"n\")\n"; 1318 1319 OS << "#endif\n\n"; 1320 } 1321 1322 void SVEEmitter::createCodeGenMap(raw_ostream &OS) { 1323 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1324 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1325 for (auto *R : RV) 1326 createIntrinsic(R, Defs); 1327 1328 // The mappings must be sorted based on BuiltinID. 1329 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, 1330 const std::unique_ptr<Intrinsic> &B) { 1331 return A->getMangledName() < B->getMangledName(); 1332 }); 1333 1334 OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n"; 1335 for (auto &Def : Defs) { 1336 // Builtins only exist for non-overloaded intrinsics, overloaded 1337 // declarations only live in the header file. 1338 if (Def->getClassKind() == ClassG) 1339 continue; 1340 1341 uint64_t Flags = Def->getFlags(); 1342 auto FlagString = std::to_string(Flags); 1343 1344 std::string LLVMName = Def->getLLVMName(); 1345 std::string Builtin = Def->getMangledName(); 1346 if (!LLVMName.empty()) 1347 OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString 1348 << "),\n"; 1349 else 1350 OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n"; 1351 } 1352 OS << "#endif\n\n"; 1353 } 1354 1355 void SVEEmitter::createRangeChecks(raw_ostream &OS) { 1356 std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); 1357 SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; 1358 for (auto *R : RV) 1359 createIntrinsic(R, Defs); 1360 1361 // The mappings must be sorted based on BuiltinID. 1362 llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, 1363 const std::unique_ptr<Intrinsic> &B) { 1364 return A->getMangledName() < B->getMangledName(); 1365 }); 1366 1367 1368 OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n"; 1369 1370 // Ensure these are only emitted once. 1371 std::set<std::string> Emitted; 1372 1373 for (auto &Def : Defs) { 1374 if (Emitted.find(Def->getMangledName()) != Emitted.end() || 1375 Def->getImmChecks().empty()) 1376 continue; 1377 1378 OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n"; 1379 for (auto &Check : Def->getImmChecks()) 1380 OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", " 1381 << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n"; 1382 OS << " break;\n"; 1383 1384 Emitted.insert(Def->getMangledName()); 1385 } 1386 1387 OS << "#endif\n\n"; 1388 } 1389 1390 /// Create the SVETypeFlags used in CGBuiltins 1391 void SVEEmitter::createTypeFlags(raw_ostream &OS) { 1392 OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n"; 1393 for (auto &KV : FlagTypes) 1394 OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n"; 1395 OS << "#endif\n\n"; 1396 1397 OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n"; 1398 for (auto &KV : EltTypes) 1399 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; 1400 OS << "#endif\n\n"; 1401 1402 OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n"; 1403 for (auto &KV : MemEltTypes) 1404 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; 1405 OS << "#endif\n\n"; 1406 1407 OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n"; 1408 for (auto &KV : MergeTypes) 1409 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; 1410 OS << "#endif\n\n"; 1411 1412 OS << "#ifdef LLVM_GET_SVE_IMMCHECKTYPES\n"; 1413 for (auto &KV : ImmCheckTypes) 1414 OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; 1415 OS << "#endif\n\n"; 1416 } 1417 1418 namespace clang { 1419 void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) { 1420 SVEEmitter(Records).createHeader(OS); 1421 } 1422 1423 void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) { 1424 SVEEmitter(Records).createBuiltins(OS); 1425 } 1426 1427 void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) { 1428 SVEEmitter(Records).createCodeGenMap(OS); 1429 } 1430 1431 void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) { 1432 SVEEmitter(Records).createRangeChecks(OS); 1433 } 1434 1435 void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) { 1436 SVEEmitter(Records).createTypeFlags(OS); 1437 } 1438 1439 } // End namespace clang 1440