xref: /freebsd/contrib/llvm-project/clang/utils/TableGen/SveEmitter.cpp (revision 7a6dacac)

//===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===//
//
//  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
//  See https://llvm.org/LICENSE.txt for license information.
//  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This tablegen backend is responsible for emitting arm_sve.h, which includes
// a declaration and definition of each function specified by the ARM C/C++
// Language Extensions (ACLE).
//
// For details, visit:
//  https://developer.arm.com/architectures/system-architectures/software-standards/acle
//
// Each SVE instruction is implemented in terms of 1 or more functions which
// are suffixed with the element type of the input vectors.  Functions may be
// implemented in terms of generic vector operations such as +, *, -, etc. or
// by calling a __builtin_-prefixed function which will be handled by clang's
// CodeGen library.
//
// See also the documentation in include/clang/Basic/arm_sve.td.
//
//===----------------------------------------------------------------------===//

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/TableGen/Error.h"
#include "llvm/TableGen/Record.h"
#include <array>
#include <cctype>
#include <set>
#include <sstream>
#include <string>
#include <tuple>

using namespace llvm;

enum ClassKind {
  ClassNone,
  ClassS,     // signed/unsigned, e.g., "_s8", "_u8" suffix
  ClassG,     // Overloaded name without type suffix
};

enum class ACLEKind { SVE, SME };

using TypeSpec = std::string;

namespace {

class ImmCheck {
  unsigned Arg;
  unsigned Kind;
  unsigned ElementSizeInBits;

public:
  ImmCheck(unsigned Arg, unsigned Kind, unsigned ElementSizeInBits = 0)
      : Arg(Arg), Kind(Kind), ElementSizeInBits(ElementSizeInBits) {}
  ImmCheck(const ImmCheck &Other) = default;
  ~ImmCheck() = default;

  unsigned getArg() const { return Arg; }
  unsigned getKind() const { return Kind; }
  unsigned getElementSizeInBits() const { return ElementSizeInBits; }
};

class SVEType {
  bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat;
  bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp,
      Svcount;
  unsigned Bitwidth, ElementBitwidth, NumVectors;

public:
  SVEType() : SVEType("", 'v') {}

  SVEType(StringRef TS, char CharMod, unsigned NumVectors = 1)
      : Float(false), Signed(true), Immediate(false), Void(false),
        Constant(false), Pointer(false), BFloat(false), DefaultType(false),
        IsScalable(true), Predicate(false), PredicatePattern(false),
        PrefetchOp(false), Svcount(false), Bitwidth(128), ElementBitwidth(~0U),
        NumVectors(NumVectors) {
    if (!TS.empty())
      applyTypespec(TS);
    applyModifier(CharMod);
  }

  SVEType(const SVEType &Base, unsigned NumV) : SVEType(Base) {
    NumVectors = NumV;
  }

  bool isPointer() const { return Pointer; }
  bool isVoidPointer() const { return Pointer && Void; }
  bool isSigned() const { return Signed; }
  bool isImmediate() const { return Immediate; }
  bool isScalar() const { return NumVectors == 0; }
  bool isVector() const { return NumVectors > 0; }
  bool isScalableVector() const { return isVector() && IsScalable; }
  bool isFixedLengthVector() const { return isVector() && !IsScalable; }
  bool isChar() const { return ElementBitwidth == 8; }
  bool isVoid() const { return Void & !Pointer; }
  bool isDefault() const { return DefaultType; }
  bool isFloat() const { return Float && !BFloat; }
  bool isBFloat() const { return BFloat && !Float; }
  bool isFloatingPoint() const { return Float || BFloat; }
  bool isInteger() const {
    return !isFloatingPoint() && !Predicate && !Svcount;
  }
  bool isScalarPredicate() const {
    return !isFloatingPoint() && Predicate && NumVectors == 0;
  }
  bool isPredicateVector() const { return Predicate; }
  bool isPredicatePattern() const { return PredicatePattern; }
  bool isPrefetchOp() const { return PrefetchOp; }
  bool isSvcount() const { return Svcount; }
  bool isConstant() const { return Constant; }
  unsigned getElementSizeInBits() const { return ElementBitwidth; }
  unsigned getNumVectors() const { return NumVectors; }

  unsigned getNumElements() const {
    assert(ElementBitwidth != ~0U);
    return Bitwidth / ElementBitwidth;
  }
  unsigned getSizeInBits() const {
    return Bitwidth;
  }

  /// Return the string representation of a type, which is an encoded
  /// string for passing to the BUILTIN() macro in Builtins.def.
  std::string builtin_str() const;

  /// Return the C/C++ string representation of a type for use in the
  /// arm_sve.h header file.
  std::string str() const;

private:
  /// Creates the type based on the typespec string in TS.
  void applyTypespec(StringRef TS);

  /// Applies a prototype modifier to the type.
  void applyModifier(char Mod);
};

class SVEEmitter;

/// The main grunt class. This represents an instantiation of an intrinsic with
/// a particular typespec and prototype.
class Intrinsic {
  /// The unmangled name.
  std::string Name;

  /// The name of the corresponding LLVM IR intrinsic.
  std::string LLVMName;

  /// Intrinsic prototype.
  std::string Proto;

  /// The base type spec for this intrinsic.
  TypeSpec BaseTypeSpec;

  /// The base class kind. Most intrinsics use ClassS, which has full type
  /// info for integers (_s32/_u32), or ClassG which is used for overloaded
  /// intrinsics.
  ClassKind Class;

  /// The architectural #ifdef guard.
  std::string Guard;

  // The merge suffix such as _m, _x or _z.
  std::string MergeSuffix;

  /// The types of return value [0] and parameters [1..].
  std::vector<SVEType> Types;

  /// The "base type", which is VarType('d', BaseTypeSpec).
  SVEType BaseType;

  uint64_t Flags;

  SmallVector<ImmCheck, 2> ImmChecks;

public:
  Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
            StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName,
            uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT,
            ClassKind Class, SVEEmitter &Emitter, StringRef Guard);

  ~Intrinsic()=default;

  std::string getName() const { return Name; }
  std::string getLLVMName() const { return LLVMName; }
  std::string getProto() const { return Proto; }
  TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; }
  SVEType getBaseType() const { return BaseType; }

  StringRef getGuard() const { return Guard; }
  ClassKind getClassKind() const { return Class; }

  SVEType getReturnType() const { return Types[0]; }
  ArrayRef<SVEType> getTypes() const { return Types; }
  SVEType getParamType(unsigned I) const { return Types[I + 1]; }
  unsigned getNumParams() const {
    return Proto.size() - (2 * llvm::count(Proto, '.')) - 1;
  }

  uint64_t getFlags() const { return Flags; }
  bool isFlagSet(uint64_t Flag) const { return Flags & Flag;}

  ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; }

  /// Return the type string for a BUILTIN() macro in Builtins.def.
  std::string getBuiltinTypeStr();

  /// Return the name, mangled with type information. The name is mangled for
  /// ClassS, so will add type suffixes such as _u32/_s32.
  std::string getMangledName() const { return mangleName(ClassS); }

  /// As above, but mangles the LLVM name instead.
  std::string getMangledLLVMName() const { return mangleLLVMName(); }

  /// Returns true if the intrinsic is overloaded, in that it should also generate
  /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of
  /// 'svld1_u32(..)'.
  static bool isOverloadedIntrinsic(StringRef Name) {
    auto BrOpen = Name.find('[');
    auto BrClose = Name.find(']');
    return BrOpen != std::string::npos && BrClose != std::string::npos;
  }

  /// Return true if the intrinsic takes a splat operand.
  bool hasSplat() const {
    // These prototype modifiers are described in arm_sve.td.
    return Proto.find_first_of("ajfrKLR@") != std::string::npos;
  }

  /// Return the parameter index of the splat operand.
  unsigned getSplatIdx() const {
    unsigned I = 1, Param = 0;
    for (; I < Proto.size(); ++I, ++Param) {
      if (Proto[I] == 'a' || Proto[I] == 'j' || Proto[I] == 'f' ||
          Proto[I] == 'r' || Proto[I] == 'K' || Proto[I] == 'L' ||
          Proto[I] == 'R' || Proto[I] == '@')
        break;

      // Multivector modifier can be skipped
      if (Proto[I] == '.')
        I += 2;
    }
    assert(I != Proto.size() && "Prototype has no splat operand");
    return Param;
  }

  /// Emits the intrinsic declaration to the ostream.
  void emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter, ACLEKind Kind) const;

private:
  std::string getMergeSuffix() const { return MergeSuffix; }
  std::string mangleName(ClassKind LocalCK) const;
  std::string mangleLLVMName() const;
  std::string replaceTemplatedArgs(std::string Name, TypeSpec TS,
                                   std::string Proto) const;
};

class SVEEmitter {
private:
  // The reinterpret builtins are generated separately because they
  // need the cross product of all types (121 functions in total),
  // which is inconvenient to specify in the arm_sve.td file or
  // generate in CGBuiltin.cpp.
  struct ReinterpretTypeInfo {
    SVEType BaseType;
    const char *Suffix;
  };

  static const std::array<ReinterpretTypeInfo, 12> Reinterprets;

  RecordKeeper &Records;
  llvm::StringMap<uint64_t> EltTypes;
  llvm::StringMap<uint64_t> MemEltTypes;
  llvm::StringMap<uint64_t> FlagTypes;
  llvm::StringMap<uint64_t> MergeTypes;
  llvm::StringMap<uint64_t> ImmCheckTypes;

public:
  SVEEmitter(RecordKeeper &R) : Records(R) {
    for (auto *RV : Records.getAllDerivedDefinitions("EltType"))
      EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
    for (auto *RV : Records.getAllDerivedDefinitions("MemEltType"))
      MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
    for (auto *RV : Records.getAllDerivedDefinitions("FlagType"))
      FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
    for (auto *RV : Records.getAllDerivedDefinitions("MergeType"))
      MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
    for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType"))
      ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value");
  }

  /// Returns the enum value for the immcheck type
  unsigned getEnumValueForImmCheck(StringRef C) const {
    auto It = ImmCheckTypes.find(C);
    if (It != ImmCheckTypes.end())
      return It->getValue();
    llvm_unreachable("Unsupported imm check");
  }

  /// Returns the enum value for the flag type
  uint64_t getEnumValueForFlag(StringRef C) const {
    auto Res = FlagTypes.find(C);
    if (Res != FlagTypes.end())
      return Res->getValue();
    llvm_unreachable("Unsupported flag");
  }

  // Returns the SVETypeFlags for a given value and mask.
  uint64_t encodeFlag(uint64_t V, StringRef MaskName) const {
    auto It = FlagTypes.find(MaskName);
    if (It != FlagTypes.end()) {
      uint64_t Mask = It->getValue();
      unsigned Shift = llvm::countr_zero(Mask);
      assert(Shift < 64 && "Mask value produced an invalid shift value");
      return (V << Shift) & Mask;
    }
    llvm_unreachable("Unsupported flag");
  }

  // Returns the SVETypeFlags for the given element type.
  uint64_t encodeEltType(StringRef EltName) {
    auto It = EltTypes.find(EltName);
    if (It != EltTypes.end())
      return encodeFlag(It->getValue(), "EltTypeMask");
    llvm_unreachable("Unsupported EltType");
  }

  // Returns the SVETypeFlags for the given memory element type.
  uint64_t encodeMemoryElementType(uint64_t MT) {
    return encodeFlag(MT, "MemEltTypeMask");
  }

  // Returns the SVETypeFlags for the given merge type.
  uint64_t encodeMergeType(uint64_t MT) {
    return encodeFlag(MT, "MergeTypeMask");
  }

  // Returns the SVETypeFlags for the given splat operand.
  unsigned encodeSplatOperand(unsigned SplatIdx) {
    assert(SplatIdx < 7 && "SplatIdx out of encodable range");
    return encodeFlag(SplatIdx + 1, "SplatOperandMask");
  }

  // Returns the SVETypeFlags value for the given SVEType.
  uint64_t encodeTypeFlags(const SVEType &T);

  /// Emit arm_sve.h.
  void createHeader(raw_ostream &o);

  // Emits core intrinsics in both arm_sme.h and arm_sve.h
  void createCoreHeaderIntrinsics(raw_ostream &o, SVEEmitter &Emitter,
                                  ACLEKind Kind);

  /// Emit all the __builtin prototypes and code needed by Sema.
  void createBuiltins(raw_ostream &o);

  /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
  void createCodeGenMap(raw_ostream &o);

  /// Emit all the range checks for the immediates.
  void createRangeChecks(raw_ostream &o);

  /// Create the SVETypeFlags used in CGBuiltins
  void createTypeFlags(raw_ostream &o);

  /// Emit arm_sme.h.
  void createSMEHeader(raw_ostream &o);

  /// Emit all the SME __builtin prototypes and code needed by Sema.
  void createSMEBuiltins(raw_ostream &o);

  /// Emit all the information needed to map builtin -> LLVM IR intrinsic.
  void createSMECodeGenMap(raw_ostream &o);

  /// Create a table for a builtin's requirement for PSTATE.SM.
  void createStreamingAttrs(raw_ostream &o, ACLEKind Kind);

  /// Emit all the range checks for the immediates.
  void createSMERangeChecks(raw_ostream &o);

  /// Create a table for a builtin's requirement for PSTATE.ZA.
  void createBuiltinZAState(raw_ostream &OS);

  /// Create intrinsic and add it to \p Out
  void createIntrinsic(Record *R,
                       SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out);
};

const std::array<SVEEmitter::ReinterpretTypeInfo, 12> SVEEmitter::Reinterprets =
    {{{SVEType("c", 'd'), "s8"},
      {SVEType("Uc", 'd'), "u8"},
      {SVEType("s", 'd'), "s16"},
      {SVEType("Us", 'd'), "u16"},
      {SVEType("i", 'd'), "s32"},
      {SVEType("Ui", 'd'), "u32"},
      {SVEType("l", 'd'), "s64"},
      {SVEType("Ul", 'd'), "u64"},
      {SVEType("h", 'd'), "f16"},
      {SVEType("b", 'd'), "bf16"},
      {SVEType("f", 'd'), "f32"},
      {SVEType("d", 'd'), "f64"}}};

} // end anonymous namespace


//===----------------------------------------------------------------------===//
// Type implementation
//===----------------------------------------------------------------------===//

std::string SVEType::builtin_str() const {
  std::string S;
  if (isVoid())
    return "v";

  if (isScalarPredicate())
    return "b";

  if (isSvcount())
    return "Qa";

  if (isVoidPointer())
    S += "v";
  else if (!isFloatingPoint())
    switch (ElementBitwidth) {
    case 1: S += "b"; break;
    case 8: S += "c"; break;
    case 16: S += "s"; break;
    case 32: S += "i"; break;
    case 64: S += "Wi"; break;
    case 128: S += "LLLi"; break;
    default: llvm_unreachable("Unhandled case!");
    }
  else if (isFloat())
    switch (ElementBitwidth) {
    case 16: S += "h"; break;
    case 32: S += "f"; break;
    case 64: S += "d"; break;
    default: llvm_unreachable("Unhandled case!");
    }
  else if (isBFloat()) {
    assert(ElementBitwidth == 16 && "Not a valid BFloat.");
    S += "y";
  }

  if (!isFloatingPoint()) {
    if ((isChar() || isPointer()) && !isVoidPointer()) {
      // Make chars and typed pointers explicitly signed.
      if (Signed)
        S = "S" + S;
      else if (!Signed)
        S = "U" + S;
    } else if (!isVoidPointer() && !Signed) {
      S = "U" + S;
    }
  }

  // Constant indices are "int", but have the "constant expression" modifier.
  if (isImmediate()) {
    assert(!isFloat() && "fp immediates are not supported");
    S = "I" + S;
  }

  if (isScalar()) {
    if (Constant) S += "C";
    if (Pointer) S += "*";
    return S;
  }

  if (isFixedLengthVector())
    return "V" + utostr(getNumElements() * NumVectors) + S;
  return "q" + utostr(getNumElements() * NumVectors) + S;
}

std::string SVEType::str() const {
  if (isPredicatePattern())
    return "enum svpattern";

  if (isPrefetchOp())
    return "enum svprfop";

  std::string S;
  if (Void)
    S += "void";
  else {
    if (isScalableVector() || isSvcount())
      S += "sv";
    if (!Signed && !isFloatingPoint())
      S += "u";

    if (Float)
      S += "float";
    else if (isSvcount())
      S += "count";
    else if (isScalarPredicate() || isPredicateVector())
      S += "bool";
    else if (isBFloat())
      S += "bfloat";
    else
      S += "int";

    if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
      S += utostr(ElementBitwidth);
    if (isFixedLengthVector())
      S += "x" + utostr(getNumElements());
    if (NumVectors > 1)
      S += "x" + utostr(NumVectors);
    if (!isScalarPredicate())
      S += "_t";
  }

  if (Constant)
    S += " const";
  if (Pointer)
    S += " *";

  return S;
}

void SVEType::applyTypespec(StringRef TS) {
  for (char I : TS) {
    switch (I) {
    case 'Q':
      Svcount = true;
      break;
    case 'P':
      Predicate = true;
      break;
    case 'U':
      Signed = false;
      break;
    case 'c':
      ElementBitwidth = 8;
      break;
    case 's':
      ElementBitwidth = 16;
      break;
    case 'i':
      ElementBitwidth = 32;
      break;
    case 'l':
      ElementBitwidth = 64;
      break;
    case 'q':
      ElementBitwidth = 128;
      break;
    case 'h':
      Float = true;
      ElementBitwidth = 16;
      break;
    case 'f':
      Float = true;
      ElementBitwidth = 32;
      break;
    case 'd':
      Float = true;
      ElementBitwidth = 64;
      break;
    case 'b':
      BFloat = true;
      Float = false;
      ElementBitwidth = 16;
      break;
    default:
      llvm_unreachable("Unhandled type code!");
    }
  }
  assert(ElementBitwidth != ~0U && "Bad element bitwidth!");
}

void SVEType::applyModifier(char Mod) {
  switch (Mod) {
  case 'v':
    Void = true;
    break;
  case 'd':
    DefaultType = true;
    break;
  case 'c':
    Constant = true;
    [[fallthrough]];
  case 'p':
    Pointer = true;
    Bitwidth = ElementBitwidth;
    NumVectors = 0;
    break;
  case 'e':
    Signed = false;
    ElementBitwidth /= 2;
    break;
  case 'h':
    ElementBitwidth /= 2;
    break;
  case 'q':
    ElementBitwidth /= 4;
    break;
  case 'b':
    Signed = false;
    Float = false;
    BFloat = false;
    ElementBitwidth /= 4;
    break;
  case 'o':
    ElementBitwidth *= 4;
    break;
  case 'P':
    Signed = true;
    Float = false;
    BFloat = false;
    Predicate = true;
    Svcount = false;
    Bitwidth = 16;
    ElementBitwidth = 1;
    break;
  case '{':
    IsScalable = false;
    Bitwidth = 128;
    NumVectors = 1;
    break;
  case 's':
  case 'a':
    Bitwidth = ElementBitwidth;
    NumVectors = 0;
    break;
  case 'R':
    ElementBitwidth /= 2;
    NumVectors = 0;
    break;
  case 'r':
    ElementBitwidth /= 4;
    NumVectors = 0;
    break;
  case '@':
    Signed = false;
    Float = false;
    BFloat = false;
    ElementBitwidth /= 4;
    NumVectors = 0;
    break;
  case 'K':
    Signed = true;
    Float = false;
    BFloat = false;
    Bitwidth = ElementBitwidth;
    NumVectors = 0;
    break;
  case 'L':
    Signed = false;
    Float = false;
    BFloat = false;
    Bitwidth = ElementBitwidth;
    NumVectors = 0;
    break;
  case 'u':
    Predicate = false;
    Svcount = false;
    Signed = false;
    Float = false;
    BFloat = false;
    break;
  case 'x':
    Predicate = false;
    Svcount = false;
    Signed = true;
    Float = false;
    BFloat = false;
    break;
  case 'i':
    Predicate = false;
    Svcount = false;
    Float = false;
    BFloat = false;
    ElementBitwidth = Bitwidth = 64;
    NumVectors = 0;
    Signed = false;
    Immediate = true;
    break;
  case 'I':
    Predicate = false;
    Svcount = false;
    Float = false;
    BFloat = false;
    ElementBitwidth = Bitwidth = 32;
    NumVectors = 0;
    Signed = true;
    Immediate = true;
    PredicatePattern = true;
    break;
  case 'J':
    Predicate = false;
    Svcount = false;
    Float = false;
    BFloat = false;
    ElementBitwidth = Bitwidth = 32;
    NumVectors = 0;
    Signed = true;
    Immediate = true;
    PrefetchOp = true;
    break;
  case 'k':
    Predicate = false;
    Svcount = false;
    Signed = true;
    Float = false;
    BFloat = false;
    ElementBitwidth = Bitwidth = 32;
    NumVectors = 0;
    break;
  case 'l':
    Predicate = false;
    Svcount = false;
    Signed = true;
    Float = false;
    BFloat = false;
    ElementBitwidth = Bitwidth = 64;
    NumVectors = 0;
    break;
  case 'm':
    Predicate = false;
    Svcount = false;
    Signed = false;
    Float = false;
    BFloat = false;
    ElementBitwidth = Bitwidth = 32;
    NumVectors = 0;
    break;
  case 'n':
    Predicate = false;
    Svcount = false;
    Signed = false;
    Float = false;
    BFloat = false;
    ElementBitwidth = Bitwidth = 64;
    NumVectors = 0;
    break;
  case 'w':
    ElementBitwidth = 64;
    break;
  case 'j':
    ElementBitwidth = Bitwidth = 64;
    NumVectors = 0;
    break;
  case 'f':
    Signed = false;
    ElementBitwidth = Bitwidth = 64;
    NumVectors = 0;
    break;
  case 'g':
    Signed = false;
    Float = false;
    BFloat = false;
    ElementBitwidth = 64;
    break;
  case '[':
    Signed = false;
    Float = false;
    BFloat = false;
    ElementBitwidth = 8;
    break;
  case 't':
    Signed = true;
    Float = false;
    BFloat = false;
    ElementBitwidth = 32;
    break;
  case 'z':
    Signed = false;
    Float = false;
    BFloat = false;
    ElementBitwidth = 32;
    break;
  case 'O':
    Predicate = false;
    Svcount = false;
    Float = true;
    ElementBitwidth = 16;
    break;
  case 'M':
    Predicate = false;
    Svcount = false;
    Float = true;
    BFloat = false;
    ElementBitwidth = 32;
    break;
  case 'N':
    Predicate = false;
    Svcount = false;
    Float = true;
    ElementBitwidth = 64;
    break;
  case 'Q':
    Constant = true;
    Pointer = true;
    Void = true;
    NumVectors = 0;
    break;
  case 'S':
    Constant = true;
    Pointer = true;
    ElementBitwidth = Bitwidth = 8;
    NumVectors = 0;
    Signed = true;
    break;
  case 'W':
    Constant = true;
    Pointer = true;
    ElementBitwidth = Bitwidth = 8;
    NumVectors = 0;
    Signed = false;
    break;
  case 'T':
    Constant = true;
    Pointer = true;
    ElementBitwidth = Bitwidth = 16;
    NumVectors = 0;
    Signed = true;
    break;
  case 'X':
    Constant = true;
    Pointer = true;
    ElementBitwidth = Bitwidth = 16;
    NumVectors = 0;
    Signed = false;
    break;
  case 'Y':
    Constant = true;
    Pointer = true;
    ElementBitwidth = Bitwidth = 32;
    NumVectors = 0;
    Signed = false;
    break;
  case 'U':
    Constant = true;
    Pointer = true;
    ElementBitwidth = Bitwidth = 32;
    NumVectors = 0;
    Signed = true;
    break;
  case '%':
    Pointer = true;
    Void = true;
    NumVectors = 0;
    break;
  case 'A':
    Pointer = true;
    ElementBitwidth = Bitwidth = 8;
    NumVectors = 0;
    Signed = true;
    break;
  case 'B':
    Pointer = true;
    ElementBitwidth = Bitwidth = 16;
    NumVectors = 0;
    Signed = true;
    break;
  case 'C':
    Pointer = true;
    ElementBitwidth = Bitwidth = 32;
    NumVectors = 0;
    Signed = true;
    break;
  case 'D':
    Pointer = true;
    ElementBitwidth = Bitwidth = 64;
    NumVectors = 0;
    Signed = true;
    break;
  case 'E':
    Pointer = true;
    ElementBitwidth = Bitwidth = 8;
    NumVectors = 0;
    Signed = false;
    break;
  case 'F':
    Pointer = true;
    ElementBitwidth = Bitwidth = 16;
    NumVectors = 0;
    Signed = false;
    break;
  case 'G':
    Pointer = true;
    ElementBitwidth = Bitwidth = 32;
    NumVectors = 0;
    Signed = false;
    break;
  case '$':
    Predicate = false;
    Svcount = false;
    Float = false;
    BFloat = true;
    ElementBitwidth = 16;
    break;
  case '}':
    Predicate = false;
    Signed = true;
    Svcount = true;
    NumVectors = 0;
    Float = false;
    BFloat = false;
    break;
  case '.':
    llvm_unreachable(". is never a type in itself");
    break;
  default:
    llvm_unreachable("Unhandled character!");
  }
}

/// Returns the modifier and number of vectors for the given operand \p Op.
std::pair<char, unsigned> getProtoModifier(StringRef Proto, unsigned Op) {
  for (unsigned P = 0; !Proto.empty(); ++P) {
    unsigned NumVectors = 1;
    unsigned CharsToSkip = 1;
    char Mod = Proto[0];
    if (Mod == '2' || Mod == '3' || Mod == '4') {
      NumVectors = Mod - '0';
      Mod = 'd';
      if (Proto.size() > 1 && Proto[1] == '.') {
        Mod = Proto[2];
        CharsToSkip = 3;
      }
    }

    if (P == Op)
      return {Mod, NumVectors};

    Proto = Proto.drop_front(CharsToSkip);
  }
  llvm_unreachable("Unexpected Op");
}

//===----------------------------------------------------------------------===//
// Intrinsic implementation
//===----------------------------------------------------------------------===//

Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
                     StringRef MergeSuffix, uint64_t MemoryElementTy,
                     StringRef LLVMName, uint64_t Flags,
                     ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class,
                     SVEEmitter &Emitter, StringRef Guard)
    : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()),
      BaseTypeSpec(BT), Class(Class), Guard(Guard.str()),
      MergeSuffix(MergeSuffix.str()), BaseType(BT, 'd'), Flags(Flags),
      ImmChecks(Checks.begin(), Checks.end()) {
  // Types[0] is the return value.
  for (unsigned I = 0; I < (getNumParams() + 1); ++I) {
    char Mod;
    unsigned NumVectors;
    std::tie(Mod, NumVectors) = getProtoModifier(Proto, I);
    SVEType T(BaseTypeSpec, Mod, NumVectors);
    Types.push_back(T);

    // Add range checks for immediates
    if (I > 0) {
      if (T.isPredicatePattern())
        ImmChecks.emplace_back(
            I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31"));
      else if (T.isPrefetchOp())
        ImmChecks.emplace_back(
            I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13"));
    }
  }

  // Set flags based on properties
  this->Flags |= Emitter.encodeTypeFlags(BaseType);
  this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy);
  this->Flags |= Emitter.encodeMergeType(MergeTy);
  if (hasSplat())
    this->Flags |= Emitter.encodeSplatOperand(getSplatIdx());
}

std::string Intrinsic::getBuiltinTypeStr() {
  std::string S = getReturnType().builtin_str();
  for (unsigned I = 0; I < getNumParams(); ++I)
    S += getParamType(I).builtin_str();

  return S;
}

std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS,
                                            std::string Proto) const {
  std::string Ret = Name;
  while (Ret.find('{') != std::string::npos) {
    size_t Pos = Ret.find('{');
    size_t End = Ret.find('}');
    unsigned NumChars = End - Pos + 1;
    assert(NumChars == 3 && "Unexpected template argument");

    SVEType T;
    char C = Ret[Pos+1];
    switch(C) {
    default:
      llvm_unreachable("Unknown predication specifier");
    case 'd':
      T = SVEType(TS, 'd');
      break;
    case '0':
    case '1':
    case '2':
    case '3':
      T = SVEType(TS, Proto[C - '0']);
      break;
    }

    // Replace templated arg with the right suffix (e.g. u32)
    std::string TypeCode;
    if (T.isInteger())
      TypeCode = T.isSigned() ? 's' : 'u';
    else if (T.isSvcount())
      TypeCode = 'c';
    else if (T.isPredicateVector())
      TypeCode = 'b';
    else if (T.isBFloat())
      TypeCode = "bf";
    else
      TypeCode = 'f';
    Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits()));
  }

  return Ret;
}

std::string Intrinsic::mangleLLVMName() const {
  std::string S = getLLVMName();

  // Replace all {d} like expressions with e.g. 'u32'
  return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto());
}

std::string Intrinsic::mangleName(ClassKind LocalCK) const {
  std::string S = getName();

  if (LocalCK == ClassG) {
    // Remove the square brackets and everything in between.
    while (S.find('[') != std::string::npos) {
      auto Start = S.find('[');
      auto End = S.find(']');
      S.erase(Start, (End-Start)+1);
    }
  } else {
    // Remove the square brackets.
    while (S.find('[') != std::string::npos) {
      auto BrPos = S.find('[');
      if (BrPos != std::string::npos)
        S.erase(BrPos, 1);
      BrPos = S.find(']');
      if (BrPos != std::string::npos)
        S.erase(BrPos, 1);
    }
  }

  // Replace all {d} like expressions with e.g. 'u32'
  return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) +
         getMergeSuffix();
}

void Intrinsic::emitIntrinsic(raw_ostream &OS, SVEEmitter &Emitter,
                              ACLEKind Kind) const {
  bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1;

  std::string FullName = mangleName(ClassS);
  std::string ProtoName = mangleName(getClassKind());
  OS << (IsOverloaded ? "__aio " : "__ai ")
     << "__attribute__((__clang_arm_builtin_alias(";

  switch (Kind) {
  case ACLEKind::SME:
    OS << "__builtin_sme_" << FullName << ")";
    break;
  case ACLEKind::SVE:
    OS << "__builtin_sve_" << FullName << ")";
    break;
  }

  OS << "))\n";

  OS << getTypes()[0].str() << " " << ProtoName << "(";
  for (unsigned I = 0; I < getTypes().size() - 1; ++I) {
    if (I != 0)
      OS << ", ";
    OS << getTypes()[I + 1].str();
  }
  OS << ");\n";
}

//===----------------------------------------------------------------------===//
// SVEEmitter implementation
//===----------------------------------------------------------------------===//
uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) {
  if (T.isFloat()) {
    switch (T.getElementSizeInBits()) {
    case 16:
      return encodeEltType("EltTyFloat16");
    case 32:
      return encodeEltType("EltTyFloat32");
    case 64:
      return encodeEltType("EltTyFloat64");
    default:
      llvm_unreachable("Unhandled float element bitwidth!");
    }
  }

  if (T.isBFloat()) {
    assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat.");
    return encodeEltType("EltTyBFloat16");
  }

  if (T.isPredicateVector() || T.isSvcount()) {
    switch (T.getElementSizeInBits()) {
    case 8:
      return encodeEltType("EltTyBool8");
    case 16:
      return encodeEltType("EltTyBool16");
    case 32:
      return encodeEltType("EltTyBool32");
    case 64:
      return encodeEltType("EltTyBool64");
    default:
      llvm_unreachable("Unhandled predicate element bitwidth!");
    }
  }

  switch (T.getElementSizeInBits()) {
  case 8:
    return encodeEltType("EltTyInt8");
  case 16:
    return encodeEltType("EltTyInt16");
  case 32:
    return encodeEltType("EltTyInt32");
  case 64:
    return encodeEltType("EltTyInt64");
  case 128:
    return encodeEltType("EltTyInt128");
  default:
    llvm_unreachable("Unhandled integer element bitwidth!");
  }
}

void SVEEmitter::createIntrinsic(
    Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) {
  StringRef Name = R->getValueAsString("Name");
  StringRef Proto = R->getValueAsString("Prototype");
  StringRef Types = R->getValueAsString("Types");
  StringRef Guard = R->getValueAsString("TargetGuard");
  StringRef LLVMName = R->getValueAsString("LLVMIntrinsic");
  uint64_t Merge = R->getValueAsInt("Merge");
  StringRef MergeSuffix = R->getValueAsString("MergeSuffix");
  uint64_t MemEltType = R->getValueAsInt("MemEltType");
  std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags");
  std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks");

  int64_t Flags = 0;
  for (auto FlagRec : FlagsList)
    Flags |= FlagRec->getValueAsInt("Value");

  // Create a dummy TypeSpec for non-overloaded builtins.
  if (Types.empty()) {
    assert((Flags & getEnumValueForFlag("IsOverloadNone")) &&
           "Expect TypeSpec for overloaded builtin!");
    Types = "i";
  }

  // Extract type specs from string
  SmallVector<TypeSpec, 8> TypeSpecs;
  TypeSpec Acc;
  for (char I : Types) {
    Acc.push_back(I);
    if (islower(I)) {
      TypeSpecs.push_back(TypeSpec(Acc));
      Acc.clear();
    }
  }

  // Remove duplicate type specs.
  llvm::sort(TypeSpecs);
  TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()),
                  TypeSpecs.end());

  // Create an Intrinsic for each type spec.
  for (auto TS : TypeSpecs) {
    // Collate a list of range/option checks for the immediates.
    SmallVector<ImmCheck, 2> ImmChecks;
    for (auto *R : ImmCheckList) {
      int64_t Arg = R->getValueAsInt("Arg");
      int64_t EltSizeArg = R->getValueAsInt("EltSizeArg");
      int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value");
      assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative");

      unsigned ElementSizeInBits = 0;
      char Mod;
      unsigned NumVectors;
      std::tie(Mod, NumVectors) = getProtoModifier(Proto, EltSizeArg + 1);
      if (EltSizeArg >= 0)
        ElementSizeInBits = SVEType(TS, Mod, NumVectors).getElementSizeInBits();
      ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits));
    }

    Out.push_back(std::make_unique<Intrinsic>(
        Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks,
        TS, ClassS, *this, Guard));

    // Also generate the short-form (e.g. svadd_m) for the given type-spec.
    if (Intrinsic::isOverloadedIntrinsic(Name))
      Out.push_back(std::make_unique<Intrinsic>(
          Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags,
          ImmChecks, TS, ClassG, *this, Guard));
  }
}

void SVEEmitter::createCoreHeaderIntrinsics(raw_ostream &OS,
                                            SVEEmitter &Emitter,
                                            ACLEKind Kind) {
  SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
  for (auto *R : RV)
    createIntrinsic(R, Defs);

  // Sort intrinsics in header file by following order/priority:
  // - Architectural guard (i.e. does it require SVE2 or SVE2_AES)
  // - Class (is intrinsic overloaded or not)
  // - Intrinsic name
  std::stable_sort(Defs.begin(), Defs.end(),
                   [](const std::unique_ptr<Intrinsic> &A,
                      const std::unique_ptr<Intrinsic> &B) {
                     auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) {
                       return std::make_tuple(I->getGuard(),
                                              (unsigned)I->getClassKind(),
                                              I->getName());
                     };
                     return ToTuple(A) < ToTuple(B);
                   });

  // Actually emit the intrinsic declarations.
  for (auto &I : Defs)
    I->emitIntrinsic(OS, Emitter, Kind);
}

void SVEEmitter::createHeader(raw_ostream &OS) {
  OS << "/*===---- arm_sve.h - ARM SVE intrinsics "
        "-----------------------------------===\n"
        " *\n"
        " *\n"
        " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
        "Exceptions.\n"
        " * See https://llvm.org/LICENSE.txt for license information.\n"
        " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
        " *\n"
        " *===-----------------------------------------------------------------"
        "------===\n"
        " */\n\n";

  OS << "#ifndef __ARM_SVE_H\n";
  OS << "#define __ARM_SVE_H\n\n";

  OS << "#if !defined(__LITTLE_ENDIAN__)\n";
  OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n";
  OS << "#endif\n";

  OS << "#include <stdint.h>\n\n";
  OS << "#ifdef  __cplusplus\n";
  OS << "extern \"C\" {\n";
  OS << "#else\n";
  OS << "#include <stdbool.h>\n";
  OS << "#endif\n\n";

  OS << "typedef __fp16 float16_t;\n";
  OS << "typedef float float32_t;\n";
  OS << "typedef double float64_t;\n";

  OS << "typedef __SVInt8_t svint8_t;\n";
  OS << "typedef __SVInt16_t svint16_t;\n";
  OS << "typedef __SVInt32_t svint32_t;\n";
  OS << "typedef __SVInt64_t svint64_t;\n";
  OS << "typedef __SVUint8_t svuint8_t;\n";
  OS << "typedef __SVUint16_t svuint16_t;\n";
  OS << "typedef __SVUint32_t svuint32_t;\n";
  OS << "typedef __SVUint64_t svuint64_t;\n";
  OS << "typedef __SVFloat16_t svfloat16_t;\n\n";

  OS << "typedef __SVBfloat16_t svbfloat16_t;\n";

  OS << "#include <arm_bf16.h>\n";
  OS << "#include <arm_vector_types.h>\n";

  OS << "typedef __SVFloat32_t svfloat32_t;\n";
  OS << "typedef __SVFloat64_t svfloat64_t;\n";
  OS << "typedef __clang_svint8x2_t svint8x2_t;\n";
  OS << "typedef __clang_svint16x2_t svint16x2_t;\n";
  OS << "typedef __clang_svint32x2_t svint32x2_t;\n";
  OS << "typedef __clang_svint64x2_t svint64x2_t;\n";
  OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n";
  OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n";
  OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n";
  OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n";
  OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n";
  OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n";
  OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n";
  OS << "typedef __clang_svint8x3_t svint8x3_t;\n";
  OS << "typedef __clang_svint16x3_t svint16x3_t;\n";
  OS << "typedef __clang_svint32x3_t svint32x3_t;\n";
  OS << "typedef __clang_svint64x3_t svint64x3_t;\n";
  OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n";
  OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n";
  OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n";
  OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n";
  OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n";
  OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n";
  OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n";
  OS << "typedef __clang_svint8x4_t svint8x4_t;\n";
  OS << "typedef __clang_svint16x4_t svint16x4_t;\n";
  OS << "typedef __clang_svint32x4_t svint32x4_t;\n";
  OS << "typedef __clang_svint64x4_t svint64x4_t;\n";
  OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n";
  OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n";
  OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n";
  OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n";
  OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n";
  OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n";
  OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n";
  OS << "typedef __SVBool_t  svbool_t;\n";
  OS << "typedef __clang_svboolx2_t  svboolx2_t;\n";
  OS << "typedef __clang_svboolx4_t  svboolx4_t;\n\n";

  OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n";
  OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n";
  OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n";

  OS << "typedef __SVCount_t svcount_t;\n\n";

  OS << "enum svpattern\n";
  OS << "{\n";
  OS << "  SV_POW2 = 0,\n";
  OS << "  SV_VL1 = 1,\n";
  OS << "  SV_VL2 = 2,\n";
  OS << "  SV_VL3 = 3,\n";
  OS << "  SV_VL4 = 4,\n";
  OS << "  SV_VL5 = 5,\n";
  OS << "  SV_VL6 = 6,\n";
  OS << "  SV_VL7 = 7,\n";
  OS << "  SV_VL8 = 8,\n";
  OS << "  SV_VL16 = 9,\n";
  OS << "  SV_VL32 = 10,\n";
  OS << "  SV_VL64 = 11,\n";
  OS << "  SV_VL128 = 12,\n";
  OS << "  SV_VL256 = 13,\n";
  OS << "  SV_MUL4 = 29,\n";
  OS << "  SV_MUL3 = 30,\n";
  OS << "  SV_ALL = 31\n";
  OS << "};\n\n";

  OS << "enum svprfop\n";
  OS << "{\n";
  OS << "  SV_PLDL1KEEP = 0,\n";
  OS << "  SV_PLDL1STRM = 1,\n";
  OS << "  SV_PLDL2KEEP = 2,\n";
  OS << "  SV_PLDL2STRM = 3,\n";
  OS << "  SV_PLDL3KEEP = 4,\n";
  OS << "  SV_PLDL3STRM = 5,\n";
  OS << "  SV_PSTL1KEEP = 8,\n";
  OS << "  SV_PSTL1STRM = 9,\n";
  OS << "  SV_PSTL2KEEP = 10,\n";
  OS << "  SV_PSTL2STRM = 11,\n";
  OS << "  SV_PSTL3KEEP = 12,\n";
  OS << "  SV_PSTL3STRM = 13\n";
  OS << "};\n\n";

  OS << "/* Function attributes */\n";
  OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
        "__nodebug__))\n\n";
  OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
        "__nodebug__, __overloadable__))\n\n";

  // Add reinterpret functions.
  for (auto [N, Suffix] :
       std::initializer_list<std::pair<unsigned, const char *>>{
           {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
    for (auto ShortForm : {false, true})
      for (const ReinterpretTypeInfo &To : Reinterprets) {
        SVEType ToV(To.BaseType, N);
        for (const ReinterpretTypeInfo &From : Reinterprets) {
          SVEType FromV(From.BaseType, N);
          if (ShortForm) {
            OS << "__aio __attribute__((target(\"sve\"))) " << ToV.str()
               << " svreinterpret_" << To.Suffix;
            OS << "(" << FromV.str() << " op) __arm_streaming_compatible {\n";
            OS << "  return __builtin_sve_reinterpret_" << To.Suffix << "_"
               << From.Suffix << Suffix << "(op);\n";
            OS << "}\n\n";
          } else
            OS << "#define svreinterpret_" << To.Suffix << "_" << From.Suffix
               << Suffix << "(...) __builtin_sve_reinterpret_" << To.Suffix
               << "_" << From.Suffix << Suffix << "(__VA_ARGS__)\n";
        }
      }
  }

  createCoreHeaderIntrinsics(OS, *this, ACLEKind::SVE);

  OS << "#define svcvtnt_bf16_x      svcvtnt_bf16_m\n";
  OS << "#define svcvtnt_bf16_f32_x  svcvtnt_bf16_f32_m\n";

  OS << "#define svcvtnt_f16_x      svcvtnt_f16_m\n";
  OS << "#define svcvtnt_f16_f32_x  svcvtnt_f16_f32_m\n";
  OS << "#define svcvtnt_f32_x      svcvtnt_f32_m\n";
  OS << "#define svcvtnt_f32_f64_x  svcvtnt_f32_f64_m\n\n";

  OS << "#define svcvtxnt_f32_x     svcvtxnt_f32_m\n";
  OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n";

  OS << "#ifdef __cplusplus\n";
  OS << "} // extern \"C\"\n";
  OS << "#endif\n\n";
  OS << "#undef __ai\n\n";
  OS << "#undef __aio\n\n";
  OS << "#endif /* __ARM_SVE_H */\n";
}

void SVEEmitter::createBuiltins(raw_ostream &OS) {
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
  SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
  for (auto *R : RV)
    createIntrinsic(R, Defs);

  // The mappings must be sorted based on BuiltinID.
  llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
                      const std::unique_ptr<Intrinsic> &B) {
    return A->getMangledName() < B->getMangledName();
  });

  OS << "#ifdef GET_SVE_BUILTINS\n";
  for (auto &Def : Defs) {
    // Only create BUILTINs for non-overloaded intrinsics, as overloaded
    // declarations only live in the header file.
    if (Def->getClassKind() != ClassG)
      OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \""
         << Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard()
         << "\")\n";
  }

  // Add reinterpret functions.
  for (auto [N, Suffix] :
       std::initializer_list<std::pair<unsigned, const char *>>{
           {1, ""}, {2, "_x2"}, {3, "_x3"}, {4, "_x4"}}) {
    for (const ReinterpretTypeInfo &To : Reinterprets) {
      SVEType ToV(To.BaseType, N);
      for (const ReinterpretTypeInfo &From : Reinterprets) {
        SVEType FromV(From.BaseType, N);
        OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << To.Suffix << "_"
           << From.Suffix << Suffix << +", \"" << ToV.builtin_str()
           << FromV.builtin_str() << "\", \"n\", \"sve\")\n";
      }
    }
  }

  OS << "#endif\n\n";
}

void SVEEmitter::createCodeGenMap(raw_ostream &OS) {
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
  SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
  for (auto *R : RV)
    createIntrinsic(R, Defs);

  // The mappings must be sorted based on BuiltinID.
  llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
                      const std::unique_ptr<Intrinsic> &B) {
    return A->getMangledName() < B->getMangledName();
  });

  OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n";
  for (auto &Def : Defs) {
    // Builtins only exist for non-overloaded intrinsics, overloaded
    // declarations only live in the header file.
    if (Def->getClassKind() == ClassG)
      continue;

    uint64_t Flags = Def->getFlags();
    auto FlagString = std::to_string(Flags);

    std::string LLVMName = Def->getMangledLLVMName();
    std::string Builtin = Def->getMangledName();
    if (!LLVMName.empty())
      OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
         << "),\n";
    else
      OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n";
  }
  OS << "#endif\n\n";
}

void SVEEmitter::createRangeChecks(raw_ostream &OS) {
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
  SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
  for (auto *R : RV)
    createIntrinsic(R, Defs);

  // The mappings must be sorted based on BuiltinID.
  llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
                      const std::unique_ptr<Intrinsic> &B) {
    return A->getMangledName() < B->getMangledName();
  });


  OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n";

  // Ensure these are only emitted once.
  std::set<std::string> Emitted;

  for (auto &Def : Defs) {
    if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
        Def->getImmChecks().empty())
      continue;

    OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n";
    for (auto &Check : Def->getImmChecks())
      OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
         << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
    OS << "  break;\n";

    Emitted.insert(Def->getMangledName());
  }

  OS << "#endif\n\n";
}

/// Create the SVETypeFlags used in CGBuiltins
void SVEEmitter::createTypeFlags(raw_ostream &OS) {
  OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n";
  for (auto &KV : FlagTypes)
    OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n";
  OS << "#endif\n\n";

  OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n";
  for (auto &KV : EltTypes)
    OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
  OS << "#endif\n\n";

  OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n";
  for (auto &KV : MemEltTypes)
    OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
  OS << "#endif\n\n";

  OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n";
  for (auto &KV : MergeTypes)
    OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
  OS << "#endif\n\n";

  OS << "#ifdef LLVM_GET_SVE_IMMCHECKTYPES\n";
  for (auto &KV : ImmCheckTypes)
    OS << "  " << KV.getKey() << " = " << KV.getValue() << ",\n";
  OS << "#endif\n\n";
}

void SVEEmitter::createSMEHeader(raw_ostream &OS) {
  OS << "/*===---- arm_sme.h - ARM SME intrinsics "
        "------===\n"
        " *\n"
        " *\n"
        " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
        "Exceptions.\n"
        " * See https://llvm.org/LICENSE.txt for license information.\n"
        " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
        " *\n"
        " *===-----------------------------------------------------------------"
        "------===\n"
        " */\n\n";

  OS << "#ifndef __ARM_SME_H\n";
  OS << "#define __ARM_SME_H\n\n";

  OS << "#if !defined(__LITTLE_ENDIAN__)\n";
  OS << "#error \"Big endian is currently not supported for arm_sme.h\"\n";
  OS << "#endif\n";

  OS << "#include <arm_sve.h>\n\n";

  OS << "/* Function attributes */\n";
  OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
        "__nodebug__))\n\n";
  OS << "#define __aio static __inline__ __attribute__((__always_inline__, "
        "__nodebug__, __overloadable__))\n\n";

  OS << "#ifdef  __cplusplus\n";
  OS << "extern \"C\" {\n";
  OS << "#endif\n\n";

  OS << "void __arm_za_disable(void) __arm_streaming_compatible;\n\n";

  OS << "__ai bool __arm_has_sme(void) __arm_streaming_compatible {\n";
  OS << "  uint64_t x0, x1;\n";
  OS << "  __builtin_arm_get_sme_state(&x0, &x1);\n";
  OS << "  return x0 & (1ULL << 63);\n";
  OS << "}\n\n";

  OS << "__ai bool __arm_in_streaming_mode(void) __arm_streaming_compatible "
        "{\n";
  OS << "  uint64_t x0, x1;\n";
  OS << "  __builtin_arm_get_sme_state(&x0, &x1);\n";
  OS << "  return x0 & 1;\n";
  OS << "}\n\n";

  OS << "__ai __attribute__((target(\"sme\"))) void svundef_za(void) "
        "__arm_streaming_compatible __arm_out(\"za\") "
        "{ }\n\n";

  createCoreHeaderIntrinsics(OS, *this, ACLEKind::SME);

  OS << "#ifdef __cplusplus\n";
  OS << "} // extern \"C\"\n";
  OS << "#endif\n\n";
  OS << "#undef __ai\n\n";
  OS << "#endif /* __ARM_SME_H */\n";
}

void SVEEmitter::createSMEBuiltins(raw_ostream &OS) {
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
  SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
  for (auto *R : RV) {
    createIntrinsic(R, Defs);
  }

  // The mappings must be sorted based on BuiltinID.
  llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
                      const std::unique_ptr<Intrinsic> &B) {
    return A->getMangledName() < B->getMangledName();
  });

  OS << "#ifdef GET_SME_BUILTINS\n";
  for (auto &Def : Defs) {
    // Only create BUILTINs for non-overloaded intrinsics, as overloaded
    // declarations only live in the header file.
    if (Def->getClassKind() != ClassG)
      OS << "TARGET_BUILTIN(__builtin_sme_" << Def->getMangledName() << ", \""
         << Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard()
         << "\")\n";
  }

  OS << "#endif\n\n";
}

void SVEEmitter::createSMECodeGenMap(raw_ostream &OS) {
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
  SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
  for (auto *R : RV) {
    createIntrinsic(R, Defs);
  }

  // The mappings must be sorted based on BuiltinID.
  llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
                      const std::unique_ptr<Intrinsic> &B) {
    return A->getMangledName() < B->getMangledName();
  });

  OS << "#ifdef GET_SME_LLVM_INTRINSIC_MAP\n";
  for (auto &Def : Defs) {
    // Builtins only exist for non-overloaded intrinsics, overloaded
    // declarations only live in the header file.
    if (Def->getClassKind() == ClassG)
      continue;

    uint64_t Flags = Def->getFlags();
    auto FlagString = std::to_string(Flags);

    std::string LLVMName = Def->getLLVMName();
    std::string Builtin = Def->getMangledName();
    if (!LLVMName.empty())
      OS << "SMEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString
         << "),\n";
    else
      OS << "SMEMAP2(" << Builtin << ", " << FlagString << "),\n";
  }
  OS << "#endif\n\n";
}

void SVEEmitter::createSMERangeChecks(raw_ostream &OS) {
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
  SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
  for (auto *R : RV) {
    createIntrinsic(R, Defs);
  }

  // The mappings must be sorted based on BuiltinID.
  llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A,
                      const std::unique_ptr<Intrinsic> &B) {
    return A->getMangledName() < B->getMangledName();
  });


  OS << "#ifdef GET_SME_IMMEDIATE_CHECK\n";

  // Ensure these are only emitted once.
  std::set<std::string> Emitted;

  for (auto &Def : Defs) {
    if (Emitted.find(Def->getMangledName()) != Emitted.end() ||
        Def->getImmChecks().empty())
      continue;

    OS << "case SME::BI__builtin_sme_" << Def->getMangledName() << ":\n";
    for (auto &Check : Def->getImmChecks())
      OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", "
         << Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n";
    OS << "  break;\n";

    Emitted.insert(Def->getMangledName());
  }

  OS << "#endif\n\n";
}

void SVEEmitter::createBuiltinZAState(raw_ostream &OS) {
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
  SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
  for (auto *R : RV)
    createIntrinsic(R, Defs);

  std::map<std::string, std::set<std::string>> IntrinsicsPerState;
  for (auto &Def : Defs) {
    std::string Key;
    auto AddToKey = [&Key](const std::string &S) -> void {
      Key = Key.empty() ? S : (Key + " | " + S);
    };

    if (Def->isFlagSet(getEnumValueForFlag("IsInZA")))
      AddToKey("ArmInZA");
    else if (Def->isFlagSet(getEnumValueForFlag("IsOutZA")))
      AddToKey("ArmOutZA");
    else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZA")))
      AddToKey("ArmInOutZA");

    if (Def->isFlagSet(getEnumValueForFlag("IsInZT0")))
      AddToKey("ArmInZT0");
    else if (Def->isFlagSet(getEnumValueForFlag("IsOutZT0")))
      AddToKey("ArmOutZT0");
    else if (Def->isFlagSet(getEnumValueForFlag("IsInOutZT0")))
      AddToKey("ArmInOutZT0");

    if (!Key.empty())
      IntrinsicsPerState[Key].insert(Def->getMangledName());
  }

  OS << "#ifdef GET_SME_BUILTIN_GET_STATE\n";
  for (auto &KV : IntrinsicsPerState) {
    for (StringRef Name : KV.second)
      OS << "case SME::BI__builtin_sme_" << Name << ":\n";
    OS << "  return " << KV.first << ";\n";
  }
  OS << "#endif\n\n";
}

void SVEEmitter::createStreamingAttrs(raw_ostream &OS, ACLEKind Kind) {
  std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst");
  SmallVector<std::unique_ptr<Intrinsic>, 128> Defs;
  for (auto *R : RV)
    createIntrinsic(R, Defs);

  StringRef ExtensionKind;
  switch (Kind) {
  case ACLEKind::SME:
    ExtensionKind = "SME";
    break;
  case ACLEKind::SVE:
    ExtensionKind = "SVE";
    break;
  }

  OS << "#ifdef GET_" << ExtensionKind << "_STREAMING_ATTRS\n";

  llvm::StringMap<std::set<std::string>> StreamingMap;

  uint64_t IsStreamingFlag = getEnumValueForFlag("IsStreaming");
  uint64_t IsStreamingOrSVE2p1Flag = getEnumValueForFlag("IsStreamingOrSVE2p1");
  uint64_t IsStreamingCompatibleFlag =
      getEnumValueForFlag("IsStreamingCompatible");
  for (auto &Def : Defs) {
    if (Def->isFlagSet(IsStreamingFlag))
      StreamingMap["ArmStreaming"].insert(Def->getMangledName());
    else if (Def->isFlagSet(IsStreamingOrSVE2p1Flag))
      StreamingMap["ArmStreamingOrSVE2p1"].insert(Def->getMangledName());
    else if (Def->isFlagSet(IsStreamingCompatibleFlag))
      StreamingMap["ArmStreamingCompatible"].insert(Def->getMangledName());
    else
      StreamingMap["ArmNonStreaming"].insert(Def->getMangledName());
  }

  for (auto BuiltinType : StreamingMap.keys()) {
    for (auto Name : StreamingMap[BuiltinType]) {
      OS << "case " << ExtensionKind << "::BI__builtin_"
         << ExtensionKind.lower() << "_";
      OS << Name << ":\n";
    }
    OS << "  BuiltinType = " << BuiltinType << ";\n";
    OS << "  break;\n";
  }

  OS << "#endif\n\n";
}

namespace clang {
void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createHeader(OS);
}

void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createBuiltins(OS);
}

void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createCodeGenMap(OS);
}

void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createRangeChecks(OS);
}

void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createTypeFlags(OS);
}

void EmitSveStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SVE);
}

void EmitSmeHeader(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createSMEHeader(OS);
}

void EmitSmeBuiltins(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createSMEBuiltins(OS);
}

void EmitSmeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createSMECodeGenMap(OS);
}

void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createSMERangeChecks(OS);
}

void EmitSmeStreamingAttrs(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createStreamingAttrs(OS, ACLEKind::SME);
}

void EmitSmeBuiltinZAState(RecordKeeper &Records, raw_ostream &OS) {
  SVEEmitter(Records).createBuiltinZAState(OS);
}
} // End namespace clang