1 //===- Token.h - MLIR Token Interface ---------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef MLIR_LIB_PARSER_TOKEN_H 10 #define MLIR_LIB_PARSER_TOKEN_H 11 12 #include "mlir/Support/LLVM.h" 13 #include "llvm/ADT/StringRef.h" 14 #include "llvm/Support/SMLoc.h" 15 16 namespace mlir { 17 18 /// This represents a token in the MLIR syntax. 19 class Token { 20 public: 21 enum Kind { 22 #define TOK_MARKER(NAME) NAME, 23 #define TOK_IDENTIFIER(NAME) NAME, 24 #define TOK_LITERAL(NAME) NAME, 25 #define TOK_PUNCTUATION(NAME, SPELLING) NAME, 26 #define TOK_KEYWORD(SPELLING) kw_##SPELLING, 27 #include "TokenKinds.def" 28 }; 29 Token(Kind kind,StringRef spelling)30 Token(Kind kind, StringRef spelling) : kind(kind), spelling(spelling) {} 31 32 // Return the bytes that make up this token. getSpelling()33 StringRef getSpelling() const { return spelling; } 34 35 // Token classification. getKind()36 Kind getKind() const { return kind; } is(Kind K)37 bool is(Kind K) const { return kind == K; } 38 isAny(Kind k1,Kind k2)39 bool isAny(Kind k1, Kind k2) const { return is(k1) || is(k2); } 40 41 /// Return true if this token is one of the specified kinds. 42 template <typename... T> isAny(Kind k1,Kind k2,Kind k3,T...others)43 bool isAny(Kind k1, Kind k2, Kind k3, T... others) const { 44 if (is(k1)) 45 return true; 46 return isAny(k2, k3, others...); 47 } 48 isNot(Kind k)49 bool isNot(Kind k) const { return kind != k; } 50 51 /// Return true if this token isn't one of the specified kinds. 52 template <typename... T> isNot(Kind k1,Kind k2,T...others)53 bool isNot(Kind k1, Kind k2, T... others) const { 54 return !isAny(k1, k2, others...); 55 } 56 57 /// Return true if this is one of the keyword token kinds (e.g. kw_if). 58 bool isKeyword() const; 59 60 // Helpers to decode specific sorts of tokens. 61 62 /// For an integer token, return its value as an unsigned. If it doesn't fit, 63 /// return None. 64 Optional<unsigned> getUnsignedIntegerValue() const; 65 66 /// For an integer token, return its value as an uint64_t. If it doesn't fit, 67 /// return None. 68 static Optional<uint64_t> getUInt64IntegerValue(StringRef spelling); getUInt64IntegerValue()69 Optional<uint64_t> getUInt64IntegerValue() const { 70 return getUInt64IntegerValue(getSpelling()); 71 } 72 73 /// For a floatliteral token, return its value as a double. Returns None in 74 /// the case of underflow or overflow. 75 Optional<double> getFloatingPointValue() const; 76 77 /// For an inttype token, return its bitwidth. 78 Optional<unsigned> getIntTypeBitwidth() const; 79 80 /// For an inttype token, return its signedness semantics: llvm::None means no 81 /// signedness semantics; true means signed integer type; false means unsigned 82 /// integer type. 83 Optional<bool> getIntTypeSignedness() const; 84 85 /// Given a hash_identifier token like #123, try to parse the number out of 86 /// the identifier, returning None if it is a named identifier like #x or 87 /// if the integer doesn't fit. 88 Optional<unsigned> getHashIdentifierNumber() const; 89 90 /// Given a token containing a string literal, return its value, including 91 /// removing the quote characters and unescaping the contents of the string. 92 std::string getStringValue() const; 93 94 /// Given a token containing a hex string literal, return its value or None if 95 /// the token does not contain a valid hex string. A hex string literal is a 96 /// string starting with `0x` and only containing hex digits. 97 Optional<std::string> getHexStringValue() const; 98 99 /// Given a token containing a symbol reference, return the unescaped string 100 /// value. 101 std::string getSymbolReference() const; 102 103 // Location processing. 104 llvm::SMLoc getLoc() const; 105 llvm::SMLoc getEndLoc() const; 106 llvm::SMRange getLocRange() const; 107 108 /// Given a punctuation or keyword token kind, return the spelling of the 109 /// token as a string. Warning: This will abort on markers, identifiers and 110 /// literal tokens since they have no fixed spelling. 111 static StringRef getTokenSpelling(Kind kind); 112 113 private: 114 /// Discriminator that indicates the sort of token this is. 115 Kind kind; 116 117 /// A reference to the entire token contents; this is always a pointer into 118 /// a memory buffer owned by the source manager. 119 StringRef spelling; 120 }; 121 122 } // end namespace mlir 123 124 #endif // MLIR_LIB_PARSER_TOKEN_H 125