14 #include "llvm/ADT/StringExtras.h"
19 SMLoc
Token::getLoc()
const {
return SMLoc::getFromPointer(spelling.data()); }
22 return SMLoc::getFromPointer(spelling.data() + spelling.size());
30 bool isHex = spelling.size() > 1 && spelling[1] ==
'x';
33 if (spelling.getAsInteger(isHex ? 0 : 10, result))
41 bool isHex = spelling.size() > 1 && spelling[1] ==
'x';
44 if (spelling.getAsInteger(isHex ? 0 : 10, result))
53 if (spelling.getAsDouble(result))
61 unsigned bitwidthStart = (spelling[0] ==
'i' ? 1 : 2);
63 if (spelling.drop_front(bitwidthStart).getAsInteger(10, result))
70 if (spelling[0] ==
'i')
72 if (spelling[0] ==
's')
74 assert(spelling[0] ==
'u');
86 if (
getKind() != Token::code_complete) {
87 bytes = bytes.drop_back();
89 bytes = bytes.drop_front();
93 result.reserve(bytes.size());
94 for (
unsigned i = 0, e = bytes.size(); i != e;) {
101 assert(i + 1 <= e &&
"invalid string should be caught by lexer");
102 auto c1 = bytes[i++];
106 result.push_back(c1);
109 result.push_back(
'\n');
112 result.push_back(
'\t');
118 assert(i + 1 <= e &&
"invalid string should be caught by lexer");
119 auto c2 = bytes[i++];
121 assert(llvm::isHexDigit(c1) && llvm::isHexDigit(c2) &&
"invalid escape");
122 result.push_back((llvm::hexDigitValue(c1) << 4) | llvm::hexDigitValue(c2));
134 StringRef bytes =
getSpelling().drop_front().drop_back();
140 if (!bytes.consume_front(
"0x") || (bytes.size() & 1) ||
141 !llvm::tryGetFromHex(bytes, hex))
149 assert(
is(Token::at_identifier) &&
"expected valid @-identifier");
153 if (nameStr.front() ==
'"')
155 return std::string(nameStr);
162 assert(
getKind() == hash_identifier);
164 if (spelling.drop_front().getAsInteger(10, result))
175 llvm_unreachable(
"This token kind has no fixed spelling");
176 #define TOK_PUNCTUATION(NAME, SPELLING) \
179 #define TOK_KEYWORD(SPELLING) \
180 case kw_##SPELLING: \
182 #include "TokenKinds.def"
191 #define TOK_KEYWORD(SPELLING) \
192 case kw_##SPELLING: \
194 #include "TokenKinds.def"
203 return spelling[0] ==
'"';
204 case Kind::hash_identifier:
205 return spelling[0] ==
'#';
206 case Kind::percent_identifier:
207 return spelling[0] ==
'%';
208 case Kind::caret_identifier:
209 return spelling[0] ==
'^';
210 case Kind::exclamation_identifier:
211 return spelling[0] ==
'!';
bool isCodeCompletionFor(Kind kind) const
Returns true if the current token represents a code completion for the "normal" token type.
SMRange getLocRange() const
bool isKeyword() const
Return true if this is one of the keyword token kinds (e.g. kw_if).
static StringRef getTokenSpelling(Kind kind)
Given a punctuation or keyword token kind, return the spelling of the token as a string.
std::string getStringValue() const
Given a token containing a string literal, return its value, including removing the quote characters ...
std::string getSymbolReference() const
Given a token containing a symbol reference, return the unescaped string value.
std::optional< unsigned > getUnsignedIntegerValue() const
For an integer token, return its value as an unsigned.
std::optional< uint64_t > getUInt64IntegerValue() const
std::optional< double > getFloatingPointValue() const
For a floatliteral token, return its value as a double.
std::optional< unsigned > getHashIdentifierNumber() const
Given a hash_identifier token like #123, try to parse the number out of the identifier,...
bool isCodeCompletion() const
Returns true if the current token represents a code completion.
StringRef getSpelling() const
std::optional< bool > getIntTypeSignedness() const
For an inttype token, return its signedness semantics: std::nullopt means no signedness semantics; tr...
std::optional< unsigned > getIntTypeBitwidth() const
For an inttype token, return its bitwidth.
std::optional< std::string > getHexStringValue() const
Given a token containing a hex string literal, return its value or std::nullopt if the token does not...
This header declares functions that assist transformations in the MemRef dialect.