13 #include "llvm/ADT/StringExtras.h"
14 #include "llvm/ADT/StringSwitch.h"
15 #include "llvm/Support/SourceMgr.h"
31 bytes = bytes.drop_front().drop_back();
33 bytes = bytes.drop_front(2).drop_back(2);
36 result.reserve(bytes.size());
37 for (
unsigned i = 0, e = bytes.size(); i != e;) {
44 assert(i + 1 <= e &&
"invalid string should be caught by lexer");
52 result.push_back(
'\n');
55 result.push_back(
'\t');
61 assert(i + 1 <= e &&
"invalid string should be caught by lexer");
64 assert(llvm::isHexDigit(c1) && llvm::isHexDigit(c2) &&
"invalid escape");
65 result.push_back((llvm::hexDigitValue(c1) << 4) | llvm::hexDigitValue(c2));
77 : srcMgr(mgr), diagEngine(diagEngine), addedHandlerToDiagEngine(false),
78 codeCompletionLocation(nullptr) {
79 curBufferID = mgr.getMainFileID();
80 curBuffer = srcMgr.getMemoryBuffer(curBufferID)->getBuffer();
81 curPtr = curBuffer.begin();
84 if (codeCompleteContext) {
85 codeCompletionLocation =
93 srcMgr.PrintMessage(
diag.getLocation().Start,
diag.getSeverity(),
96 srcMgr.PrintMessage(note.getLocation().Start, note.getSeverity(),
99 addedHandlerToDiagEngine =
true;
104 if (addedHandlerToDiagEngine)
109 std::string includedFile;
111 srcMgr.AddIncludeFile(filename.str(), includeLoc.End, includedFile);
115 curBufferID = bufferID;
116 curBuffer = srcMgr.getMemoryBuffer(curBufferID)->getBuffer();
117 curPtr = curBuffer.begin();
132 SMRange(SMLoc::getFromPointer(loc), SMLoc::getFromPointer(loc + 1)), msg);
135 int Lexer::getNextChar() {
136 char curChar = *curPtr++;
139 return static_cast<unsigned char>(curChar);
143 if (curPtr - 1 != curBuffer.end())
155 if ((*curPtr ==
'\n' || (*curPtr ==
'\r')) && *curPtr != curChar)
163 const char *tokStart = curPtr;
166 if (tokStart == codeCompletionLocation)
170 int curChar = getNextChar();
174 if (isalpha(curChar) || curChar ==
'_')
175 return lexIdentifier(tokStart);
178 return emitError(tokStart,
"unexpected character");
184 SMLoc parentIncludeLoc = srcMgr.getParentIncludeLoc(curBufferID);
185 if (parentIncludeLoc.isValid()) {
186 curBufferID = srcMgr.FindBufferContainingLoc(parentIncludeLoc);
187 curBuffer = srcMgr.getMemoryBuffer(curBufferID)->getBuffer();
188 curPtr = parentIncludeLoc.getPointer();
196 if (*curPtr ==
'>') {
200 return emitError(tokStart,
"unexpected character");
208 if (*curPtr ==
'>') {
216 if (*curPtr ==
'{') {
218 return lexString(tokStart,
true);
237 if (*curPtr ==
'/') {
241 return emitError(tokStart,
"unexpected character");
251 return lexDirective(tokStart);
253 return lexString(tokStart,
false);
265 return lexNumber(tokStart);
271 void Lexer::lexComment() {
273 assert(*curPtr ==
'/');
284 if (curPtr - 1 == curBuffer.end()) {
296 Token Lexer::lexDirective(
const char *tokStart) {
298 while (isalnum(*curPtr) || *curPtr ==
'_')
301 StringRef str(tokStart, curPtr - tokStart);
305 Token Lexer::lexIdentifier(
const char *tokStart) {
307 while (isalnum(*curPtr) || *curPtr ==
'_')
311 StringRef str(tokStart, curPtr - tokStart);
335 return Token(kind, str);
338 Token Lexer::lexNumber(
const char *tokStart) {
339 assert(isdigit(curPtr[-1]));
342 while (isdigit(*curPtr))
348 Token Lexer::lexString(
const char *tokStart,
bool isStringBlock) {
355 if (curPtr == codeCompletionLocation) {
357 tokStart + (isStringBlock ? 2 : 1));
370 if (!isStringBlock || *curPtr !=
']')
377 if (curPtr - 1 != curBuffer.end())
381 StringRef expectedEndStr = isStringBlock ?
"}]" :
"\"";
383 "expected '" + expectedEndStr +
"' in string literal");
391 return emitError(curPtr - 1,
"expected '\"' in string literal");
396 if (*curPtr ==
'"' || *curPtr ==
'\\' || *curPtr ==
'n' ||
399 }
else if (llvm::isHexDigit(*curPtr) && llvm::isHexDigit(curPtr[1])) {
403 return emitError(curPtr - 1,
"unknown escape in string literal");
static std::string diag(const llvm::Value &value)
This represents a token in the MLIR syntax.
This class provides an abstract interface into the parser for hooking in code completion events.
SMLoc getCodeCompleteLoc() const
Return the location used to provide code completion.
Token emitError(SMRange loc, const Twine &msg)
Emit an error to the lexer with the given location and message.
Token emitErrorAndNote(SMRange loc, const Twine &msg, SMRange noteLoc, const Twine ¬e)
Token lexToken()
Lex the next token and return it.
LogicalResult pushInclude(StringRef filename, SMRange includeLoc)
Push an include of the given file.
Lexer(llvm::SourceMgr &mgr, ast::DiagnosticEngine &diagEngine, CodeCompleteContext *codeCompleteContext)
bool is(Kind k) const
Return if the token has the given kind.
StringRef getSpelling() const
Return the bytes that make up this token.
@ code_complete_string
Token signifying a code completion location within a string.
@ code_complete
Token signifying a code completion location.
@ less
Paired punctuation.
@ kw_Attr
General keywords.
Kind getKind() const
Return the kind of this token.
std::string getStringValue() const
Given a token containing a string literal, return its value, including removing the quote characters ...
This class manages the construction and emission of PDLL diagnostics.
void setHandlerFn(HandlerFn &&newHandler)
Set the handler function for this diagnostic engine.
InFlightDiagnostic emitError(SMRange loc, const Twine &msg)
Emit an error to the diagnostic engine.
const HandlerFn & getHandlerFn() const
Get the current handler function of this diagnostic engine.
This class provides a simple implementation of a PDLL diagnostic.
Diagnostic & attachNote(const Twine &msg, std::optional< SMRange > noteLoc=std::nullopt)
Attach a note to this diagnostic.
Include the generated interface declarations.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
This class represents an efficient way to signal success or failure.