Token.h

Go to the documentation of this file.

//===- Token.h - MLIR Token Interface ---------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#ifndef MLIR_LIB_PARSER_TOKEN_H
#define MLIR_LIB_PARSER_TOKEN_H
 
#include "mlir/Support/LLVM.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/SMLoc.h"
#include <optional>
 
namespace mlir {
 
/// This represents a token in the MLIR syntax.
class Token {
public:
  enum Kind {
#define TOK_MARKER(NAME) NAME,
#define TOK_IDENTIFIER(NAME) NAME,
#define TOK_LITERAL(NAME) NAME,
#define TOK_PUNCTUATION(NAME, SPELLING) NAME,
#define TOK_KEYWORD(SPELLING) kw_##SPELLING,
#include "TokenKinds.def"
  };
 
  Token(Kind kind, StringRef spelling) : kind(kind), spelling(spelling) {}
 
  // Return the bytes that make up this token.
  StringRef getSpelling() const { return spelling; }
 
  // Token classification.
  Kind getKind() const { return kind; }
  bool is(Kind k) const { return kind == k; }
 
  bool isAny(Kind k1, Kind k2) const { return is(k1) || is(k2); }
 
  /// Return true if this token is one of the specified kinds.
  template <typename... T>
  bool isAny(Kind k1, Kind k2, Kind k3, T... others) const {
    if (is(k1))
      return true;
    return isAny(k2, k3, others...);
  }
 
  bool isNot(Kind k) const { return kind != k; }
 
  /// Return true if this token isn't one of the specified kinds.
  template <typename... T>
  bool isNot(Kind k1, Kind k2, T... others) const {
    return !isAny(k1, k2, others...);
  }
 
  /// Return true if this is one of the keyword token kinds (e.g. kw_if).
  bool isKeyword() const;
 
  /// Returns true if the current token represents a code completion.
  bool isCodeCompletion() const { return is(code_complete); }
 
  /// Returns true if the current token represents a code completion for the
  /// "normal" token type.
  bool isCodeCompletionFor(Kind kind) const;
 
  /// Returns true if the current token is the given type, or represents a code
  /// completion for that type.
  bool isOrIsCodeCompletionFor(Kind kind) const {
    return is(kind) || isCodeCompletionFor(kind);
  }
 
  // Helpers to decode specific sorts of tokens.
 
  /// For an integer token, return its value as an unsigned.  If it doesn't fit,
  /// return std::nullopt.
  std::optional<unsigned> getUnsignedIntegerValue() const;
 
  /// For an integer token, return its value as an uint64_t.  If it doesn't fit,
  /// return std::nullopt.
  static std::optional<uint64_t> getUInt64IntegerValue(StringRef spelling);
  std::optional<uint64_t> getUInt64IntegerValue() const {
    return getUInt64IntegerValue(getSpelling());
  }
 
  /// For a floatliteral token, return its value as a double. Returns
  /// std::nullopt in the case of underflow or overflow.
  std::optional<double> getFloatingPointValue() const;
 
  /// For an inttype token, return its bitwidth.
  std::optional<unsigned> getIntTypeBitwidth() const;
 
  /// For an inttype token, return its signedness semantics: std::nullopt means
  /// no signedness semantics; true means signed integer type; false means
  /// unsigned integer type.
  std::optional<bool> getIntTypeSignedness() const;
 
  /// Given a hash_identifier token like #123, try to parse the number out of
  /// the identifier, returning std::nullopt if it is a named identifier like #x
  /// or if the integer doesn't fit.
  std::optional<unsigned> getHashIdentifierNumber() const;
 
  /// Given a token containing a string literal, return its value, including
  /// removing the quote characters and unescaping the contents of the string.
  std::string getStringValue() const;
 
  /// Given a token containing a hex string literal, return its value or
  /// std::nullopt if the token does not contain a valid hex string. A hex
  /// string literal is a string starting with `0x` and only containing hex
  /// digits.
  std::optional<std::string> getHexStringValue() const;
 
  /// Given a token containing a symbol reference, return the unescaped string
  /// value.
  std::string getSymbolReference() const;
 
  // Location processing.
  SMLoc getLoc() const;
  SMLoc getEndLoc() const;
  SMRange getLocRange() const;
 
  /// Given a punctuation or keyword token kind, return the spelling of the
  /// token as a string.  Warning: This will abort on markers, identifiers and
  /// literal tokens since they have no fixed spelling.
  static StringRef getTokenSpelling(Kind kind);
 
private:
  /// Discriminator that indicates the sort of token this is.
  Kind kind;
 
  /// A reference to the entire token contents; this is always a pointer into
  /// a memory buffer owned by the source manager.
  StringRef spelling;
};
 
} // namespace mlir
 
#endif // MLIR_LIB_PARSER_TOKEN_H