7
enum TokenType {TOKEN_EOF, TOKEN_ERROR, TOKEN_IDENTIFIER, TOKEN_NUMBER, TOKEN_STRING,
8
TOKEN_LEFT_PAREN, TOKEN_RIGHT_PAREN,
9
TOKEN_LEFT_BRACKET, TOKEN_RIGHT_BRACKET};
14
// typically only one of the following has a meaningful value
16
std::string string_value;
18
Token() : type(TOKEN_ERROR), number_value(1e+30), string_value("ERROR") {}
21
Token(const Token &source);
22
Token &operator=(const Token &source);
24
void set(TokenType type_);
25
void set(TokenType type_, const std::string &value);
26
void set(TokenType type_, double value);
27
void clear(); // free up string storage after finished with a TOKEN_ERROR or TOKEN_IDENTIFIER
30
std::ostream& operator<<(std::ostream& out, const Token& t);
36
Lexer(std::istream& input_) : input(input_) {}
38
void read(Token &tok);