diff options
Diffstat (limited to 'tokenizer.c')
-rw-r--r-- | tokenizer.c | 110 |
1 files changed, 0 insertions, 110 deletions
diff --git a/tokenizer.c b/tokenizer.c index 520edd0..3393e0d 100644 --- a/tokenizer.c +++ b/tokenizer.c @@ -1,71 +1,3 @@ -typedef enum { - TOKEN_KW, - TOKEN_IDENT, - TOKEN_DIRECT, - TOKEN_NUM_LITERAL, - TOKEN_CHAR_LITERAL, - TOKEN_STR_LITERAL, - TOKEN_EOF -} TokenKind; - -typedef enum { - DIRECT_C, - DIRECT_COUNT -} Directive; - -typedef enum { - KW_SEMICOLON, - KW_COLON, - KW_AT, - KW_COMMA, - KW_LPAREN, - KW_RPAREN, - KW_LBRACE, - KW_RBRACE, - KW_LSQUARE, - KW_RSQUARE, - KW_EQEQ, - KW_NE, - KW_LT, - KW_LE, - KW_GT, - KW_GE, - KW_PLUS, - KW_MINUS, - KW_ASTERISK, - KW_EXCLAMATION, - KW_AMPERSAND, - KW_SLASH, - KW_EQ, - KW_LAST_SYMBOL = KW_EQ, /* last one entirely consisting of symbols */ - KW_IF, - KW_ELIF, - KW_ELSE, - KW_WHILE, - KW_RETURN, - KW_FN, - KW_AS, - KW_NEW, - KW_DEL, - KW_INT, - KW_I8, - KW_I16, - KW_I32, - KW_I64, - KW_U8, - KW_U16, - KW_U32, - KW_U64, - KW_FLOAT, - KW_F32, - KW_F64, - KW_CHAR, - KW_BOOL, - KW_TRUE, - KW_FALSE, - KW_COUNT -} Keyword; - static const char *keywords[KW_COUNT] = {";", ":", "@", ",", "(", ")", "{", "}", "[", "]", "==", "!=", "<", "<=", ">", ">=", "+", "-", "*", "!", "&", "/", @@ -120,48 +52,6 @@ static Directive tokenize_direct(char **s) { return DIRECT_COUNT; } -typedef enum { - NUM_LITERAL_INT, - NUM_LITERAL_FLOAT -} NumLiteralKind; - -typedef struct { - NumLiteralKind kind; - union { - UInteger intval; - Floating floatval; - }; -} NumLiteral; - -typedef struct { - char *str; - size_t len; -} StrLiteral; - -/* NOTE: Location is typedef'd in util/err.c */ -typedef struct { - TokenKind kind; - Location where; - union { - Keyword kw; - Directive direct; - Identifier ident; - NumLiteral num; - char chr; - StrLiteral str; - }; -} Token; - -typedef struct { - Allocator allocr; - Array tokens; - char *s; /* string being parsed */ - const char *filename; - LineNo line; - Token *token; /* token currently being processed */ - Identifiers *idents; -} Tokenizer; - static inline bool token_is_kw(Token *t, Keyword kw) { return t->kind == TOKEN_KW && t->kw == kw; } |