summaryrefslogtreecommitdiff
path: root/tokenizer.c
diff options
context:
space:
mode:
Diffstat (limited to 'tokenizer.c')
-rw-r--r--tokenizer.c110
1 files changed, 0 insertions, 110 deletions
diff --git a/tokenizer.c b/tokenizer.c
index 520edd0..3393e0d 100644
--- a/tokenizer.c
+++ b/tokenizer.c
@@ -1,71 +1,3 @@
-typedef enum {
- TOKEN_KW,
- TOKEN_IDENT,
- TOKEN_DIRECT,
- TOKEN_NUM_LITERAL,
- TOKEN_CHAR_LITERAL,
- TOKEN_STR_LITERAL,
- TOKEN_EOF
-} TokenKind;
-
-typedef enum {
- DIRECT_C,
- DIRECT_COUNT
-} Directive;
-
-typedef enum {
- KW_SEMICOLON,
- KW_COLON,
- KW_AT,
- KW_COMMA,
- KW_LPAREN,
- KW_RPAREN,
- KW_LBRACE,
- KW_RBRACE,
- KW_LSQUARE,
- KW_RSQUARE,
- KW_EQEQ,
- KW_NE,
- KW_LT,
- KW_LE,
- KW_GT,
- KW_GE,
- KW_PLUS,
- KW_MINUS,
- KW_ASTERISK,
- KW_EXCLAMATION,
- KW_AMPERSAND,
- KW_SLASH,
- KW_EQ,
- KW_LAST_SYMBOL = KW_EQ, /* last one entirely consisting of symbols */
- KW_IF,
- KW_ELIF,
- KW_ELSE,
- KW_WHILE,
- KW_RETURN,
- KW_FN,
- KW_AS,
- KW_NEW,
- KW_DEL,
- KW_INT,
- KW_I8,
- KW_I16,
- KW_I32,
- KW_I64,
- KW_U8,
- KW_U16,
- KW_U32,
- KW_U64,
- KW_FLOAT,
- KW_F32,
- KW_F64,
- KW_CHAR,
- KW_BOOL,
- KW_TRUE,
- KW_FALSE,
- KW_COUNT
-} Keyword;
-
static const char *keywords[KW_COUNT] =
{";", ":", "@", ",", "(", ")", "{", "}", "[", "]", "==", "!=", "<", "<=", ">", ">=",
"+", "-", "*", "!", "&", "/",
@@ -120,48 +52,6 @@ static Directive tokenize_direct(char **s) {
return DIRECT_COUNT;
}
-typedef enum {
- NUM_LITERAL_INT,
- NUM_LITERAL_FLOAT
-} NumLiteralKind;
-
-typedef struct {
- NumLiteralKind kind;
- union {
- UInteger intval;
- Floating floatval;
- };
-} NumLiteral;
-
-typedef struct {
- char *str;
- size_t len;
-} StrLiteral;
-
-/* NOTE: Location is typedef'd in util/err.c */
-typedef struct {
- TokenKind kind;
- Location where;
- union {
- Keyword kw;
- Directive direct;
- Identifier ident;
- NumLiteral num;
- char chr;
- StrLiteral str;
- };
-} Token;
-
-typedef struct {
- Allocator allocr;
- Array tokens;
- char *s; /* string being parsed */
- const char *filename;
- LineNo line;
- Token *token; /* token currently being processed */
- Identifiers *idents;
-} Tokenizer;
-
static inline bool token_is_kw(Token *t, Keyword kw) {
return t->kind == TOKEN_KW && t->kw == kw;
}