Added string constants and improved tokenization

9 files changed, 430 insertions, 173 deletions

diff --git a/build.sh b/build.sh
index e8bd52e..abdd474 100755
--- a/build.sh
+++ b/build.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-gcc -o toc main.c -g -o toc -Wall -Wextra -Wpedantic -Wconversion -std=c11 || exit 1
+gcc -o toc main.c -g -o toc -Wall -Wextra -Wpedantic -Wconversion -Wno-unused-function -std=c11 || exit 1
diff --git a/identifiers.c b/identifiers.c
index 5615452..572aa4e 100644
--- a/identifiers.c
+++ b/identifiers.c
@@ -21,7 +21,7 @@ static int isident(int c) {
 
 /* can this character be used as the first character in an identifier? */
 static int isidentstart(int c) {
-	return isident(c) && c != '_' && c != '.';
+	return isident(c) && c != '.';
 }
 
 typedef struct IdentTree {
@@ -40,11 +40,12 @@ static long ident_curr_id; /* NOTE: you should eventually add something to reset
 /* moves s to the char after the identifier */
 static Identifier ident_tree_insert(IdentTree *t, char **s) {
 	while (1) {
-		char c = *((*s)++);
+		char c = **s;
 		if (!isident(c)) {
 			if (t->id == 0) t->id = ++ident_curr_id;
 			return t;
 		}
+		
 		if (!t->children) {
 			/* allocate children */
 			t->children = err_calloc(NIDENTIFIER_CHARS, sizeof *t->children);
@@ -52,6 +53,7 @@ static Identifier ident_tree_insert(IdentTree *t, char **s) {
 				t->children[i].parent = t; /* child's parent = self */
 		}
 		t = &t->children[ident_char_index(c)];
+		(*s)++;
 	}
 }
 
diff --git a/main.c b/main.c
index 20407a1..fe5fb14 100644
--- a/main.c
+++ b/main.c
@@ -1,23 +1,26 @@
 #include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <stdint.h>
 #include <stdarg.h>
 #include <string.h>
 #include <ctype.h>
 #include <limits.h>
-#include "util/colored_text.c"
+#include <stdint.h>
+#include <stdbool.h>
 #include "util/err.c"
+#include "util/arr.c"
 #include "identifiers.c"
 #include "tokenizer.c"
+#include "parse.c"
 
 int main(int argc, char **argv) {
 	if (argc < 2) {
 		fprintf(stderr, "Please specify an input file.\n");
 		return EXIT_FAILURE;
 	}
-	
-	FILE *in = fopen(argv[1], "r");
+
+	const char *in_filename = argv[1];
+	FILE *in = fopen(in_filename, "r");
 	if (!in) {
 		fprintf(stderr, "Could not open file: %s.\n", argv[1]);
 		return EXIT_FAILURE;
@@ -38,18 +41,28 @@ int main(int argc, char **argv) {
 		fprintf(stderr, "Error reading input file: %s.\n", argv[1]);
 		return EXIT_FAILURE;
 	}
+
+	err_filename = in_filename;
+	Tokenizer t;
+	if (!tokenize_string(&t, contents)) {
+		err_fprint(TEXT_IMPORTANT("Errors occured while preprocessing.\n"));
+		return EXIT_FAILURE;
+	}
 	
-	Tokenizer t = tokenize_string(contents);
-	
-	for (size_t i = 0; i < t.ntokens; i++) {
-		if (i)
+	arr_foreach(t.tokens, Token, token) {
+		if (token != t.tokens.data)
 			printf("    ");
-		token_fprint(stdout, &t.tokens[i]);
+		token_fprint(stdout, token);
 	}
 	printf("\n");
+	
+   	ParsedFile f;
+	parse_file(&f, &t);
 
+	parsed_file_fprint(stdout, &f);
+	
+	tokr_free(&t);
 	free(contents);
-	tokenizer_free(&t);
 	
 	fclose(in);
 	idents_free();
diff --git a/parse.c b/parse.c
new file mode 100644
index 0000000..b025d08
--- /dev/null
+++ b/parse.c
@@ -0,0 +1,66 @@
+typedef struct {
+	LineNo line;
+	LineNo col;
+} Location;
+
+typedef struct {
+	Location where;
+	char *var;
+} Declaration;
+
+arr_declaration(Declarations, Declaration, decls_)
+
+typedef struct {
+	int type;
+	Location where;
+	union {
+		Declarations decls;
+	};
+} Statement;
+
+arr_declaration(Statements, Statement, stmts_)
+
+typedef struct {
+	Statements stmts;
+} ParsedFile;
+
+/* TODO: Add newline tokens back in; give tokens pointer to text */
+static bool parse_decls(Declarations *ds, Tokenizer *t) {
+	if (t->token->kind != TOKEN_IDENT) {
+		tokr_err(t, "Cannot declare non-identifier.");
+		return false;
+	}
+	t->token++;
+	return true;
+}
+
+static bool parse_stmt(Statement *s, Tokenizer *t) {
+	if (token_is_kw(t->token + 1, KW_COLON)) {
+		return parse_decls(&s->decls, t);
+	} else {
+		t->token++;	/* TODO: This is temporary */
+	}
+	return true;
+}
+
+static bool parse_file(ParsedFile *f, Tokenizer *t) {
+	stmts_create(&f->stmts);
+	bool ret = true;
+	while (t->token->kind != TOKEN_EOF) {
+		Statement stmt = {0};
+		if (!parse_stmt(&stmt, t))
+			ret = false;
+		stmts_add(&f->stmts, &stmt);
+	}
+	return ret;
+}
+
+static void stmt_fprint(FILE *out, Statement *s) {
+	fprintf(out, "statement!\n");
+}
+
+static void parsed_file_fprint(FILE *out, ParsedFile *f) {
+	arr_foreach(f->stmts, Statement, stmt) {
+		stmt_fprint(out, stmt);
+	}
+}
diff --git a/test.toc b/test.toc
index ae9ceef..2163792 100644
--- a/test.toc
+++ b/test.toc
@@ -1,4 +1,3 @@
-   
-0x3f3a == 0777
-923.5808 == 2e-33
-38942187381273e+102 '\''
+P :-
+Q :-
+R :-
\ No newline at end of file
diff --git a/tokenizer.c b/tokenizer.c
index 5fafd3b..7782ca3 100644
--- a/tokenizer.c
+++ b/tokenizer.c
@@ -3,28 +3,35 @@ typedef enum {
 			  TOKEN_IDENT,
 			  TOKEN_NUM_CONST,
 			  TOKEN_CHAR_CONST,
+			  TOKEN_STR_CONST,
 			  TOKEN_EOF
-			  /* TODO: char constnats, str constants */
 } TokenKind;
 
 typedef enum {
 			  KW_SEMICOLON,
+			  KW_EQ,
+			  KW_COLON,
+			  KW_FN,
+			  KW_LPAREN,
+			  KW_RPAREN,
+			  KW_LBRACE,
+			  KW_RBRACE,
 			  KW_EQEQ,
 			  KW_LT,
 			  KW_LE,
-			  KW_EQ,
+			  KW_MINUS,
 			  KW_COUNT
 } Keyword;
 
 /* OPTIM: Use a trie or just a function if this gets too long */
 static const char *keywords[KW_COUNT] =
-	{";", "==", "<", "<=", "="}; 
+	{";", "=", ":", "fn", "(", ")", "{", "}", "==", "<", "<=", "-"}; 
 
-#define TOKENIZER_USE_LLONG 1
+#define TOKR_USE_LLONG 1
 
 typedef unsigned long long IntConst;
 
-typedef long double RealConst; /* OPTIM: Maybe only use double */
+typedef long double RealConst; /* OPTIM: Switch to double */
 
 typedef enum {
 			  NUM_CONST_INT,
@@ -39,28 +46,40 @@ typedef struct {
 	};
 } NumConst;
 
+typedef struct {
+	char *str;
+	size_t len;
+} StrConst;
+
 /* NOTE: LineNo is typedef'd in util/err.c */
 typedef struct {
 	TokenKind kind;
 	LineNo line;
-    LineNo col;
+	char *code;
 	union {
 		Keyword kw;
 		Identifier ident;
 		NumConst num;
 		char chr;
+		StrConst str;
 	};
 } Token;
 
+arr_declaration(Tokens, Token, tokens_)
+
 typedef struct {
-	Token *tokens;
-	size_t ntokens;
-	size_t cap;	/* used internally */
+	Tokens tokens;
+	char *s; /* string being parsed */
+	LineNo line;
 	Token *token; /* token currently being processed */
 } Tokenizer;
 
+static bool token_is_kw(Token *t, Keyword kw) {
+	return t->kind == TOKEN_KW && t->kw == kw;
+}
+
 static void token_fprint(FILE *out, Token *t) {
-	fprintf(out, "l%luc%lu-", (unsigned long)t->line, (unsigned long)t->col);
+	fprintf(out, "l%lu-", (unsigned long)t->line);
 	switch (t->kind) {
 	case TOKEN_KW:
 		fprintf(out, "keyword: %s", keywords[t->kw]);
@@ -83,129 +102,177 @@ static void token_fprint(FILE *out, Token *t) {
 	case TOKEN_CHAR_CONST:
 		fprintf(out, "char: '%c' (%d)", t->chr, t->chr);
 		break;
+	case TOKEN_STR_CONST:
+		fprintf(out, "str: \"%s\"", t->str.str);
+		break;
 	case TOKEN_EOF:
 		fprintf(out, "eof");
 		break;
 	}
 }
 
-static void tokenizer_add(Tokenizer *t, Token *token, LineNo line, LineNo col) {
-	if (t->ntokens >= t->cap) {
-		t->cap *= 2;
-		t->tokens = err_realloc(t->tokens, t->cap * sizeof(*t->tokens));
+static void tokr_add(Tokenizer *t, Token *token) {
+	if (!token->line)
+		token->line = t->line;
+	if (!token->code)
+		token->code = t->s;
+	tokens_add(&t->tokens, token);
+}
+
+static void tokr_nextchar(Tokenizer *t) {
+	if (*(t->s) == '\n') {
+		t->line++;
+	}
+	t->s++;
+}
+
+static char tokr_esc_seq(Tokenizer *t) {
+	/* TODO: add more of these incl. \x41, \100 */
+	switch (*t->s) {
+	case '\'':
+		tokr_nextchar(t);
+		return '\'';
+	case '"':
+		tokr_nextchar(t);
+		return '"';
+	case '\\':
+		tokr_nextchar(t);
+		return '\\';
+	case 'n':
+		tokr_nextchar(t);
+		return '\n';
+	default:
+		return 0;
 	}
-	token->line = line;
-	token->col = col;
-	t->tokens[t->ntokens++] = *token;
+
 }
 
-static Tokenizer tokenize_string(char *s) {	/* NOTE: May modify string. Don't even try to pass it a string literal.*/
+/* to be used during tokenization */
+static void tokenization_err(Tokenizer *t, const char *fmt, ...) {
+	va_list args;
+	va_start(args, fmt);
+	err_vprint(t->line, t->s, fmt, args);
+	va_end(args);
+	
+	char *end_of_line = strchr(t->s, '\n');
+	if (end_of_line) {
+		t->s = end_of_line;
+		t->s++; /* move past newline */
+	} else {
+		t->s = strchr(t->s, '\0');
+	}
+	t->line++;
+}
+
+/* to be used after tokenization */
+static void tokr_err(Tokenizer *t, const char *fmt, ...) {
+	LineNo line = t->token->line;
+	va_list args;
+	va_start(args, fmt);
+	err_vprint(line, t->token->code, fmt, args);
+	va_end(args);
+	while (1) {
+		if (t->token->line != line) break;
+		if (t->token->kind == TOKEN_EOF) break;
+		t->token++;
+	}
+}
+
+static bool tokenize_string(Tokenizer *tokr, char *str) {
 	int has_err = 0;
 	Tokenizer t;
-	t.cap = 256;
-	t.ntokens = 0;
-	t.tokens = err_malloc(t.cap * sizeof(*t.tokens));
-
-	LineNo line = 1;
-	LineNo col = 1;
+	tokens_create(&t.tokens);
+	tokens_reserve(&t.tokens, 256);
+	t.s = str;
+	t.line = 1;
 	
 	while (1) {
-	    if (*s == 0) break;
-		if (isspace(*s)) {
-			if (*s == '\n') {
-				line++;
-				col = 0;
-			}
-			s++; col++;
+	    if (*t.s == 0) break;
+		if (isspace(*t.s)) {
+			tokr_nextchar(&t);
 	    	continue;
 		}
 
-		if (*s == '/') {
+		if (*t.s == '/') {
 			/* maybe it's a comment */
 			int is_comment = 1;
-			s++; col++;
-			switch (*s) {
+			switch (t.s[1]) {
 			case '/': /* single line comment */
-				for (s++; *s != '\n' && *s; s++);
-				line++;
-				col = 1;
+				tokr_nextchar(&t);
+				for (t.s++; *t.s != '\n' && *t.s; t.s++);
+				t.line++;
 				break;
 			case '*': { /* multi line comment */
+				tokr_nextchar(&t);
 				int comment_level = 1; /* allow nested multi-line comments */
-			    while (*s) {
-					if (*s == '\n') {
-						line++;
-						col = 1;
-						s++;
-						continue;
-					}
-					if (s[0] == '*' && s[1] == '/') {
-						s += 2; col += 2;
+			    while (*t.s) {
+					if (t.s[0] == '*' && t.s[1] == '/') {
+						t.s += 2;
 						comment_level--;
 						if (comment_level == 0) {
 							break;
 						}
-					} else if (s[0] == '/' && s[1] == '*') {
-						s += 2; col += 2;
+					} else if (t.s[0] == '/' && t.s[1] == '*') {
+						t.s += 2;
 						comment_level++;
 					} else {
-						s++; col++;
+						tokr_nextchar(&t);
 					}
 				}
-				if (*s == 0) {
-					err_print(line, col, "End of file reached inside multi-line comment.");
+				if (*t.s == 0) {
+					tokenization_err(&t, "End of file reached inside multi-line comment.");
 					abort(); /* there won't be any further errors, of course */
 				}
 			} break;
 			default:
 				is_comment = 0;
-				s--; /* go back */
 				break;
 			}
 			if (is_comment) continue;
 		}
 		Keyword kw;
 		for (kw = 0; kw < KW_COUNT; kw++) {
-			if (strncmp(s, keywords[kw], strlen(keywords[kw])) == 0) {
+			if (strncmp(t.s, keywords[kw], strlen(keywords[kw])) == 0) {
 				break;
 			}
 		}
 		if (kw != KW_COUNT) {
 			/* it's a keyword */
-			Token token;
+			Token token = {0};
 			token.kind = TOKEN_KW;
 			token.kw = kw;
-			tokenizer_add(&t, &token, line, col);
-			col += (LineNo)strlen(keywords[kw]);
-			s += (LineNo)strlen(keywords[kw]);
+			tokr_add(&t, &token);
+			t.s += (LineNo)strlen(keywords[kw]);
 			continue;
 		}
 		
 		/* check if it's a number */
 
-		if (isdigit(*s)) {
+		if (isdigit(*t.s)) {
 			/* it's a numeric constant */
 			int base = 10;
 			RealConst decimal_pow10;
 			NumConst n;
 			n.kind = NUM_CONST_INT;
 			n.intval = 0;
-			LineNo line_start = line, col_start = col;
-			if (*s == '0') {
-				s++; col++;
+			Token token = {0};
+			token.line = t.line;
+			token.code = t.s;
+			if (*t.s == '0') {
+				tokr_nextchar(&t);
 				/* octal/hexadecimal/binary (or zero) */
-				char format = *s;
+				char format = *t.s;
 				if (isdigit(format)) /* octal */
 					base = 8;
 				else {
 					switch (format) {
 					case 'b':
 						base = 2;
-						s++; col++;
+						tokr_nextchar(&t);
 						break;
 					case 'x':
 						base = 16;
-						s++; col++;
+						tokr_nextchar(&t);
 						break;
 					default:
 						/* it's 0/0.something etc.  */
@@ -215,40 +282,39 @@ static Tokenizer tokenize_string(char *s) {	/* NOTE: May modify string. Don't ev
 			}
 
 			while (1) {
-				if (*s == '.') {
+				if (*t.s == '.') {
 					if (n.kind == NUM_CONST_REAL) {
-						err_print(line, col, "Double . in number.");
+						tokenization_err(&t, "Double . in number.");
 						goto err;
 					}
 					if (base != 10) {
-						err_print(line, col, "Decimal point in non base 10 number.");
+						tokenization_err(&t, "Decimal point in non base 10 number.");
 						goto err;
 					}
 				    n.kind = NUM_CONST_REAL;
 					decimal_pow10 = 0.1;
 					n.realval = (RealConst)n.intval;
-					s++, col++;
+					tokr_nextchar(&t);
 					continue;
-				} else if (*s == 'e') {
-					s++; col++;
+				} else if (*t.s == 'e') {
+					tokr_nextchar(&t);
 					if (n.kind == NUM_CONST_INT) {
 						n.kind = NUM_CONST_REAL;
 						n.realval = (RealConst)n.intval;
 					}
 					/* TODO: check if exceeding maximum exponent */
 					int exponent = 0;
-					if (*s == '+') {
-						s++; col++;
-					}
+					if (*t.s == '+')
+						tokr_nextchar(&t); /* ignore + after e */
 					
 					int negative_exponent = 0;
-					if (*s == '-') {
-						s++; col++;
+					if (*t.s == '-') {
+						tokr_nextchar(&t);
 						negative_exponent = 1;
 					}
-					for (; isdigit(*s); s++, col++) {
+					for (; isdigit(*t.s); tokr_nextchar(&t)) {
 						exponent *= 10;
-						exponent += *s - '0';
+						exponent += *t.s - '0';
 					}
 					/* OPTIM: Slow for very large exponents (unlikely to happen) */
 					for (int i = 0; i < exponent; i++) {
@@ -262,19 +328,19 @@ static Tokenizer tokenize_string(char *s) {	/* NOTE: May modify string. Don't ev
 				}
 				int digit = -1;
 				if (base == 16) {
-					if (*s >= 'a' && *s <= 'f')
-						digit = 10 + *s - 'a';
-					else if (*s >= 'A' && *s <= 'F')
-						digit = *s - 'A';
+					if (*t.s >= 'a' && *t.s <= 'f')
+						digit = 10 + *t.s - 'a';
+					else if (*t.s >= 'A' && *t.s <= 'F')
+						digit = *t.s - 'A';
 				}
 				if (digit == -1) {
-					if (*s >= '0' && *s <= '9')
-						digit = *s - '0';
+					if (*t.s >= '0' && *t.s <= '9')
+						digit = *t.s - '0';
 				}
 				if (digit < 0 || digit >= base) {
-					if (isdigit(*s)) {
+					if (isdigit(*t.s)) {
 						/* something like 0b011012 */
-						err_print(line, col, "Digit %d cannot appear in a base %d number.", digit, base);
+						tokenization_err(&t, "Digit %d cannot appear in a base %d number.", digit, base);
 						goto err;
 					}
 					/* end of numeric constant */
@@ -282,9 +348,10 @@ static Tokenizer tokenize_string(char *s) {	/* NOTE: May modify string. Don't ev
 				}
 				switch (n.kind) {
 				case NUM_CONST_INT:
-					if (n.intval > ULLONG_MAX / (IntConst)base) {
+					if (n.intval > ULLONG_MAX / (IntConst)base ||
+						n.intval * (IntConst)base > ULLONG_MAX - (IntConst)digit) {
 						/* too big! */
-						err_print(line, col, "Number too big to fit in a numeric constant.");
+						tokenization_err(&t, "Number too big to fit in a numeric constant.");
 						goto err;
 					}
 					n.intval *= (IntConst)base;
@@ -295,88 +362,128 @@ static Tokenizer tokenize_string(char *s) {	/* NOTE: May modify string. Don't ev
 					decimal_pow10 /= 10;
 					break;
 				}
-				s++; col++;
+				tokr_nextchar(&t);
 			}
-			Token token;
 			token.kind = TOKEN_NUM_CONST;
 			token.num = n;
-			tokenizer_add(&t, &token, line_start, col_start);
+			tokr_add(&t, &token);
 			continue;
 		}
 
-		if (*s == '\'') {
+		if (*t.s == '\'') {
 			/* it's a character constant! */
-			s++; col++;
+			tokr_nextchar(&t);
+			Token token = {0};
+			token.line = t.line;
+			token.code = t.s;
 			char c;
-			if (*s == '\\') {
+			if (*t.s == '\\') {
 				/* escape sequence */
-				s++; col++;
-				/* TODO: Separate into function when string literals are added; add more of these */
-				switch (*s) {
-				case '\'':
-					c = '\'';
-					break;
-				case '\\':
-					c = '\\';
-					break;
-				case 'n':
-					c = '\n';
-					break;
-				default:
-					err_print(line, col, "Unrecognized escape character: '%c'.", *s);
+				tokr_nextchar(&t);
+				c = tokr_esc_seq(&t);
+				if (c == 0) {
+					tokenization_err(&t, "Unrecognized escape character: '\\%c'.", *t.s);
 					goto err;
 				}
 			} else {
-				c = *s;
+				c = *t.s;
+				tokr_nextchar(&t);
 			}
-			s++; col++;
-			if (*s != '\'') {
-				err_print(line, col, "End of character constant expected.");
+			if (*t.s != '\'') {
+				tokenization_err(&t, "End of character constant expected.");
 				goto err;
 			}
-			s++; col++;
-			Token token;
+			tokr_nextchar(&t);
 			token.kind = TOKEN_CHAR_CONST;
 			token.chr = c;
-			tokenizer_add(&t, &token, line, col);
+			tokr_add(&t, &token);
+			continue;
+		}
+
+		if (*t.s == '"') {
+			/* it's a string constant! */
+			Token token;
+			token.line = t.line;
+			token.code = t.s;
+			tokr_nextchar(&t);
+			size_t len = 0;
+			size_t backslashes = 0;
+			while (*t.s != '"' || backslashes % 2 == 1) {
+				if (*t.s == '\\') {
+					backslashes++;
+				} else if (*t.s == 0) {
+					/* return t to opening " so that we go to the next line */
+					t.line = token.line;
+					t.s = token.code;
+					tokenization_err(&t, "No matching \" found.");
+					goto err;
+				} else {
+					backslashes = 0;
+				}
+				len++;
+				tokr_nextchar(&t);
+			}
+			char *str = malloc(len + 1);
+		    char *strptr = str;
+			t.s = token.code;
+			t.line = token.line;
+			tokr_nextchar(&t); /* past opening " */
+			while (*t.s != '"') {
+				assert(*t.s);
+				if (*t.s == '\\') {
+					tokr_nextchar(&t);
+					char c = tokr_esc_seq(&t);
+					if (c == 0) {
+						tokenization_err(&t, "Unrecognized escape character: '\\%c'.", *t.s);
+						goto err;
+					}
+					*strptr++ = c;
+				} else {
+					*strptr++ = *t.s;
+					tokr_nextchar(&t);
+				}
+			}
+			*strptr = 0;
+			token.kind = TOKEN_STR_CONST;
+			token.str.len = len;
+			token.str.str = str;
+			tokr_add(&t, &token);
+			tokr_nextchar(&t); /* move past closing " */
 			continue;
 		}
 		
-		if (isidentstart(*s)) {
+		if (isidentstart(*t.s)) {
 			/* it's an identifier */
-			Identifier ident = ident_insert(&s);
-			Token token;
+			Token token = {0};
+			token.line = t.line;
+			token.code = t.s;
+			Identifier ident = ident_insert(&t.s);
 			token.kind = TOKEN_IDENT;
 			token.ident = ident;
-			tokenizer_add(&t, &token, line, col);			
+			tokr_add(&t, &token);			
 			continue;
-		}
-		int has_newline;
-		char *end_of_line = strchr(s, '\n');
-		has_newline = end_of_line != NULL;
-		if (has_newline)
-			*end_of_line = 0;
-		
-		err_print(line, col, TEXT_IMPORTANT("Unrecognized token:") "\n\there --> %s\n", s);
-		if (has_newline)
-			*end_of_line = '\n';
+		}		
+		tokenization_err(&t, "Token not recognized");
 	err:
 		has_err = 1;
-		s = strchr(s, '\n');
-		if (s == NULL) break;
-		s++; /* move past newline */
-		col = 1;
-		line++;
-				
-	}
-	if (has_err) {
-		fprintf(stderr, TEXT_IMPORTANT("Errors occured while preprocessing.\n"));
-		abort();
 	}
-	t.token = t.tokens;
-	return t;
+	Token token = {0};
+	token.kind = TOKEN_EOF;
+	tokr_add(&t, &token);
+	
+	t.token = t.tokens.data;
+	*tokr = t;
+	return !has_err;
 }
 
-static void tokenizer_free(Tokenizer *t) {
-	free(t->tokens);
+static void tokr_free(Tokenizer *t) {
+	arr_foreach(t->tokens, Token, token) {
+		switch (token->kind) {
+		case TOKEN_STR_CONST:
+			free(token->str.str);
+			break;
+		default: break;
+		}
+	}
+	tokens_clear(&t->tokens);
 }
diff --git a/util/arr.c b/util/arr.c
new file mode 100644
index 0000000..7595b4b
--- /dev/null
+++ b/util/arr.c
@@ -0,0 +1,28 @@
+#define arr_declaration(arr_type, type, prefix) typedef struct {	\
+		type *data;													\
+		size_t cap;													\
+		size_t len;													\
+	} arr_type;														\
+	static void prefix##create(arr_type *arr) {						\
+		arr->data = NULL;											\
+		arr->cap = 0;												\
+		arr->len = 0;												\
+	}																\
+	static void prefix##reserve(arr_type *arr, size_t n) {			\
+		arr->data = err_realloc(arr->data, n * sizeof(*arr->data)); \
+		arr->cap = n;												\
+	}																\
+	static void prefix##add(arr_type *arr, type *item) {			\
+		if (arr->len >= arr->cap) {									\
+			prefix##reserve(arr, 2 * arr->len + 2);					\
+		}															\
+	    arr->data[arr->len++] = *item;								\
+	}																\
+	static void prefix##clear(arr_type *arr) {						\
+		free(arr->data);											\
+		arr->data = NULL;											\
+		arr->cap = 0;												\
+		arr->len = 0;												\
+	}
+
+#define arr_foreach(arr, type, var) for (type *var = (arr).data, *arr_iterate_last_ = (arr).data + ((arr).len - 1); var; (var == arr_iterate_last_) ? var = NULL : var++)
diff --git a/util/colored_text.c b/util/colored_text.c
deleted file mode 100644
index 836f357..0000000
--- a/util/colored_text.c
+++ /dev/null
@@ -1,10 +0,0 @@
-
-#define USE_COLORED_TEXT 1
-
-#if USE_COLORED_TEXT
-#define TEXT_ERROR(x) "\x1b[91m" x "\x1b[0m"
-#define TEXT_IMPORTANT(x) "\x1b[1m" x "\x1b[0m"
-#else
-#define TEXT_ERROR(x) x
-#define TEXT_IMPORTANT(x) x
-#endif
diff --git a/util/err.c b/util/err.c
index 6fd2ac3..5e296da 100644
--- a/util/err.c
+++ b/util/err.c
@@ -1,13 +1,65 @@
+#define USE_COLORED_TEXT 1
+
+#if USE_COLORED_TEXT
+#define TEXT_ERROR(x) "\x1b[91m" x "\x1b[0m"
+#define TEXT_IMPORTANT(x) "\x1b[1m" x "\x1b[0m"
+#else
+#define TEXT_ERROR(x) x
+#define TEXT_IMPORTANT(x) x
+#endif
+
 typedef uint32_t LineNo;
 
-static void err_print(LineNo line, LineNo col, const char *fmt, ...) {
-	/* TODO: Color */
+/* file name of file being processed */
+static const char *err_filename;
+
+/* Write directly to the error file */
+static void err_fwrite(const void *data, size_t size, size_t n) {
+	fwrite(data, size, n, stderr);
+}
+
+static void err_fprint(const char *fmt, ...) {
 	va_list args;
-	fprintf(stderr, TEXT_ERROR("error:") " at line %lu col %lu:\n", (unsigned long)line, (unsigned long)col);
 	va_start(args, fmt);
 	vfprintf(stderr, fmt, args);
 	va_end(args);
-	fprintf(stderr, "\n");
+}
+
+static void err_vfprint(const char *fmt, va_list args) {
+	vfprintf(stderr, fmt, args);
+}
+
+static void err_print_header_(LineNo line) {
+	err_fprint(TEXT_ERROR("error:") " at line %lu of %s:\n", (unsigned long)line, err_filename);
+}
+
+static void err_print_footer_(const char *context) {
+	err_fprint("\n\there --> ");
+	const char *end = strchr(context, '\n');
+	int has_newline = end != NULL;
+	if (!has_newline)
+		end = strchr(context, '\0');
+	assert(end);
+	err_fwrite(context, 1, (size_t)(end - context));
+	if (!has_newline)
+		err_fprint("<end of file>");
+	err_fprint("\n");
+}
+
+/* Write nicely-formatted errors to the error file */
+static void err_print(LineNo line, const char *context, const char *fmt, ...) {
+	err_print_header_(line);
+	va_list args;
+	va_start(args, fmt);
+	err_vfprint(fmt, args);
+	va_end(args);
+	err_print_footer_(context);
+}
+
+static void err_vprint(LineNo line, const char *context, const char *fmt, va_list args) {
+	err_print_header_(line);
+	err_vfprint(fmt, args);
+	err_print_footer_(context);
 }
 
 static void *err_malloc(size_t size) {