1 files changed, 14 insertions, 7 deletions

diff --git a/tokenizer.c b/tokenizer.c
index 7adb7d5..8807b33 100644
--- a/tokenizer.c
+++ b/tokenizer.c
@@ -153,6 +153,7 @@ typedef struct {
 } Token;
 
 typedef struct {
+	Allocator allocr;
 	Array tokens;
 	char *s; /* string being parsed */
 	const char *filename;
@@ -161,14 +162,10 @@ typedef struct {
 	Identifiers *idents;
 } Tokenizer;
 
-
-
 static inline bool token_is_kw(Token *t, Keyword kw) {
 	return t->kind == TOKEN_KW && t->kw == kw;
 }
 
-
-
 static const char *token_kind_to_str(TokenKind t) {
 	switch (t) {
 	case TOKEN_KW: return "keyword";
@@ -288,10 +285,15 @@ static void tokr_get_location(Tokenizer *tokr, Token *t) {
 static void tokr_create(Tokenizer *t, Identifiers *idents, const char *filename) {
 	arr_create(&t->tokens, sizeof(Token));
 	arr_reserve(&t->tokens, 256);
+	allocr_create(&t->allocr);
 	t->idents = idents;
 	t->filename = filename;
 }
 
+static inline void *tokr_malloc(Tokenizer *t, size_t bytes) {
+	return allocr_malloc(&t->allocr, bytes);
+}
+
 static Token *tokr_add(Tokenizer *t) {
 	Token *token = arr_add(&t->tokens);
 	tokr_put_location(t, token);
@@ -552,7 +554,7 @@ static bool tokenize_string(Tokenizer *t, char *str) {
 				len++;
 				tokr_nextchar(t);
 			}
-			char *strlit = err_malloc(len + 1);
+			char *strlit = tokr_malloc(t, len + 1);
 		    char *strptr = strlit;
 			tokr_get_location(t, token);
 			tokr_nextchar(t); /* past opening " */
@@ -599,7 +601,12 @@ static bool tokenize_string(Tokenizer *t, char *str) {
 	return !has_err;
 }
 
-/* Does NOT free string literals!!! */
-static void tokr_free(Tokenizer *t) {
+/* ONLY frees tokens, not string literals. You can call this followed by tokr_free. */
+static void tokr_free_tokens(Tokenizer *t) {
 	arr_clear(&t->tokens);
 }
+
+static void tokr_free(Tokenizer *t) {
+	tokr_free_tokens(t);
+	allocr_free_all(&t->allocr);
+}