move string32.c into util.c

7 files changed, 393 insertions, 230 deletions

diff --git a/buffer.c b/buffer.c
index 441f3f5..2d80191 100644
--- a/buffer.c
+++ b/buffer.c
@@ -1252,9 +1252,9 @@ i64 buffer_pos_move_words(TextBuffer *buffer, BufferPos *pos, i64 nwords) {
 					pos->index = 0;
 				}
 			} else {
-				bool starting_isword = is_word(str[index]) != 0;
+				bool starting_isword = is32_word(str[index]) != 0;
 				for (; index < line->len; ++index) {
-					bool this_isword = is_word(str[index]) != 0;
+					bool this_isword = is32_word(str[index]) != 0;
 					if (this_isword != starting_isword) {
 						// either the position *was* on an alphanumeric character and now it's not
 						// or it wasn't and now it is.
@@ -1284,9 +1284,9 @@ i64 buffer_pos_move_words(TextBuffer *buffer, BufferPos *pos, i64 nwords) {
 			} else {
 				--index;
 				if (index > 0) {
-					bool starting_isword = is_word(str[index]) != 0;
+					bool starting_isword = is32_word(str[index]) != 0;
 					while (true) {
-						bool this_isword = is_word(str[index]) != 0;
+						bool this_isword = is32_word(str[index]) != 0;
 						if (this_isword != starting_isword) {
 							++index;
 							break;
@@ -1333,11 +1333,11 @@ String32 buffer_word_at_pos(TextBuffer *buffer, BufferPos pos) {
 	char32_t *str = line->str;
 	i64 word_start, word_end;
 	for (word_start = pos.index; word_start > 0; --word_start) {
-		if (!is_word(str[word_start - 1]))
+		if (!is32_word(str[word_start - 1]))
 			break;
 	}
 	for (word_end = pos.index; word_end < line->len; ++word_end) {
-		if (!is_word(str[word_end]))
+		if (!is32_word(str[word_end]))
 			break;
 	}
 	u32 len = (u32)(word_end - word_start);
@@ -1527,7 +1527,7 @@ BufferPos buffer_insert_text_at_pos(TextBuffer *buffer, BufferPos pos, String32
 		// close completions if a non-word character is typed
 		bool close_completions = false;
 		for (u32 i = 0; i < str.len; ++i) {
-			if (!is_word(str.str[i])) {
+			if (!is32_word(str.str[i])) {
 				close_completions = true;
 				break;
 			}
@@ -1801,7 +1801,7 @@ void buffer_delete_chars_at_pos(TextBuffer *buffer, BufferPos pos, i64 nchars_)
 			(void)n;
 			assert(n == nchars);
 			for (u32 i = 0; i < nchars; ++i) {
-				if (!is_word(text[i])) {
+				if (!is32_word(text[i])) {
 					close_completions = true;
 					break;
 				}
diff --git a/ide-autocomplete.c b/ide-autocomplete.c
index 76871af..1b38816 100644
--- a/ide-autocomplete.c
+++ b/ide-autocomplete.c
@@ -18,7 +18,7 @@ static void autocomplete_clear_completions(Ted *ted) {
 static void autocomplete_complete(Ted *ted, Autocompletion completion) {
 	TextBuffer *buffer = ted->active_buffer;
 	buffer_start_edit_chain(buffer); // don't merge with other edits
-	if (is_word(buffer_char_before_cursor(buffer)))
+	if (is32_word(buffer_char_before_cursor(buffer)))
 		buffer_backspace_words_at_cursor(buffer, 1); // delete whatever text was already typed
 	buffer_insert_utf8_at_cursor(buffer, completion.text);
 	buffer_end_edit_chain(buffer);
diff --git a/main.c b/main.c
index 97e4103..bbe7aea 100644
--- a/main.c
+++ b/main.c
@@ -100,7 +100,6 @@ no_warn_end
 #include "ted.h"
 #include "gl.c"
 #include "text.c"
-#include "string32.c"
 #include "colors.c"
 #include "syntax.c"
 #include "buffer.c"
@@ -766,8 +765,9 @@ int main(int argc, char **argv) {
 						// characters because currently we ask for signature
 						// help any time a character is inserted.
 						
-						if (settings->identifier_trigger_characters && is_word(last_char)
-							&& !is_digit(last_char))
+						if (settings->identifier_trigger_characters
+							&& is32_word(last_char)
+							&& !is32_digit(last_char))
 							autocomplete_open(ted, last_char);
 					}
 					 
diff --git a/string32.c b/string32.c
deleted file mode 100644
index 1b4f1ac..0000000
--- a/string32.c
+++ /dev/null
@@ -1,193 +0,0 @@
-// UTF-32 string
-typedef struct {
-	char32_t *str;
-	size_t len;
-} String32;
-
-String32 str32(char32_t *str, size_t len) {
-	String32 s = {str, len};
-	return s;
-}
-
-String32 str32_substr(String32 s, size_t from, size_t len) {
-	return str32(s.str + from, len);
-}
-
-// frees string and sets it to ""
-void str32_free(String32 *s) {
-	free(s->str);
-	s->str = NULL;
-	s->len = 0;
-}
-
-// the string returned should be str32_free'd.
-// this will return an empty string if the allocation failed or the string is invalid UTF-8
-String32 str32_from_utf8(char const *utf8) {
-	String32 string = {NULL, 0};
-	size_t len = strlen(utf8);
-	if (len) {
-		// the wide string uses at most as many "characters" (elements?) as the UTF-8 string
-		char32_t *widestr = calloc(len, sizeof *widestr);
-		if (widestr) {
-			char32_t *wide_p = widestr;
-			char const *utf8_p = utf8;
-			char const *utf8_end = utf8_p + len;
-			while (utf8_p < utf8_end) {
-				char32_t c = 0;
-				size_t n = unicode_utf8_to_utf32(&c, utf8_p, (size_t)(utf8_end - utf8_p));
-				if (n == 0 // null character. this shouldn't happen.
-					|| n >= (size_t)(-2) // invalid UTF-8
-					) {
-					free(widestr);
-					widestr = wide_p = NULL;
-					break;
-				} else {
-					// n bytes consumed
-					*wide_p++ = c;
-					utf8_p += n;
-				}
-			}
-			string.str = widestr;
-			string.len = (size_t)(wide_p - widestr);
-		}
-	}
-	return string;
-}
-
-// returns a null-terminated UTF-8 string
-// the string returned should be free'd
-// this will return NULL on failure
-static char *str32_to_utf8_cstr(String32 s) {
-	char *utf8 = calloc(4 * s.len + 1, 1); // each codepoint takes up at most 4 bytes in UTF-8, + we need a terminating null byte
-	if (utf8) {
-		char *p = utf8;
-		for (size_t i = 0; i < s.len; ++i) {
-			size_t bytes = unicode_utf32_to_utf8(p, s.str[i]);
-			if (bytes == (size_t)-1) {
-				// invalid UTF-32 code point
-				free(utf8);
-				return NULL;
-			} else {
-				p += bytes;
-			}
-		}
-		*p = '\0';
-	}
-	return utf8;
-}
-
-// compare s to the ASCII string `ascii`
-static int str32_cmp_ascii(String32 s, char const *ascii) {
-	for (size_t i = 0; i < s.len; ++i) {
-		assert((char32_t)ascii[i] < 128);
-		if ((char32_t)ascii[i] == '\0')
-			return -1; // ascii is a prefix of s
-		if (s.str[i] > (char32_t)ascii[i])
-			return +1;
-		if (s.str[i] < (char32_t)ascii[i])
-			return -1;
-	}
-	if (ascii[s.len]) {
-		// s is a prefix of ascii
-		return +1;
-	}
-	return 0;
-}
-
-// check if s starts with the ASCII string `ascii`
-static int str32_has_ascii_prefix(String32 s, char const *ascii) {
-	for (size_t i = 0; i < s.len; ++i) {
-		assert((char32_t)ascii[i] < 128);
-		if ((char32_t)ascii[i] == '\0')
-			return true; // ascii is a prefix of s
-		if (s.str[i] > (char32_t)ascii[i])
-			return false;
-		if (s.str[i] < (char32_t)ascii[i])
-			return false;
-	}
-	if (ascii[s.len]) {
-		// s is a prefix of ascii
-		return false;
-	}
-	// s is the same as ascii
-	return true;
-}
-
-// returns the index of the given character in the string, or the length of the string if it's not found.
-size_t str32chr(String32 s, char32_t c) {
-	for (size_t i = 0; i < s.len; ++i) {
-		if (s.str[i] == c)
-			return i;
-	}
-	return s.len;
-}
-
-// returns number of instances of c in s
-size_t str32_count_char(String32 s, char32_t c) {
-	size_t total = 0;
-	for (size_t i = 0; i < s.len; ++i) {
-		total += s.str[i] == c;
-	}
-	return total;
-}
-
-// returns number of characters deleted from s
-size_t str32_remove_all_instances_of_char(String32 *s, char32_t c) {
-	char32_t *str = s->str;
-	size_t ndeleted = 0;
-	size_t len = s->len;
-	size_t out = 0;
-	for (size_t in = 0; in < len; ++in) {
-		if (str[in] == c) {
-			++ndeleted;
-		} else {
-			str[out++] = str[in];
-		}
-	}
-	s->len = out;
-	return ndeleted;
-}
-
-// returns the length of the longest prefix of `s` containing only
-// ASCII characters in the C-string `charset`.
-size_t str32_ascii_spn(String32 s, char const *charset) {
-	for (u32 i = 0; i < s.len; ++i) {
-		if (s.str[i] >= 128)
-			return i; // non-ASCII character in s, so that can't be in charset.
-		bool found = false;
-		for (char const *p = charset; *p; ++p) {
-			assert((char32_t)*p < 128);
-			if ((char32_t)*p == s.str[i]) {
-				found = true;
-				break;
-			}
-		}
-		if (!found) return i;
-	}
-	return s.len;
-}
-
-bool is32_space(char32_t c) {
-	return c <= WINT_MAX && iswspace((wint_t)c);
-}
-
-bool is32_alpha(char32_t c) {
-	return c <= WINT_MAX && iswalpha((wint_t)c);
-}
-
-bool is32_alnum(char32_t c) {
-	return c <= WINT_MAX && iswalnum((wint_t)c);
-}
-
-bool is32_digit(char32_t c) {
-	return c <= WINT_MAX && iswdigit((wint_t)c);
-}
-
-bool is32_graph(char32_t c) {
-	return c <= WINT_MAX && iswgraph((wint_t)c);
-}
-
-// could this character appear in a C-style identifier?
-bool is32_ident(char32_t c) {
-	return c <= WINT_MAX && (iswalnum((wint_t)c) || c == '_');
-}
diff --git a/syntax.c b/syntax.c
index 1525e1d..cddebc2 100644
--- a/syntax.c
+++ b/syntax.c
@@ -182,7 +182,7 @@ static inline bool syntax_number_continues(Language lang, char32_t const *line,
 
 static bool is_keyword(Language lang, char32_t c) {
 	if (c == '_' && lang == LANG_TEX) return false;
-	if (is32_ident(c)) return true;
+	if (is32_word(c)) return true;
 	switch (lang) {
 	case LANG_RUST:
 		// Rust builtin macros
@@ -297,7 +297,7 @@ static void syntax_highlight_c_cpp(SyntaxState *state_ptr, char32_t const *line,
 					if (line[i - 1] == '.') {
 						// support .6, for example
 						char_types[i - 1] = SYNTAX_CONSTANT;
-					} else if (is32_ident(line[i - 1])) {
+					} else if (is32_word(line[i - 1])) {
 						// actually, this isn't a number. it's something like a*6* or u3*2*.
 						in_number = false;
 					}
@@ -305,7 +305,7 @@ static void syntax_highlight_c_cpp(SyntaxState *state_ptr, char32_t const *line,
 			}
 			break;
 		default: {
-			if ((i && is32_ident(line[i - 1])) || !is32_ident(c))
+			if ((i && is32_word(line[i - 1])) || !is32_word(c))
 				break; // can't be a keyword on its own.
 			
 			if (!in_single_line_comment && !in_multi_line_comment && !in_string && c == 'R' && has_2_chars && line[i + 1] == '"' && line[i + 2] == '(') {
@@ -507,7 +507,7 @@ static void syntax_highlight_rust(SyntaxState *state, char32_t const *line, u32
 			// a number!
 			if (char_types && !comment_depth && !in_string && !in_number) {
 				in_number = true;
-				if (i && (is32_ident(line[i - 1])
+				if (i && (is32_word(line[i - 1])
 					|| (line[i-1] == '.' && !(i >= 2 && line[i-2] == '.')))
 				) {
 					// actually, this isn't a number. it's something like a*6* or u3*2*.
@@ -550,7 +550,7 @@ static void syntax_highlight_rust(SyntaxState *state, char32_t const *line, u32
 			break;
 		default:
 		keyword_check: {
-			if ((i && is32_ident(line[i - 1])) || !is32_ident(c))
+			if ((i && is32_word(line[i - 1])) || !is32_word(c))
 				break; // can't be a keyword on its own.
 			if (i >= 2 && line[i-2] == 'r' && line[i-1] == '#') {
 				// raw identifier
@@ -669,7 +669,7 @@ static void syntax_highlight_python(SyntaxState *state, char32_t const *line, u3
 					if (line[i - 1] == '.') {
 						// support .6, for example
 						char_types[i - 1] = SYNTAX_CONSTANT;
-					} else if (is32_ident(line[i - 1])) {
+					} else if (is32_word(line[i - 1])) {
 						// actually, this isn't a number. it's something like a*6* or u3*2*.
 						in_number = false;
 					}
@@ -681,7 +681,7 @@ static void syntax_highlight_python(SyntaxState *state, char32_t const *line, u3
 			break;
 		default:
 		keyword_check:
-			if ((i && is32_ident(line[i - 1])) || !is32_ident(c))
+			if ((i && is32_word(line[i - 1])) || !is32_word(c))
 				break; // can't be a keyword on its own.
 			
 			if (char_types && !in_string && !in_number) {
@@ -723,7 +723,7 @@ static void syntax_highlight_python(SyntaxState *state, char32_t const *line, u3
 
 static bool is_tex_ident(char32_t c) {
 	// digits and underscores cannot appear in tex identifiers
-	return is32_ident(c) && !is32_digit(c) && c != '_';
+	return is32_word(c) && !is32_digit(c) && c != '_';
 }
 
 static void syntax_highlight_tex(SyntaxState *state, char32_t const *line, u32 line_len, SyntaxCharType *char_types) {
@@ -1098,7 +1098,7 @@ static void syntax_highlight_xml(SyntaxState *state, char32_t const *line, u32 l
 			default:
 				if (char_types) {
 				
-					if ((i && is32_ident(line[i - 1])) || !is32_ident(line[i]))
+					if ((i && is32_word(line[i - 1])) || !is32_word(line[i]))
 						break; // can't be a keyword on its own.
 					
 					if (lang == LANG_XML)
@@ -1170,7 +1170,7 @@ static void syntax_highlight_config(SyntaxState *state, char32_t const *line, u3
 			break;
 		case ANY_DIGIT:
 			if (char_types && i > 0 && !string) {
-				if (is32_ident(line[i-1]) // something like e5
+				if (is32_word(line[i-1]) // something like e5
 					|| line[i-1] == '+') // key combinations, e.g. Alt+0
 					break;
 				while (i < line_len && syntax_number_continues(LANG_CONFIG, line, line_len, i)) {
@@ -1183,7 +1183,7 @@ static void syntax_highlight_config(SyntaxState *state, char32_t const *line, u3
 				break; // don't care
 			if (i == 0) // none of the keywords in syntax_all_keywords_config should appear at the start of the line
 				break;
-			if (is32_ident(line[i-1]) || line[i-1] == '-' || !is32_ident(line[i]))
+			if (is32_word(line[i-1]) || line[i-1] == '-' || !is32_word(line[i]))
 				break; // can't be a keyword on its own.
 			u32 keyword_len = syntax_keyword_len(LANG_CONFIG, line, i, line_len);
 			Keyword const *keyword = syntax_keyword_lookup(syntax_all_keywords_config, &line[i], keyword_len);
@@ -1309,7 +1309,7 @@ static void syntax_highlight_javascript_like(
 					if (line[i - 1] == '.') {
 						// support .6, for example
 						char_types[i - 1] = SYNTAX_CONSTANT;
-					} else if (is32_ident(line[i - 1])) {
+					} else if (is32_word(line[i - 1])) {
 						// actually, this isn't a number. it's something like a*6* or u3*2*.
 						in_number = false;
 					}
@@ -1320,7 +1320,7 @@ static void syntax_highlight_javascript_like(
 			++backslashes;
 			break;
 		default:
-			if ((i && is32_ident(line[i - 1])) || !is32_ident(c))
+			if ((i && is32_word(line[i - 1])) || !is32_word(c))
 				break; // can't be a keyword on its own.
 			
 			if (char_types && !in_string && !in_number && !in_multiline_comment) {
@@ -1445,7 +1445,7 @@ static void syntax_highlight_java(SyntaxState *state_ptr, char32_t const *line,
 					if (line[i - 1] == '.') {
 						// support .6, for example
 						char_types[i - 1] = SYNTAX_CONSTANT;
-					} else if (is32_ident(line[i - 1])) {
+					} else if (is32_word(line[i - 1])) {
 						// actually, this isn't a number. it's something like a*6* or u3*2*.
 						in_number = false;
 					}
@@ -1453,7 +1453,7 @@ static void syntax_highlight_java(SyntaxState *state_ptr, char32_t const *line,
 			}
 			break;
 		default: {
-			if ((i && is32_ident(line[i - 1])) || !is32_ident(c))
+			if ((i && is32_word(line[i - 1])) || !is32_word(c))
 				break; // can't be a keyword on its own.
 			
 			// keywords don't matter for advancing the state
@@ -1585,7 +1585,7 @@ static void syntax_highlight_go(SyntaxState *state_ptr, char32_t const *line, u3
 					if (line[i - 1] == '.') {
 						// support .6, for example
 						char_types[i - 1] = SYNTAX_CONSTANT;
-					} else if (is32_ident(line[i - 1])) {
+					} else if (is32_word(line[i - 1])) {
 						// actually, this isn't a number. it's something like a*6* or u3*2*.
 						in_number = false;
 					}
@@ -1593,7 +1593,7 @@ static void syntax_highlight_go(SyntaxState *state_ptr, char32_t const *line, u3
 			}
 			break;
 		default: {
-			if ((i && is32_ident(line[i - 1])) || !is32_ident(c))
+			if ((i && is32_word(line[i - 1])) || !is32_word(c))
 				break; // can't be a keyword on its own.
 			
 			// keywords don't matter for advancing the state
diff --git a/util.c b/util.c
index f8bdab5..6751a72 100644
--- a/util.c
+++ b/util.c
@@ -14,16 +14,28 @@
 // it is unusual to have a string that long.
 #define STRLEN_SAFE_MAX (UINT_MAX >> 2)
 
-bool is_word(char32_t c) {
+bool is32_word(char32_t c) {
 	return c > WCHAR_MAX || c == '_' || iswalnum((wint_t)c);
 }
 
-bool is_digit(char32_t c) {
-	return c < WCHAR_MAX && iswdigit((wint_t)c);
+bool is32_space(char32_t c) {
+	return c <= WINT_MAX && iswspace((wint_t)c);
 }
 
-bool is_space(char32_t c) {
-	return c < WCHAR_MAX && iswspace((wint_t)c);
+bool is32_alpha(char32_t c) {
+	return c <= WINT_MAX && iswalpha((wint_t)c);
+}
+
+bool is32_alnum(char32_t c) {
+	return c <= WINT_MAX && iswalnum((wint_t)c);
+}
+
+bool is32_digit(char32_t c) {
+	return c <= WINT_MAX && iswdigit((wint_t)c);
+}
+
+bool is32_graph(char32_t c) {
+	return c <= WINT_MAX && iswgraph((wint_t)c);
 }
 
 bool is_a_tty(FILE *out) {
@@ -1250,3 +1262,167 @@ double timespec_to_seconds(struct timespec ts) {
 	return (double)ts.tv_sec
 		+ (double)ts.tv_nsec * 1e-9;
 }
+
+
+String32 str32(char32_t *str, size_t len) {
+	String32 s = {str, len};
+	return s;
+}
+
+String32 str32_substr(String32 s, size_t from, size_t len) {
+	return str32(s.str + from, len);
+}
+
+// frees string and sets it to ""
+void str32_free(String32 *s) {
+	free(s->str);
+	s->str = NULL;
+	s->len = 0;
+}
+
+// the string returned should be str32_free'd.
+// this will return an empty string if the allocation failed or the string is invalid UTF-8
+String32 str32_from_utf8(char const *utf8) {
+	String32 string = {NULL, 0};
+	size_t len = strlen(utf8);
+	if (len) {
+		// the wide string uses at most as many "characters" (elements?) as the UTF-8 string
+		char32_t *widestr = calloc(len, sizeof *widestr);
+		if (widestr) {
+			char32_t *wide_p = widestr;
+			char const *utf8_p = utf8;
+			char const *utf8_end = utf8_p + len;
+			while (utf8_p < utf8_end) {
+				char32_t c = 0;
+				size_t n = unicode_utf8_to_utf32(&c, utf8_p, (size_t)(utf8_end - utf8_p));
+				if (n == 0 // null character. this shouldn't happen.
+					|| n >= (size_t)(-2) // invalid UTF-8
+					) {
+					free(widestr);
+					widestr = wide_p = NULL;
+					break;
+				} else {
+					// n bytes consumed
+					*wide_p++ = c;
+					utf8_p += n;
+				}
+			}
+			string.str = widestr;
+			string.len = (size_t)(wide_p - widestr);
+		}
+	}
+	return string;
+}
+
+// returns a null-terminated UTF-8 string
+// the string returned should be free'd
+// this will return NULL on failure
+char *str32_to_utf8_cstr(String32 s) {
+	char *utf8 = calloc(4 * s.len + 1, 1); // each codepoint takes up at most 4 bytes in UTF-8, + we need a terminating null byte
+	if (utf8) {
+		char *p = utf8;
+		for (size_t i = 0; i < s.len; ++i) {
+			size_t bytes = unicode_utf32_to_utf8(p, s.str[i]);
+			if (bytes == (size_t)-1) {
+				// invalid UTF-32 code point
+				free(utf8);
+				return NULL;
+			} else {
+				p += bytes;
+			}
+		}
+		*p = '\0';
+	}
+	return utf8;
+}
+
+// compare s to the ASCII string `ascii`
+int str32_cmp_ascii(String32 s, char const *ascii) {
+	for (size_t i = 0; i < s.len; ++i) {
+		assert((char32_t)ascii[i] < 128);
+		if ((char32_t)ascii[i] == '\0')
+			return -1; // ascii is a prefix of s
+		if (s.str[i] > (char32_t)ascii[i])
+			return +1;
+		if (s.str[i] < (char32_t)ascii[i])
+			return -1;
+	}
+	if (ascii[s.len]) {
+		// s is a prefix of ascii
+		return +1;
+	}
+	return 0;
+}
+
+// check if s starts with the ASCII string `ascii`
+bool str32_has_ascii_prefix(String32 s, char const *ascii) {
+	for (size_t i = 0; i < s.len; ++i) {
+		assert((char32_t)ascii[i] < 128);
+		if ((char32_t)ascii[i] == '\0')
+			return true; // ascii is a prefix of s
+		if (s.str[i] > (char32_t)ascii[i])
+			return false;
+		if (s.str[i] < (char32_t)ascii[i])
+			return false;
+	}
+	if (ascii[s.len]) {
+		// s is a prefix of ascii
+		return false;
+	}
+	// s is the same as ascii
+	return true;
+}
+
+// returns the index of the given character in the string, or the length of the string if it's not found.
+size_t str32chr(String32 s, char32_t c) {
+	for (size_t i = 0; i < s.len; ++i) {
+		if (s.str[i] == c)
+			return i;
+	}
+	return s.len;
+}
+
+// returns number of instances of c in s
+size_t str32_count_char(String32 s, char32_t c) {
+	size_t total = 0;
+	for (size_t i = 0; i < s.len; ++i) {
+		total += s.str[i] == c;
+	}
+	return total;
+}
+
+// returns number of characters deleted from s
+size_t str32_remove_all_instances_of_char(String32 *s, char32_t c) {
+	char32_t *str = s->str;
+	size_t ndeleted = 0;
+	size_t len = s->len;
+	size_t out = 0;
+	for (size_t in = 0; in < len; ++in) {
+		if (str[in] == c) {
+			++ndeleted;
+		} else {
+			str[out++] = str[in];
+		}
+	}
+	s->len = out;
+	return ndeleted;
+}
+
+// returns the length of the longest prefix of `s` containing only
+// ASCII characters in the C-string `charset`.
+size_t str32_ascii_spn(String32 s, char const *charset) {
+	for (u32 i = 0; i < s.len; ++i) {
+		if (s.str[i] >= 128)
+			return i; // non-ASCII character in s, so that can't be in charset.
+		bool found = false;
+		for (char const *p = charset; *p; ++p) {
+			assert((char32_t)*p < 128);
+			if ((char32_t)*p == s.str[i]) {
+				found = true;
+				break;
+			}
+		}
+		if (!found) return i;
+	}
+	return s.len;
+}
diff --git a/util.h b/util.h
index 5645341..7437c9d 100644
--- a/util.h
+++ b/util.h
@@ -56,11 +56,20 @@ typedef struct {
 	v2 pos, size;
 } Rect;
 
+// UTF-32 string
+typedef struct {
+	char32_t *str;
+	size_t len;
+} String32;
+
 
 // ctype functions for 32-bit chars.
-bool is_word(char32_t c);
-bool is_digit(char32_t c);
-bool is_space(char32_t c);
+bool is32_word(char32_t c);
+bool is32_space(char32_t c);
+bool is32_alpha(char32_t c);
+bool is32_alnum(char32_t c);
+bool is32_digit(char32_t c);
+bool is32_graph(char32_t c);
 bool is_a_tty(FILE *out);
 // terminal colors. if `out` is a TTY, these will return the appropriate escape sequences.
 // if `out` is not a TTY, these will return "".
@@ -253,5 +262,176 @@ int timespec_cmp(struct timespec a, struct timespec b);
 bool timespec_eq(struct timespec a, struct timespec b);
 struct timespec timespec_max(struct timespec a, struct timespec b);
 double timespec_to_seconds(struct timespec ts);
+bool is32_word(char32_t c);
+bool is32_space(char32_t c);
+bool is32_alpha(char32_t c);
+bool is32_alnum(char32_t c);
+bool is32_digit(char32_t c);
+bool is32_graph(char32_t c);
+bool is_a_tty(FILE *out);
+const char *term_italics(FILE *out);
+const char *term_bold(FILE *out);
+const char *term_yellow(FILE *out);
+const char *term_clear(FILE *out);
+u8 util_popcount(u64 x);
+u8 util_count_leading_zeroes32(u32 x);
+bool util_is_power_of_2(u64 x);
+char32_t *util_mem32chr(char32_t *s, char32_t c, size_t n);
+const char32_t *util_mem32chr_const(const char32_t *s, char32_t c, size_t n);
+bool str_has_prefix(const char *str, const char *prefix);
+bool str_has_path_prefix(const char *path, const char *prefix);
+bool streq(const char *a, const char *b);
+size_t strn_len(const char *src, size_t n);
+char *strn_dup(const char *src, size_t n);
+char *str_dup(const char *src);
+void strn_cat(char *dst, size_t dst_sz, const char *src, size_t src_len);
+void str_cat(char *dst, size_t dst_sz, const char *src);
+void strn_cpy(char *dst, size_t dst_sz, const char *src, size_t src_len);
+void str_cpy(char *dst, size_t dst_sz, const char *src);
+char *a_sprintf(const char *fmt, ...);
+char *strstr_case_insensitive(const char *haystack, const char *needle);
+void print_bytes(const u8 *bytes, size_t n);
+int strcmp_case_insensitive(const char *a, const char *b);
+int str_qsort_case_insensitive_cmp(const void *av, const void *bv);
+int qsort_with_context_cmp(const void *a, const void *b, void *context);
+const char *path_filename(const char *path);
+bool path_is_absolute(const char *path);
+void path_full(const char *dir, const char *relpath, char *abspath, size_t abspath_size);
+bool paths_eq(const char *path1, const char *path2);
+void change_directory(const char *path);
+bool copy_file(const char *src, const char *dst);
+float degrees(float r);
+float radians(float r);
+float lerpf(float x, float a, float b);
+float normf(float x, float a, float b);
+float clampf(float x, float a, float b);
+int clampi(int x, int a, int b);
+i16 clamp_i16(i16 x, i16 a, i16 b);
+u16 clamp_u16(u16 x, u16 a, u16 b);
+i32 clamp_i32(i32 x, i32 a, i32 b);
+u32 clamp_u32(u32 x, u32 a, u32 b);
+u8 ndigits_u64(u64 x);
+float remapf(float x, float from_a, float from_b, float to_a, float to_b);
+float minf(float a, float b);
+float maxf(float a, float b);
+double maxd(double a, double b);
+double mind(double a, double b);
+u32 min_u32(u32 a, u32 b);
+u32 max_u32(u32 a, u32 b);
+void sort2_u32(u32 *a, u32 *b);
+i32 min_i32(i32 a, i32 b);
+i32 max_i32(i32 a, i32 b);
+u64 min_u64(u64 a, u64 b);
+u64 max_u64(u64 a, u64 b);
+i64 min_i64(i64 a, i64 b);
+i64 max_i64(i64 a, i64 b);
+i64 mod_i64(i64 a, i64 b);
+i32 mod_i32(i32 a, i32 b);
+i64 abs_i64(i64 x);
+i64 sgn_i64(i64 x);
+float sgnf(float x);
+float smoothstepf(float x);
+float randf(void);
+float rand_gauss(void);
+u32 rand_u32(void);
+float rand_uniform(float from, float to);
+float sigmoidf(float x);
+i32 ceildivi32(i32 x, i32 y);
+v2 V2(float x, float y);
+v2 v2_add(v2 a, v2 b);
+v2 v2_add_const(v2 a, float c);
+v2 v2_sub(v2 a, v2 b);
+v2 v2_scale(v2 v, float s);
+v2 v2_mul(v2 a, v2 b);
+v2 v2_clamp(v2 x, v2 a, v2 b);
+float v2_dot(v2 a, v2 b);
+float v2_len(v2 v);
+v2 v2_lerp(float x, v2 a, v2 b);
+v2 v2_rotate(v2 v, float theta);
+v2 v2_normalize(v2 v);
+float v2_dist(v2 a, v2 b);
+float v2_dist_squared(v2 a, v2 b);
+void v2_print(v2 v);
+v2 v2_rand_unit(void);
+v2 v2_polar(float r, float theta);
+v3 V3(float x, float y, float z);
+v3 v3_from_v2(v2 v);
+v3 v3_add(v3 a, v3 b);
+v3 v3_sub(v3 a, v3 b);
+v3 v3_scale(v3 v, float s);
+v3 v3_lerp(float x, v3 a, v3 b);
+float v3_dot(v3 u, v3 v);
+v3 v3_cross(v3 u, v3 v);
+float v3_len(v3 v);
+float v3_dist(v3 a, v3 b);
+float v3_dist_squared(v3 a, v3 b);
+v3 v3_normalize(v3 v);
+v2 v3_xy(v3 v);
+v3 v3_on_sphere(float yaw, float pitch);
+void v3_print(v3 v);
+v3 v3_rand(void);
+v3 v3_rand_unit(void);
+v4 V4(float x, float y, float z, float w);
+v4 v4_add(v4 a, v4 b);
+v4 v4_sub(v4 a, v4 b);
+v4 v4_scale(v4 v, float s);
+v4 v4_scale_xyz(v4 v, float s);
+v4 v4_lerp(float x, v4 a, v4 b);
+float v4_dot(v4 u, v4 v);
+v4 v4_mul(v4 u, v4 v);
+float v4_len(v4 v);
+v4 v4_normalize(v4 v);
+v3 v4_xyz(v4 v);
+v4 v4_rand(void);
+void v4_print(v4 v);
+v2d V2D(double x, double y);
+void m4_print(m4 m);
+m4 m4_yaw(float yaw);
+m4 m4_pitch(float pitch);
+m4 m4_translate(v3 t);
+v3 m4_mul_v3(m4 m, v3 v);
+m4 m4_perspective(float fov, float aspect, float z_near, float z_far);
+m4 m4_ortho(float left, float right, float bottom, float top, float near_, float far_);
+m4 m4_mul(m4 a, m4 b);
+m4 m4_inv(m4 mat);
+v2i V2I(int x, int y);
+void rgba_u32_to_floats(u32 rgba, float floats[4]);
+v4 rgba_u32_to_v4(u32 rgba);
+u32 rgba_v4_to_u32(v4 color);
+float rgba_brightness(u32 color);
+bool rect_contains_point_v2(v2 pos, v2 size, v2 point);
+bool centered_rect_contains_point(v2 center, v2 size, v2 point);
+Rect rect(v2 pos, v2 size);
+Rect rect_endpoints(v2 e1, v2 e2);
+Rect rect4(float x1, float y1, float x2, float y2);
+Rect rect_xywh(float x, float y, float w, float h);
+Rect rect_centered(v2 center, v2 size);
+v2 rect_center(Rect r);
+bool rect_contains_point(Rect r, v2 point);
+Rect rect_translate(Rect r, v2 by);
+void rect_coords(Rect r, float *x1, float *y1, float *x2, float *y2);
+void rect_print(Rect r);
+float rects_intersect(Rect r1, Rect r2);
+bool rect_clip_to_rect(Rect *clipped, Rect clipper);
+Rect rect_shrink(Rect r, float amount);
+Rect rect_grow(Rect r, float amount);
+v4 color_rgba_to_hsva(v4 rgba);
+v4 color_hsva_to_rgba(v4 hsva);
+u32 color_interpolate(float x, u32 color1, u32 color2);
+int timespec_cmp(struct timespec a, struct timespec b);
+bool timespec_eq(struct timespec a, struct timespec b);
+struct timespec timespec_max(struct timespec a, struct timespec b);
+double timespec_to_seconds(struct timespec ts);
+String32 str32(char32_t *str, size_t len);
+String32 str32_substr(String32 s, size_t from, size_t len);
+void str32_free(String32 *s);
+String32 str32_from_utf8(char const *utf8);
+char *str32_to_utf8_cstr(String32 s);
+int str32_cmp_ascii(String32 s, char const *ascii);
+bool str32_has_ascii_prefix(String32 s, char const *ascii);
+size_t str32chr(String32 s, char32_t c);
+size_t str32_count_char(String32 s, char32_t c);
+size_t str32_remove_all_instances_of_char(String32 *s, char32_t c);
+size_t str32_ascii_spn(String32 s, char const *charset);
 
 #endif // UTIL_H_