fix(core): Tokenizer doesn't assume input is NUL-terminated

1 files changed, 3 insertions, 2 deletions

diff --git a/src/core/tokenizer.c b/src/core/tokenizer.c
index 2fc366b..3fc26c0 100644
--- a/src/core/tokenizer.c
+++ b/src/core/tokenizer.c
@@ -7,6 +7,8 @@
 SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, const char *filename) {
 	return (SandTokenizer) {
 		.filename = filename,
+		.source = source,
+		.source_length = source_length,
 
 		.start = source,
 		.start_line = 0,
@@ -19,8 +21,7 @@ SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, co
 }
 
 static bool is_at_end(const SandTokenizer *tokenizer) {
-	// FIXME: This seems bad. In principle the NUL character should not be special. We're leaking our implementation language.
-	return *tokenizer->current == '\0';
+	return tokenizer->current == tokenizer->source + tokenizer->source_length;
 }
 
 static SandToken make_token(const SandTokenizer *tokenizer, SandTokenKind kind) {