fix(core): Tokenizer doesn't assume input is NUL-terminated

2 files changed, 5 insertions, 2 deletions

diff --git a/src/core/tokenizer.c b/src/core/tokenizer.c
index 2fc366b..3fc26c0 100644
--- a/src/core/tokenizer.c
+++ b/src/core/tokenizer.c
@@ -7,6 +7,8 @@
 SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, const char *filename) {
 	return (SandTokenizer) {
 		.filename = filename,
+		.source = source,
+		.source_length = source_length,
 
 		.start = source,
 		.start_line = 0,
@@ -19,8 +21,7 @@ SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, co
 }
 
 static bool is_at_end(const SandTokenizer *tokenizer) {
-	// FIXME: This seems bad. In principle the NUL character should not be special. We're leaking our implementation language.
-	return *tokenizer->current == '\0';
+	return tokenizer->current == tokenizer->source + tokenizer->source_length;
 }
 
 static SandToken make_token(const SandTokenizer *tokenizer, SandTokenKind kind) {
diff --git a/src/core/tokenizer.h b/src/core/tokenizer.h
index 6105c3e..1c435a9 100644
--- a/src/core/tokenizer.h
+++ b/src/core/tokenizer.h
@@ -72,6 +72,8 @@ typedef struct {
 
 typedef struct {
 	const char *const filename;
+	const char *const source;
+	const size_t source_length;
 
 	const char *start;
 	unsigned start_line;