From a08a23885d1c80b19680b907357aaf6a0faf6d62 Mon Sep 17 00:00:00 2001 From: Linnnus Date: Fri, 11 Apr 2025 23:27:48 +0000 Subject: fix(core): Tokenizer doesn't assume input is NUL-terminated --- src/core/tokenizer.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/core/tokenizer.c') diff --git a/src/core/tokenizer.c b/src/core/tokenizer.c index 2fc366b..3fc26c0 100644 --- a/src/core/tokenizer.c +++ b/src/core/tokenizer.c @@ -7,6 +7,8 @@ SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, const char *filename) { return (SandTokenizer) { .filename = filename, + .source = source, + .source_length = source_length, .start = source, .start_line = 0, @@ -19,8 +21,7 @@ SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, co } static bool is_at_end(const SandTokenizer *tokenizer) { - // FIXME: This seems bad. In principle the NUL character should not be special. We're leaking our implementation language. - return *tokenizer->current == '\0'; + return tokenizer->current == tokenizer->source + tokenizer->source_length; } static SandToken make_token(const SandTokenizer *tokenizer, SandTokenKind kind) { -- cgit v1.2.3