From a08a23885d1c80b19680b907357aaf6a0faf6d62 Mon Sep 17 00:00:00 2001 From: Linnnus Date: Fri, 11 Apr 2025 23:27:48 +0000 Subject: fix(core): Tokenizer doesn't assume input is NUL-terminated --- src/core/tokenizer.c | 5 +++-- src/core/tokenizer.h | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'src/core') diff --git a/src/core/tokenizer.c b/src/core/tokenizer.c index 2fc366b..3fc26c0 100644 --- a/src/core/tokenizer.c +++ b/src/core/tokenizer.c @@ -7,6 +7,8 @@ SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, const char *filename) { return (SandTokenizer) { .filename = filename, + .source = source, + .source_length = source_length, .start = source, .start_line = 0, @@ -19,8 +21,7 @@ SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, co } static bool is_at_end(const SandTokenizer *tokenizer) { - // FIXME: This seems bad. In principle the NUL character should not be special. We're leaking our implementation language. - return *tokenizer->current == '\0'; + return tokenizer->current == tokenizer->source + tokenizer->source_length; } static SandToken make_token(const SandTokenizer *tokenizer, SandTokenKind kind) { diff --git a/src/core/tokenizer.h b/src/core/tokenizer.h index 6105c3e..1c435a9 100644 --- a/src/core/tokenizer.h +++ b/src/core/tokenizer.h @@ -72,6 +72,8 @@ typedef struct { typedef struct { const char *const filename; + const char *const source; + const size_t source_length; const char *start; unsigned start_line; -- cgit v1.2.3