diff options
Diffstat (limited to 'src/core/tokenizer.c')
| -rw-r--r-- | src/core/tokenizer.c | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/src/core/tokenizer.c b/src/core/tokenizer.c index 2fc366b..3fc26c0 100644 --- a/src/core/tokenizer.c +++ b/src/core/tokenizer.c @@ -7,6 +7,8 @@ SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, const char *filename) { return (SandTokenizer) { .filename = filename, + .source = source, + .source_length = source_length, .start = source, .start_line = 0, @@ -19,8 +21,7 @@ SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, co } static bool is_at_end(const SandTokenizer *tokenizer) { - // FIXME: This seems bad. In principle the NUL character should not be special. We're leaking our implementation language. - return *tokenizer->current == '\0'; + return tokenizer->current == tokenizer->source + tokenizer->source_length; } static SandToken make_token(const SandTokenizer *tokenizer, SandTokenKind kind) { |