diff options
| -rw-r--r-- | src/core/tokenizer.c | 5 | ||||
| -rw-r--r-- | src/core/tokenizer.h | 2 |
2 files changed, 5 insertions, 2 deletions
diff --git a/src/core/tokenizer.c b/src/core/tokenizer.c index 2fc366b..3fc26c0 100644 --- a/src/core/tokenizer.c +++ b/src/core/tokenizer.c @@ -7,6 +7,8 @@ SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, const char *filename) { return (SandTokenizer) { .filename = filename, + .source = source, + .source_length = source_length, .start = source, .start_line = 0, @@ -19,8 +21,7 @@ SandTokenizer sand_create_tokenizer(const char *source, size_t source_length, co } static bool is_at_end(const SandTokenizer *tokenizer) { - // FIXME: This seems bad. In principle the NUL character should not be special. We're leaking our implementation language. - return *tokenizer->current == '\0'; + return tokenizer->current == tokenizer->source + tokenizer->source_length; } static SandToken make_token(const SandTokenizer *tokenizer, SandTokenKind kind) { diff --git a/src/core/tokenizer.h b/src/core/tokenizer.h index 6105c3e..1c435a9 100644 --- a/src/core/tokenizer.h +++ b/src/core/tokenizer.h @@ -72,6 +72,8 @@ typedef struct { typedef struct { const char *const filename; + const char *const source; + const size_t source_length; const char *start; unsigned start_line; |