- n = segmenter_push (&s, input + offset, length - offset, &type);
- assert (n >= 0);
- assert (offset + n <= length);
+ struct token *tokens = NULL;
+ size_t n_tokens = 0;
+ size_t allocated_tokens = 0;
+ string_lexer_init (&slex, input, length, mode, false);
+ for (;;)
+ {
+ if (n_tokens >= allocated_tokens)
+ tokens = x2nrealloc (tokens, &allocated_tokens, sizeof *tokens);
+ enum string_lexer_result result
+ = string_lexer_next (&slex, &tokens[n_tokens]);