+ while (deque_count (&src->deque) <= n)
+ {
+ if (!deque_is_empty (&src->deque))
+ {
+ struct lex_token *front;
+
+ front = &src->tokens[deque_front (&src->deque, 0)];
+ if (front->token.type == T_STOP || front->token.type == T_ENDCMD)
+ return front;
+ }
+
+ lex_source_get__ (src);
+ }
+
+ return &src->tokens[deque_back (&src->deque, n)];
+}
+
+/* Returns the "struct token" of the token N after the current one in LEXER.
+ The returned pointer can be invalidated by pretty much any succeeding call
+ into the lexer, although the string pointer within the returned token is
+ only invalidated by consuming the token (e.g. with lex_get()). */
+const struct token *
+lex_next (const struct lexer *lexer, int n)
+{
+ return &lex_next__ (lexer, n)->token;
+}
+
+/* Returns the type of the token N after the current one in LEXER. */
+enum token_type
+lex_next_token (const struct lexer *lexer, int n)
+{
+ return lex_next (lexer, n)->type;
+}
+
+/* Returns the number in the tokn N after the current one in LEXER.
+
+ Only T_NEG_NUM and T_POS_NUM tokens have meaningful values. For other
+ tokens this function will always return zero. */
+double
+lex_next_tokval (const struct lexer *lexer, int n)
+{
+ const struct token *token = lex_next (lexer, n);
+ return token->number;
+}
+
+/* Returns the null-terminated string in the token N after the current one, in
+ UTF-8 encoding.
+
+ Only T_ID and T_STRING tokens have meaningful strings. For other tokens
+ this functions this function will always return NULL.
+
+ The UTF-8 encoding of the returned string is correct for variable names and
+ other identifiers. Use filename_to_utf8() to use it as a filename. Use
+ data_in() to use it in a "union value". */
+const char *
+lex_next_tokcstr (const struct lexer *lexer, int n)
+{
+ return lex_next_tokss (lexer, n).string;
+}
+
+/* Returns the string in the token N after the current one, in UTF-8 encoding.
+ The string is null-terminated (but the null terminator is not included in
+ the returned substring's 'length').
+
+ Only T_ID and T_STRING tokens have meaningful strings. For other tokens
+ this functions this function will always return NULL.
+
+ The UTF-8 encoding of the returned string is correct for variable names and
+ other identifiers. Use filename_to_utf8() to use it as a filename. Use
+ data_in() to use it in a "union value". */
+struct substring
+lex_next_tokss (const struct lexer *lexer, int n)
+{
+ return lex_next (lexer, n)->string;
+}
+
+/* If LEXER is positioned at the (pseudo)identifier S, skips it and returns
+ true. Otherwise, returns false.
+
+ S may consist of an arbitrary number of identifiers, integers, and
+ punctuation e.g. "KRUSKAL-WALLIS", "2SLS", or "END INPUT PROGRAM".
+ Identifiers may be abbreviated to their first three letters. Currently only
+ hyphens, slashes, and equals signs are supported as punctuation (but it
+ would be easy to add more).
+
+ S must be an ASCII string. */
+bool
+lex_match_phrase (struct lexer *lexer, const char *s)
+{
+ int tok_idx;
+
+ for (tok_idx = 0; ; tok_idx++)
+ {
+ enum token_type token;
+ unsigned char c;
+
+ while (c_isspace (*s))
+ s++;
+
+ c = *s;
+ if (c == '\0')
+ {
+ int i;
+
+ for (i = 0; i < tok_idx; i++)
+ lex_get (lexer);
+ return true;
+ }
+
+ token = lex_next_token (lexer, tok_idx);
+ switch (c)
+ {
+ case '-':
+ if (token != T_DASH)
+ return false;
+ s++;
+ break;
+
+ case '/':
+ if (token != T_SLASH)
+ return false;
+ s++;
+ break;
+
+ case '=':
+ if (token != T_EQUALS)
+ return false;
+ s++;
+ break;
+
+ case '0': case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ {
+ unsigned int value;
+
+ if (token != T_POS_NUM)
+ return false;
+
+ value = 0;
+ do
+ {
+ value = value * 10 + (*s++ - '0');
+ }
+ while (c_isdigit (*s));
+
+ if (lex_next_tokval (lexer, tok_idx) != value)
+ return false;
+ }
+ break;
+
+ default:
+ if (lex_is_id1 (c))
+ {
+ int len;
+
+ if (token != T_ID)
+ return false;
+
+ len = lex_id_get_length (ss_cstr (s));
+ if (!lex_id_match (ss_buffer (s, len),
+ lex_next_tokss (lexer, tok_idx)))
+ return false;
+
+ s += len;
+ }
+ else
+ NOT_REACHED ();
+ }
+ }
+}
+
+static int
+lex_source_get_first_line_number (const struct lex_source *src, int n)
+{
+ return lex_source_next__ (src, n)->first_line;
+}
+
+static int
+count_newlines (char *s, size_t length)
+{
+ int n_newlines = 0;
+ char *newline;
+
+ while ((newline = memchr (s, '\n', length)) != NULL)
+ {
+ n_newlines++;
+ length -= (newline + 1) - s;
+ s = newline + 1;
+ }
+
+ return n_newlines;
+}
+
+static int
+lex_source_get_last_line_number (const struct lex_source *src, int n)
+{
+ const struct lex_token *token = lex_source_next__ (src, n);
+
+ if (token->first_line == 0)
+ return 0;