X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Flexer%2Flexer.c;h=6d3549a82782134f62c8603e76ac207f54fe6fef;hb=cd221d80fafb54e550398c6de105d4c1b7f02ba0;hp=9a27d867b98cb87e15420671819ab24ccd1ca1fa;hpb=9ade26c8349b4434008c46cf09bc7473ec743972;p=pspp diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c index 9a27d867b9..6d3549a827 100644 --- a/src/language/lexer/lexer.c +++ b/src/language/lexer/lexer.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 1997-9, 2000, 2006, 2009, 2010, 2011 Free Software Foundation, Inc. + Copyright (C) 1997-9, 2000, 2006, 2009, 2010, 2011, 2013, 2016 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -30,7 +30,6 @@ #include #include -#include "data/file-name.h" #include "language/command.h" #include "language/lexer/scan.h" #include "language/lexer/segment.h" @@ -45,7 +44,7 @@ #include "libpspp/str.h" #include "libpspp/u8-istream.h" #include "output/journal.h" -#include "output/text-item.h" +#include "output/output-item.h" #include "gl/c-ctype.h" #include "gl/minmax.h" @@ -129,9 +128,11 @@ lex_reader_init (struct lex_reader *reader, { reader->class = class; reader->syntax = LEX_SYNTAX_AUTO; - reader->error = LEX_ERROR_INTERACTIVE; + reader->error = LEX_ERROR_CONTINUE; reader->file_name = NULL; + reader->encoding = NULL; reader->line_number = 0; + reader->eof = false; } /* Frees any file name already in READER and replaces it by a copy of @@ -140,7 +141,7 @@ void lex_reader_set_file_name (struct lex_reader *reader, const char *file_name) { free (reader->file_name); - reader->file_name = file_name != NULL ? xstrdup (file_name) : NULL; + reader->file_name = xstrdup_if_nonnull (file_name); } /* Creates and returns a new lexer. */ @@ -184,7 +185,7 @@ lex_append (struct lexer *lexer, struct lex_reader *reader) ll_push_tail (&lexer->sources, &lex_source_create (reader)->ll); } -/* Advacning. */ +/* Advancing. */ static struct lex_token * lex_push_token__ (struct lex_source *src) @@ -268,20 +269,133 @@ lex_next_error (struct lexer *lexer, int n0, int n1, const char *format, ...) va_end (args); } -/* Reports an error to the effect that subcommand SBC may only be - specified once. */ +/* Prints a syntax error message saying that one of the strings provided as + varargs, up to the first NULL, is expected. */ +void +(lex_error_expecting) (struct lexer *lexer, ...) +{ + va_list args; + + va_start (args, lexer); + lex_error_expecting_valist (lexer, args); + va_end (args); +} + +/* Prints a syntax error message saying that one of the options provided in + ARGS, up to the first NULL, is expected. */ +void +lex_error_expecting_valist (struct lexer *lexer, va_list args) +{ + enum { MAX_OPTIONS = 9 }; + const char *options[MAX_OPTIONS]; + int n = 0; + while (n < MAX_OPTIONS) + { + const char *option = va_arg (args, const char *); + if (!option) + break; + + options[n++] = option; + } + lex_error_expecting_array (lexer, options, n); +} + +void +lex_error_expecting_array (struct lexer *lexer, const char **options, size_t n) +{ + switch (n) + { + case 0: + lex_error (lexer, NULL); + break; + + case 1: + lex_error (lexer, _("expecting %s"), options[0]); + break; + + case 2: + lex_error (lexer, _("expecting %s or %s"), options[0], options[1]); + break; + + case 3: + lex_error (lexer, _("expecting %s, %s, or %s"), options[0], options[1], + options[2]); + break; + + case 4: + lex_error (lexer, _("expecting %s, %s, %s, or %s"), + options[0], options[1], options[2], options[3]); + break; + + case 5: + lex_error (lexer, _("expecting %s, %s, %s, %s, or %s"), + options[0], options[1], options[2], options[3], options[4]); + break; + + case 6: + lex_error (lexer, _("expecting %s, %s, %s, %s, %s, or %s"), + options[0], options[1], options[2], options[3], options[4], + options[5]); + break; + + case 7: + lex_error (lexer, _("expecting %s, %s, %s, %s, %s, %s, or %s"), + options[0], options[1], options[2], options[3], options[4], + options[5], options[6]); + break; + + case 8: + lex_error (lexer, _("expecting %s, %s, %s, %s, %s, %s, %s, or %s"), + options[0], options[1], options[2], options[3], options[4], + options[5], options[6], options[7]); + break; + + default: + lex_error (lexer, NULL); + } +} + +/* Reports an error to the effect that subcommand SBC may only be specified + once. + + This function does not take a lexer as an argument or use lex_error(), + because the result would ordinarily just be redundant: "Syntax error at + SUBCOMMAND: Subcommand SUBCOMMAND may only be specified once.", which does + not help the user find the error. */ void lex_sbc_only_once (const char *sbc) { msg (SE, _("Subcommand %s may only be specified once."), sbc); } -/* Reports an error to the effect that subcommand SBC is - missing. */ +/* Reports an error to the effect that subcommand SBC is missing. + + This function does not take a lexer as an argument or use lex_error(), + because a missing subcommand can normally be detected only after the whole + command has been parsed, and so lex_error() would always report "Syntax + error at end of command", which does not help the user find the error. */ void -lex_sbc_missing (struct lexer *lexer, const char *sbc) +lex_sbc_missing (const char *sbc) { - lex_error (lexer, _("missing required subcommand %s"), sbc); + msg (SE, _("Required subcommand %s was not specified."), sbc); +} + +/* Reports an error to the effect that specification SPEC may only be specified + once within subcommand SBC. */ +void +lex_spec_only_once (struct lexer *lexer, const char *sbc, const char *spec) +{ + lex_error (lexer, _("%s may only be specified once within subcommand %s"), + spec, sbc); +} + +/* Reports an error to the effect that specification SPEC is missing within + subcommand SBC. */ +void +lex_spec_missing (struct lexer *lexer, const char *sbc, const char *spec) +{ + lex_error (lexer, _("Required %s specification missing from %s subcommand"), + sbc, spec); } /* Prints a syntax error message containing the current token and @@ -331,14 +445,14 @@ lex_end_of_command (struct lexer *lexer) /* Returns true if the current token is a number. */ bool -lex_is_number (struct lexer *lexer) +lex_is_number (const struct lexer *lexer) { return lex_next_is_number (lexer, 0); } /* Returns true if the current token is a string. */ bool -lex_is_string (struct lexer *lexer) +lex_is_string (const struct lexer *lexer) { return lex_next_is_string (lexer, 0); } @@ -346,14 +460,14 @@ lex_is_string (struct lexer *lexer) /* Returns the value of the current token, which must be a floating point number. */ double -lex_number (struct lexer *lexer) +lex_number (const struct lexer *lexer) { return lex_next_number (lexer, 0); } /* Returns true iff the current token is an integer. */ bool -lex_is_integer (struct lexer *lexer) +lex_is_integer (const struct lexer *lexer) { return lex_next_is_integer (lexer, 0); } @@ -361,7 +475,7 @@ lex_is_integer (struct lexer *lexer) /* Returns the value of the current token, which must be an integer. */ long -lex_integer (struct lexer *lexer) +lex_integer (const struct lexer *lexer) { return lex_next_integer (lexer, 0); } @@ -375,7 +489,7 @@ lex_integer (struct lexer *lexer) /* Returns true if the token N ahead of the current token is a number. */ bool -lex_next_is_number (struct lexer *lexer, int n) +lex_next_is_number (const struct lexer *lexer, int n) { enum token_type next_token = lex_next_token (lexer, n); return next_token == T_POS_NUM || next_token == T_NEG_NUM; @@ -383,7 +497,7 @@ lex_next_is_number (struct lexer *lexer, int n) /* Returns true if the token N ahead of the current token is a string. */ bool -lex_next_is_string (struct lexer *lexer, int n) +lex_next_is_string (const struct lexer *lexer, int n) { return lex_next_token (lexer, n) == T_STRING; } @@ -391,7 +505,7 @@ lex_next_is_string (struct lexer *lexer, int n) /* Returns the value of the token N ahead of the current token, which must be a floating point number. */ double -lex_next_number (struct lexer *lexer, int n) +lex_next_number (const struct lexer *lexer, int n) { assert (lex_next_is_number (lexer, n)); return lex_next_tokval (lexer, n); @@ -399,7 +513,7 @@ lex_next_number (struct lexer *lexer, int n) /* Returns true if the token N ahead of the current token is an integer. */ bool -lex_next_is_integer (struct lexer *lexer, int n) +lex_next_is_integer (const struct lexer *lexer, int n) { double value; @@ -413,7 +527,7 @@ lex_next_is_integer (struct lexer *lexer, int n) /* Returns the value of the token N ahead of the current token, which must be an integer. */ long -lex_next_integer (struct lexer *lexer, int n) +lex_next_integer (const struct lexer *lexer, int n) { assert (lex_next_is_integer (lexer, n)); return lex_next_tokval (lexer, n); @@ -491,7 +605,7 @@ lex_force_match_id (struct lexer *lexer, const char *identifier) return true; else { - lex_error (lexer, _("expecting `%s'"), identifier); + lex_error_expecting (lexer, identifier); return false; } } @@ -508,7 +622,16 @@ lex_force_match (struct lexer *lexer, enum token_type type) } else { - lex_error (lexer, _("expecting `%s'"), token_type_to_string (type)); + const char *type_string = token_type_to_string (type); + if (type_string) + { + char *s = xasprintf ("`%s'", type_string); + lex_error_expecting (lexer, s); + free (s); + } + else + lex_error_expecting (lexer, token_type_to_name (type)); + return false; } } @@ -527,6 +650,21 @@ lex_force_string (struct lexer *lexer) } } +/* If the current token is a string or an identifier, does nothing and returns + true. Otherwise, reports an error and returns false. + + This is meant for use in syntactic situations where we want to encourage the + user to supply a quoted string, but for compatibility we also accept + identifiers. (One example of such a situation is file names.) Therefore, + the error message issued when the current token is wrong only says that a + string is expected and doesn't mention that an identifier would also be + accepted. */ +bool +lex_force_string_or_id (struct lexer *lexer) +{ + return lex_token (lexer) == T_ID || lex_force_string (lexer); +} + /* If the current token is an integer, does nothing and returns true. Otherwise, reports an error and returns false. */ bool @@ -541,6 +679,96 @@ lex_force_int (struct lexer *lexer) } } +/* If the current token is an integer in the range MIN...MAX (inclusive), does + nothing and returns true. Otherwise, reports an error and returns false. + If NAME is nonnull, then it is used in the error message. */ +bool +lex_force_int_range (struct lexer *lexer, const char *name, long min, long max) +{ + bool is_integer = lex_is_integer (lexer); + bool too_small = is_integer && lex_integer (lexer) < min; + bool too_big = is_integer && lex_integer (lexer) > max; + if (is_integer && !too_small && !too_big) + return true; + + if (min > max) + { + /* Weird, maybe a bug in the caller. Just report that we needed an + integer. */ + if (name) + lex_error (lexer, _("Integer expected for %s."), name); + else + lex_error (lexer, _("Integer expected.")); + } + else if (min == max) + { + if (name) + lex_error (lexer, _("Expected %ld for %s."), min, name); + else + lex_error (lexer, _("Expected %ld."), min); + } + else if (min + 1 == max) + { + if (name) + lex_error (lexer, _("Expected %ld or %ld for %s."), min, min + 1, name); + else + lex_error (lexer, _("Expected %ld or %ld."), min, min + 1); + } + else + { + bool report_lower_bound = (min > INT_MIN / 2) || too_small; + bool report_upper_bound = (max < INT_MAX / 2) || too_big; + + if (report_lower_bound && report_upper_bound) + { + if (name) + lex_error (lexer, + _("Expected integer between %ld and %ld for %s."), + min, max, name); + else + lex_error (lexer, _("Expected integer between %ld and %ld."), + min, max); + } + else if (report_lower_bound) + { + if (min == 0) + { + if (name) + lex_error (lexer, _("Expected non-negative integer for %s."), + name); + else + lex_error (lexer, _("Expected non-negative integer.")); + } + else if (min == 1) + { + if (name) + lex_error (lexer, _("Expected positive integer for %s."), + name); + else + lex_error (lexer, _("Expected positive integer.")); + } + } + else if (report_upper_bound) + { + if (name) + lex_error (lexer, + _("Expected integer less than or equal to %ld for %s."), + max, name); + else + lex_error (lexer, _("Expected integer less than or equal to %ld."), + max); + } + else + { + if (name) + lex_error (lexer, _("Integer expected for %s."), name); + else + lex_error (lexer, _("Integer expected.")); + } + } + return false; +} + /* If the current token is a number, does nothing and returns true. Otherwise, reports an error and returns false. */ bool @@ -717,99 +945,58 @@ lex_next_tokss (const struct lexer *lexer, int n) return lex_next (lexer, n)->string; } -/* If LEXER is positioned at the (pseudo)identifier S, skips it and returns - true. Otherwise, returns false. - - S may consist of an arbitrary number of identifiers, integers, and - punctuation e.g. "KRUSKAL-WALLIS", "2SLS", or "END INPUT PROGRAM". - Identifiers may be abbreviated to their first three letters. Currently only - hyphens, slashes, and equals signs are supported as punctuation (but it - would be easy to add more). - - S must be an ASCII string. */ -bool -lex_match_phrase (struct lexer *lexer, const char *s) +static bool +lex_tokens_match (const struct token *actual, const struct token *expected) { - int tok_idx; + if (actual->type != expected->type) + return false; - for (tok_idx = 0; ; tok_idx++) + switch (actual->type) { - enum token_type token; - unsigned char c; + case T_POS_NUM: + case T_NEG_NUM: + return actual->number == expected->number; - while (c_isspace (*s)) - s++; + case T_ID: + return lex_id_match (expected->string, actual->string); - c = *s; - if (c == '\0') - { - int i; - - for (i = 0; i < tok_idx; i++) - lex_get (lexer); - return true; - } + case T_STRING: + return (actual->string.length == expected->string.length + && !memcmp (actual->string.string, expected->string.string, + actual->string.length)); - token = lex_next_token (lexer, tok_idx); - switch (c) - { - case '-': - if (token != T_DASH) - return false; - s++; - break; - - case '/': - if (token != T_SLASH) - return false; - s++; - break; - - case '=': - if (token != T_EQUALS) - return false; - s++; - break; - - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - { - unsigned int value; - - if (token != T_POS_NUM) - return false; - - value = 0; - do - { - value = value * 10 + (*s++ - '0'); - } - while (c_isdigit (*s)); - - if (lex_next_tokval (lexer, tok_idx) != value) - return false; - } - break; + default: + return true; + } +} - default: - if (lex_is_id1 (c)) - { - int len; +/* If LEXER is positioned at the sequence of tokens that may be parsed from S, + skips it and returns true. Otherwise, returns false. - if (token != T_ID) - return false; + S may consist of an arbitrary sequence of tokens, e.g. "KRUSKAL-WALLIS", + "2SLS", or "END INPUT PROGRAM". Identifiers may be abbreviated to their + first three letters. */ +bool +lex_match_phrase (struct lexer *lexer, const char *s) +{ + struct string_lexer slex; + struct token token; + int i; - len = lex_id_get_length (ss_cstr (s)); - if (!lex_id_match (ss_buffer (s, len), - lex_next_tokss (lexer, tok_idx))) - return false; + i = 0; + string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE); + while (string_lexer_next (&slex, &token)) + if (token.type != SCAN_SKIP) + { + bool match = lex_tokens_match (lex_next (lexer, i++), &token); + token_destroy (&token); + if (!match) + return false; + } - s += len; - } - else - NOT_REACHED (); - } - } + while (i-- > 0) + lex_get (lexer); + return true; } static int @@ -966,6 +1153,14 @@ lex_get_file_name (const struct lexer *lexer) return src == NULL ? NULL : src->reader->file_name; } +const char * +lex_get_encoding (const struct lexer *lexer) +{ + struct lex_source *src = lex_source__ (lexer); + return src == NULL ? NULL : src->reader->encoding; +} + + /* Returns the syntax mode for the syntax file from which the current drawn is drawn. Returns LEX_SYNTAX_AUTO for a T_STOP token or if the command's source does not have line numbers. @@ -981,7 +1176,7 @@ lex_get_syntax_mode (const struct lexer *lexer) } /* Returns the error mode for the syntax file from which the current drawn is - drawn. Returns LEX_ERROR_INTERACTIVE for a T_STOP token or if the command's + drawn. Returns LEX_ERROR_TERMINAL for a T_STOP token or if the command's source does not have line numbers. There is no version of this function that takes an N argument because @@ -991,13 +1186,12 @@ enum lex_error_mode lex_get_error_mode (const struct lexer *lexer) { struct lex_source *src = lex_source__ (lexer); - return src == NULL ? LEX_ERROR_INTERACTIVE : src->reader->error; + return src == NULL ? LEX_ERROR_TERMINAL : src->reader->error; } /* If the source that LEXER is currently reading has error mode - LEX_ERROR_INTERACTIVE, discards all buffered input and tokens, so that the - next token to be read comes directly from whatever is next read from the - stream. + LEX_ERROR_TERMINAL, discards all buffered input and tokens, so that the next + token to be read comes directly from whatever is next read from the stream. It makes sense to call this function after encountering an error in a command entered on the console, because usually the user would prefer not to @@ -1006,7 +1200,7 @@ void lex_interactive_reset (struct lexer *lexer) { struct lex_source *src = lex_source__ (lexer); - if (src != NULL && src->reader->error == LEX_ERROR_INTERACTIVE) + if (src != NULL && src->reader->error == LEX_ERROR_TERMINAL) { src->head = src->tail = 0; src->journal_pos = src->seg_pos = src->line_pos = 0; @@ -1028,7 +1222,7 @@ lex_discard_rest_of_command (struct lexer *lexer) } /* Discards all lookahead tokens in LEXER, then discards all input sources - until it encounters one with error mode LEX_ERROR_INTERACTIVE or until it + until it encounters one with error mode LEX_ERROR_TERMINAL or until it runs out of input sources. */ void lex_discard_noninteractive (struct lexer *lexer) @@ -1040,7 +1234,7 @@ lex_discard_noninteractive (struct lexer *lexer) while (!deque_is_empty (&src->deque)) lex_source_pop__ (src); - for (; src != NULL && src->reader->error != LEX_ERROR_INTERACTIVE; + for (; src != NULL && src->reader->error != LEX_ERROR_TERMINAL; src = lex_source__ (lexer)) lex_source_destroy (src); } @@ -1095,29 +1289,20 @@ lex_source_read__ (struct lex_source *src) { do { - size_t head_ofs; - size_t n; - lex_source_expand__ (src); - head_ofs = src->head - src->tail; - n = src->reader->class->read (src->reader, &src->buffer[head_ofs], - src->allocated - head_ofs, - segmenter_get_prompt (&src->segmenter)); + size_t head_ofs = src->head - src->tail; + size_t space = src->allocated - head_ofs; + enum prompt_style prompt = segmenter_get_prompt (&src->segmenter); + size_t n = src->reader->class->read (src->reader, &src->buffer[head_ofs], + space, prompt); + assert (n <= space); + if (n == 0) { - /* End of input. - - Ensure that the input always ends in a new-line followed by a null - byte, as required by the segmenter library. */ - - if (src->head == src->tail - || src->buffer[src->head - src->tail - 1] != '\n') - src->buffer[src->head++ - src->tail] = '\n'; - + /* End of input. */ + src->reader->eof = true; lex_source_expand__ (src); - src->buffer[src->head++ - src->tail] = '\0'; - return; } @@ -1153,10 +1338,14 @@ lex_ellipsize__ (struct substring in, char *out, size_t out_size) int mblen; assert (out_size >= 16); - out_maxlen = out_size - (in.length >= out_size ? 3 : 0) - 1; + out_maxlen = out_size - 1; + if (in.length > out_maxlen - 3) + out_maxlen -= 3; + for (out_len = 0; out_len < in.length; out_len += mblen) { if (in.string[out_len] == '\n' + || in.string[out_len] == '\0' || (in.string[out_len] == '\r' && out_len + 1 < in.length && in.string[out_len + 1] == '\n')) @@ -1164,6 +1353,10 @@ lex_ellipsize__ (struct substring in, char *out, size_t out_size) mblen = u8_mblen (CHAR_CAST (const uint8_t *, in.string + out_len), in.length - out_len); + + if (mblen < 0) + break; + if (out_len + mblen > out_maxlen) break; } @@ -1178,7 +1371,6 @@ lex_source_error_valist (struct lex_source *src, int n0, int n1, { const struct lex_token *token; struct string s; - struct msg m; ds_init_empty (&s); @@ -1204,16 +1396,19 @@ lex_source_error_valist (struct lex_source *src, int n0, int n1, ds_put_cstr (&s, ": "); ds_put_vformat (&s, format, args); } - ds_put_byte (&s, '.'); - - m.category = MSG_C_SYNTAX; - m.severity = MSG_S_ERROR; - m.file_name = src->reader->file_name; - m.first_line = lex_source_get_first_line_number (src, n0); - m.last_line = lex_source_get_last_line_number (src, n1); - m.first_column = lex_source_get_first_column (src, n0); - m.last_column = lex_source_get_last_column (src, n1); - m.text = ds_steal_cstr (&s); + if (ds_last (&s) != '.') + ds_put_byte (&s, '.'); + + struct msg m = { + .category = MSG_C_SYNTAX, + .severity = MSG_S_ERROR, + .file_name = src->reader->file_name, + .first_line = lex_source_get_first_line_number (src, n0), + .last_line = lex_source_get_last_line_number (src, n1), + .first_column = lex_source_get_first_column (src, n0), + .last_column = lex_source_get_last_column (src, n1), + .text = ds_steal_cstr (&s), + }; msg_emit (&m); } @@ -1232,37 +1427,44 @@ lex_get_error (struct lex_source *src, const char *format, ...) va_end (args); } +/* Attempts to append an additional token into SRC's deque, reading more from + the underlying lex_reader if necessary.. Returns true if successful, false + if the deque already represents (a suffix of) the whole lex_reader's + contents, */ static bool lex_source_get__ (const struct lex_source *src_) { struct lex_source *src = CONST_CAST (struct lex_source *, src_); + if (src->eof) + return false; + /* State maintained while scanning tokens. Usually we only need a single + state, but scanner_push() can return SCAN_SAVE to indicate that the state + needs to be saved and possibly restored later with SCAN_BACK. */ struct state { struct segmenter segmenter; enum segment_type last_segment; - int newlines; + int newlines; /* Number of newlines encountered so far. */ + /* Maintained here so we can update lex_source's similar members when we + finish. */ size_t line_pos; size_t seg_pos; }; - struct state state, saved; - enum scan_result result; - struct scanner scanner; - struct lex_token *token; - int n_lines; - int i; - - if (src->eof) - return false; - - state.segmenter = src->segmenter; - state.newlines = 0; - state.seg_pos = src->seg_pos; - state.line_pos = src->line_pos; - saved = state; + /* Initialize state. */ + struct state state = + { + .segmenter = src->segmenter, + .newlines = 0, + .seg_pos = src->seg_pos, + .line_pos = src->line_pos, + }; + struct state saved = state; - token = lex_push_token__ (src); + /* Append a new token to SRC and initialize it. */ + struct lex_token *token = lex_push_token__ (src); + struct scanner scanner; scanner_init (&scanner, &token->token); token->line_pos = src->line_pos; token->token_pos = src->seg_pos; @@ -1271,22 +1473,25 @@ lex_source_get__ (const struct lex_source *src_) else token->first_line = 0; + /* Extract segments and pass them through the scanner until we obtain a + token. */ for (;;) { + /* Extract a segment. */ + const char *segment = &src->buffer[state.seg_pos - src->tail]; + size_t seg_maxlen = src->head - state.seg_pos; enum segment_type type; - const char *segment; - size_t seg_maxlen; - int seg_len; - - segment = &src->buffer[state.seg_pos - src->tail]; - seg_maxlen = src->head - state.seg_pos; - seg_len = segmenter_push (&state.segmenter, segment, seg_maxlen, &type); + int seg_len = segmenter_push (&state.segmenter, segment, seg_maxlen, + src->reader->eof, &type); if (seg_len < 0) { + /* The segmenter needs more input to produce a segment. */ + assert (!src->reader->eof); lex_source_read__ (src); continue; } + /* Update state based on the segment. */ state.last_segment = type; state.seg_pos += seg_len; if (type == SEG_NEWLINE) @@ -1295,8 +1500,10 @@ lex_source_get__ (const struct lex_source *src_) state.line_pos = state.seg_pos; } - result = scanner_push (&scanner, type, ss_buffer (segment, seg_len), - &token->token); + /* Pass the segment into the scanner and try to get a token out. */ + enum scan_result result = scanner_push (&scanner, type, + ss_buffer (segment, seg_len), + &token->token); if (result == SCAN_SAVE) saved = state; else if (result == SCAN_BACK) @@ -1308,7 +1515,9 @@ lex_source_get__ (const struct lex_source *src_) break; } - n_lines = state.newlines; + /* If we've reached the end of a line, or the end of a command, then pass + the line to the output engine as a syntax text item. */ + int n_lines = state.newlines; if (state.last_segment == SEG_END_COMMAND && !src->suppress_next_newline) { n_lines++; @@ -1319,22 +1528,34 @@ lex_source_get__ (const struct lex_source *src_) n_lines--; src->suppress_next_newline = false; } - for (i = 0; i < n_lines; i++) + for (int i = 0; i < n_lines; i++) { - const char *newline; - const char *line; - size_t line_len; - - line = &src->buffer[src->journal_pos - src->tail]; - newline = rawmemchr (line, '\n'); - line_len = newline - line; - if (line_len > 0 && line[line_len - 1] == '\r') - line_len--; - - text_item_submit (text_item_create_nocopy (TEXT_ITEM_SYNTAX, - xmemdup0 (line, line_len))); - - src->journal_pos += newline - line + 1; + /* Beginning of line. */ + const char *line = &src->buffer[src->journal_pos - src->tail]; + + /* Calculate line length, including \n or \r\n end-of-line if present. + + We use src->head even though that may be beyond what we've actually + converted to tokens (which is only through state.line_pos). That's + because, if we're emitting the line due to SEG_END_COMMAND, we want to + take the whole line through the newline, not just through the '.'. */ + size_t max_len = src->head - src->journal_pos; + const char *newline = memchr (line, '\n', max_len); + size_t line_len = newline ? newline - line + 1 : max_len; + + /* Calculate line length excluding end-of-line. */ + size_t copy_len = line_len; + if (copy_len > 0 && line[copy_len - 1] == '\n') + copy_len--; + if (copy_len > 0 && line[copy_len - 1] == '\r') + copy_len--; + + /* Submit the line as syntax. */ + output_item_submit (text_item_create_nocopy (TEXT_ITEM_SYNTAX, + xmemdup0 (line, copy_len), + NULL)); + + src->journal_pos += line_len; } token->token_len = state.seg_pos - src->seg_pos; @@ -1447,9 +1668,11 @@ static void lex_source_destroy (struct lex_source *src) { char *file_name = src->reader->file_name; - if (src->reader->class->close != NULL) - src->reader->class->close (src->reader); + char *encoding = src->reader->encoding; + if (src->reader->class->destroy != NULL) + src->reader->class->destroy (src->reader); free (file_name); + free (encoding); free (src->buffer); while (!deque_is_empty (&src->deque)) lex_source_pop__ (src); @@ -1462,7 +1685,6 @@ struct lex_file_reader { struct lex_reader reader; struct u8_istream *istream; - char *file_name; }; static struct lex_reader_class lex_file_reader_class; @@ -1496,9 +1718,9 @@ lex_reader_for_file (const char *file_name, const char *encoding, r->reader.syntax = syntax; r->reader.error = error; r->reader.file_name = xstrdup (file_name); + r->reader.encoding = xstrdup_if_nonnull (encoding); r->reader.line_number = 1; r->istream = istream; - r->file_name = xstrdup (file_name); return &r->reader; } @@ -1517,7 +1739,7 @@ lex_file_read (struct lex_reader *r_, char *buf, size_t n, ssize_t n_read = u8_istream_read (r->istream, buf, n); if (n_read < 0) { - msg (ME, _("Error reading `%s': %s."), r->file_name, strerror (errno)); + msg (ME, _("Error reading `%s': %s."), r_->file_name, strerror (errno)); return 0; } return n_read; @@ -1531,12 +1753,11 @@ lex_file_close (struct lex_reader *r_) if (u8_istream_fileno (r->istream) != STDIN_FILENO) { if (u8_istream_close (r->istream) != 0) - msg (ME, _("Error closing `%s': %s."), r->file_name, strerror (errno)); + msg (ME, _("Error closing `%s': %s."), r_->file_name, strerror (errno)); } else u8_istream_free (r->istream); - free (r->file_name); free (r); } @@ -1556,16 +1777,17 @@ struct lex_string_reader static struct lex_reader_class lex_string_reader_class; /* Creates and returns a new lex_reader for the contents of S, which must be - encoded in UTF-8. The new reader takes ownership of S and will free it + encoded in the given ENCODING. The new reader takes ownership of S and will free it with ss_dealloc() when it is closed. */ struct lex_reader * -lex_reader_for_substring_nocopy (struct substring s) +lex_reader_for_substring_nocopy (struct substring s, const char *encoding) { struct lex_string_reader *r; r = xmalloc (sizeof *r); lex_reader_init (&r->reader, &lex_string_reader_class); - r->reader.syntax = LEX_SYNTAX_INTERACTIVE; + r->reader.syntax = LEX_SYNTAX_AUTO; + r->reader.encoding = xstrdup_if_nonnull (encoding); r->s = s; r->offset = 0; @@ -1573,25 +1795,25 @@ lex_reader_for_substring_nocopy (struct substring s) } /* Creates and returns a new lex_reader for a copy of null-terminated string S, - which must be encoded in UTF-8. The caller retains ownership of S. */ + which must be encoded in ENCODING. The caller retains ownership of S. */ struct lex_reader * -lex_reader_for_string (const char *s) +lex_reader_for_string (const char *s, const char *encoding) { struct substring ss; ss_alloc_substring (&ss, ss_cstr (s)); - return lex_reader_for_substring_nocopy (ss); + return lex_reader_for_substring_nocopy (ss, encoding); } /* Formats FORMAT as a printf()-like format string and creates and returns a new lex_reader for the formatted result. */ struct lex_reader * -lex_reader_for_format (const char *format, ...) +lex_reader_for_format (const char *format, const char *encoding, ...) { struct lex_reader *r; va_list args; - va_start (args, format); - r = lex_reader_for_substring_nocopy (ss_cstr (xvasprintf (format, args))); + va_start (args, encoding); + r = lex_reader_for_substring_nocopy (ss_cstr (xvasprintf (format, args)), encoding); va_end (args); return r;