/* PSPP - a program for statistical analysis.
- Copyright (C) 1997-9, 2000, 2006, 2009, 2010, 2011 Free Software Foundation, Inc.
+ Copyright (C) 1997-9, 2000, 2006, 2009, 2010, 2011, 2013, 2016 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#include <unistr.h>
#include <uniwidth.h>
-#include "data/file-name.h"
#include "language/command.h"
#include "language/lexer/scan.h"
#include "language/lexer/segment.h"
#include "libpspp/str.h"
#include "libpspp/u8-istream.h"
#include "output/journal.h"
-#include "output/text-item.h"
+#include "output/output-item.h"
#include "gl/c-ctype.h"
#include "gl/minmax.h"
{
reader->class = class;
reader->syntax = LEX_SYNTAX_AUTO;
- reader->error = LEX_ERROR_INTERACTIVE;
+ reader->error = LEX_ERROR_CONTINUE;
reader->file_name = NULL;
+ reader->encoding = NULL;
reader->line_number = 0;
+ reader->eof = false;
}
/* Frees any file name already in READER and replaces it by a copy of
lex_reader_set_file_name (struct lex_reader *reader, const char *file_name)
{
free (reader->file_name);
- reader->file_name = file_name != NULL ? xstrdup (file_name) : NULL;
+ reader->file_name = xstrdup_if_nonnull (file_name);
}
\f
/* Creates and returns a new lexer. */
ll_push_tail (&lexer->sources, &lex_source_create (reader)->ll);
}
\f
-/* Advacning. */
+/* Advancing. */
static struct lex_token *
lex_push_token__ (struct lex_source *src)
va_end (args);
}
-/* Reports an error to the effect that subcommand SBC may only be
- specified once. */
+/* Prints a syntax error message saying that one of the strings provided as
+ varargs, up to the first NULL, is expected. */
+void
+(lex_error_expecting) (struct lexer *lexer, ...)
+{
+ va_list args;
+
+ va_start (args, lexer);
+ lex_error_expecting_valist (lexer, args);
+ va_end (args);
+}
+
+/* Prints a syntax error message saying that one of the options provided in
+ ARGS, up to the first NULL, is expected. */
+void
+lex_error_expecting_valist (struct lexer *lexer, va_list args)
+{
+ enum { MAX_OPTIONS = 9 };
+ const char *options[MAX_OPTIONS];
+ int n = 0;
+ while (n < MAX_OPTIONS)
+ {
+ const char *option = va_arg (args, const char *);
+ if (!option)
+ break;
+
+ options[n++] = option;
+ }
+ lex_error_expecting_array (lexer, options, n);
+}
+
+void
+lex_error_expecting_array (struct lexer *lexer, const char **options, size_t n)
+{
+ switch (n)
+ {
+ case 0:
+ lex_error (lexer, NULL);
+ break;
+
+ case 1:
+ lex_error (lexer, _("expecting %s"), options[0]);
+ break;
+
+ case 2:
+ lex_error (lexer, _("expecting %s or %s"), options[0], options[1]);
+ break;
+
+ case 3:
+ lex_error (lexer, _("expecting %s, %s, or %s"), options[0], options[1],
+ options[2]);
+ break;
+
+ case 4:
+ lex_error (lexer, _("expecting %s, %s, %s, or %s"),
+ options[0], options[1], options[2], options[3]);
+ break;
+
+ case 5:
+ lex_error (lexer, _("expecting %s, %s, %s, %s, or %s"),
+ options[0], options[1], options[2], options[3], options[4]);
+ break;
+
+ case 6:
+ lex_error (lexer, _("expecting %s, %s, %s, %s, %s, or %s"),
+ options[0], options[1], options[2], options[3], options[4],
+ options[5]);
+ break;
+
+ case 7:
+ lex_error (lexer, _("expecting %s, %s, %s, %s, %s, %s, or %s"),
+ options[0], options[1], options[2], options[3], options[4],
+ options[5], options[6]);
+ break;
+
+ case 8:
+ lex_error (lexer, _("expecting %s, %s, %s, %s, %s, %s, %s, or %s"),
+ options[0], options[1], options[2], options[3], options[4],
+ options[5], options[6], options[7]);
+ break;
+
+ default:
+ lex_error (lexer, NULL);
+ }
+}
+
+/* Reports an error to the effect that subcommand SBC may only be specified
+ once.
+
+ This function does not take a lexer as an argument or use lex_error(),
+ because the result would ordinarily just be redundant: "Syntax error at
+ SUBCOMMAND: Subcommand SUBCOMMAND may only be specified once.", which does
+ not help the user find the error. */
void
lex_sbc_only_once (const char *sbc)
{
msg (SE, _("Subcommand %s may only be specified once."), sbc);
}
-/* Reports an error to the effect that subcommand SBC is
- missing. */
+/* Reports an error to the effect that subcommand SBC is missing.
+
+ This function does not take a lexer as an argument or use lex_error(),
+ because a missing subcommand can normally be detected only after the whole
+ command has been parsed, and so lex_error() would always report "Syntax
+ error at end of command", which does not help the user find the error. */
void
-lex_sbc_missing (struct lexer *lexer, const char *sbc)
+lex_sbc_missing (const char *sbc)
{
- lex_error (lexer, _("missing required subcommand %s"), sbc);
+ msg (SE, _("Required subcommand %s was not specified."), sbc);
+}
+
+/* Reports an error to the effect that specification SPEC may only be specified
+ once within subcommand SBC. */
+void
+lex_spec_only_once (struct lexer *lexer, const char *sbc, const char *spec)
+{
+ lex_error (lexer, _("%s may only be specified once within subcommand %s"),
+ spec, sbc);
+}
+
+/* Reports an error to the effect that specification SPEC is missing within
+ subcommand SBC. */
+void
+lex_spec_missing (struct lexer *lexer, const char *sbc, const char *spec)
+{
+ lex_error (lexer, _("Required %s specification missing from %s subcommand"),
+ sbc, spec);
}
/* Prints a syntax error message containing the current token and
/* Returns true if the current token is a number. */
bool
-lex_is_number (struct lexer *lexer)
+lex_is_number (const struct lexer *lexer)
{
return lex_next_is_number (lexer, 0);
}
/* Returns true if the current token is a string. */
bool
-lex_is_string (struct lexer *lexer)
+lex_is_string (const struct lexer *lexer)
{
return lex_next_is_string (lexer, 0);
}
/* Returns the value of the current token, which must be a
floating point number. */
double
-lex_number (struct lexer *lexer)
+lex_number (const struct lexer *lexer)
{
return lex_next_number (lexer, 0);
}
/* Returns true iff the current token is an integer. */
bool
-lex_is_integer (struct lexer *lexer)
+lex_is_integer (const struct lexer *lexer)
{
return lex_next_is_integer (lexer, 0);
}
/* Returns the value of the current token, which must be an
integer. */
long
-lex_integer (struct lexer *lexer)
+lex_integer (const struct lexer *lexer)
{
return lex_next_integer (lexer, 0);
}
/* Returns true if the token N ahead of the current token is a number. */
bool
-lex_next_is_number (struct lexer *lexer, int n)
+lex_next_is_number (const struct lexer *lexer, int n)
{
enum token_type next_token = lex_next_token (lexer, n);
return next_token == T_POS_NUM || next_token == T_NEG_NUM;
/* Returns true if the token N ahead of the current token is a string. */
bool
-lex_next_is_string (struct lexer *lexer, int n)
+lex_next_is_string (const struct lexer *lexer, int n)
{
return lex_next_token (lexer, n) == T_STRING;
}
/* Returns the value of the token N ahead of the current token, which must be a
floating point number. */
double
-lex_next_number (struct lexer *lexer, int n)
+lex_next_number (const struct lexer *lexer, int n)
{
assert (lex_next_is_number (lexer, n));
return lex_next_tokval (lexer, n);
/* Returns true if the token N ahead of the current token is an integer. */
bool
-lex_next_is_integer (struct lexer *lexer, int n)
+lex_next_is_integer (const struct lexer *lexer, int n)
{
double value;
/* Returns the value of the token N ahead of the current token, which must be
an integer. */
long
-lex_next_integer (struct lexer *lexer, int n)
+lex_next_integer (const struct lexer *lexer, int n)
{
assert (lex_next_is_integer (lexer, n));
return lex_next_tokval (lexer, n);
return true;
else
{
- lex_error (lexer, _("expecting `%s'"), identifier);
+ lex_error_expecting (lexer, identifier);
return false;
}
}
}
else
{
- lex_error (lexer, _("expecting `%s'"), token_type_to_string (type));
+ const char *type_string = token_type_to_string (type);
+ if (type_string)
+ {
+ char *s = xasprintf ("`%s'", type_string);
+ lex_error_expecting (lexer, s);
+ free (s);
+ }
+ else
+ lex_error_expecting (lexer, token_type_to_name (type));
+
return false;
}
}
}
}
+/* If the current token is a string or an identifier, does nothing and returns
+ true. Otherwise, reports an error and returns false.
+
+ This is meant for use in syntactic situations where we want to encourage the
+ user to supply a quoted string, but for compatibility we also accept
+ identifiers. (One example of such a situation is file names.) Therefore,
+ the error message issued when the current token is wrong only says that a
+ string is expected and doesn't mention that an identifier would also be
+ accepted. */
+bool
+lex_force_string_or_id (struct lexer *lexer)
+{
+ return lex_token (lexer) == T_ID || lex_force_string (lexer);
+}
+
/* If the current token is an integer, does nothing and returns true.
Otherwise, reports an error and returns false. */
bool
}
}
+/* If the current token is an integer in the range MIN...MAX (inclusive), does
+ nothing and returns true. Otherwise, reports an error and returns false.
+ If NAME is nonnull, then it is used in the error message. */
+bool
+lex_force_int_range (struct lexer *lexer, const char *name, long min, long max)
+{
+ bool is_integer = lex_is_integer (lexer);
+ bool too_small = is_integer && lex_integer (lexer) < min;
+ bool too_big = is_integer && lex_integer (lexer) > max;
+ if (is_integer && !too_small && !too_big)
+ return true;
+
+ if (min > max)
+ {
+ /* Weird, maybe a bug in the caller. Just report that we needed an
+ integer. */
+ if (name)
+ lex_error (lexer, _("Integer expected for %s."), name);
+ else
+ lex_error (lexer, _("Integer expected."));
+ }
+ else if (min == max)
+ {
+ if (name)
+ lex_error (lexer, _("Expected %ld for %s."), min, name);
+ else
+ lex_error (lexer, _("Expected %ld."), min);
+ }
+ else if (min + 1 == max)
+ {
+ if (name)
+ lex_error (lexer, _("Expected %ld or %ld for %s."), min, min + 1, name);
+ else
+ lex_error (lexer, _("Expected %ld or %ld."), min, min + 1);
+ }
+ else
+ {
+ bool report_lower_bound = (min > INT_MIN / 2) || too_small;
+ bool report_upper_bound = (max < INT_MAX / 2) || too_big;
+
+ if (report_lower_bound && report_upper_bound)
+ {
+ if (name)
+ lex_error (lexer,
+ _("Expected integer between %ld and %ld for %s."),
+ min, max, name);
+ else
+ lex_error (lexer, _("Expected integer between %ld and %ld."),
+ min, max);
+ }
+ else if (report_lower_bound)
+ {
+ if (min == 0)
+ {
+ if (name)
+ lex_error (lexer, _("Expected non-negative integer for %s."),
+ name);
+ else
+ lex_error (lexer, _("Expected non-negative integer."));
+ }
+ else if (min == 1)
+ {
+ if (name)
+ lex_error (lexer, _("Expected positive integer for %s."),
+ name);
+ else
+ lex_error (lexer, _("Expected positive integer."));
+ }
+ }
+ else if (report_upper_bound)
+ {
+ if (name)
+ lex_error (lexer,
+ _("Expected integer less than or equal to %ld for %s."),
+ max, name);
+ else
+ lex_error (lexer, _("Expected integer less than or equal to %ld."),
+ max);
+ }
+ else
+ {
+ if (name)
+ lex_error (lexer, _("Integer expected for %s."), name);
+ else
+ lex_error (lexer, _("Integer expected."));
+ }
+ }
+ return false;
+}
+
/* If the current token is a number, does nothing and returns true.
Otherwise, reports an error and returns false. */
bool
The string is null-terminated (but the null terminator is not included in
the returned substring's 'length').
- Only T_ID and T_STRING tokens have meaningful strings. For other tokens
- this functions this function will always return NULL.
+ Only T_ID, T_MACRO_ID, T_STRING tokens have meaningful strings. For other
+ tokens this functions this function will always return NULL.
The UTF-8 encoding of the returned string is correct for variable names and
other identifiers. Use filename_to_utf8() to use it as a filename. Use
return lex_next (lexer, n)->string;
}
-/* If LEXER is positioned at the (pseudo)identifier S, skips it and returns
- true. Otherwise, returns false.
-
- S may consist of an arbitrary number of identifiers, integers, and
- punctuation e.g. "KRUSKAL-WALLIS", "2SLS", or "END INPUT PROGRAM".
- Identifiers may be abbreviated to their first three letters. Currently only
- hyphens, slashes, and equals signs are supported as punctuation (but it
- would be easy to add more).
-
- S must be an ASCII string. */
-bool
-lex_match_phrase (struct lexer *lexer, const char *s)
+static bool
+lex_tokens_match (const struct token *actual, const struct token *expected)
{
- int tok_idx;
+ if (actual->type != expected->type)
+ return false;
- for (tok_idx = 0; ; tok_idx++)
+ switch (actual->type)
{
- enum token_type token;
- unsigned char c;
+ case T_POS_NUM:
+ case T_NEG_NUM:
+ return actual->number == expected->number;
- while (c_isspace (*s))
- s++;
+ case T_ID:
+ return lex_id_match (expected->string, actual->string);
- c = *s;
- if (c == '\0')
- {
- int i;
-
- for (i = 0; i < tok_idx; i++)
- lex_get (lexer);
- return true;
- }
-
- token = lex_next_token (lexer, tok_idx);
- switch (c)
- {
- case '-':
- if (token != T_DASH)
- return false;
- s++;
- break;
+ case T_STRING:
+ return (actual->string.length == expected->string.length
+ && !memcmp (actual->string.string, expected->string.string,
+ actual->string.length));
- case '/':
- if (token != T_SLASH)
- return false;
- s++;
- break;
-
- case '=':
- if (token != T_EQUALS)
- return false;
- s++;
- break;
-
- case '0': case '1': case '2': case '3': case '4':
- case '5': case '6': case '7': case '8': case '9':
- {
- unsigned int value;
-
- if (token != T_POS_NUM)
- return false;
-
- value = 0;
- do
- {
- value = value * 10 + (*s++ - '0');
- }
- while (c_isdigit (*s));
-
- if (lex_next_tokval (lexer, tok_idx) != value)
- return false;
- }
- break;
+ default:
+ return true;
+ }
+}
- default:
- if (lex_is_id1 (c))
- {
- int len;
+/* If LEXER is positioned at the sequence of tokens that may be parsed from S,
+ skips it and returns true. Otherwise, returns false.
- if (token != T_ID)
- return false;
+ S may consist of an arbitrary sequence of tokens, e.g. "KRUSKAL-WALLIS",
+ "2SLS", or "END INPUT PROGRAM". Identifiers may be abbreviated to their
+ first three letters. */
+bool
+lex_match_phrase (struct lexer *lexer, const char *s)
+{
+ struct string_lexer slex;
+ struct token token;
+ int i;
- len = lex_id_get_length (ss_cstr (s));
- if (!lex_id_match (ss_buffer (s, len),
- lex_next_tokss (lexer, tok_idx)))
- return false;
+ i = 0;
+ string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE);
+ while (string_lexer_next (&slex, &token))
+ if (token.type != SCAN_SKIP)
+ {
+ bool match = lex_tokens_match (lex_next (lexer, i++), &token);
+ token_destroy (&token);
+ if (!match)
+ return false;
+ }
- s += len;
- }
- else
- NOT_REACHED ();
- }
- }
+ while (i-- > 0)
+ lex_get (lexer);
+ return true;
}
static int
return src == NULL ? NULL : src->reader->file_name;
}
+const char *
+lex_get_encoding (const struct lexer *lexer)
+{
+ struct lex_source *src = lex_source__ (lexer);
+ return src == NULL ? NULL : src->reader->encoding;
+}
+
+
/* Returns the syntax mode for the syntax file from which the current drawn is
drawn. Returns LEX_SYNTAX_AUTO for a T_STOP token or if the command's
source does not have line numbers.
}
/* Returns the error mode for the syntax file from which the current drawn is
- drawn. Returns LEX_ERROR_INTERACTIVE for a T_STOP token or if the command's
+ drawn. Returns LEX_ERROR_TERMINAL for a T_STOP token or if the command's
source does not have line numbers.
There is no version of this function that takes an N argument because
lex_get_error_mode (const struct lexer *lexer)
{
struct lex_source *src = lex_source__ (lexer);
- return src == NULL ? LEX_ERROR_INTERACTIVE : src->reader->error;
+ return src == NULL ? LEX_ERROR_TERMINAL : src->reader->error;
}
/* If the source that LEXER is currently reading has error mode
- LEX_ERROR_INTERACTIVE, discards all buffered input and tokens, so that the
- next token to be read comes directly from whatever is next read from the
- stream.
+ LEX_ERROR_TERMINAL, discards all buffered input and tokens, so that the next
+ token to be read comes directly from whatever is next read from the stream.
It makes sense to call this function after encountering an error in a
command entered on the console, because usually the user would prefer not to
lex_interactive_reset (struct lexer *lexer)
{
struct lex_source *src = lex_source__ (lexer);
- if (src != NULL && src->reader->error == LEX_ERROR_INTERACTIVE)
+ if (src != NULL && src->reader->error == LEX_ERROR_TERMINAL)
{
src->head = src->tail = 0;
src->journal_pos = src->seg_pos = src->line_pos = 0;
}
/* Discards all lookahead tokens in LEXER, then discards all input sources
- until it encounters one with error mode LEX_ERROR_INTERACTIVE or until it
+ until it encounters one with error mode LEX_ERROR_TERMINAL or until it
runs out of input sources. */
void
lex_discard_noninteractive (struct lexer *lexer)
while (!deque_is_empty (&src->deque))
lex_source_pop__ (src);
- for (; src != NULL && src->reader->error != LEX_ERROR_INTERACTIVE;
+ for (; src != NULL && src->reader->error != LEX_ERROR_TERMINAL;
src = lex_source__ (lexer))
lex_source_destroy (src);
}
{
do
{
- size_t head_ofs;
- size_t n;
-
lex_source_expand__ (src);
- head_ofs = src->head - src->tail;
- n = src->reader->class->read (src->reader, &src->buffer[head_ofs],
- src->allocated - head_ofs,
- segmenter_get_prompt (&src->segmenter));
+ size_t head_ofs = src->head - src->tail;
+ size_t space = src->allocated - head_ofs;
+ enum prompt_style prompt = segmenter_get_prompt (&src->segmenter);
+ size_t n = src->reader->class->read (src->reader, &src->buffer[head_ofs],
+ space, prompt);
+ assert (n <= space);
+
if (n == 0)
{
- /* End of input.
-
- Ensure that the input always ends in a new-line followed by a null
- byte, as required by the segmenter library. */
-
- if (src->head == src->tail
- || src->buffer[src->head - src->tail - 1] != '\n')
- src->buffer[src->head++ - src->tail] = '\n';
-
+ /* End of input. */
+ src->reader->eof = true;
lex_source_expand__ (src);
- src->buffer[src->head++ - src->tail] = '\0';
-
return;
}
int mblen;
assert (out_size >= 16);
- out_maxlen = out_size - (in.length >= out_size ? 3 : 0) - 1;
+ out_maxlen = out_size - 1;
+ if (in.length > out_maxlen - 3)
+ out_maxlen -= 3;
+
for (out_len = 0; out_len < in.length; out_len += mblen)
{
if (in.string[out_len] == '\n'
+ || in.string[out_len] == '\0'
|| (in.string[out_len] == '\r'
&& out_len + 1 < in.length
&& in.string[out_len + 1] == '\n'))
mblen = u8_mblen (CHAR_CAST (const uint8_t *, in.string + out_len),
in.length - out_len);
+
+ if (mblen < 0)
+ break;
+
if (out_len + mblen > out_maxlen)
break;
}
{
const struct lex_token *token;
struct string s;
- struct msg m;
ds_init_empty (&s);
ds_put_cstr (&s, ": ");
ds_put_vformat (&s, format, args);
}
- ds_put_byte (&s, '.');
-
- m.category = MSG_C_SYNTAX;
- m.severity = MSG_S_ERROR;
- m.file_name = src->reader->file_name;
- m.first_line = lex_source_get_first_line_number (src, n0);
- m.last_line = lex_source_get_last_line_number (src, n1);
- m.first_column = lex_source_get_first_column (src, n0);
- m.last_column = lex_source_get_last_column (src, n1);
- m.text = ds_steal_cstr (&s);
+ if (ds_last (&s) != '.')
+ ds_put_byte (&s, '.');
+
+ struct msg m = {
+ .category = MSG_C_SYNTAX,
+ .severity = MSG_S_ERROR,
+ .file_name = src->reader->file_name,
+ .first_line = lex_source_get_first_line_number (src, n0),
+ .last_line = lex_source_get_last_line_number (src, n1),
+ .first_column = lex_source_get_first_column (src, n0),
+ .last_column = lex_source_get_last_column (src, n1),
+ .text = ds_steal_cstr (&s),
+ };
msg_emit (&m);
}
va_end (args);
}
+/* Attempts to append an additional token into SRC's deque, reading more from
+ the underlying lex_reader if necessary. Returns true if successful, false
+ if the deque already represents (a suffix of) the whole lex_reader's
+ contents, */
static bool
lex_source_get__ (const struct lex_source *src_)
{
struct lex_source *src = CONST_CAST (struct lex_source *, src_);
+ if (src->eof)
+ return false;
+ /* State maintained while scanning tokens. Usually we only need a single
+ state, but scanner_push() can return SCAN_SAVE to indicate that the state
+ needs to be saved and possibly restored later with SCAN_BACK. */
struct state
{
struct segmenter segmenter;
enum segment_type last_segment;
- int newlines;
+ int newlines; /* Number of newlines encountered so far. */
+ /* Maintained here so we can update lex_source's similar members when we
+ finish. */
size_t line_pos;
size_t seg_pos;
};
- struct state state, saved;
- enum scan_result result;
- struct scanner scanner;
- struct lex_token *token;
- int n_lines;
- int i;
-
- if (src->eof)
- return false;
-
- state.segmenter = src->segmenter;
- state.newlines = 0;
- state.seg_pos = src->seg_pos;
- state.line_pos = src->line_pos;
- saved = state;
+ /* Initialize state. */
+ struct state state =
+ {
+ .segmenter = src->segmenter,
+ .newlines = 0,
+ .seg_pos = src->seg_pos,
+ .line_pos = src->line_pos,
+ };
+ struct state saved = state;
- token = lex_push_token__ (src);
+ /* Append a new token to SRC and initialize it. */
+ struct lex_token *token = lex_push_token__ (src);
+ struct scanner scanner;
scanner_init (&scanner, &token->token);
token->line_pos = src->line_pos;
token->token_pos = src->seg_pos;
else
token->first_line = 0;
+ /* Extract segments and pass them through the scanner until we obtain a
+ token. */
for (;;)
{
+ /* Extract a segment. */
+ const char *segment = &src->buffer[state.seg_pos - src->tail];
+ size_t seg_maxlen = src->head - state.seg_pos;
enum segment_type type;
- const char *segment;
- size_t seg_maxlen;
- int seg_len;
-
- segment = &src->buffer[state.seg_pos - src->tail];
- seg_maxlen = src->head - state.seg_pos;
- seg_len = segmenter_push (&state.segmenter, segment, seg_maxlen, &type);
+ int seg_len = segmenter_push (&state.segmenter, segment, seg_maxlen,
+ src->reader->eof, &type);
if (seg_len < 0)
{
+ /* The segmenter needs more input to produce a segment. */
+ assert (!src->reader->eof);
lex_source_read__ (src);
continue;
}
+ /* Update state based on the segment. */
state.last_segment = type;
state.seg_pos += seg_len;
if (type == SEG_NEWLINE)
state.line_pos = state.seg_pos;
}
- result = scanner_push (&scanner, type, ss_buffer (segment, seg_len),
- &token->token);
+ /* Pass the segment into the scanner and try to get a token out. */
+ enum scan_result result = scanner_push (&scanner, type,
+ ss_buffer (segment, seg_len),
+ &token->token);
if (result == SCAN_SAVE)
saved = state;
else if (result == SCAN_BACK)
break;
}
- n_lines = state.newlines;
+ /* If we've reached the end of a line, or the end of a command, then pass
+ the line to the output engine as a syntax text item. */
+ int n_lines = state.newlines;
if (state.last_segment == SEG_END_COMMAND && !src->suppress_next_newline)
{
n_lines++;
n_lines--;
src->suppress_next_newline = false;
}
- for (i = 0; i < n_lines; i++)
+ for (int i = 0; i < n_lines; i++)
{
- const char *newline;
- const char *line;
- size_t line_len;
- char *syntax;
-
- line = &src->buffer[src->journal_pos - src->tail];
- newline = rawmemchr (line, '\n');
- line_len = newline - line;
- if (line_len > 0 && line[line_len - 1] == '\r')
- line_len--;
-
- syntax = malloc (line_len + 2);
- memcpy (syntax, line, line_len);
- syntax[line_len] = '\n';
- syntax[line_len + 1] = '\0';
-
- text_item_submit (text_item_create_nocopy (TEXT_ITEM_SYNTAX, syntax));
-
- src->journal_pos += newline - line + 1;
+ /* Beginning of line. */
+ const char *line = &src->buffer[src->journal_pos - src->tail];
+
+ /* Calculate line length, including \n or \r\n end-of-line if present.
+
+ We use src->head even though that may be beyond what we've actually
+ converted to tokens (which is only through state.line_pos). That's
+ because, if we're emitting the line due to SEG_END_COMMAND, we want to
+ take the whole line through the newline, not just through the '.'. */
+ size_t max_len = src->head - src->journal_pos;
+ const char *newline = memchr (line, '\n', max_len);
+ size_t line_len = newline ? newline - line + 1 : max_len;
+
+ /* Calculate line length excluding end-of-line. */
+ size_t copy_len = line_len;
+ if (copy_len > 0 && line[copy_len - 1] == '\n')
+ copy_len--;
+ if (copy_len > 0 && line[copy_len - 1] == '\r')
+ copy_len--;
+
+ /* Submit the line as syntax. */
+ output_item_submit (text_item_create_nocopy (TEXT_ITEM_SYNTAX,
+ xmemdup0 (line, copy_len),
+ NULL));
+
+ src->journal_pos += line_len;
}
token->token_len = state.seg_pos - src->seg_pos;
token->token.string.string);
break;
- case SCAN_UNEXPECTED_DOT:
- lex_get_error (src, _("Unexpected `.' in middle of command"));
- break;
-
case SCAN_UNEXPECTED_CHAR:
{
char c_name[16];
lex_source_destroy (struct lex_source *src)
{
char *file_name = src->reader->file_name;
- if (src->reader->class->close != NULL)
- src->reader->class->close (src->reader);
+ char *encoding = src->reader->encoding;
+ if (src->reader->class->destroy != NULL)
+ src->reader->class->destroy (src->reader);
free (file_name);
+ free (encoding);
free (src->buffer);
while (!deque_is_empty (&src->deque))
lex_source_pop__ (src);
{
struct lex_reader reader;
struct u8_istream *istream;
- char *file_name;
};
static struct lex_reader_class lex_file_reader_class;
r->reader.syntax = syntax;
r->reader.error = error;
r->reader.file_name = xstrdup (file_name);
+ r->reader.encoding = xstrdup_if_nonnull (encoding);
r->reader.line_number = 1;
r->istream = istream;
- r->file_name = xstrdup (file_name);
return &r->reader;
}
ssize_t n_read = u8_istream_read (r->istream, buf, n);
if (n_read < 0)
{
- msg (ME, _("Error reading `%s': %s."), r->file_name, strerror (errno));
+ msg (ME, _("Error reading `%s': %s."), r_->file_name, strerror (errno));
return 0;
}
return n_read;
if (u8_istream_fileno (r->istream) != STDIN_FILENO)
{
if (u8_istream_close (r->istream) != 0)
- msg (ME, _("Error closing `%s': %s."), r->file_name, strerror (errno));
+ msg (ME, _("Error closing `%s': %s."), r_->file_name, strerror (errno));
}
else
u8_istream_free (r->istream);
- free (r->file_name);
free (r);
}
static struct lex_reader_class lex_string_reader_class;
/* Creates and returns a new lex_reader for the contents of S, which must be
- encoded in UTF-8. The new reader takes ownership of S and will free it
+ encoded in the given ENCODING. The new reader takes ownership of S and will free it
with ss_dealloc() when it is closed. */
struct lex_reader *
-lex_reader_for_substring_nocopy (struct substring s)
+lex_reader_for_substring_nocopy (struct substring s, const char *encoding)
{
struct lex_string_reader *r;
r = xmalloc (sizeof *r);
lex_reader_init (&r->reader, &lex_string_reader_class);
- r->reader.syntax = LEX_SYNTAX_INTERACTIVE;
+ r->reader.syntax = LEX_SYNTAX_AUTO;
+ r->reader.encoding = xstrdup_if_nonnull (encoding);
r->s = s;
r->offset = 0;
}
/* Creates and returns a new lex_reader for a copy of null-terminated string S,
- which must be encoded in UTF-8. The caller retains ownership of S. */
+ which must be encoded in ENCODING. The caller retains ownership of S. */
struct lex_reader *
-lex_reader_for_string (const char *s)
+lex_reader_for_string (const char *s, const char *encoding)
{
struct substring ss;
ss_alloc_substring (&ss, ss_cstr (s));
- return lex_reader_for_substring_nocopy (ss);
+ return lex_reader_for_substring_nocopy (ss, encoding);
}
/* Formats FORMAT as a printf()-like format string and creates and returns a
new lex_reader for the formatted result. */
struct lex_reader *
-lex_reader_for_format (const char *format, ...)
+lex_reader_for_format (const char *format, const char *encoding, ...)
{
struct lex_reader *r;
va_list args;
- va_start (args, format);
- r = lex_reader_for_substring_nocopy (ss_cstr (xvasprintf (format, args)));
+ va_start (args, encoding);
+ r = lex_reader_for_substring_nocopy (ss_cstr (xvasprintf (format, args)), encoding);
va_end (args);
return r;
projects / pspp / blobdiff