reader->file_name = NULL;
reader->encoding = NULL;
reader->line_number = 0;
+ reader->eof = false;
}
/* Frees any file name already in READER and replaces it by a copy of
/* Prints a syntax error message saying that OPTION0 or one of the other
strings following it, up to the first NULL, is expected. */
void
-lex_error_expecting (struct lexer *lexer, const char *option0, ...)
+(lex_error_expecting) (struct lexer *lexer, const char *option0, ...)
{
enum { MAX_OPTIONS = 8 };
const char *options[MAX_OPTIONS + 1];
/* Returns true if the current token is a number. */
bool
-lex_is_number (struct lexer *lexer)
+lex_is_number (const struct lexer *lexer)
{
return lex_next_is_number (lexer, 0);
}
/* Returns true if the current token is a string. */
bool
-lex_is_string (struct lexer *lexer)
+lex_is_string (const struct lexer *lexer)
{
return lex_next_is_string (lexer, 0);
}
/* Returns the value of the current token, which must be a
floating point number. */
double
-lex_number (struct lexer *lexer)
+lex_number (const struct lexer *lexer)
{
return lex_next_number (lexer, 0);
}
/* Returns true iff the current token is an integer. */
bool
-lex_is_integer (struct lexer *lexer)
+lex_is_integer (const struct lexer *lexer)
{
return lex_next_is_integer (lexer, 0);
}
/* Returns the value of the current token, which must be an
integer. */
long
-lex_integer (struct lexer *lexer)
+lex_integer (const struct lexer *lexer)
{
return lex_next_integer (lexer, 0);
}
/* Returns true if the token N ahead of the current token is a number. */
bool
-lex_next_is_number (struct lexer *lexer, int n)
+lex_next_is_number (const struct lexer *lexer, int n)
{
enum token_type next_token = lex_next_token (lexer, n);
return next_token == T_POS_NUM || next_token == T_NEG_NUM;
/* Returns true if the token N ahead of the current token is a string. */
bool
-lex_next_is_string (struct lexer *lexer, int n)
+lex_next_is_string (const struct lexer *lexer, int n)
{
return lex_next_token (lexer, n) == T_STRING;
}
/* Returns the value of the token N ahead of the current token, which must be a
floating point number. */
double
-lex_next_number (struct lexer *lexer, int n)
+lex_next_number (const struct lexer *lexer, int n)
{
assert (lex_next_is_number (lexer, n));
return lex_next_tokval (lexer, n);
/* Returns true if the token N ahead of the current token is an integer. */
bool
-lex_next_is_integer (struct lexer *lexer, int n)
+lex_next_is_integer (const struct lexer *lexer, int n)
{
double value;
/* Returns the value of the token N ahead of the current token, which must be
an integer. */
long
-lex_next_integer (struct lexer *lexer, int n)
+lex_next_integer (const struct lexer *lexer, int n)
{
assert (lex_next_is_integer (lexer, n));
return lex_next_tokval (lexer, n);
return true;
else
{
- lex_error_expecting (lexer, identifier, NULL_SENTINEL);
+ lex_error_expecting (lexer, identifier);
return false;
}
}
if (type_string)
{
char *s = xasprintf ("`%s'", type_string);
- lex_error_expecting (lexer, s, NULL_SENTINEL);
+ lex_error_expecting (lexer, s);
free (s);
}
else
- lex_error_expecting (lexer, token_type_to_name (type), NULL_SENTINEL);
+ lex_error_expecting (lexer, token_type_to_name (type));
return false;
}
int i;
i = 0;
- string_lexer_init (&slex, s, SEG_MODE_INTERACTIVE);
+ string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE);
while (string_lexer_next (&slex, &token))
if (token.type != SCAN_SKIP)
{
space, prompt);
assert (n <= space);
- for (char *p = &src->buffer[head_ofs]; p < &src->buffer[head_ofs + n];
- p++)
- if (*p == '\0')
- {
- struct msg m;
- m.category = MSG_C_SYNTAX;
- m.severity = MSG_S_ERROR;
- m.file_name = src->reader->file_name;
- m.first_line = 0;
- m.last_line = 0;
- m.first_column = 0;
- m.last_column = 0;
- m.text = xstrdup ("Bad character U+0000 in input.");
- msg_emit (&m);
-
- *p = ' ';
- }
-
if (n == 0)
{
- /* End of input.
-
- Ensure that the input always ends in a new-line followed by a null
- byte, as required by the segmenter library. */
-
- if (src->head == src->tail
- || src->buffer[src->head - src->tail - 1] != '\n')
- src->buffer[src->head++ - src->tail] = '\n';
-
+ /* End of input. */
+ src->reader->eof = true;
lex_source_expand__ (src);
- src->buffer[src->head++ - src->tail] = '\0';
-
return;
}
int mblen;
assert (out_size >= 16);
- out_maxlen = out_size - (in.length >= out_size ? 3 : 0) - 1;
+ out_maxlen = out_size - 1;
+ if (in.length > out_maxlen - 3)
+ out_maxlen -= 3;
+
for (out_len = 0; out_len < in.length; out_len += mblen)
{
if (in.string[out_len] == '\n'
+ || in.string[out_len] == '\0'
|| (in.string[out_len] == '\r'
&& out_len + 1 < in.length
&& in.string[out_len + 1] == '\n'))
mblen = u8_mblen (CHAR_CAST (const uint8_t *, in.string + out_len),
in.length - out_len);
+
+ if (mblen < 0)
+ break;
+
if (out_len + mblen > out_maxlen)
break;
}
{
const struct lex_token *token;
struct string s;
- struct msg m;
ds_init_empty (&s);
}
ds_put_byte (&s, '.');
- m.category = MSG_C_SYNTAX;
- m.severity = MSG_S_ERROR;
- m.file_name = src->reader->file_name;
- m.first_line = lex_source_get_first_line_number (src, n0);
- m.last_line = lex_source_get_last_line_number (src, n1);
- m.first_column = lex_source_get_first_column (src, n0);
- m.last_column = lex_source_get_last_column (src, n1);
- m.text = ds_steal_cstr (&s);
+ struct msg m = {
+ .category = MSG_C_SYNTAX,
+ .severity = MSG_S_ERROR,
+ .file_name = src->reader->file_name,
+ .first_line = lex_source_get_first_line_number (src, n0),
+ .last_line = lex_source_get_last_line_number (src, n1),
+ .first_column = lex_source_get_first_column (src, n0),
+ .last_column = lex_source_get_last_column (src, n1),
+ .text = ds_steal_cstr (&s),
+ };
msg_emit (&m);
}
size_t seg_maxlen = src->head - state.seg_pos;
enum segment_type type;
int seg_len = segmenter_push (&state.segmenter, segment, seg_maxlen,
- &type);
+ src->reader->eof, &type);
if (seg_len < 0)
{
/* The segmenter needs more input to produce a segment. */
+ assert (!src->reader->eof);
lex_source_read__ (src);
continue;
}
}
for (int i = 0; i < n_lines; i++)
{
+ /* Beginning of line. */
const char *line = &src->buffer[src->journal_pos - src->tail];
- const char *newline = rawmemchr (line, '\n');
- size_t line_len = newline - line;
- if (line_len > 0 && line[line_len - 1] == '\r')
- line_len--;
-
- char *syntax = malloc (line_len + 2);
- memcpy (syntax, line, line_len);
- syntax[line_len] = '\n';
- syntax[line_len + 1] = '\0';
-
- text_item_submit (text_item_create_nocopy (TEXT_ITEM_SYNTAX, syntax));
- src->journal_pos += newline - line + 1;
+ /* Calculate line length, including \n or \r\n end-of-line if present.
+
+ We use src->head even though that may be beyond what we've actually
+ converted to tokens (which is only through state.line_pos). That's
+ because, if we're emitting the line due to SEG_END_COMMAND, we want to
+ take the whole line through the newline, not just through the '.'. */
+ size_t max_len = src->head - src->journal_pos;
+ const char *newline = memchr (line, '\n', max_len);
+ size_t line_len = newline ? newline - line + 1 : max_len;
+
+ /* Calculate line length excluding end-of-line. */
+ size_t copy_len = line_len;
+ if (copy_len > 0 && line[copy_len - 1] == '\n')
+ copy_len--;
+ if (copy_len > 0 && line[copy_len - 1] == '\r')
+ copy_len--;
+
+ /* Submit the line as syntax. */
+ text_item_submit (text_item_create_nocopy (TEXT_ITEM_SYNTAX,
+ xmemdup0 (line, copy_len),
+ NULL));
+
+ src->journal_pos += line_len;
}
token->token_len = state.seg_pos - src->seg_pos;
projects / pspp / blobdiff