};
static struct lex_source *lex_source__ (const struct lexer *);
+static struct substring lex_source_get_syntax__ (const struct lex_source *,
+ int n0, int n1);
static const struct lex_token *lex_next__ (const struct lexer *, int n);
static void lex_source_push_endcmd__ (struct lex_source *);
const struct lex_reader_class *class)
{
reader->class = class;
- reader->syntax = LEX_SYNTAX_AUTO;
+ reader->syntax = SEG_MODE_AUTO;
reader->error = LEX_ERROR_CONTINUE;
reader->file_name = NULL;
reader->encoding = NULL;
static void
lex_source_pop__ (struct lex_source *src)
{
- token_destroy (&src->tokens[deque_pop_back (&src->deque)].token);
+ token_uninit (&src->tokens[deque_pop_back (&src->deque)].token);
}
static void
lex_source_pop_front (struct lex_source *src)
{
- token_destroy (&src->tokens[deque_pop_front (&src->deque)].token);
+ token_uninit (&src->tokens[deque_pop_front (&src->deque)].token);
}
/* Advances LEXER to the next token, consuming the current token. */
bool
lex_next_is_number (const struct lexer *lexer, int n)
{
- enum token_type next_token = lex_next_token (lexer, n);
- return next_token == T_POS_NUM || next_token == T_NEG_NUM;
+ return token_is_number (lex_next (lexer, n));
}
/* Returns true if the token N ahead of the current token is a string. */
bool
lex_next_is_string (const struct lexer *lexer, int n)
{
- return lex_next_token (lexer, n) == T_STRING;
+ return token_is_string (lex_next (lexer, n));
}
/* Returns the value of the token N ahead of the current token, which must be a
double
lex_next_number (const struct lexer *lexer, int n)
{
- assert (lex_next_is_number (lexer, n));
- return lex_next_tokval (lexer, n);
+ return token_number (lex_next (lexer, n));
}
/* Returns true if the token N ahead of the current token is an integer. */
bool
lex_next_is_integer (const struct lexer *lexer, int n)
{
- double value;
-
- if (!lex_next_is_number (lexer, n))
- return false;
-
- value = lex_next_tokval (lexer, n);
- return value > LONG_MIN && value <= LONG_MAX && floor (value) == value;
+ return token_is_integer (lex_next (lexer, n));
}
/* Returns the value of the token N ahead of the current token, which must be
long
lex_next_integer (const struct lexer *lexer, int n)
{
- assert (lex_next_is_integer (lexer, n));
- return lex_next_tokval (lexer, n);
+ return token_integer (lex_next (lexer, n));
}
\f
/* Token matching functions. */
double
lex_next_tokval (const struct lexer *lexer, int n)
{
- const struct token *token = lex_next (lexer, n);
- return token->number;
+ return token_number (lex_next (lexer, n));
}
/* Returns the null-terminated string in the token N after the current one, in
return lex_next (lexer, n)->string;
}
+/* Returns the text of the syntax in tokens N0 ahead of the current one,
+ through N1 ahead of the current one, inclusive. (For example, if N0 and N1
+ are both zero, this requests the syntax for the current token.) The caller
+ must not modify or free the returned string. The syntax is encoded in UTF-8
+ and in the original form supplied to the lexer so that, for example, it may
+ include comments, spaces, and new-lines if it spans multiple tokens. */
+struct substring
+lex_next_representation (const struct lexer *lexer, int n0, int n1)
+{
+ return lex_source_get_syntax__ (lex_source__ (lexer), n0, n1);
+}
+
static bool
lex_tokens_match (const struct token *actual, const struct token *expected)
{
int i;
i = 0;
- string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE);
+ string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE, true);
while (string_lexer_next (&slex, &token))
if (token.type != SCAN_SKIP)
{
bool match = lex_tokens_match (lex_next (lexer, i++), &token);
- token_destroy (&token);
+ token_uninit (&token);
if (!match)
return false;
}
return src == NULL ? NULL : src->reader->file_name;
}
+/* Returns a newly allocated msg_location for the syntax that represents tokens
+ with 0-based offsets N0...N1, inclusive, from the current token. The caller
+ must eventually free the location (with msg_location_destroy()). */
+struct msg_location *
+lex_get_location (const struct lexer *lexer, int n0, int n1)
+{
+ struct msg_location *loc = lex_get_lines (lexer, n0, n1);
+ loc->first_column = lex_get_first_column (lexer, n0);
+ loc->last_column = lex_get_last_column (lexer, n1);
+ return loc;
+}
+
+/* Returns a newly allocated msg_location for the syntax that represents tokens
+ with 0-based offsets N0...N1, inclusive, from the current token. The
+ location only covers the tokens' lines, not the columns. The caller must
+ eventually free the location (with msg_location_destroy()). */
+struct msg_location *
+lex_get_lines (const struct lexer *lexer, int n0, int n1)
+{
+ struct msg_location *loc = xmalloc (sizeof *loc);
+ *loc = (struct msg_location) {
+ .file_name = xstrdup_if_nonnull (lex_get_file_name (lexer)),
+ .first_line = lex_get_first_line_number (lexer, n0),
+ .last_line = lex_get_last_line_number (lexer, n1),
+ };
+ return loc;
+}
+
const char *
lex_get_encoding (const struct lexer *lexer)
{
/* Returns the syntax mode for the syntax file from which the current drawn is
- drawn. Returns LEX_SYNTAX_AUTO for a T_STOP token or if the command's
- source does not have line numbers.
+ drawn. Returns SEG_MODE_AUTO for a T_STOP token or if the command's source
+ does not have line numbers.
There is no version of this function that takes an N argument because
lookahead only works to the end of a command and any given command is always
within a single syntax file. */
-enum lex_syntax_mode
+enum segmenter_mode
lex_get_syntax_mode (const struct lexer *lexer)
{
struct lex_source *src = lex_source__ (lexer);
- return src == NULL ? LEX_SYNTAX_AUTO : src->reader->syntax;
+ return src == NULL ? SEG_MODE_AUTO : src->reader->syntax;
}
/* Returns the error mode for the syntax file from which the current drawn is
src->journal_pos = src->seg_pos = src->line_pos = 0;
src->n_newlines = 0;
src->suppress_next_newline = false;
- segmenter_init (&src->segmenter, segmenter_get_mode (&src->segmenter));
+ src->segmenter = segmenter_init (segmenter_get_mode (&src->segmenter),
+ false);
while (!deque_is_empty (&src->deque))
lex_source_pop__ (src);
lex_source_push_endcmd__ (src);
}
static struct substring
-lex_source_get_syntax__ (const struct lex_source *src, int n0, int n1)
+lex_tokens_get_syntax__ (const struct lex_source *src,
+ const struct lex_token *token0,
+ const struct lex_token *token1)
{
- const struct lex_token *token0 = lex_source_next__ (src, n0);
- const struct lex_token *token1 = lex_source_next__ (src, MAX (n0, n1));
size_t start = token0->token_pos;
size_t end = token1->token_pos + token1->token_len;
return ss_buffer (&src->buffer[start - src->tail], end - start);
}
+static struct substring
+lex_source_get_syntax__ (const struct lex_source *src, int n0, int n1)
+{
+ return lex_tokens_get_syntax__ (src,
+ lex_source_next__ (src, n0),
+ lex_source_next__ (src, MAX (n0, n1)));
+}
+
static void
lex_ellipsize__ (struct substring in, char *out, size_t out_size)
{
if (ds_last (&s) != '.')
ds_put_byte (&s, '.');
- struct msg m = {
- .category = MSG_C_SYNTAX,
- .severity = MSG_S_ERROR,
- .file_name = src->reader->file_name,
+ struct msg_location *location = xmalloc (sizeof *location);
+ *location = (struct msg_location) {
+ .file_name = xstrdup_if_nonnull (src->reader->file_name),
.first_line = lex_source_get_first_line_number (src, n0),
.last_line = lex_source_get_last_line_number (src, n1),
.first_column = lex_source_get_first_column (src, n0),
.last_column = lex_source_get_last_column (src, n1),
+ };
+ struct msg *m = xmalloc (sizeof *m);
+ *m = (struct msg) {
+ .category = MSG_C_SYNTAX,
+ .severity = MSG_S_ERROR,
+ .location = location,
.text = ds_steal_cstr (&s),
};
- msg_emit (&m);
+ msg_emit (m);
}
-static void PRINTF_FORMAT (2, 3)
-lex_get_error (struct lex_source *src, const char *format, ...)
+static void PRINTF_FORMAT (4, 5)
+lex_source_error (struct lex_source *src, int n0, int n1,
+ const char *format, ...)
{
va_list args;
- int n;
-
va_start (args, format);
+ lex_source_error_valist (src, n0, n1, format, args);
+ va_end (args);
+}
- n = deque_count (&src->deque) - 1;
- lex_source_error_valist (src, n, n, format, args);
+static void
+lex_get_error (struct lex_source *src, const char *s)
+{
+ int n = deque_count (&src->deque) - 1;
+ lex_source_error (src, n, n, "%s", s);
lex_source_pop_front (src);
-
- va_end (args);
}
/* Attempts to append an additional token into SRC's deque, reading more from
- the underlying lex_reader if necessary.. Returns true if successful, false
+ the underlying lex_reader if necessary. Returns true if successful, false
if the deque already represents (a suffix of) the whole lex_reader's
contents, */
static bool
break;
case SCAN_BAD_HEX_LENGTH:
- lex_get_error (src, _("String of hex digits has %d characters, which "
- "is not a multiple of 2"),
- (int) token->token.number);
- break;
-
case SCAN_BAD_HEX_DIGIT:
case SCAN_BAD_UNICODE_DIGIT:
- lex_get_error (src, _("`%c' is not a valid hex digit"),
- (int) token->token.number);
- break;
-
case SCAN_BAD_UNICODE_LENGTH:
- lex_get_error (src, _("Unicode string contains %d bytes, which is "
- "not in the valid range of 1 to 8 bytes"),
- (int) token->token.number);
- break;
-
case SCAN_BAD_UNICODE_CODE_POINT:
- lex_get_error (src, _("U+%04X is not a valid Unicode code point"),
- (int) token->token.number);
- break;
-
case SCAN_EXPECTED_QUOTE:
- lex_get_error (src, _("Unterminated string constant"));
- break;
-
case SCAN_EXPECTED_EXPONENT:
- lex_get_error (src, _("Missing exponent following `%s'"),
- token->token.string.string);
- break;
-
- case SCAN_UNEXPECTED_DOT:
- lex_get_error (src, _("Unexpected `.' in middle of command"));
- break;
-
case SCAN_UNEXPECTED_CHAR:
- {
- char c_name[16];
- lex_get_error (src, _("Bad character %s in input"),
- uc_name (token->token.number, c_name));
- }
+ char *msg = scan_token_to_error (&token->token);
+ lex_get_error (src, msg);
+ free (msg);
break;
case SCAN_SKIP:
static struct lex_source *
lex_source_create (struct lex_reader *reader)
{
- struct lex_source *src;
- enum segmenter_mode mode;
-
- src = xzalloc (sizeof *src);
- src->reader = reader;
-
- if (reader->syntax == LEX_SYNTAX_AUTO)
- mode = SEG_MODE_AUTO;
- else if (reader->syntax == LEX_SYNTAX_INTERACTIVE)
- mode = SEG_MODE_INTERACTIVE;
- else if (reader->syntax == LEX_SYNTAX_BATCH)
- mode = SEG_MODE_BATCH;
- else
- NOT_REACHED ();
- segmenter_init (&src->segmenter, mode);
-
- src->tokens = deque_init (&src->deque, 4, sizeof *src->tokens);
+ struct lex_source *src = xmalloc (sizeof *src);
+ *src = (struct lex_source) {
+ .reader = reader,
+ .segmenter = segmenter_init (reader->syntax, false),
+ .tokens = deque_init (&src->deque, 4, sizeof *src->tokens),
+ };
lex_source_push_endcmd__ (src);
Returns a null pointer if FILE_NAME cannot be opened. */
struct lex_reader *
lex_reader_for_file (const char *file_name, const char *encoding,
- enum lex_syntax_mode syntax,
+ enum segmenter_mode syntax,
enum lex_error_mode error)
{
struct lex_file_reader *r;
r = xmalloc (sizeof *r);
lex_reader_init (&r->reader, &lex_string_reader_class);
- r->reader.syntax = LEX_SYNTAX_AUTO;
+ r->reader.syntax = SEG_MODE_AUTO;
r->reader.encoding = xstrdup_if_nonnull (encoding);
r->s = s;
r->offset = 0;
projects / pspp / blobdiff