segment: Add support for segmenting "snippets".

[pspp] / src / language / lexer / lexer.c

diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index 534ed077e1b2a6ae3d54f13c7241100a9f583bd9..1148cc29e0856752d071b784207944c5d95fa536 100644 (file)
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -511,15 +511,14 @@ lex_integer (const struct lexer *lexer)
 bool
 lex_next_is_number (const struct lexer *lexer, int n)
 {
-  enum token_type next_token = lex_next_token (lexer, n);
-  return next_token == T_POS_NUM || next_token == T_NEG_NUM;
+  return token_is_number (lex_next (lexer, n));
 }
 
 /* Returns true if the token N ahead of the current token is a string. */
 bool
 lex_next_is_string (const struct lexer *lexer, int n)
 {
-  return lex_next_token (lexer, n) == T_STRING;
+  return token_is_string (lex_next (lexer, n));
 }
 
 /* Returns the value of the token N ahead of the current token, which must be a
@@ -527,21 +526,14 @@ lex_next_is_string (const struct lexer *lexer, int n)
 double
 lex_next_number (const struct lexer *lexer, int n)
 {
-  assert (lex_next_is_number (lexer, n));
-  return lex_next_tokval (lexer, n);
+  return token_number (lex_next (lexer, n));
 }
 
 /* Returns true if the token N ahead of the current token is an integer. */
 bool
 lex_next_is_integer (const struct lexer *lexer, int n)
 {
-  double value;
-
-  if (!lex_next_is_number (lexer, n))
-    return false;
-
-  value = lex_next_tokval (lexer, n);
-  return value > LONG_MIN && value <= LONG_MAX && floor (value) == value;
+  return token_is_integer (lex_next (lexer, n));
 }
 
 /* Returns the value of the token N ahead of the current token, which must be
@@ -549,8 +541,7 @@ lex_next_is_integer (const struct lexer *lexer, int n)
 long
 lex_next_integer (const struct lexer *lexer, int n)
 {
-  assert (lex_next_is_integer (lexer, n));
-  return lex_next_tokval (lexer, n);
+  return token_integer (lex_next (lexer, n));
 }
 \f
 /* Token matching functions. */
@@ -932,8 +923,7 @@ lex_next_token (const struct lexer *lexer, int n)
 double
 lex_next_tokval (const struct lexer *lexer, int n)
 {
-  const struct token *token = lex_next (lexer, n);
-  return token->number;
+  return token_number (lex_next (lexer, n));
 }
 
 /* Returns the null-terminated string in the token N after the current one, in
@@ -1239,7 +1229,8 @@ lex_interactive_reset (struct lexer *lexer)
       src->journal_pos = src->seg_pos = src->line_pos = 0;
       src->n_newlines = 0;
       src->suppress_next_newline = false;
-      segmenter_init (&src->segmenter, segmenter_get_mode (&src->segmenter));
+      src->segmenter = segmenter_init (segmenter_get_mode (&src->segmenter),
+                                       false);
       while (!deque_is_empty (&src->deque))
         lex_source_pop__ (src);
       lex_source_push_endcmd__ (src);
@@ -1758,12 +1749,7 @@ lex_source_get (const struct lex_source *src_)
 static void
 lex_source_push_endcmd__ (struct lex_source *src)
 {
-  struct lex_token *token = lex_push_token__ (src);
-  token->token.type = T_ENDCMD;
-  token->token_pos = 0;
-  token->token_len = 0;
-  token->line_pos = 0;
-  token->first_line = 0;
+  *lex_push_token__ (src) = (struct lex_token) { .token = { .type = T_ENDCMD } };
 }
 
 static struct lex_source *
@@ -1773,7 +1759,7 @@ lex_source_create (struct lexer *lexer, struct lex_reader *reader)
 
   src = xzalloc (sizeof *src);
   src->reader = reader;
-  segmenter_init (&src->segmenter, reader->syntax);
+  src->segmenter = segmenter_init (reader->syntax, false);
   src->lexer = lexer;
   src->tokens = deque_init (&src->deque, 4, sizeof *src->tokens);