output: Move text_item and group_item usage closer to the SPV model.

[pspp] / src / language / lexer / lexer.c

diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index 08d35d8c00312d98d2baeb7e80ff04141037b9f9..11de4897724f1e4f8631e41c715e88f54c61ebd5 100644 (file)
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -132,6 +132,7 @@ lex_reader_init (struct lex_reader *reader,
   reader->file_name = NULL;
   reader->encoding = NULL;
   reader->line_number = 0;
+  reader->eof = false;
 }
 
 /* Frees any file name already in READER and replaces it by a copy of
@@ -613,7 +614,7 @@ lex_force_match (struct lexer *lexer, enum token_type type)
        }
       else
        lex_error_expecting (lexer, token_type_to_name (type), NULL_SENTINEL);
-      
+
       return false;
     }
 }
@@ -644,7 +645,7 @@ lex_force_string (struct lexer *lexer)
 bool
 lex_force_string_or_id (struct lexer *lexer)
 {
-  return lex_is_integer (lexer) || lex_force_string (lexer);
+  return lex_token (lexer) == T_ID || lex_force_string (lexer);
 }
 
 /* If the current token is an integer, does nothing and returns true.
@@ -876,7 +877,7 @@ lex_match_phrase (struct lexer *lexer, const char *s)
   int i;
 
   i = 0;
-  string_lexer_init (&slex, s, SEG_MODE_INTERACTIVE);
+  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE);
   while (string_lexer_next (&slex, &token))
     if (token.type != SCAN_SKIP)
       {
@@ -1190,38 +1191,11 @@ lex_source_read__ (struct lex_source *src)
                                            space, prompt);
       assert (n <= space);
 
-      for (char *p = &src->buffer[head_ofs]; p < &src->buffer[head_ofs + n];
-           p++)
-        if (*p == '\0')
-          {
-            struct msg m;
-            m.category = MSG_C_SYNTAX;
-            m.severity = MSG_S_ERROR;
-            m.file_name = src->reader->file_name;
-            m.first_line = 0;
-            m.last_line = 0;
-            m.first_column = 0;
-            m.last_column = 0;
-            m.text = xstrdup ("Bad character U+0000 in input.");
-            msg_emit (&m);
-
-            *p = ' ';
-          }
-
       if (n == 0)
         {
-          /* End of input.
-
-             Ensure that the input always ends in a new-line followed by a null
-             byte, as required by the segmenter library. */
-
-          if (src->head == src->tail
-              || src->buffer[src->head - src->tail - 1] != '\n')
-            src->buffer[src->head++ - src->tail] = '\n';
-
+          /* End of input. */
+          src->reader->eof = true;
           lex_source_expand__ (src);
-          src->buffer[src->head++ - src->tail] = '\0';
-
           return;
         }
 
@@ -1261,6 +1235,7 @@ lex_ellipsize__ (struct substring in, char *out, size_t out_size)
   for (out_len = 0; out_len < in.length; out_len += mblen)
     {
       if (in.string[out_len] == '\n'
+          || in.string[out_len] == '\0'
           || (in.string[out_len] == '\r'
               && out_len + 1 < in.length
               && in.string[out_len + 1] == '\n'))
@@ -1282,7 +1257,6 @@ lex_source_error_valist (struct lex_source *src, int n0, int n1,
 {
   const struct lex_token *token;
   struct string s;
-  struct msg m;
 
   ds_init_empty (&s);
 
@@ -1310,14 +1284,16 @@ lex_source_error_valist (struct lex_source *src, int n0, int n1,
     }
   ds_put_byte (&s, '.');
 
-  m.category = MSG_C_SYNTAX;
-  m.severity = MSG_S_ERROR;
-  m.file_name = src->reader->file_name;
-  m.first_line = lex_source_get_first_line_number (src, n0);
-  m.last_line = lex_source_get_last_line_number (src, n1);
-  m.first_column = lex_source_get_first_column (src, n0);
-  m.last_column = lex_source_get_last_column (src, n1);
-  m.text = ds_steal_cstr (&s);
+  struct msg m = {
+    .category = MSG_C_SYNTAX,
+    .severity = MSG_S_ERROR,
+    .file_name = src->reader->file_name,
+    .first_line = lex_source_get_first_line_number (src, n0),
+    .last_line = lex_source_get_last_line_number (src, n1),
+    .first_column = lex_source_get_first_column (src, n0),
+    .last_column = lex_source_get_last_column (src, n1),
+    .text = ds_steal_cstr (&s),
+  };
   msg_emit (&m);
 }
 
@@ -1391,10 +1367,11 @@ lex_source_get__ (const struct lex_source *src_)
       size_t seg_maxlen = src->head - state.seg_pos;
       enum segment_type type;
       int seg_len = segmenter_push (&state.segmenter, segment, seg_maxlen,
-                                    &type);
+                                    src->reader->eof, &type);
       if (seg_len < 0)
         {
           /* The segmenter needs more input to produce a segment. */
+          assert (!src->reader->eof);
           lex_source_read__ (src);
           continue;
         }
@@ -1438,20 +1415,31 @@ lex_source_get__ (const struct lex_source *src_)
     }
   for (int i = 0; i < n_lines; i++)
     {
+      /* Beginning of line. */
       const char *line = &src->buffer[src->journal_pos - src->tail];
-      const char *newline = rawmemchr (line, '\n');
-      size_t line_len = newline - line;
-      if (line_len > 0 && line[line_len - 1] == '\r')
-        line_len--;
 
-      char *syntax = malloc (line_len + 2);
-      memcpy (syntax, line, line_len);
-      syntax[line_len] = '\n';
-      syntax[line_len + 1] = '\0';
+      /* Calculate line length, including \n or \r\n end-of-line if present.
+
+         We use src->head even though that may be beyond what we've actually
+         converted to tokens (which is only through state.line_pos).  That's
+         because, if we're emitting the line due to SEG_END_COMMAND, we want to
+         take the whole line through the newline, not just through the '.'. */
+      size_t max_len = src->head - src->journal_pos;
+      const char *newline = memchr (line, '\n', max_len);
+      size_t line_len = newline ? newline - line + 1 : max_len;
+
+      /* Calculate line length excluding end-of-line. */
+      size_t copy_len = line_len;
+      if (copy_len > 0 && line[copy_len - 1] == '\n')
+        copy_len--;
+      if (copy_len > 0 && line[copy_len - 1] == '\r')
+        copy_len--;
 
-      text_item_submit (text_item_create_nocopy (TEXT_ITEM_SYNTAX, syntax));
+      /* Submit the line as syntax. */
+      text_item_submit (text_item_create_nocopy (TEXT_ITEM_SYNTAX,
+                                                 xmemdup0 (line, copy_len)));
 
-      src->journal_pos += newline - line + 1;
+      src->journal_pos += line_len;
     }
 
   token->token_len = state.seg_pos - src->seg_pos;