lexer: Move lex_ellipsize() into string module, as str_ellipsize().

[pspp] / src / language / lexer / lexer.c

diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c
index 8eac18066f7b36346192743ed3ab6702c0a5a179..21309f5f975f688693e0ea09534cd750cf29dab9 100644 (file)
--- a/src/language/lexer/lexer.c
+++ b/src/language/lexer/lexer.c
@@ -198,7 +198,7 @@ lex_push_token__ (struct lex_source *src)
     src->tokens = deque_expand (&src->deque, src->tokens, sizeof *src->tokens);
 
   token = &src->tokens[deque_push_front (&src->deque)];
-  token_init (&token->token);
+  token->token = (struct token) { .type = T_STOP };
   return token;
 }
 
@@ -852,9 +852,7 @@ lex_next__ (const struct lexer *lexer_, int n)
     return lex_source_next__ (src, n);
   else
     {
-      static const struct lex_token stop_token =
-        { TOKEN_INITIALIZER (T_STOP, 0.0, ""), 0, 0, 0, 0 };
-
+      static const struct lex_token stop_token = { .token = { .type = T_STOP } };
       return &stop_token;
     }
 }
@@ -988,7 +986,7 @@ lex_match_phrase (struct lexer *lexer, const char *s)
   int i;
 
   i = 0;
-  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE);
+  string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE, true);
   while (string_lexer_next (&slex, &token))
     if (token.type != SCAN_SKIP)
       {
@@ -1238,7 +1236,8 @@ lex_interactive_reset (struct lexer *lexer)
       src->journal_pos = src->seg_pos = src->line_pos = 0;
       src->n_newlines = 0;
       src->suppress_next_newline = false;
-      segmenter_init (&src->segmenter, segmenter_get_mode (&src->segmenter));
+      src->segmenter = segmenter_init (segmenter_get_mode (&src->segmenter),
+                                       false);
       while (!deque_is_empty (&src->deque))
         lex_source_pop__ (src);
       lex_source_push_endcmd__ (src);
@@ -1370,41 +1369,6 @@ lex_source_get_syntax__ (const struct lex_source *src, int n0, int n1)
                                   lex_source_next__ (src, MAX (n0, n1)));
 }
 
-static void
-lex_ellipsize__ (struct substring in, char *out, size_t out_size)
-{
-  size_t out_maxlen;
-  size_t out_len;
-  int mblen;
-
-  assert (out_size >= 16);
-  out_maxlen = out_size - 1;
-  if (in.length > out_maxlen - 3)
-    out_maxlen -= 3;
-
-  for (out_len = 0; out_len < in.length; out_len += mblen)
-    {
-      if (in.string[out_len] == '\n'
-          || in.string[out_len] == '\0'
-          || (in.string[out_len] == '\r'
-              && out_len + 1 < in.length
-              && in.string[out_len + 1] == '\n'))
-        break;
-
-      mblen = u8_mblen (CHAR_CAST (const uint8_t *, in.string + out_len),
-                        in.length - out_len);
-
-      if (mblen < 0)
-        break;
-
-      if (out_len + mblen > out_maxlen)
-        break;
-    }
-
-  memcpy (out, in.string, out_len);
-  strcpy (&out[out_len], out_len < in.length ? "..." : "");
-}
-
 static void
 lex_source_error_valist (struct lex_source *src, int n0, int n1,
                          const char *format, va_list args)
@@ -1424,7 +1388,7 @@ lex_source_error_valist (struct lex_source *src, int n0, int n1,
         {
           char syntax_cstr[64];
 
-          lex_ellipsize__ (syntax, syntax_cstr, sizeof syntax_cstr);
+          str_ellipsize (syntax, syntax_cstr, sizeof syntax_cstr);
           ds_put_format (&s, _("Syntax error at `%s'"), syntax_cstr);
         }
       else
@@ -1457,19 +1421,22 @@ lex_source_error_valist (struct lex_source *src, int n0, int n1,
   msg_emit (m);
 }
 
-static void PRINTF_FORMAT (2, 3)
-lex_get_error (struct lex_source *src, const char *format, ...)
+static void PRINTF_FORMAT (4, 5)
+lex_source_error (struct lex_source *src, int n0, int n1,
+                  const char *format, ...)
 {
   va_list args;
-  int n;
-
   va_start (args, format);
+  lex_source_error_valist (src, n0, n1, format, args);
+  va_end (args);
+}
 
-  n = deque_count (&src->deque) - 1;
-  lex_source_error_valist (src, n, n, format, args);
+static void
+lex_get_error (struct lex_source *src, const char *s)
+{
+  int n = deque_count (&src->deque) - 1;
+  lex_source_error (src, n, n, "%s", s);
   lex_source_pop_front (src);
-
-  va_end (args);
 }
 
 /* Attempts to append an additional token into SRC's deque, reading more from
@@ -1621,43 +1588,16 @@ lex_source_get__ (const struct lex_source *src_)
       break;
 
     case SCAN_BAD_HEX_LENGTH:
-      lex_get_error (src, _("String of hex digits has %d characters, which "
-                            "is not a multiple of 2"),
-                     (int) token->token.number);
-      break;
-
     case SCAN_BAD_HEX_DIGIT:
     case SCAN_BAD_UNICODE_DIGIT:
-      lex_get_error (src, _("`%c' is not a valid hex digit"),
-                     (int) token->token.number);
-      break;
-
     case SCAN_BAD_UNICODE_LENGTH:
-      lex_get_error (src, _("Unicode string contains %d bytes, which is "
-                            "not in the valid range of 1 to 8 bytes"),
-                     (int) token->token.number);
-      break;
-
     case SCAN_BAD_UNICODE_CODE_POINT:
-      lex_get_error (src, _("U+%04X is not a valid Unicode code point"),
-                     (int) token->token.number);
-      break;
-
     case SCAN_EXPECTED_QUOTE:
-      lex_get_error (src, _("Unterminated string constant"));
-      break;
-
     case SCAN_EXPECTED_EXPONENT:
-      lex_get_error (src, _("Missing exponent following `%s'"),
-                     token->token.string.string);
-      break;
-
     case SCAN_UNEXPECTED_CHAR:
-      {
-        char c_name[16];
-        lex_get_error (src, _("Bad character %s in input"),
-                       uc_name (token->token.number, c_name));
-      }
+      char *msg = scan_token_to_error (&token->token);
+      lex_get_error (src, msg);
+      free (msg);
       break;
 
     case SCAN_SKIP:
@@ -1682,12 +1622,12 @@ lex_source_push_endcmd__ (struct lex_source *src)
 static struct lex_source *
 lex_source_create (struct lex_reader *reader)
 {
-  struct lex_source *src;
-
-  src = xzalloc (sizeof *src);
-  src->reader = reader;
-  segmenter_init (&src->segmenter, reader->syntax);
-  src->tokens = deque_init (&src->deque, 4, sizeof *src->tokens);
+  struct lex_source *src = xmalloc (sizeof *src);
+  *src = (struct lex_source) {
+    .reader = reader,
+    .segmenter = segmenter_init (reader->syntax, false),
+    .tokens = deque_init (&src->deque, 4, sizeof *src->tokens),
+  };
 
   lex_source_push_endcmd__ (src);