X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Flexer%2Fsegment.c;h=346910898ce3898b4bec6c62563a1a6f8dfe10bd;hb=e0cbdf0daefcca81be9572aab0deedf945687f5a;hp=62e9970bb1cff0a6709278303230b1f1dbbca0cd;hpb=e546c5d3c1ef7bc06c8d7bb6752a863ba3b041e5;p=pspp

diff --git a/src/language/lexer/segment.c b/src/language/lexer/segment.c
index 62e9970bb1..346910898c 100644
--- a/src/language/lexer/segment.c
+++ b/src/language/lexer/segment.c
@@ -38,7 +38,9 @@ enum segmenter_state
     S_DOCUMENT_1,
     S_DOCUMENT_2,
     S_DOCUMENT_3,
-    S_FILE_LABEL,
+    S_FILE_LABEL_1,
+    S_FILE_LABEL_2,
+    S_FILE_LABEL_3,
     S_DO_REPEAT_1,
     S_DO_REPEAT_2,
     S_DO_REPEAT_3,
@@ -51,8 +53,6 @@ enum segmenter_state
     S_BEGIN_DATA_2,
     S_BEGIN_DATA_3,
     S_BEGIN_DATA_4,
-    S_TITLE_1,
-    S_TITLE_2
   };
 
 #define SS_START_OF_LINE (1u << 0)
@@ -219,6 +219,22 @@ at_end_of_line (const char *input, size_t n, bool eof, int ofs)
   return is_end_of_line (input, n, eof, ofs);
 }
 
+static bool
+is_all_spaces (const char *input_, size_t n)
+{
+  const uint8_t *input = CHAR_CAST (const uint8_t *, input_);
+
+  int mblen;
+  for (int ofs = 0; ofs < n; ofs += mblen)
+    {
+      ucs4_t uc;
+      mblen = u8_mbtouc (&uc, input + ofs, n - ofs);
+      if (!lex_uc_is_space (uc))
+        return false;
+    }
+  return true;
+}
+
 static int
 segmenter_parse_newline__ (const char *input, size_t n, bool eof,
                            enum segment_type *type)
@@ -276,13 +292,11 @@ skip_digits (const char *input, size_t n, bool eof, int ofs)
 
 static int
 segmenter_parse_number__ (struct segmenter *s, const char *input, size_t n,
-                          bool eof, enum segment_type *type)
+                          bool eof, enum segment_type *type, int ofs)
 {
-  int ofs;
-
   assert (s->state == S_GENERAL);
 
-  ofs = skip_digits (input, n, eof, 0);
+  ofs = skip_digits (input, n, eof, ofs);
   if (ofs < 0)
     return -1;
 
@@ -667,6 +681,7 @@ next_id_in_command (const struct segmenter *s, const char *input, size_t n,
         case SEG_DO_REPEAT_COMMAND:
         case SEG_INLINE_DATA:
         case SEG_MACRO_ID:
+        case SEG_MACRO_NAME:
         case SEG_MACRO_BODY:
         case SEG_START_DOCUMENT:
         case SEG_DOCUMENT:
@@ -744,18 +759,6 @@ segmenter_parse_id__ (struct segmenter *s, const char *input, size_t n,
           *type = SEG_START_DOCUMENT;
           return 0;
         }
-      else if (lex_id_match (ss_cstr ("TITLE"), word)
-               || lex_id_match (ss_cstr ("SUBTITLE"), word))
-        {
-          int result = segmenter_unquoted (input, n, eof, ofs);
-          if (result < 0)
-            return -1;
-          else if (result)
-            {
-              s->state = S_TITLE_1;
-              return ofs;
-            }
-        }
       else if (lex_id_match_n (ss_cstr ("DEFINE"), word, 6))
         {
           s->state = S_DEFINE_1;
@@ -769,7 +772,7 @@ segmenter_parse_id__ (struct segmenter *s, const char *input, size_t n,
             return -1;
           else if (lex_id_match (ss_cstr ("LABEL"), ss_cstr (id)))
             {
-              s->state = S_FILE_LABEL;
+              s->state = S_FILE_LABEL_1;
               s->substate = 0;
               return ofs;
             }
@@ -936,8 +939,25 @@ segmenter_parse_mid_command__ (struct segmenter *s,
       *type = SEG_PUNCT;
       return 1;
 
-    case '(': case ')': case ',': case '=': case '-':
-    case '[': case ']': case '&': case '|': case '+':
+    case '-':
+      ofs = skip_spaces (input, n, eof, 1);
+      if (ofs < 0)
+        return -1;
+      else if (ofs < n && c_isdigit (input[ofs]))
+        return segmenter_parse_number__ (s, input, n, eof, type, ofs);
+      else if (ofs < n && input[ofs] == '.')
+        {
+          if (ofs + 1 >= n)
+            {
+              if (!eof)
+                return -1;
+            }
+          else if (c_isdigit (input[ofs + 1]))
+            return segmenter_parse_number__ (s, input, n, eof, type, ofs);
+        }
+      /* Fall through. */
+    case '(': case ')': case '{': case ',': case '=': case ';': case ':':
+    case '[': case ']': case '}': case '&': case '|': case '+':
       *type = SEG_PUNCT;
       s->substate = 0;
       return 1;
@@ -968,7 +988,7 @@ segmenter_parse_mid_command__ (struct segmenter *s,
             return -1;
         }
       else if (c_isdigit (input[1]))
-        return segmenter_parse_number__ (s, input, n, eof, type);
+        return segmenter_parse_number__ (s, input, n, eof, type, 0);
 
       int eol = at_end_of_line (input, n, eof, 1);
       if (eol < 0)
@@ -985,7 +1005,7 @@ segmenter_parse_mid_command__ (struct segmenter *s,
 
     case '0': case '1': case '2': case '3': case '4':
     case '5': case '6': case '7': case '8': case '9':
-      return segmenter_parse_number__ (s, input, n, eof, type);
+      return segmenter_parse_number__ (s, input, n, eof, type, 0);
 
     case 'u': case 'U':
       return segmenter_maybe_parse_string__ (SEG_UNICODE_STRING,
@@ -1000,7 +1020,20 @@ segmenter_parse_mid_command__ (struct segmenter *s,
                                        s, input, n, eof, type);
 
     case '!':
-      return segmenter_parse_id__ (s, input, n, eof, type);
+      if (n < 2)
+        {
+          if (!eof)
+            return -1;
+          *type = SEG_PUNCT;
+          return 1;
+        }
+      else if (input[1] == '*')
+        {
+          *type = SEG_MACRO_ID;
+          return 2;
+        }
+      else
+        return segmenter_parse_id__ (s, input, n, eof, type);
 
     default:
       if (lex_uc_is_space (uc))
@@ -1009,7 +1042,7 @@ segmenter_parse_mid_command__ (struct segmenter *s,
           if (ofs < 0)
             return -1;
 
-          if (input[ofs - 1] == '\r' && input[ofs] == '\n')
+          if (ofs < n && input[ofs - 1] == '\r' && input[ofs] == '\n')
             {
               if (ofs == 1)
                 {
@@ -1242,9 +1275,9 @@ segmenter_parse_start_of_line__ (struct segmenter *s,
 }
 
 static int
-segmenter_parse_file_label__ (struct segmenter *s,
-                              const char *input, size_t n, bool eof,
-                              enum segment_type *type)
+segmenter_parse_file_label_1__ (struct segmenter *s,
+                                const char *input, size_t n, bool eof,
+                                enum segment_type *type)
 {
   struct segmenter sub;
   int ofs;
@@ -1267,7 +1300,7 @@ segmenter_parse_file_label__ (struct segmenter *s,
       else
         {
           if (result)
-            s->state = S_TITLE_1;
+            s->state = S_FILE_LABEL_2;
           else
             *s = sub;
           return ofs;
@@ -1280,6 +1313,70 @@ segmenter_parse_file_label__ (struct segmenter *s,
     }
 }
 
+static int
+segmenter_parse_file_label_2__ (struct segmenter *s,
+                                const char *input, size_t n, bool eof,
+                                enum segment_type *type)
+{
+  int ofs;
+
+  ofs = skip_spaces (input, n, eof, 0);
+  if (ofs < 0)
+    return -1;
+  s->state = S_FILE_LABEL_3;
+  *type = SEG_SPACES;
+  return ofs;
+}
+
+static int
+segmenter_parse_file_label_3__ (struct segmenter *s,
+                                const char *input, size_t n, bool eof,
+                                enum segment_type *type)
+{
+  int endcmd;
+  int ofs;
+
+  endcmd = -1;
+  ofs = 0;
+  while (ofs < n)
+    {
+      ucs4_t uc;
+      int mblen;
+
+      mblen = segmenter_u8_to_uc__ (&uc, input, n, eof, ofs);
+      if (mblen < 0)
+        return -1;
+
+      switch (uc)
+        {
+        case '\n':
+          goto end_of_line;
+
+        case '.':
+          endcmd = ofs;
+          break;
+
+        default:
+          if (!lex_uc_is_space (uc))
+            endcmd = -1;
+          break;
+        }
+
+      ofs += mblen;
+    }
+
+  if (eof)
+    {
+    end_of_line:
+      s->state = S_GENERAL;
+      s->substate = 0;
+      *type = SEG_UNQUOTED_STRING;
+      return endcmd >= 0 ? endcmd : ofs;
+    }
+
+  return -1;
+}
+
 static int
 segmenter_subparse (struct segmenter *s,
                     const char *input, size_t n, bool eof,
@@ -1430,29 +1527,37 @@ segmenter_parse_do_repeat_3__ (struct segmenter *s,
 
   - The DEFINE keyword.
 
+  - An identifier.  We transform this into SEG_MACRO_NAME instead of
+    SEG_IDENTIFIER or SEG_MACRO_NAME because this identifier must never be
+    macro-expanded.
+
   - Anything but "(".
 
   - "(" followed by a sequence of tokens possibly including balanced parentheses
     up to a final ")".
 
-  - A newline.
-
-  - A sequence of lines that don't start with "!ENDDEFINE", one string per line,
-    each ending in a newline.
-
-  - "!ENDDEFINE".
-
+  - A sequence of any number of lines, one string per line, ending with
+    "!ENDDEFINE".  The first line is usually blank (that is, a newline follows
+    the "(").  The last line usually just has "!ENDDEFINE." on it, but it can
+    start with other tokens.  The whole DEFINE...!ENDDEFINE can be on a single
+    line, even.
    */
 static int
-segmenter_parse_define_1__ (struct segmenter *s,
-                            const char *input, size_t n, bool eof,
-                            enum segment_type *type)
+segmenter_parse_define_1_2__ (struct segmenter *s,
+                              const char *input, size_t n, bool eof,
+                              enum segment_type *type)
 {
   int ofs = segmenter_subparse (s, input, n, eof, type);
   if (ofs < 0)
     return -1;
 
-  if (*type == SEG_SEPARATE_COMMANDS
+  if (s->state == S_DEFINE_1
+      && (*type == SEG_IDENTIFIER || *type == SEG_MACRO_ID))
+    {
+      *type = SEG_MACRO_NAME;
+      s->state = S_DEFINE_2;
+    }
+  else if (*type == SEG_SEPARATE_COMMANDS
       || *type == SEG_END_COMMAND
       || *type == SEG_START_COMMAND)
     {
@@ -1463,7 +1568,7 @@ segmenter_parse_define_1__ (struct segmenter *s,
     }
   else if (*type == SEG_PUNCT && input[0] == '(')
     {
-      s->state = S_DEFINE_2;
+      s->state = S_DEFINE_3;
       s->nest = 1;
       return ofs;
     }
@@ -1472,7 +1577,7 @@ segmenter_parse_define_1__ (struct segmenter *s,
 }
 
 static int
-segmenter_parse_define_2__ (struct segmenter *s,
+segmenter_parse_define_3__ (struct segmenter *s,
                             const char *input, size_t n, bool eof,
                             enum segment_type *type)
 {
@@ -1499,79 +1604,108 @@ segmenter_parse_define_2__ (struct segmenter *s,
     {
       s->nest--;
       if (!s->nest)
-        s->state = S_DEFINE_3;
+        {
+          s->state = S_DEFINE_4;
+          s->substate = 0;
+        }
       return ofs;
     }
 
   return ofs;
 }
 
-static int
-segmenter_parse_define_3__ (struct segmenter *s,
-                            const char *input, size_t n, bool eof,
-                            enum segment_type *type)
+static size_t
+find_enddefine (struct substring input)
 {
-  int ofs = segmenter_subparse (s, input, n, eof, type);
-  if (ofs < 0)
-    return -1;
-
-  if (*type == SEG_END_COMMAND)
+  size_t n = input.length;
+  const struct substring enddefine = ss_cstr ("!ENDDEFINE");
+  for (int ofs = 0;;)
     {
-      /* The DEFINE command is malformed because there was a command terminator
-         before the first line of the body.  Transition back to general
-         parsing. */
-      s->state = S_GENERAL;
-      return ofs;
+      /* Skip !ENDDEFINE in comment. */
+      ofs = skip_spaces_and_comments (input.string, n, true, ofs);
+      if (ofs + enddefine.length > n)
+        return SIZE_MAX;
+
+      char c = input.string[ofs];
+      if (c == '!'
+               && ss_equals_case (ss_substr (input, ofs, enddefine.length),
+                                  enddefine))
+        return ofs;
+      else if (c == '\'' || c == '"')
+        {
+          /* Skip quoted !ENDDEFINE. */
+          ofs++;
+          for (;;)
+            {
+              if (ofs >= n)
+                return SIZE_MAX;
+              else if (input.string[ofs++] == c)
+                break;
+            }
+        }
+      else
+        ofs++;
     }
-  else if (*type == SEG_NEWLINE)
-    s->state = S_DEFINE_4;
-
-  return ofs;
-}
-
-static bool
-is_enddefine (const char *input, size_t n)
-{
-  int ofs = skip_spaces_and_comments (input, n, true, 0);
-  assert (ofs >= 0);
-
-  const struct substring enddefine = ss_cstr ("!ENDDEFINE");
-  if (n - ofs < enddefine.length)
-    return false;
-
-  if (!ss_equals_case (ss_buffer (input + ofs, enddefine.length), enddefine))
-    return false;
-
-  if (ofs + enddefine.length >= n)
-    return true;
-
-  const uint8_t *u_input = CHAR_CAST (const uint8_t *, input);
-  ucs4_t uc;
-  u8_mbtouc (&uc, u_input + ofs, n - ofs);
-  return uc == '.' || !lex_uc_is_idn (uc);
 }
 
+/* We are in the body of a macro definition, looking for additional lines of
+   the body or !ENDDEFINE. */
 static int
 segmenter_parse_define_4__ (struct segmenter *s,
                             const char *input, size_t n, bool eof,
                             enum segment_type *type)
 {
-  int ofs;
-
-  ofs = segmenter_parse_full_line__ (input, n, eof, type);
+  /* Gather a whole line. */
+  const char *newline = memchr (input, '\n', n);
+  int ofs = (newline ? newline - input - (newline > input && newline[-1] == '\r')
+             : eof ? n
+             : -1);
   if (ofs < 0)
     return -1;
-  else if (is_enddefine (input, ofs))
+
+  /* Does the line contain !ENDDEFINE? */
+  size_t end = find_enddefine (ss_buffer (input, ofs));
+  if (end == SIZE_MAX)
     {
-      s->state = S_GENERAL;
-      s->substate = SS_START_OF_COMMAND | SS_START_OF_LINE;
-      return segmenter_push (s, input, n, eof, type);
+      /* No !ENDDEFINE.  We have a full line of macro body.
+
+         The line might be blank, whether completely empty or just spaces and
+         comments.  That's OK: we need to report blank lines because they can
+         have significance.
+
+         However, if the first line of the macro body (the same line as the
+         closing parenthesis in the argument definition) is blank, we just
+         report it as spaces because it's not significant. */
+      *type = (s->substate == 0 && is_all_spaces (input, ofs)
+               ? SEG_SPACES : SEG_MACRO_BODY);
+      s->state = S_DEFINE_5;
+      s->substate = 1;
+      return ofs;
     }
   else
     {
-      *type = SEG_MACRO_BODY;
-      s->state = S_DEFINE_5;
-      return input[ofs - 1] == '\n' ? 0 : ofs;
+      /* Macro ends at the !ENDDEFINE on this line. */
+      s->state = S_GENERAL;
+      s->substate = 0;
+      if (!end)
+        {
+          /* Line starts with !ENDDEFINE. */
+          return segmenter_push (s, input, n, eof, type);
+        }
+      else
+        {
+          if (is_all_spaces (input, end))
+            {
+              /* Line starts with spaces followed by !ENDDEFINE. */
+              *type = SEG_SPACES;
+            }
+          else
+            {
+              /* Line starts with some content followed by !ENDDEFINE. */
+              *type = SEG_MACRO_BODY;
+            }
+          return end;
+        }
     }
 }
 
@@ -1580,9 +1714,7 @@ segmenter_parse_define_5__ (struct segmenter *s,
                             const char *input, size_t n, bool eof,
                             enum segment_type *type)
 {
-  int ofs;
-
-  ofs = segmenter_parse_newline__ (input, n, eof, type);
+  int ofs = segmenter_parse_newline__ (input, n, eof, type);
   if (ofs < 0)
     return -1;
 
@@ -1699,70 +1831,6 @@ segmenter_parse_begin_data_4__ (struct segmenter *s,
   return ofs;
 }
 
-static int
-segmenter_parse_title_1__ (struct segmenter *s,
-                           const char *input, size_t n, bool eof,
-                           enum segment_type *type)
-{
-  int ofs;
-
-  ofs = skip_spaces (input, n, eof, 0);
-  if (ofs < 0)
-    return -1;
-  s->state = S_TITLE_2;
-  *type = SEG_SPACES;
-  return ofs;
-}
-
-static int
-segmenter_parse_title_2__ (struct segmenter *s,
-                           const char *input, size_t n, bool eof,
-                           enum segment_type *type)
-{
-  int endcmd;
-  int ofs;
-
-  endcmd = -1;
-  ofs = 0;
-  while (ofs < n)
-    {
-      ucs4_t uc;
-      int mblen;
-
-      mblen = segmenter_u8_to_uc__ (&uc, input, n, eof, ofs);
-      if (mblen < 0)
-        return -1;
-
-      switch (uc)
-        {
-        case '\n':
-          goto end_of_line;
-
-        case '.':
-          endcmd = ofs;
-          break;
-
-        default:
-          if (!lex_uc_is_space (uc))
-            endcmd = -1;
-          break;
-        }
-
-      ofs += mblen;
-    }
-
-  if (eof)
-    {
-    end_of_line:
-      s->state = S_GENERAL;
-      s->substate = 0;
-      *type = SEG_UNQUOTED_STRING;
-      return endcmd >= 0 ? endcmd : ofs;
-    }
-
-  return -1;
-}
-
 /* Returns the name of segment TYPE as a string.  The caller must not modify
    or free the returned string.
 
@@ -1780,17 +1848,28 @@ segment_type_to_string (enum segment_type type)
     }
 }
 
-/* Initializes S as a segmenter with the given syntax MODE.
+/* Returns a segmenter with the given syntax MODE.
+
+   If IS_SNIPPET is false, then the segmenter will parse as if it's being given
+   a whole file.  This means, for example, that it will interpret - or + at the
+   beginning of the syntax as a separator between commands (since - or + at the
+   beginning of a line has this meaning).
+
+   If IS_SNIPPET is true, then the segmenter will parse as if it's being given
+   an isolated piece of syntax.  This means that, for example, that it will
+   interpret - or + at the beginning of the syntax as an operator token or (if
+   followed by a digit) as part of a number.
 
    A segmenter does not contain any external references, so nothing needs to be
    done to destroy one.  For the same reason, segmenters may be copied with
    plain struct assignment (or memcpy). */
-void
-segmenter_init (struct segmenter *s, enum segmenter_mode mode)
+struct segmenter
+segmenter_init (enum segmenter_mode mode, bool is_snippet)
 {
-  s->state = S_SHBANG;
-  s->substate = 0;
-  s->mode = mode;
+  return (struct segmenter) {
+    .state = is_snippet ? S_GENERAL : S_SHBANG,
+    .mode = mode,
+  };
 }
 
 /* Returns the mode passed to segmenter_init() for S. */
@@ -1814,6 +1893,9 @@ segmenter_get_mode (const struct segmenter *s)
    bytes as part of INPUT, because they have (figuratively) been consumed by
    the segmenter.
 
+   Segments can have zero length, including segment types SEG_END,
+   SEG_SEPARATE_COMMANDS, SEG_START_DOCUMENT, SEG_INLINE_DATA, and SEG_SPACES.
+
    Failure occurs only if the segment type of the N bytes in INPUT cannot yet
    be determined.  In this case segmenter_push() returns -1.  If more input is
    available, the caller should obtain some more, then call again with a larger
@@ -1863,8 +1945,12 @@ segmenter_push (struct segmenter *s, const char *input, size_t n, bool eof,
     case S_DOCUMENT_3:
       return segmenter_parse_document_3__ (s, type);
 
-    case S_FILE_LABEL:
-      return segmenter_parse_file_label__ (s, input, n, eof, type);
+    case S_FILE_LABEL_1:
+      return segmenter_parse_file_label_1__ (s, input, n, eof, type);
+    case S_FILE_LABEL_2:
+      return segmenter_parse_file_label_2__ (s, input, n, eof, type);
+    case S_FILE_LABEL_3:
+      return segmenter_parse_file_label_3__ (s, input, n, eof, type);
 
     case S_DO_REPEAT_1:
       return segmenter_parse_do_repeat_1__ (s, input, n, eof, type);
@@ -1874,9 +1960,8 @@ segmenter_push (struct segmenter *s, const char *input, size_t n, bool eof,
       return segmenter_parse_do_repeat_3__ (s, input, n, eof, type);
 
     case S_DEFINE_1:
-      return segmenter_parse_define_1__ (s, input, n, eof, type);
     case S_DEFINE_2:
-      return segmenter_parse_define_2__ (s, input, n, eof, type);
+      return segmenter_parse_define_1_2__ (s, input, n, eof, type);
     case S_DEFINE_3:
       return segmenter_parse_define_3__ (s, input, n, eof, type);
     case S_DEFINE_4:
@@ -1892,11 +1977,6 @@ segmenter_push (struct segmenter *s, const char *input, size_t n, bool eof,
       return segmenter_parse_begin_data_3__ (s, input, n, eof, type);
     case S_BEGIN_DATA_4:
       return segmenter_parse_begin_data_4__ (s, input, n, eof, type);
-
-    case S_TITLE_1:
-      return segmenter_parse_title_1__ (s, input, n, eof, type);
-    case S_TITLE_2:
-      return segmenter_parse_title_2__ (s, input, n, eof, type);
     }
 
   NOT_REACHED ();
@@ -1927,8 +2007,11 @@ segmenter_get_prompt (const struct segmenter *s)
     case S_DOCUMENT_3:
       return PROMPT_FIRST;
 
-    case S_FILE_LABEL:
+    case S_FILE_LABEL_1:
       return PROMPT_LATER;
+    case S_FILE_LABEL_2:
+    case S_FILE_LABEL_3:
+      return PROMPT_FIRST;
 
     case S_DO_REPEAT_1:
     case S_DO_REPEAT_2:
@@ -1952,9 +2035,6 @@ segmenter_get_prompt (const struct segmenter *s)
     case S_BEGIN_DATA_4:
       return PROMPT_DATA;
 
-    case S_TITLE_1:
-    case S_TITLE_2:
-      return PROMPT_FIRST;
     }
 
   NOT_REACHED ();