lexer: Add tokens for '{', '}', ':', ';' for use in the matrix language.

[pspp] / src / language / lexer / segment.c

diff --git a/src/language/lexer/segment.c b/src/language/lexer/segment.c
index 4a6fefb4c25d1e7c1441cd1a6a8d43f5b76b1873..346910898ce3898b4bec6c62563a1a6f8dfe10bd 100644 (file)
--- a/src/language/lexer/segment.c
+++ b/src/language/lexer/segment.c
@@ -943,9 +943,9 @@ segmenter_parse_mid_command__ (struct segmenter *s,
       ofs = skip_spaces (input, n, eof, 1);
       if (ofs < 0)
         return -1;
-      else if (c_isdigit (input[ofs]))
+      else if (ofs < n && c_isdigit (input[ofs]))
         return segmenter_parse_number__ (s, input, n, eof, type, ofs);
-      else if (input[ofs] == '.')
+      else if (ofs < n && input[ofs] == '.')
         {
           if (ofs + 1 >= n)
             {
@@ -956,8 +956,8 @@ segmenter_parse_mid_command__ (struct segmenter *s,
             return segmenter_parse_number__ (s, input, n, eof, type, ofs);
         }
       /* Fall through. */
-    case '(': case ')': case ',': case '=':
-    case '[': case ']': case '&': case '|': case '+':
+    case '(': case ')': case '{': case ',': case '=': case ';': case ':':
+    case '[': case ']': case '}': case '&': case '|': case '+':
       *type = SEG_PUNCT;
       s->substate = 0;
       return 1;
@@ -1893,6 +1893,9 @@ segmenter_get_mode (const struct segmenter *s)
    bytes as part of INPUT, because they have (figuratively) been consumed by
    the segmenter.
 
+   Segments can have zero length, including segment types SEG_END,
+   SEG_SEPARATE_COMMANDS, SEG_START_DOCUMENT, SEG_INLINE_DATA, and SEG_SPACES.
+
    Failure occurs only if the segment type of the N bytes in INPUT cannot yet
    be determined.  In this case segmenter_push() returns -1.  If more input is
    available, the caller should obtain some more, then call again with a larger