segment: Properly handle DO REPEAT in batch mode.

In batch mode, a line that begins with a non-blank starts a new command,
but the segmenter didn't properly support this.  Without this commit,
the following wasn't properly handled:

DO REPEAT #x=1
ECHO 'hi'
END REPEAT

This commit also adds a test against regression, which also checks that
a blank line to separate DO REPEAT from the repeated syntax works (it
did before too, but there was no test).

diff --git a/src/language/lexer/segment.c b/src/language/lexer/segment.c
index c607c4bd1ffc52061ab7cf7d6d6632d3e4a8b249..93eb1e8f8594b564d768737f5e84057da8085009 100644 (file)
--- a/src/language/lexer/segment.c
+++ b/src/language/lexer/segment.c
@@ -1274,6 +1274,9 @@ segmenter_subparse (struct segmenter *s,
   return ofs;
 }
 
+/* We are segmenting a DO REPEAT command, currently reading the syntax that
+   defines the stand-in variables (the head) before the lines of syntax to be
+   repeated (the body). */
 static int
 segmenter_parse_do_repeat_1__ (struct segmenter *s,
                                const char *input, size_t n, bool eof,
@@ -1283,10 +1286,14 @@ segmenter_parse_do_repeat_1__ (struct segmenter *s,
   if (ofs < 0)
     return -1;
 
-  if (*type == SEG_START_COMMAND || *type == SEG_SEPARATE_COMMANDS)
-    s->state = S_DO_REPEAT_2;
-  else if (*type == SEG_END_COMMAND)
+  if (*type == SEG_SEPARATE_COMMANDS)
     {
+      /* We reached a blank line that separates the head from the body. */
+      s->state = S_DO_REPEAT_2;
+    }
+  else if (*type == SEG_END_COMMAND || *type == SEG_START_COMMAND)
+    {
+      /* We reached the body. */
       s->state = S_DO_REPEAT_3;
       s->substate = 1;
     }
@@ -1294,6 +1301,8 @@ segmenter_parse_do_repeat_1__ (struct segmenter *s,
   return ofs;
 }
 
+/* We are segmenting a DO REPEAT command, currently reading a blank line that
+   separates the head from the body. */
 static int
 segmenter_parse_do_repeat_2__ (struct segmenter *s,
                                const char *input, size_t n, bool eof,
@@ -1305,6 +1314,7 @@ segmenter_parse_do_repeat_2__ (struct segmenter *s,
 
   if (*type == SEG_NEWLINE)
     {
+      /* We reached the body. */
       s->state = S_DO_REPEAT_3;
       s->substate = 1;
     }
@@ -1361,6 +1371,12 @@ segmenter_parse_full_line__ (const char *input, size_t n, bool eof,
     return ofs - (input[ofs - 1] == '\r');
 }
 
+/* We are in the body of DO REPEAT, segmenting the lines of syntax that are to
+   be repeated.  Report each line of syntax as a single SEG_DO_REPEAT_COMMAND.
+
+   DO REPEAT can be nested, so we look for DO REPEAT...END REPEAT blocks inside
+   the lines we're segmenting.  s->substate counts the nesting level, starting
+   at 1. */
 static int
 segmenter_parse_do_repeat_3__ (struct segmenter *s,
                                const char *input, size_t n, bool eof,
@@ -1375,6 +1391,8 @@ segmenter_parse_do_repeat_3__ (struct segmenter *s,
     return -1;
   else if (s->substate == 0)
     {
+      /* Nesting level dropped to 0, so we've finished reading the DO REPEAT
+         body. */
       s->state = S_GENERAL;
       s->substate = SS_START_OF_COMMAND | SS_START_OF_LINE;
       return segmenter_push (s, input, n, eof, type);

diff --git a/tests/language/lexer/scan.at b/tests/language/lexer/scan.at
index 9eded70f8756c789aa5c04e45d656fcd6b8242fb..3da89484c44e1e0145d406269840d21843d47816 100644 (file)
--- a/tests/language/lexer/scan.at
+++ b/tests/language/lexer/scan.at
@@ -781,6 +781,85 @@ STOP
 PSPP_CHECK_SCAN([-i])
 AT_CLEANUP
 \f
+AT_SETUP([DO REPEAT command in batch mode])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+do repeat x=a b c
+          y=d e f
+do repeat a=1 thru 5
+another command
+second command
++ third command
+end /* x */ /* y */ repeat print
+end
+ repeat
+do
+  repeat #a=1
+
+  inner command
+end repeat
+])
+AT_DATA([expout-base], [dnl
+ID "do"
+SKIP
+ID "repeat"
+SKIP
+ID "x"
+EQUALS
+ID "a"
+SKIP
+ID "b"
+SKIP
+ID "c"
+SKIP
+SKIP
+ID "y"
+EQUALS
+ID "d"
+SKIP
+ID "e"
+SKIP
+ID "f"
+SKIP
+ENDCMD
+STRING "do repeat a=1 thru 5"
+SKIP
+STRING "another command"
+SKIP
+STRING "second command"
+SKIP
+STRING "+ third command"
+SKIP
+STRING "end /* x */ /* y */ repeat print"
+SKIP
+ID "end"
+SKIP
+SKIP
+ID "repeat"
+SKIP
+ENDCMD
+ID "do"
+SKIP
+SKIP
+ID "repeat"
+SKIP
+ID "#a"
+EQUALS
+POS_NUM 1
+SKIP
+ENDCMD
+SKIP
+STRING "  inner command"
+SKIP
+ID "end"
+SKIP
+ID "repeat"
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-b])
+AT_CLEANUP
+\f
 AT_SETUP([batch mode])
 AT_KEYWORDS([scan])
 AT_DATA([input], [dnl

diff --git a/tests/language/lexer/segment.at b/tests/language/lexer/segment.at
index 22d1bdee5b79cd8aed834b76227d607ea5887104..04a77a9b8fd103ebc0b09e3a2533cc7e13d7d32d 100644 (file)
--- a/tests/language/lexer/segment.at
+++ b/tests/language/lexer/segment.at
@@ -949,6 +949,90 @@ end
 PSPP_CHECK_SEGMENT([-i])
 AT_CLEANUP
 \f
+AT_SETUP([DO REPEAT command in batch mode])
+AT_KEYWORDS([segment])
+AT_DATA([input], [dnl
+do repeat x=a b c
+          y=d e f
+do repeat a=1 thru 5
+another command
+second command
++ third command
+end /* x */ /* y */ repeat print
+end
+ repeat
+do
+  repeat #a=1
+
+  inner command
+end repeat
+])
+AT_DATA([expout-base], [dnl
+identifier      do    space
+identifier      repeat    space
+identifier      x
+punct           =
+identifier      a    space
+identifier      b    space
+identifier      c
+newline         \n (later)
+
+spaces          __________
+identifier      y
+punct           =
+identifier      d    space
+identifier      e    space
+identifier      f
+newline         \n (later)
+
+start_command
+do_repeat_command do_repeat_a=1_thru_5
+newline         \n (DO REPEAT)
+
+do_repeat_command another_command
+newline         \n (DO REPEAT)
+
+do_repeat_command second_command
+newline         \n (DO REPEAT)
+
+do_repeat_command +_third_command
+newline         \n (DO REPEAT)
+
+do_repeat_command end_/*_x_*/_/*_y_*/_repeat_print
+newline         \n (DO REPEAT)
+
+identifier      end
+newline         \n (later)
+    space
+identifier      repeat
+newline         \n (later)
+
+start_command
+identifier      do
+newline         \n (later)
+
+spaces          __
+identifier      repeat    space
+identifier      #a
+punct           =
+number          1
+newline         \n (later)
+
+separate_commands
+newline         \n (DO REPEAT)
+
+do_repeat_command __inner_command
+newline         \n (DO REPEAT)
+
+identifier      end    space
+identifier      repeat
+-newline         \n (later)
+-
+end
+])
+PSPP_CHECK_SEGMENT([-b])
+AT_CLEANUP
+\f
 AT_SETUP([batch mode])
 AT_KEYWORDS([segment])
 AT_DATA([input], [dnl