segment: Refine treatment of start of macro body.

[pspp] / tests / language / lexer / scan.at

diff --git a/tests/language/lexer/scan.at b/tests/language/lexer/scan.at
index a6b0e62881663c00ba87075203e08d067d3ef2c0..d1fb66fcbeb5dd90deefefc81c62a32f851149e4 100644 (file)
--- a/tests/language/lexer/scan.at
+++ b/tests/language/lexer/scan.at
@@ -1,34 +1,38 @@
 dnl PSPP - a program for statistical analysis.
 dnl Copyright (C) 2017 Free Software Foundation, Inc.
-dnl 
+dnl
 dnl This program is free software: you can redistribute it and/or modify
 dnl it under the terms of the GNU General Public License as published by
 dnl the Free Software Foundation, either version 3 of the License, or
 dnl (at your option) any later version.
-dnl 
+dnl
 dnl This program is distributed in the hope that it will be useful,
 dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
 dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 dnl GNU General Public License for more details.
-dnl 
+dnl
 dnl You should have received a copy of the GNU General Public License
 dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
 dnl
 AT_BANNER([syntax scanning])
 m4_define([PSPP_CHECK_SCAN],
-  [AT_CHECK([scan-test $1 input], [0], [expout])])
+  [sed 's/^-//' < expout-base > expout
+   AT_CHECK([scan-test $1 input], [0], [expout])
+
+   sed '/^-/d' < expout-base > expout
+   AT_CHECK([scan-test -s $1 input], [0], [expout])])
 \f
 AT_SETUP([identifiers])
 AT_KEYWORDS([scan])
 AT_DATA([input], [dnl
-a aB i5 $x @efg @@. #.# .x _z.
+a aB i5 $x @efg @@. !abcd #.# .x _z.
 abcd. abcd.
 QRSTUV./* end of line comment */
 QrStUv./* end of line comment */ @&t@
 WXYZ. /* unterminated end of line comment
 �. /* U+FFFD is not valid in an identifier
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 ID "a"
 SKIP
 ID "aB"
@@ -41,12 +45,14 @@ ID "@efg"
 SKIP
 ID "@@."
 SKIP
+MACRO_ID "!abcd"
+SKIP
 ID "#.#"
 SKIP
-UNEXPECTED_DOT
+MACRO_PUNCT "."
 ID "x"
 SKIP
-UNEXPECTED_CHAR 95
+MACRO_PUNCT "_"
 ID "z"
 ENDCMD
 SKIP
@@ -73,7 +79,7 @@ UNEXPECTED_CHAR 65533
 ENDCMD
 SKIP
 SKIP
-SKIP
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -87,7 +93,7 @@ AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH
 andx orx notx eqx gex gtx lex ltx nex allx byx tox withx
 and. with.
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 AND
 SKIP
 OR
@@ -170,7 +176,7 @@ ID "and."
 SKIP
 WITH
 ENDCMD
-SKIP
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -181,8 +187,9 @@ AT_KEYWORDS([scan])
 AT_DATA([input], [dnl
 ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] **
 ~&|=>=><=<~=<>(),-+*/[[]]**
+% : ; ? _ ` { } ~
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 NOT
 SKIP
 AND
@@ -244,6 +251,24 @@ LBRACK
 RBRACK
 EXP
 SKIP
+MACRO_PUNCT "%"
+SKIP
+MACRO_PUNCT ":"
+SKIP
+MACRO_PUNCT ";"
+SKIP
+MACRO_PUNCT "?"
+SKIP
+MACRO_PUNCT "_"
+SKIP
+MACRO_PUNCT "`"
+SKIP
+MACRO_PUNCT "{"
+SKIP
+MACRO_PUNCT "}"
+SKIP
+NOT
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -260,7 +285,7 @@ AT_DATA([input], [dnl
 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03
 . 1e e1 1e+ 1e-
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 POS_NUM
 SKIP
 POS_NUM 1
@@ -328,7 +353,7 @@ SKIP
 EXPECTED_EXPONENT "1e+"
 SKIP
 EXPECTED_EXPONENT "1e-"
-SKIP
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -367,7 +392,7 @@ x"4142"
 "�あいうえお"
 "abc"+U"FFFD"+u'3048'+"xyz"
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 STRING "x"
 SKIP
 STRING "y"
@@ -423,7 +448,7 @@ SKIP
 STRING "�あいうえお"
 SKIP
 STRING "abc�えxyz"
-SKIP
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -435,11 +460,11 @@ AT_DATA([input], [dnl
 #! /usr/bin/pspp
 #! /usr/bin/pspp
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 SKIP
 SKIP
 ID "#"
-UNEXPECTED_CHAR 33
+MACRO_ID "!"
 SKIP
 SLASH
 ID "usr"
@@ -447,7 +472,7 @@ SLASH
 ID "bin"
 SLASH
 ID "pspp"
-SKIP
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -473,7 +498,7 @@ com is ambiguous with COMPUTE.
 next command.
 
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 SKIP
 SKIP
 SKIP
@@ -523,8 +548,8 @@ SKIP
 ID "command"
 ENDCMD
 SKIP
-ENDCMD
-SKIP
+-ENDCMD
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -544,7 +569,7 @@ isn't parsed as tokens
 
 second paragraph.
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 ID "DOCUMENT"
 STRING "DOCUMENT one line."
 ENDCMD
@@ -571,9 +596,9 @@ SKIP
 STRING ""
 SKIP
 STRING "second paragraph."
-ENDCMD
-ENDCMD
-SKIP
+-ENDCMD
+-ENDCMD
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -598,7 +623,7 @@ FILE /*
 /**/  lab not quoted here either
 
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 ID "title"
 SKIP
 STRING "Quoted string title"
@@ -656,8 +681,8 @@ ID "lab"
 SKIP
 STRING "not quoted here either"
 SKIP
-ENDCMD
-SKIP
+-ENDCMD
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -678,7 +703,7 @@ end  data
 end data
 .
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 ID "begin"
 SKIP
 ID "data"
@@ -714,7 +739,7 @@ SKIP
 ID "data"
 SKIP
 ENDCMD
-SKIP
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -733,7 +758,7 @@ end /* x */ /* y */ repeat print.
 end
  repeat.
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 ID "do"
 SKIP
 ID "repeat"
@@ -771,7 +796,485 @@ SKIP
 SKIP
 ID "repeat"
 ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DO REPEAT command in batch mode])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+do repeat x=a b c
+          y=d e f
+do repeat a=1 thru 5
+another command
+second command
++ third command
+end /* x */ /* y */ repeat print
+end
+ repeat
+do
+  repeat #a=1
+
+  inner command
+end repeat
+])
+AT_DATA([expout-base], [dnl
+ID "do"
+SKIP
+ID "repeat"
+SKIP
+ID "x"
+EQUALS
+ID "a"
+SKIP
+ID "b"
+SKIP
+ID "c"
+SKIP
+SKIP
+ID "y"
+EQUALS
+ID "d"
+SKIP
+ID "e"
+SKIP
+ID "f"
+SKIP
+ENDCMD
+STRING "do repeat a=1 thru 5"
+SKIP
+STRING "another command"
+SKIP
+STRING "second command"
+SKIP
+STRING "+ third command"
+SKIP
+STRING "end /* x */ /* y */ repeat print"
+SKIP
+ID "end"
+SKIP
+SKIP
+ID "repeat"
+SKIP
+ENDCMD
+ID "do"
+SKIP
+SKIP
+ID "repeat"
+SKIP
+ID "#a"
+EQUALS
+POS_NUM 1
+SKIP
+ENDCMD
+SKIP
+STRING "  inner command"
+SKIP
+ID "end"
+SKIP
+ID "repeat"
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-b])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - simple])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1()
+var1 var2 var3
+!enddefine.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+LPAREN
+RPAREN
+SKIP
+SKIP
+STRING "var1 var2 var3"
+SKIP
+MACRO_ID "!enddefine"
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - no newline after parentheses])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1() var1 var2 var3
+!enddefine.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+LPAREN
+RPAREN
+STRING " var1 var2 var3"
+SKIP
+MACRO_ID "!enddefine"
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - no newline before !ENDDEFINE])
+AT_KEYWORDS([scan ENDDEFINE])
+AT_DATA([input], [dnl
+define !macro1()
+var1 var2 var3!enddefine.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+LPAREN
+RPAREN
+SKIP
+SKIP
+STRING "var1 var2 var3"
+MACRO_ID "!enddefine"
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - all on one line])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1()var1 var2 var3!enddefine.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+LPAREN
+RPAREN
+STRING "var1 var2 var3"
+MACRO_ID "!enddefine"
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - empty])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1()
+!enddefine.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+LPAREN
+RPAREN
+SKIP
+SKIP
+MACRO_ID "!enddefine"
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - blank lines])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1()
+
+
+!enddefine.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+LPAREN
+RPAREN
+SKIP
+SKIP
+STRING ""
+SKIP
+STRING ""
+SKIP
+MACRO_ID "!enddefine"
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - arguments])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1(a(), b(), c())
+!enddefine.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+LPAREN
+ID "a"
+LPAREN
+RPAREN
+COMMA
+SKIP
+ID "b"
+LPAREN
+RPAREN
+COMMA
+SKIP
+ID "c"
+LPAREN
+RPAREN
+RPAREN
+SKIP
+SKIP
+MACRO_ID "!enddefine"
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - multiline arguments])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1(
+  a(), b(
+  ),
+  c()
+)
+!enddefine.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+LPAREN
+SKIP
+SKIP
+ID "a"
+LPAREN
+RPAREN
+COMMA
+SKIP
+ID "b"
+LPAREN
+SKIP
+SKIP
+RPAREN
+COMMA
+SKIP
+SKIP
+ID "c"
+LPAREN
+RPAREN
+SKIP
+RPAREN
+SKIP
+SKIP
+MACRO_ID "!enddefine"
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - arguments start on second line])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1
+(x,y,z
+)
+content 1
+content 2
+!enddefine.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+SKIP
+LPAREN
+ID "x"
+COMMA
+ID "y"
+COMMA
+ID "z"
+SKIP
+RPAREN
+SKIP
+SKIP
+STRING "content 1"
+SKIP
+STRING "content 2"
+SKIP
+MACRO_ID "!enddefine"
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - early end of command 1])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1.
+data list /x 1.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+ENDCMD
+SKIP
+ID "data"
+SKIP
+ID "list"
+SKIP
+SLASH
+ID "x"
+SKIP
+POS_NUM 1
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - early end of command 2])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1
+x.
+data list /x 1.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+SKIP
+ID "x"
+ENDCMD
+SKIP
+ID "data"
+SKIP
+ID "list"
+SKIP
+SLASH
+ID "x"
+SKIP
+POS_NUM 1
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - early end of command 3])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1(.
+x.
+data list /x 1.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+LPAREN
+ENDCMD
 SKIP
+ID "x"
+ENDCMD
+SKIP
+ID "data"
+SKIP
+ID "list"
+SKIP
+SLASH
+ID "x"
+SKIP
+POS_NUM 1
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - early end of command 4])
+AT_KEYWORDS([segment])
+AT_DATA([input], [dnl
+dnl Notice the command terminator at the end of the DEFINE command,
+dnl which should not be there and ends it early.
+define !macro1.
+data list /x 1.
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+ENDCMD
+SKIP
+ID "data"
+SKIP
+ID "list"
+SKIP
+SLASH
+ID "x"
+SKIP
+POS_NUM 1
+ENDCMD
+-SKIP
+STOP
+])
+PSPP_CHECK_SCAN([-i])
+AT_CLEANUP
+\f
+AT_SETUP([DEFINE command - missing !ENDDEFINE])
+AT_KEYWORDS([scan])
+AT_DATA([input], [dnl
+define !macro1()
+content line 1
+content line 2
+])
+AT_DATA([expout-base], [dnl
+ID "define"
+SKIP
+MACRO_ID "!macro1"
+LPAREN
+RPAREN
+SKIP
+SKIP
+STRING "content line 1"
+SKIP
+STRING "content line 2"
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-i])
@@ -788,7 +1291,7 @@ third command
 fourth command.
    fifth command.
 ])
-AT_DATA([expout], [dnl
+AT_DATA([expout-base], [dnl
 ID "first"
 SKIP
 ID "command"
@@ -827,7 +1330,7 @@ ID "fifth"
 SKIP
 ID "command"
 ENDCMD
-SKIP
+-SKIP
 STOP
 ])
 PSPP_CHECK_SCAN([-b])