dnl PSPP - a program for statistical analysis. dnl Copyright (C) 2017 Free Software Foundation, Inc. dnl dnl This program is free software: you can redistribute it and/or modify dnl it under the terms of the GNU General Public License as published by dnl the Free Software Foundation, either version 3 of the License, or dnl (at your option) any later version. dnl dnl This program is distributed in the hope that it will be useful, dnl but WITHOUT ANY WARRANTY; without even the implied warranty of dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the dnl GNU General Public License for more details. dnl dnl You should have received a copy of the GNU General Public License dnl along with this program. If not, see . dnl AT_BANNER([syntax scanning]) m4_define([PSPP_CHECK_SCAN], [sed 's/^-//' < expout-base > expout AT_CHECK([scan-test $1 input], [0], [expout]) sed '/^-/d' < expout-base > expout AT_CHECK([scan-test -s $1 input], [0], [expout])]) AT_SETUP([identifiers]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl a aB i5 $x @efg @@. !abcd #.# .x _z. abcd. abcd. QRSTUV./* end of line comment */ QrStUv./* end of line comment */ @&t@ WXYZ. /* unterminated end of line comment �. /* U+FFFD is not valid in an identifier ]) AT_DATA([expout-base], [dnl ID "a" SKIP ID "aB" SKIP ID "i5" SKIP ID "$x" SKIP ID "@efg" SKIP ID "@@." SKIP MACRO_ID "!abcd" SKIP ID "#.#" SKIP UNEXPECTED_DOT ID "x" SKIP MACRO_PUNCT "_" ID "z" ENDCMD SKIP ID "abcd." SKIP ID "abcd" ENDCMD SKIP ID "QRSTUV" ENDCMD SKIP SKIP ID "QrStUv" ENDCMD SKIP SKIP SKIP ID "WXYZ" ENDCMD SKIP SKIP SKIP UNEXPECTED_CHAR 65533 ENDCMD SKIP SKIP -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([reserved words]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl and or not eq ge gt le lt ne all by to with AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH andx orx notx eqx gex gtx lex ltx nex allx byx tox withx and. with. ]) AT_DATA([expout-base], [dnl AND SKIP OR SKIP NOT SKIP EQ SKIP GE SKIP GT SKIP LE SKIP LT SKIP NE SKIP ALL SKIP BY SKIP TO SKIP WITH SKIP AND SKIP OR SKIP NOT SKIP EQ SKIP GE SKIP GT SKIP LE SKIP LT SKIP NE SKIP ALL SKIP BY SKIP TO SKIP WITH SKIP ID "andx" SKIP ID "orx" SKIP ID "notx" SKIP ID "eqx" SKIP ID "gex" SKIP ID "gtx" SKIP ID "lex" SKIP ID "ltx" SKIP ID "nex" SKIP ID "allx" SKIP ID "byx" SKIP ID "tox" SKIP ID "withx" SKIP ID "and." SKIP WITH ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([punctuation]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] ** ~&|=>=><=<~=<>(),-+*/[[]]** % : ; ? _ ` { } ~ ]) AT_DATA([expout-base], [dnl NOT SKIP AND SKIP OR SKIP EQUALS SKIP GE SKIP GT SKIP LE SKIP LT SKIP NE SKIP NE SKIP LPAREN SKIP RPAREN SKIP COMMA SKIP DASH SKIP PLUS SKIP ASTERISK SKIP SLASH SKIP LBRACK SKIP RBRACK SKIP EXP SKIP NOT AND OR EQUALS GE GT LE LT NE NE LPAREN RPAREN COMMA DASH PLUS ASTERISK SLASH LBRACK RBRACK EXP SKIP MACRO_PUNCT "%" SKIP MACRO_PUNCT ":" SKIP MACRO_PUNCT ";" SKIP MACRO_PUNCT "?" SKIP MACRO_PUNCT "_" SKIP MACRO_PUNCT "`" SKIP MACRO_PUNCT "{" SKIP MACRO_PUNCT "}" SKIP NOT -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([numbers]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl 0 1 01 001. 1. 123. /* comment 1 */ /* comment 2 */ .1 0.1 00.1 00.10 5e1 6E-1 7e+1 6E+01 6e-03 .3E1 .4e-1 .5E+1 .6e+01 .7E-03 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03 . 1e e1 1e+ 1e- ]) AT_DATA([expout-base], [dnl POS_NUM SKIP POS_NUM 1 SKIP POS_NUM 1 SKIP POS_NUM 1 SKIP POS_NUM 1 ENDCMD SKIP POS_NUM 123 ENDCMD SKIP SKIP SKIP SKIP SKIP ENDCMD POS_NUM 1 SKIP POS_NUM 0.1 SKIP POS_NUM 0.1 SKIP POS_NUM 0.1 SKIP POS_NUM 50 SKIP POS_NUM 0.6 SKIP POS_NUM 70 SKIP POS_NUM 60 SKIP POS_NUM 0.006 SKIP ENDCMD POS_NUM 30 SKIP POS_NUM 0.04 SKIP POS_NUM 5 SKIP POS_NUM 6 SKIP POS_NUM 0.0007 SKIP POS_NUM 12.3 SKIP POS_NUM 4.56 SKIP POS_NUM 789 SKIP POS_NUM 999 SKIP POS_NUM 0.0112 SKIP ENDCMD SKIP EXPECTED_EXPONENT "1e" SKIP ID "e1" SKIP EXPECTED_EXPONENT "1e+" SKIP EXPECTED_EXPONENT "1e-" -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([strings]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl 'x' "y" 'abc' 'Don''t' "Can't" 'Won''t' """quoted""" '"quoted"' '' "" '''' """" 'missing end quote "missing double quote 'x' + "y" + 'z' + 'a' /* abc */ + "b" /* + 'c' +/* */"d"/* */+'e' 'foo' + /* special case: + in column 0 would ordinarily start a new command 'bar' 'foo' + 'bar' 'foo' + 'bar' + x"4142"+'5152' "4142"+ x'5152' x"4142" +u'304a' "�あいうえお" "abc"+U"FFFD"+u'3048'+"xyz" ]) AT_DATA([expout-base], [dnl STRING "x" SKIP STRING "y" SKIP STRING "abc" SKIP STRING "Don't" SKIP STRING "Can't" SKIP STRING "Won't" SKIP STRING ""quoted"" SKIP STRING ""quoted"" SKIP STRING "" SKIP STRING "" SKIP STRING "'" SKIP STRING """ SKIP EXPECTED_QUOTE SKIP EXPECTED_QUOTE SKIP STRING "xyzabcde" SKIP STRING "foobar" SKIP STRING "foobar" SKIP STRING "foo" SKIP PLUS SKIP ENDCMD SKIP STRING "bar" SKIP ENDCMD SKIP PLUS SKIP STRING "AB5152" SKIP STRING "4142QR" SKIP STRING "ABお" SKIP STRING "�あいうえお" SKIP STRING "abc�えxyz" -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([@%:@! construct]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl #! /usr/bin/pspp #! /usr/bin/pspp ]) AT_DATA([expout-base], [dnl SKIP SKIP ID "#" MACRO_ID "!" SKIP SLASH ID "usr" SLASH ID "bin" SLASH ID "pspp" -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([* and COMMENT commands]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl * Comment commands "don't have to contain valid tokens. ** Check ambiguity with ** token. ****************. comment keyword works too. COMM also. com is ambiguous with COMPUTE. * Comment need not start at left margin. * Comment ends with blank line next command. ]) AT_DATA([expout-base], [dnl SKIP SKIP SKIP ENDCMD SKIP ENDCMD SKIP SKIP ENDCMD SKIP SKIP ENDCMD SKIP ENDCMD SKIP SKIP ENDCMD SKIP SKIP ENDCMD SKIP ID "com" SKIP ID "is" SKIP ID "ambiguous" SKIP WITH SKIP ID "COMPUTE" ENDCMD SKIP ENDCMD SKIP SKIP SKIP ENDCMD SKIP ENDCMD SKIP SKIP SKIP ENDCMD SKIP ID "next" SKIP ID "command" ENDCMD SKIP -ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DOCUMENT command]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl DOCUMENT one line. DOC more than one line. docu first.paragraph isn't parsed as tokens second paragraph. ]) AT_DATA([expout-base], [dnl ID "DOCUMENT" STRING "DOCUMENT one line." ENDCMD ENDCMD SKIP ID "DOCUMENT" STRING "DOC more" SKIP STRING " than" SKIP STRING " one" SKIP STRING " line." ENDCMD ENDCMD SKIP ID "DOCUMENT" STRING "docu" SKIP STRING "first.paragraph" SKIP STRING "isn't parsed as tokens" SKIP STRING "" SKIP STRING "second paragraph." -ENDCMD -ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl title/**/'Quoted string title'. tit /* "Quoted string on second line". sub "Quoted string subtitle" . TITL /* Not a */ quoted string title. SUBT Not a quoted string /* subtitle FIL label isn't quoted. FILE lab 'is quoted'. FILE /* /**/ lab not quoted here either ]) AT_DATA([expout-base], [dnl ID "title" SKIP STRING "Quoted string title" ENDCMD SKIP ID "tit" SKIP SKIP SKIP STRING "Quoted string on second line" ENDCMD SKIP ID "sub" SKIP STRING "Quoted string subtitle" SKIP SKIP ENDCMD SKIP ENDCMD SKIP ID "TITL" SKIP STRING "/* Not a */ quoted string title" ENDCMD SKIP ID "SUBT" SKIP STRING "Not a quoted string /* subtitle" SKIP ENDCMD SKIP ID "FIL" SKIP ID "label" SKIP STRING "isn't quoted" ENDCMD SKIP ID "FILE" SKIP SKIP ID "lab" SKIP STRING "is quoted" ENDCMD SKIP ID "FILE" SKIP SKIP SKIP SKIP SKIP ID "lab" SKIP STRING "not quoted here either" SKIP -ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([BEGIN DATA command]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl begin data. 123 xxx end data. BEG /**/ DAT /* 5 6 7 /* x end data end data . ]) AT_DATA([expout-base], [dnl ID "begin" SKIP ID "data" ENDCMD SKIP STRING "123" SKIP STRING "xxx" SKIP ID "end" SKIP ID "data" ENDCMD SKIP ENDCMD SKIP ID "BEG" SKIP SKIP SKIP ID "DAT" SKIP SKIP SKIP STRING "5 6 7 /* x" SKIP STRING "" SKIP STRING "end data" SKIP ID "end" SKIP ID "data" SKIP ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DO REPEAT command]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl do repeat x=a b c y=d e f. do repeat a=1 thru 5. another command. second command + third command. end /* x */ /* y */ repeat print. end repeat. ]) AT_DATA([expout-base], [dnl ID "do" SKIP ID "repeat" SKIP ID "x" EQUALS ID "a" SKIP ID "b" SKIP ID "c" SKIP SKIP ID "y" EQUALS ID "d" SKIP ID "e" SKIP ID "f" ENDCMD SKIP STRING " do repeat a=1 thru 5." SKIP STRING "another command." SKIP STRING "second command" SKIP STRING "+ third command." SKIP STRING "end /* x */ /* y */ repeat print." SKIP ID "end" SKIP SKIP ID "repeat" ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DO REPEAT command in batch mode]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl do repeat x=a b c y=d e f do repeat a=1 thru 5 another command second command + third command end /* x */ /* y */ repeat print end repeat do repeat #a=1 inner command end repeat ]) AT_DATA([expout-base], [dnl ID "do" SKIP ID "repeat" SKIP ID "x" EQUALS ID "a" SKIP ID "b" SKIP ID "c" SKIP SKIP ID "y" EQUALS ID "d" SKIP ID "e" SKIP ID "f" SKIP ENDCMD STRING "do repeat a=1 thru 5" SKIP STRING "another command" SKIP STRING "second command" SKIP STRING "+ third command" SKIP STRING "end /* x */ /* y */ repeat print" SKIP ID "end" SKIP SKIP ID "repeat" SKIP ENDCMD ID "do" SKIP SKIP ID "repeat" SKIP ID "#a" EQUALS POS_NUM 1 SKIP ENDCMD SKIP STRING " inner command" SKIP ID "end" SKIP ID "repeat" -SKIP STOP ]) PSPP_CHECK_SCAN([-b]) AT_CLEANUP AT_SETUP([DEFINE command - simple]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl define !macro1() var1 var2 var3 !enddefine. ]) AT_DATA([expout-base], [dnl ID "define" SKIP MACRO_ID "!macro1" LPAREN RPAREN SKIP STRING "var1 var2 var3" SKIP MACRO_ID "!enddefine" ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DEFINE command - empty]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl define !macro1() !enddefine. ]) AT_DATA([expout-base], [dnl ID "define" SKIP MACRO_ID "!macro1" LPAREN RPAREN SKIP MACRO_ID "!enddefine" ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DEFINE command - arguments]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl define !macro1(a(), b(), c()) !enddefine. ]) AT_DATA([expout-base], [dnl ID "define" SKIP MACRO_ID "!macro1" LPAREN ID "a" LPAREN RPAREN COMMA SKIP ID "b" LPAREN RPAREN COMMA SKIP ID "c" LPAREN RPAREN RPAREN SKIP MACRO_ID "!enddefine" ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DEFINE command - multiline arguments]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl define !macro1( a(), b( ), c() ) !enddefine. ]) AT_DATA([expout-base], [dnl ID "define" SKIP MACRO_ID "!macro1" LPAREN SKIP SKIP ID "a" LPAREN RPAREN COMMA SKIP ID "b" LPAREN SKIP SKIP RPAREN COMMA SKIP SKIP ID "c" LPAREN RPAREN SKIP RPAREN SKIP MACRO_ID "!enddefine" ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DEFINE command - arguments start on second line]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl define !macro1 (x,y,z ) content 1 content 2 !enddefine. ]) AT_DATA([expout-base], [dnl ID "define" SKIP MACRO_ID "!macro1" SKIP LPAREN ID "x" COMMA ID "y" COMMA ID "z" SKIP RPAREN SKIP STRING "content 1" SKIP STRING "content 2" SKIP MACRO_ID "!enddefine" ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DEFINE command - early end of command 1]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl define !macro1. data list /x 1. ]) AT_DATA([expout-base], [dnl ID "define" SKIP MACRO_ID "!macro1" ENDCMD SKIP ID "data" SKIP ID "list" SKIP SLASH ID "x" SKIP POS_NUM 1 ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DEFINE command - early end of command 2]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl define !macro1 x. data list /x 1. ]) AT_DATA([expout-base], [dnl ID "define" SKIP MACRO_ID "!macro1" SKIP ID "x" ENDCMD SKIP ID "data" SKIP ID "list" SKIP SLASH ID "x" SKIP POS_NUM 1 ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DEFINE command - early end of command 3]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl define !macro1(. x. data list /x 1. ]) AT_DATA([expout-base], [dnl ID "define" SKIP MACRO_ID "!macro1" LPAREN ENDCMD SKIP ID "x" ENDCMD SKIP ID "data" SKIP ID "list" SKIP SLASH ID "x" SKIP POS_NUM 1 ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([DEFINE command - missing !ENDDEFINE]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl define !macro1(). content line 1 content line 2 ]) AT_DATA([expout-base], [dnl ID "define" SKIP MACRO_ID "!macro1" LPAREN RPAREN ENDCMD SKIP STRING "content line 1" SKIP STRING "content line 2" -SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP AT_SETUP([batch mode]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl first command another line of first command + second command third command fourth command. fifth command. ]) AT_DATA([expout-base], [dnl ID "first" SKIP ID "command" SKIP SKIP ID "another" SKIP ID "line" SKIP ID "of" SKIP ID "first" SKIP ID "command" SKIP ENDCMD SKIP ID "second" SKIP ID "command" SKIP ENDCMD ID "third" SKIP ID "command" SKIP ENDCMD SKIP ID "fourth" SKIP ID "command" ENDCMD SKIP SKIP ID "fifth" SKIP ID "command" ENDCMD -SKIP STOP ]) PSPP_CHECK_SCAN([-b]) AT_CLEANUP