X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=tests%2Flanguage%2Flexer%2Fscan.at;h=c877628fdf9e43f420f4e6cd268c003d5e21d5ba;hb=760c82efea3926983c50e0dab624505b20a39053;hp=8eb48059e9aff32d215ff063408ff5f665df5989;hpb=e0f9210e814d03bc43b6a9b30a402e403d5666b9;p=pspp diff --git a/tests/language/lexer/scan.at b/tests/language/lexer/scan.at index 8eb48059e9..c877628fdf 100644 --- a/tests/language/lexer/scan.at +++ b/tests/language/lexer/scan.at @@ -1,16 +1,16 @@ dnl PSPP - a program for statistical analysis. dnl Copyright (C) 2017 Free Software Foundation, Inc. -dnl +dnl dnl This program is free software: you can redistribute it and/or modify dnl it under the terms of the GNU General Public License as published by dnl the Free Software Foundation, either version 3 of the License, or dnl (at your option) any later version. -dnl +dnl dnl This program is distributed in the hope that it will be useful, dnl but WITHOUT ANY WARRANTY; without even the implied warranty of dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the dnl GNU General Public License for more details. -dnl +dnl dnl You should have received a copy of the GNU General Public License dnl along with this program. If not, see . dnl @@ -25,7 +25,7 @@ m4_define([PSPP_CHECK_SCAN], AT_SETUP([identifiers]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl -a aB i5 $x @efg @@. #.# .x _z. +a aB i5 $x @efg @@. !abcd !* !*a #.# .x _z. abcd. abcd. QRSTUV./* end of line comment */ QrStUv./* end of line comment */ @&t@ @@ -34,50 +34,32 @@ WXYZ. /* unterminated end of line comment ]) AT_DATA([expout-base], [dnl ID "a" -SKIP ID "aB" -SKIP ID "i5" -SKIP ID "$x" -SKIP ID "@efg" -SKIP ID "@@." -SKIP +MACRO_ID "!abcd" +MACRO_ID "!*" +MACRO_ID "!*" +ID "a" ID "#.#" -SKIP -UNEXPECTED_DOT +MACRO_PUNCT "." ID "x" -SKIP -UNEXPECTED_CHAR 95 +MACRO_PUNCT "_" ID "z" ENDCMD -SKIP ID "abcd." -SKIP ID "abcd" ENDCMD -SKIP ID "QRSTUV" ENDCMD -SKIP -SKIP ID "QrStUv" ENDCMD -SKIP -SKIP -SKIP ID "WXYZ" ENDCMD -SKIP -SKIP -SKIP -UNEXPECTED_CHAR 65533 +STOP "Bad character U+FFFD in input." ENDCMD -SKIP -SKIP --SKIP STOP ]) PSPP_CHECK_SCAN([-i]) @@ -93,88 +75,47 @@ and. with. ]) AT_DATA([expout-base], [dnl AND -SKIP OR -SKIP NOT -SKIP EQ -SKIP GE -SKIP GT -SKIP LE -SKIP LT -SKIP NE -SKIP ALL -SKIP BY -SKIP TO -SKIP WITH -SKIP AND -SKIP OR -SKIP NOT -SKIP EQ -SKIP GE -SKIP GT -SKIP LE -SKIP LT -SKIP NE -SKIP ALL -SKIP BY -SKIP TO -SKIP WITH -SKIP ID "andx" -SKIP ID "orx" -SKIP ID "notx" -SKIP ID "eqx" -SKIP ID "gex" -SKIP ID "gtx" -SKIP ID "lex" -SKIP ID "ltx" -SKIP ID "nex" -SKIP ID "allx" -SKIP ID "byx" -SKIP ID "tox" -SKIP ID "withx" -SKIP ID "and." -SKIP WITH ENDCMD --SKIP STOP ]) PSPP_CHECK_SCAN([-i]) @@ -185,48 +126,29 @@ AT_KEYWORDS([scan]) AT_DATA([input], [dnl ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] ** ~&|=>=><=<~=<>(),-+*/[[]]** +% : ; ? _ ` { } ~ ]) AT_DATA([expout-base], [dnl NOT -SKIP AND -SKIP OR -SKIP EQUALS -SKIP GE -SKIP GT -SKIP LE -SKIP LT -SKIP NE -SKIP NE -SKIP LPAREN -SKIP RPAREN -SKIP COMMA -SKIP DASH -SKIP PLUS -SKIP ASTERISK -SKIP SLASH -SKIP LBRACK -SKIP RBRACK -SKIP EXP -SKIP NOT AND OR @@ -247,13 +169,21 @@ SLASH LBRACK RBRACK EXP --SKIP +MACRO_PUNCT "%" +COLON +SEMICOLON +MACRO_PUNCT "?" +MACRO_PUNCT "_" +MACRO_PUNCT "`" +LCURLY +RCURLY +NOT STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP -AT_SETUP([numbers]) +AT_SETUP([positive numbers]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl 0 1 01 001. 1. @@ -266,78 +196,106 @@ AT_DATA([input], [dnl ]) AT_DATA([expout-base], [dnl POS_NUM -SKIP POS_NUM 1 -SKIP POS_NUM 1 -SKIP POS_NUM 1 -SKIP POS_NUM 1 ENDCMD -SKIP POS_NUM 123 ENDCMD -SKIP -SKIP -SKIP -SKIP -SKIP ENDCMD POS_NUM 1 -SKIP POS_NUM 0.1 -SKIP POS_NUM 0.1 -SKIP POS_NUM 0.1 -SKIP POS_NUM 50 -SKIP POS_NUM 0.6 -SKIP POS_NUM 70 -SKIP POS_NUM 60 -SKIP POS_NUM 0.006 -SKIP ENDCMD POS_NUM 30 -SKIP POS_NUM 0.04 -SKIP POS_NUM 5 -SKIP POS_NUM 6 -SKIP POS_NUM 0.0007 -SKIP POS_NUM 12.3 -SKIP POS_NUM 4.56 -SKIP POS_NUM 789 -SKIP POS_NUM 999 -SKIP POS_NUM 0.0112 -SKIP ENDCMD -SKIP -EXPECTED_EXPONENT "1e" -SKIP +STOP "Missing exponent following `1e'." ID "e1" -SKIP -EXPECTED_EXPONENT "1e+" -SKIP -EXPECTED_EXPONENT "1e-" --SKIP +STOP "Missing exponent following `1e+'." +STOP "Missing exponent following `1e-'." STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP +AT_SETUP([negative numbers]) +AT_KEYWORDS([scan]) +AT_DATA([input-base], [dnl + -0 -1 -01 -001. -1. + -123. /* comment 1 */ /* comment 2 */ + -.1 -0.1 -00.1 -00.10 + -5e1 -6E-1 -7e+1 -6E+01 -6e-03 + -.3E1 -.4e-1 -.5E+1 -.6e+01 -.7E-03 + -1.23e1 -45.6E-1 -78.9e+1 -99.9E+01 -11.2e-03 + -/**/1 + -. -1e -e1 -1e+ -1e- -1. +]) +AT_DATA([expout-base0], [dnl +NEG_NUM +NEG_NUM -1 +NEG_NUM -1 +NEG_NUM -1 +NEG_NUM -1 +ENDCMD +NEG_NUM -123 +ENDCMD +NEG_NUM -0.1 +NEG_NUM -0.1 +NEG_NUM -0.1 +NEG_NUM -0.1 +NEG_NUM -50 +NEG_NUM -0.6 +NEG_NUM -70 +NEG_NUM -60 +NEG_NUM -0.006 +NEG_NUM -3 +NEG_NUM -0.04 +NEG_NUM -5 +NEG_NUM -6 +NEG_NUM -0.0007 +NEG_NUM -12.3 +NEG_NUM -4.56 +NEG_NUM -789 +NEG_NUM -999 +NEG_NUM -0.0112 +NEG_NUM -1 +DASH +MACRO_PUNCT "." +STOP "Missing exponent following `-1e'." +DASH +ID "e1" +STOP "Missing exponent following `-1e+'." +STOP "Missing exponent following `-1e-'." +NEG_NUM -1 +ENDCMD +STOP +]) + +cp input-base input +cp expout-base0 expout-base +PSPP_CHECK_SCAN([-i]) + +sed 's/ -/ - /g' < input-base > input +sed 's/following `-/following `- /' < expout-base0 > expout-base +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + AT_SETUP([strings]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl @@ -373,61 +331,33 @@ x"4142" ]) AT_DATA([expout-base], [dnl STRING "x" -SKIP STRING "y" -SKIP STRING "abc" -SKIP STRING "Don't" -SKIP STRING "Can't" -SKIP STRING "Won't" -SKIP STRING ""quoted"" -SKIP STRING ""quoted"" -SKIP STRING "" -SKIP STRING "" -SKIP STRING "'" -SKIP STRING """ -SKIP -EXPECTED_QUOTE -SKIP -EXPECTED_QUOTE -SKIP +STOP "Unterminated string constant." +STOP "Unterminated string constant." STRING "xyzabcde" -SKIP STRING "foobar" -SKIP STRING "foobar" -SKIP STRING "foo" -SKIP PLUS -SKIP ENDCMD -SKIP STRING "bar" -SKIP ENDCMD -SKIP PLUS -SKIP STRING "AB5152" -SKIP STRING "4142QR" -SKIP STRING "ABお" -SKIP STRING "�あいうえお" -SKIP STRING "abc�えxyz" --SKIP STOP ]) PSPP_CHECK_SCAN([-i]) @@ -440,18 +370,14 @@ AT_DATA([input], [dnl #! /usr/bin/pspp ]) AT_DATA([expout-base], [dnl -SKIP -SKIP ID "#" -UNEXPECTED_CHAR 33 -SKIP +MACRO_ID "!" SLASH ID "usr" SLASH ID "bin" SLASH ID "pspp" --SKIP STOP ]) PSPP_CHECK_SCAN([-i]) @@ -478,57 +404,27 @@ next command. ]) AT_DATA([expout-base], [dnl -SKIP -SKIP -SKIP ENDCMD -SKIP ENDCMD -SKIP -SKIP ENDCMD -SKIP -SKIP ENDCMD -SKIP ENDCMD -SKIP -SKIP ENDCMD -SKIP -SKIP ENDCMD -SKIP ID "com" -SKIP ID "is" -SKIP ID "ambiguous" -SKIP WITH -SKIP ID "COMPUTE" ENDCMD -SKIP ENDCMD -SKIP -SKIP -SKIP ENDCMD -SKIP ENDCMD -SKIP -SKIP -SKIP ENDCMD -SKIP ID "next" -SKIP ID "command" ENDCMD -SKIP -ENDCMD --SKIP STOP ]) PSPP_CHECK_SCAN([-i]) @@ -553,48 +449,29 @@ ID "DOCUMENT" STRING "DOCUMENT one line." ENDCMD ENDCMD -SKIP ID "DOCUMENT" STRING "DOC more" -SKIP STRING " than" -SKIP STRING " one" -SKIP STRING " line." ENDCMD ENDCMD -SKIP ID "DOCUMENT" STRING "docu" -SKIP STRING "first.paragraph" -SKIP STRING "isn't parsed as tokens" -SKIP STRING "" -SKIP STRING "second paragraph." -ENDCMD -ENDCMD --SKIP STOP ]) PSPP_CHECK_SCAN([-i]) AT_CLEANUP -AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands]) +AT_SETUP([FILE LABEL commands]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl -title/**/'Quoted string title'. -tit /* -"Quoted string on second line". -sub "Quoted string subtitle" - . - -TITL /* Not a */ quoted string title. -SUBT Not a quoted string /* subtitle - FIL label isn't quoted. FILE lab 'is quoted'. @@ -603,65 +480,18 @@ FILE /* ]) AT_DATA([expout-base], [dnl -ID "title" -SKIP -STRING "Quoted string title" -ENDCMD -SKIP -ID "tit" -SKIP -SKIP -SKIP -STRING "Quoted string on second line" -ENDCMD -SKIP -ID "sub" -SKIP -STRING "Quoted string subtitle" -SKIP -SKIP -ENDCMD -SKIP -ENDCMD -SKIP -ID "TITL" -SKIP -STRING "/* Not a */ quoted string title" -ENDCMD -SKIP -ID "SUBT" -SKIP -STRING "Not a quoted string /* subtitle" -SKIP -ENDCMD -SKIP ID "FIL" -SKIP ID "label" -SKIP STRING "isn't quoted" ENDCMD -SKIP ID "FILE" -SKIP -SKIP ID "lab" -SKIP STRING "is quoted" ENDCMD -SKIP ID "FILE" -SKIP -SKIP -SKIP -SKIP -SKIP ID "lab" -SKIP STRING "not quoted here either" -SKIP -ENDCMD --SKIP STOP ]) PSPP_CHECK_SCAN([-i]) @@ -684,41 +514,22 @@ end data ]) AT_DATA([expout-base], [dnl ID "begin" -SKIP ID "data" ENDCMD -SKIP STRING "123" -SKIP STRING "xxx" -SKIP ID "end" -SKIP ID "data" ENDCMD -SKIP ENDCMD -SKIP ID "BEG" -SKIP -SKIP -SKIP ID "DAT" -SKIP -SKIP -SKIP STRING "5 6 7 /* x" -SKIP STRING "" -SKIP STRING "end data" -SKIP ID "end" -SKIP ID "data" -SKIP ENDCMD --SKIP STOP ]) PSPP_CHECK_SCAN([-i]) @@ -739,43 +550,398 @@ end ]) AT_DATA([expout-base], [dnl ID "do" -SKIP ID "repeat" -SKIP ID "x" EQUALS ID "a" -SKIP ID "b" -SKIP ID "c" -SKIP -SKIP ID "y" EQUALS ID "d" -SKIP ID "e" -SKIP ID "f" ENDCMD -SKIP STRING " do repeat a=1 thru 5." -SKIP STRING "another command." -SKIP STRING "second command" -SKIP STRING "+ third command." -SKIP STRING "end /* x */ /* y */ repeat print." -SKIP ID "end" -SKIP -SKIP ID "repeat" ENDCMD --SKIP +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DO REPEAT command in batch mode]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +do repeat x=a b c + y=d e f +do repeat a=1 thru 5 +another command +second command ++ third command +end /* x */ /* y */ repeat print +end + repeat +do + repeat #a=1 + + inner command +end repeat +]) +AT_DATA([expout-base], [dnl +ID "do" +ID "repeat" +ID "x" +EQUALS +ID "a" +ID "b" +ID "c" +ID "y" +EQUALS +ID "d" +ID "e" +ID "f" +ENDCMD +STRING "do repeat a=1 thru 5" +STRING "another command" +STRING "second command" +STRING "+ third command" +STRING "end /* x */ /* y */ repeat print" +ID "end" +ID "repeat" +ENDCMD +ID "do" +ID "repeat" +ID "#a" +EQUALS +POS_NUM 1 +ENDCMD +STRING " inner command" +ID "end" +ID "repeat" +STOP +]) +PSPP_CHECK_SCAN([-b]) +AT_CLEANUP + +AT_SETUP([DEFINE command - simple]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1() +var1 var2 var3 +!enddefine. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +RPAREN +STRING "var1 var2 var3" +MACRO_ID "!enddefine" +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - no newline after parentheses]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1() var1 var2 var3 +!enddefine. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +RPAREN +STRING " var1 var2 var3" +MACRO_ID "!enddefine" +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - no newline before !ENDDEFINE]) +AT_KEYWORDS([scan ENDDEFINE]) +AT_DATA([input], [dnl +define !macro1() +var1 var2 var3!enddefine. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +RPAREN +STRING "var1 var2 var3" +MACRO_ID "!enddefine" +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - all on one line]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1()var1 var2 var3!enddefine. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +RPAREN +STRING "var1 var2 var3" +MACRO_ID "!enddefine" +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - empty]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1() +!enddefine. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +RPAREN +MACRO_ID "!enddefine" +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - blank lines]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1() + + +!enddefine. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +RPAREN +STRING "" +STRING "" +MACRO_ID "!enddefine" +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - arguments]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1(a(), b(), c()) +!enddefine. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +ID "a" +LPAREN +RPAREN +COMMA +ID "b" +LPAREN +RPAREN +COMMA +ID "c" +LPAREN +RPAREN +RPAREN +MACRO_ID "!enddefine" +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - multiline arguments]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1( + a(), b( + ), + c() +) +!enddefine. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +ID "a" +LPAREN +RPAREN +COMMA +ID "b" +LPAREN +RPAREN +COMMA +ID "c" +LPAREN +RPAREN +RPAREN +MACRO_ID "!enddefine" +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - arguments start on second line]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1 +(x,y,z +) +content 1 +content 2 +!enddefine. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +ID "x" +COMMA +ID "y" +COMMA +ID "z" +RPAREN +STRING "content 1" +STRING "content 2" +MACRO_ID "!enddefine" +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - early end of command 1]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1. +data list /x 1. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +ENDCMD +ID "data" +ID "list" +SLASH +ID "x" +POS_NUM 1 +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - early end of command 2]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1 +x. +data list /x 1. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +ID "x" +ENDCMD +ID "data" +ID "list" +SLASH +ID "x" +POS_NUM 1 +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - early end of command 3]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1(. +x. +data list /x 1. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +ENDCMD +ID "x" +ENDCMD +ID "data" +ID "list" +SLASH +ID "x" +POS_NUM 1 +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - early end of command 4]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +dnl Notice the command terminator at the end of the DEFINE command, +dnl which should not be there and ends it early. +define !macro1. +data list /x 1. +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +ENDCMD +ID "data" +ID "list" +SLASH +ID "x" +POS_NUM 1 +ENDCMD +STOP +]) +PSPP_CHECK_SCAN([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - missing !ENDDEFINE]) +AT_KEYWORDS([scan]) +AT_DATA([input], [dnl +define !macro1() +content line 1 +content line 2 +]) +AT_DATA([expout-base], [dnl +ID "define" +STRING "!macro1" +LPAREN +RPAREN +STRING "content line 1" +STRING "content line 2" STOP ]) PSPP_CHECK_SCAN([-i]) @@ -794,44 +960,25 @@ fourth command. ]) AT_DATA([expout-base], [dnl ID "first" -SKIP ID "command" -SKIP -SKIP ID "another" -SKIP ID "line" -SKIP ID "of" -SKIP ID "first" -SKIP ID "command" -SKIP ENDCMD -SKIP ID "second" -SKIP ID "command" -SKIP ENDCMD ID "third" -SKIP ID "command" -SKIP ENDCMD -SKIP ID "fourth" -SKIP ID "command" ENDCMD -SKIP -SKIP ID "fifth" -SKIP ID "command" ENDCMD --SKIP STOP ]) PSPP_CHECK_SCAN([-b])