dnl PSPP - a program for statistical analysis. dnl Copyright (C) 2017 Free Software Foundation, Inc. dnl dnl This program is free software: you can redistribute it and/or modify dnl it under the terms of the GNU General Public License as published by dnl the Free Software Foundation, either version 3 of the License, or dnl (at your option) any later version. dnl dnl This program is distributed in the hope that it will be useful, dnl but WITHOUT ANY WARRANTY; without even the implied warranty of dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the dnl GNU General Public License for more details. dnl dnl You should have received a copy of the GNU General Public License dnl along with this program. If not, see . dnl AT_BANNER([syntax segmentation]) m4_define([PSPP_CHECK_SEGMENT], [for strip in "" "-s"; do case $strip in # ( '') sed 's/^-//' < expout-base > expout ;; # ( -s) sed '/^-/d' < expout-base > expout ;; esac AT_CHECK([segment-test $1 $strip input], [0], [expout]) AT_CHECK([segment-test -1 $strip $1 input], [0], [expout]) AT_CHECK([segment-test -0 $strip $1 input]) AT_CHECK([segment-test -01 $strip $1 input]) done]) AT_SETUP([identifiers]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl a ab abc abcd !abcd A AB ABC ABCD !ABCD aB aBC aBcD !aBcD $x $y $z !$z grève@<00A0>@Ângstrom@<00A0>@poté #a #b #c ## #d !#d @efg @ @@. @#@ !@ @&t@ ## # #12345 #.# f@#_.#6 GhIjK .x 1y _z ]) AT_DATA([expout-base], [dnl identifier a space identifier ab space identifier abc space identifier abcd space macro_id !abcd newline \n (later) identifier A space identifier AB space identifier ABC space identifier ABCD space macro_id !ABCD newline \n (later) identifier aB space identifier aBC space identifier aBcD space macro_id !aBcD newline \n (later) identifier $x space identifier $y space identifier $z space macro_id !$z newline \n (later) identifier grève spaces identifier Ângstrom spaces identifier poté newline \n (later) identifier #a space identifier #b space identifier #c space identifier ## space identifier #d space macro_id !#d newline \n (later) identifier @efg space identifier @ space identifier @@. space identifier @#@ space macro_id !@ space newline \n (later) identifier ## space identifier # space identifier #12345 space identifier #.# newline \n (later) identifier f@#\_.#6 newline \n (later) identifier GhIjK newline \n (later) start_command . identifier x space number 1 identifier y space punct \_ identifier z -newline \n (later) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([identifiers that end in '.']) AT_KEYWORDS([segment]) AT_DATA([input], [dnl abcd. abcd. ABCD. ABCD. aBcD. aBcD. @&t@ $y. $z. あいうえお. #c. #d.. @@. @@.... #.#. #abcd. . . @&t@ LMNOP. @&t@ QRSTUV./* end of line comment */ qrstuv. /* end of line comment */ QrStUv./* end of line comment */ @&t@ wxyz./* unterminated end of line comment WXYZ. /* unterminated end of line comment WxYz./* unterminated end of line comment @&t@ ]) AT_DATA([expout-base], [dnl identifier abcd. space identifier abcd end_command . newline \n (first) identifier ABCD. space identifier ABCD end_command . newline \n (first) identifier aBcD. space identifier aBcD end_command . space newline \n (first) identifier $y. space identifier $z. space identifier あいうえお end_command . newline \n (first) identifier #c. space identifier #d. end_command . newline \n (first) identifier @@. space identifier @@... end_command . newline \n (first) identifier #.# end_command . newline \n (first) identifier #abcd end_command . newline \n (first) start_command . newline \n (first) start_command . space newline \n (first) identifier LMNOP end_command . space newline \n (first) identifier QRSTUV end_command . comment /*_end_of_line_comment_*/ newline \n (first) identifier qrstuv end_command . space comment /*_end_of_line_comment_*/ newline \n (first) identifier QrStUv end_command . comment /*_end_of_line_comment_*/ space newline \n (first) identifier wxyz end_command . comment /*_unterminated_end_of_line_comment newline \n (first) identifier WXYZ end_command . space comment /*_unterminated_end_of_line_comment newline \n (first) identifier WxYz end_command . comment /*_unterminated_end_of_line_comment_ -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([reserved words]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl and or not eq ge gt le lt ne all by to with AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH andx orx notx eqx gex gtx lex ltx nex allx byx tox withx and. with. ]) AT_DATA([expout-base], [dnl reserved_word and space reserved_word or space reserved_word not space reserved_word eq space reserved_word ge space reserved_word gt space reserved_word le space reserved_word lt space reserved_word ne space reserved_word all space reserved_word by space reserved_word to space reserved_word with newline \n (later) reserved_word AND space reserved_word OR space reserved_word NOT space reserved_word EQ space reserved_word GE space reserved_word GT space reserved_word LE space reserved_word LT space reserved_word NE space reserved_word ALL space reserved_word BY space reserved_word TO space reserved_word WITH newline \n (later) identifier andx space identifier orx space identifier notx space identifier eqx space identifier gex space identifier gtx space identifier lex space identifier ltx space identifier nex space identifier allx space identifier byx space identifier tox space identifier withx newline \n (later) identifier and. space reserved_word with end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([punctuation]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] ** ~&|=>=><=<~=<>(),-+*/[[]]** % : ; ? _ ` { } ~ ]) AT_DATA([expout-base], [dnl punct ~ space punct & space punct | space punct = space punct >= space punct > space punct <= space punct < space punct ~= space punct <> space punct ( space punct ) space punct , space punct - space punct + space punct * space punct / space punct [[ space punct ]] space punct ** newline \n (later) punct ~ punct & punct | punct = punct >= punct > punct <= punct < punct ~= punct <> punct ( punct ) punct , punct - punct + punct * punct / punct [[ punct ]] punct ** newline \n (later) punct % space punct : space punct ; space punct ? space punct \_ space punct ` space punct { space punct } space punct ~ -newline \n (later) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([numbers]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl 0 1 01 001. 1. 123. /* comment 1 */ /* comment 2 */ .1 0.1 00.1 00.10 5e1 6E-1 7e+1 6E+01 6e-03 .3E1 .4e-1 .5E+1 .6e+01 .7E-03 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03 . 1e e1 1e+ 1e- 1. ]) AT_DATA([expout-base], [dnl number 0 space number 1 space number 01 space number 001. space number 1 end_command . newline \n (first) number 123 end_command . space comment /*_comment_1_*/ space comment /*_comment_2_*/ newline \n (first) start_command . number 1 space number 0.1 space number 00.1 space number 00.10 newline \n (later) number 5e1 space number 6E-1 space number 7e+1 space number 6E+01 space number 6e-03 newline \n (later) start_command . number 3E1 space number .4e-1 space number .5E+1 space number .6e+01 space number .7E-03 newline \n (later) number 1.23e1 space number 45.6E-1 space number 78.9e+1 space number 99.9E+01 space number 11.2e-03 newline \n (later) start_command . space expected_exponent 1e space identifier e1 space expected_exponent 1e+ space expected_exponent 1e- space number 1 end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([strings]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl 'x' "y" 'abc' 'Don''t' "Can't" 'Won''t' """quoted""" '"quoted"' '' "" 'missing end quote "missing double quote x"4142" X'5152' u'fffd' U"041" + new command + /* comment */ 'string continuation' + /* also a punctuator on blank line - 'new command' ]) AT_DATA([expout-base], [dnl quoted_string 'x' space quoted_string "y" space quoted_string 'abc' newline \n (later) quoted_string 'Don''t' space quoted_string "Can't" space quoted_string 'Won''t' newline \n (later) quoted_string """quoted""" space quoted_string '"quoted"' newline \n (later) quoted_string '' space quoted_string "" newline \n (later) expected_quote 'missing_end_quote newline \n (later) expected_quote "missing_double_quote newline \n (later) hex_string x"4142" space hex_string X'5152' newline \n (later) unicode_string u'fffd' space unicode_string U"041" newline \n (later) start_command + space identifier new space identifier command newline \n (later) punct + space comment /*_comment_*/ space quoted_string 'string_continuation' newline \n (later) punct + space comment /*_also_a_punctuator_on_blank_line newline \n (later) start_command - space quoted_string 'new_command' -newline \n (later) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([@%:@! construct]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl #! /usr/bin/pspp title my title. #! /usr/bin/pspp ]) AT_DATA([expout-base], [dnl shbang #!_/usr/bin/pspp newline \n (first) identifier title space unquoted_string my_title end_command . newline \n (first) identifier # macro_id ! space punct / identifier usr punct / identifier bin punct / identifier pspp -newline \n (later) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([* and COMMENT commands]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl * Comment commands "don't have to contain valid tokens. ** Check ambiguity with ** token. ****************. comment keyword works too. COMM also. com is ambiguous with COMPUTE. * Comment need not start at left margin. * Comment ends with blank line next command. ]) AT_DATA([expout-base], [dnl comment_command *_Comment_commands_"don't newline \n (COMMENT) comment_command have_to_contain_valid_tokens end_command . newline \n (first) separate_commands newline \n (first) comment_command **_Check_ambiguity_with_**_token end_command . newline \n (first) comment_command **************** end_command . newline \n (first) separate_commands newline \n (first) comment_command comment_keyword_works_too end_command . newline \n (first) comment_command COMM_also end_command . newline \n (first) identifier com space identifier is space identifier ambiguous space reserved_word with space identifier COMPUTE end_command . newline \n (first) separate_commands newline \n (first) spaces ___ comment_command *_Comment_need_not_start_at_left_margin end_command . newline \n (first) separate_commands newline \n (first) comment_command *_Comment_ends_with_blank_line newline \n (COMMENT) separate_commands newline \n (first) identifier next space identifier command end_command . newline \n (first) -separate_commands -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DOCUMENT command]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl DOCUMENT one line. DOC more than one line. docu first.paragraph isn't parsed as tokens second paragraph. ]) AT_DATA([expout-base], [dnl start_document document DOCUMENT_one_line. end_command separate_commands newline \n (first) start_document document DOC_more newline \n (DOCUMENT) document ____than newline \n (DOCUMENT) document ________one newline \n (DOCUMENT) document ____________line. end_command separate_commands newline \n (first) start_document document docu newline \n (DOCUMENT) document first.paragraph newline \n (DOCUMENT) document isn't_parsed_as_tokens newline \n (DOCUMENT) document newline \n (DOCUMENT) document second_paragraph. -end_command -separate_commands -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl title/**/'Quoted string title'. tit /* "Quoted string on second line". sub "Quoted string subtitle" . TITL /* Not a */ quoted string title. SUBT Not a quoted string /* subtitle FIL label isn't quoted. FILE lab 'is quoted'. FILE /* /**/ lab not quoted here either ]) AT_DATA([expout-base], [dnl identifier title comment /**/ quoted_string 'Quoted_string_title' end_command . newline \n (first) identifier tit space comment /* newline \n (later) quoted_string "Quoted_string_on_second_line" end_command . newline \n (first) identifier sub space quoted_string "Quoted_string_subtitle" newline \n (later) space end_command . newline \n (first) separate_commands newline \n (first) identifier TITL space unquoted_string /*_Not_a_*/_quoted_string_title end_command . newline \n (first) identifier SUBT space unquoted_string Not_a_quoted_string_/*_subtitle newline \n (later) separate_commands newline \n (first) identifier FIL space identifier label space unquoted_string isn't_quoted end_command . newline \n (first) identifier FILE newline \n (later) spaces __ identifier lab space quoted_string 'is_quoted' end_command . newline \n (first) identifier FILE space comment /* newline \n (later) comment /**/ spaces __ identifier lab space unquoted_string not_quoted_here_either newline \n (later) -separate_commands -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([BEGIN DATA command]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl begin data. end data. begin data. /* 123 xxx end data. BEG /**/ DAT /* 5 6 7 /* x end data end data . begin data. data end data. begin data "xxx". begin data 123. not data ]) AT_DATA([expout-base], [dnl identifier begin space identifier data end_command . newline \n (data) identifier end space identifier data end_command . newline \n (first) separate_commands newline \n (first) identifier begin space identifier data end_command . space comment /* newline \n (data) inline_data 123 newline \n (data) inline_data xxx newline \n (data) identifier end space identifier data end_command . newline \n (first) separate_commands newline \n (first) identifier BEG space comment /**/ space identifier DAT space comment /* newline \n (data) inline_data 5_6_7_/*_x newline \n (data) inline_data newline \n (data) inline_data end__data newline \n (data) identifier end space identifier data newline \n (later) start_command . newline \n (first) separate_commands newline \n (first) identifier begin newline \n (later) space identifier data end_command . newline \n (data) inline_data data newline \n (data) identifier end space identifier data end_command . newline \n (first) separate_commands newline \n (first) identifier begin space identifier data space quoted_string "xxx" end_command . newline \n (first) identifier begin space identifier data space number 123 end_command . newline \n (first) reserved_word not space identifier data -newline \n (later) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DO REPEAT command]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl do repeat x=a b c y=d e f. do repeat a=1 thru 5. another command. second command + third command. end /* x */ /* y */ repeat print. end repeat. do repeat #a=1. inner command. end repeat. ]) AT_DATA([expout-base], [dnl identifier do space identifier repeat space identifier x punct = identifier a space identifier b space identifier c newline \n (later) spaces __________ identifier y punct = identifier d space identifier e space identifier f end_command . newline \n (DO REPEAT) do_repeat_command __do_repeat_a=1_thru_5. newline \n (DO REPEAT) do_repeat_command another_command. newline \n (DO REPEAT) do_repeat_command second_command newline \n (DO REPEAT) do_repeat_command +_third_command. newline \n (DO REPEAT) do_repeat_command end_/*_x_*/_/*_y_*/_repeat_print. newline \n (DO REPEAT) identifier end newline \n (later) space identifier repeat end_command . newline \n (first) identifier do newline \n (later) spaces __ identifier repeat space identifier #a punct = number 1 end_command . newline \n (DO REPEAT) do_repeat_command __inner_command. newline \n (DO REPEAT) identifier end space identifier repeat end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DO REPEAT command in batch mode]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl do repeat x=a b c y=d e f do repeat a=1 thru 5 another command second command + third command end /* x */ /* y */ repeat print end repeat do repeat #a=1 inner command end repeat ]) AT_DATA([expout-base], [dnl identifier do space identifier repeat space identifier x punct = identifier a space identifier b space identifier c newline \n (later) spaces __________ identifier y punct = identifier d space identifier e space identifier f newline \n (later) start_command do_repeat_command do_repeat_a=1_thru_5 newline \n (DO REPEAT) do_repeat_command another_command newline \n (DO REPEAT) do_repeat_command second_command newline \n (DO REPEAT) do_repeat_command +_third_command newline \n (DO REPEAT) do_repeat_command end_/*_x_*/_/*_y_*/_repeat_print newline \n (DO REPEAT) identifier end newline \n (later) space identifier repeat newline \n (later) start_command identifier do newline \n (later) spaces __ identifier repeat space identifier #a punct = number 1 newline \n (later) separate_commands newline \n (DO REPEAT) do_repeat_command __inner_command newline \n (DO REPEAT) identifier end space identifier repeat -newline \n (later) - end ]) PSPP_CHECK_SEGMENT([-b]) AT_CLEANUP AT_SETUP([DEFINE command - simple]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl define !macro1() var1 var2 var3 !enddefine. ]) AT_DATA([expout-base], [dnl identifier define space macro_id !macro1 punct ( punct ) newline \n (DEFINE) macro_body var1_var2_var3 newline \n (DEFINE) macro_id !enddefine end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DEFINE command - empty]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl define !macro1() !enddefine. ]) AT_DATA([expout-base], [dnl identifier define space macro_id !macro1 punct ( punct ) newline \n (DEFINE) macro_id !enddefine end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DEFINE command - arguments]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl define !macro1(a(), b(), c()) !enddefine. ]) AT_DATA([expout-base], [dnl identifier define space macro_id !macro1 punct ( identifier a punct ( punct ) punct , space identifier b punct ( punct ) punct , space identifier c punct ( punct ) punct ) newline \n (DEFINE) macro_id !enddefine end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DEFINE command - multiline arguments]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl define !macro1( a(), b( ), c() ) !enddefine. ]) AT_DATA([expout-base], [dnl identifier define space macro_id !macro1 punct ( newline \n (later) spaces __ identifier a punct ( punct ) punct , space identifier b punct ( newline \n (later) spaces __ punct ) punct , newline \n (later) spaces __ identifier c punct ( punct ) newline \n (later) punct ) newline \n (DEFINE) macro_id !enddefine end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DEFINE command - arguments start on second line]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl define !macro1 (x,y,z ) content 1 content 2 !enddefine. ]) AT_DATA([expout-base], [dnl identifier define space macro_id !macro1 newline \n (later) punct ( identifier x punct , identifier y punct , identifier z newline \n (later) punct ) newline \n (DEFINE) macro_body content_1 newline \n (DEFINE) macro_body content_2 newline \n (DEFINE) macro_id !enddefine end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DEFINE command - early end of command 1]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl define !macro1. data list /x 1. ]) AT_DATA([expout-base], [dnl identifier define space macro_id !macro1 end_command . newline \n (first) identifier data space identifier list space punct / identifier x space number 1 end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DEFINE command - early end of command 2]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl define !macro1 x. data list /x 1. ]) AT_DATA([expout-base], [dnl identifier define space macro_id !macro1 newline \n (later) identifier x end_command . newline \n (first) identifier data space identifier list space punct / identifier x space number 1 end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DEFINE command - early end of command 3]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl define !macro1(. x. data list /x 1. ]) AT_DATA([expout-base], [dnl identifier define space macro_id !macro1 punct ( end_command . newline \n (first) identifier x end_command . newline \n (first) identifier data space identifier list space punct / identifier x space number 1 end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DEFINE command - early end of command 4]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl dnl Notice the command terminator at the end of the DEFINE command, dnl which should not be there and ends it early. define !macro1. data list /x 1. ]) AT_DATA([expout-base], [dnl identifier define space macro_id !macro1 end_command . newline \n (first) identifier data space identifier list space punct / identifier x space number 1 end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([DEFINE command - missing !ENDDEFINE]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl define !macro1() content line 1 content line 2 ]) AT_DATA([expout-base], [dnl identifier define space macro_id !macro1 punct ( punct ) newline \n (DEFINE) macro_body content_line_1 newline \n (DEFINE) macro_body content_line_2 -newline \n (DEFINE) - end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP AT_SETUP([batch mode]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl first command another line of first command + second command third command fourth command. fifth command. ]) AT_DATA([expout-base], [dnl identifier first space identifier command newline \n (later) spaces _____ identifier another space identifier line space identifier of space identifier first space identifier command newline \n (later) start_command + spaces __ identifier second space identifier command newline \n (later) start_command identifier third space identifier command newline \n (later) separate_commands newline \n (first) identifier fourth space identifier command end_command . newline \n (first) spaces ___ identifier fifth space identifier command end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-b]) AT_CLEANUP AT_SETUP([auto mode]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl command another line of command 2sls + another command another line of second command data list /x 1 aggregate. print eject. twostep cluster fourth command. fifth command. ]) AT_DATA([expout-base], [dnl identifier command newline \n (later) spaces _____ identifier another space identifier line space identifier of space identifier command newline \n (later) start_command number 2 identifier sls newline \n (later) start_command + spaces __ identifier another space identifier command newline \n (later) identifier another space identifier line space identifier of space identifier second space identifier command newline \n (later) start_command identifier data space identifier list space punct / identifier x space number 1 newline \n (later) start_command identifier aggregate end_command . newline \n (first) identifier print space identifier eject end_command . newline \n (first) identifier twostep space identifier cluster newline \n (later) separate_commands newline \n (first) separate_commands newline \n (first) identifier fourth space identifier command end_command . newline \n (first) spaces ___ identifier fifth space identifier command end_command . -newline \n (first) - end ]) PSPP_CHECK_SEGMENT([-a]) AT_CLEANUP