X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=tests%2Flanguage%2Flexer%2Fsegment.at;h=abbc08c8cd4b4a037b7f7155ccc799b0cdde6593;hb=134b0f8bcfadd9d4ae051d665f30c3227fae1c75;hp=e1dd0b5eb4f313a519aa26a13352508db259394d;hpb=fe8dc2171009e90d2335f159d05f7e6660e24780;p=pspp diff --git a/tests/language/lexer/segment.at b/tests/language/lexer/segment.at index e1dd0b5eb4..abbc08c8cd 100644 --- a/tests/language/lexer/segment.at +++ b/tests/language/lexer/segment.at @@ -1,44 +1,73 @@ +dnl PSPP - a program for statistical analysis. +dnl Copyright (C) 2017 Free Software Foundation, Inc. +dnl +dnl This program is free software: you can redistribute it and/or modify +dnl it under the terms of the GNU General Public License as published by +dnl the Free Software Foundation, either version 3 of the License, or +dnl (at your option) any later version. +dnl +dnl This program is distributed in the hope that it will be useful, +dnl but WITHOUT ANY WARRANTY; without even the implied warranty of +dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +dnl GNU General Public License for more details. +dnl +dnl You should have received a copy of the GNU General Public License +dnl along with this program. If not, see . +dnl AT_BANNER([syntax segmentation]) m4_define([PSPP_CHECK_SEGMENT], - [AT_CHECK([segment-test $1 input], [0], [expout]) - AT_CHECK([segment-test -1 $1 input], [0], [expout])]) + [AT_CAPTURE_FILE([input]) + for strip in "" "-s"; do + case $strip in # ( + '') sed 's/^-//' < expout-base > expout ;; # ( + -s) sed '/^-/d' < expout-base > expout ;; + esac + AT_CHECK([segment-test $1 $strip input], [0], [expout]) + AT_CHECK([segment-test -1 $strip $1 input], [0], [expout]) + AT_CHECK([segment-test -0 $strip $1 input]) + AT_CHECK([segment-test -01 $strip $1 input]) + done]) AT_SETUP([identifiers]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl -a ab abc abcd -A AB ABC ABCD -aB aBC aBcD -$x $y $z +a ab abc abcd !abcd +A AB ABC ABCD !ABCD +aB aBC aBcD !aBcD +$x $y $z !$z grève@<00A0>@Ângstrom@<00A0>@poté -#a #b #c ## #d -@efg @ @@. @#@ @&t@ +#a #b #c ## #d !#d +@efg @ @@. @#@ !@ @&t@ ## # #12345 #.# f@#_.#6 GhIjK .x 1y _z ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl identifier a space identifier ab space identifier abc space -identifier abcd +identifier abcd space +macro_id !abcd newline \n (later) identifier A space identifier AB space identifier ABC space -identifier ABCD +identifier ABCD space +macro_id !ABCD newline \n (later) identifier aB space identifier aBC space -identifier aBcD +identifier aBcD space +macro_id !aBcD newline \n (later) identifier $x space identifier $y space -identifier $z +identifier $z space +macro_id !$z newline \n (later) identifier grève @@ -52,13 +81,15 @@ identifier #a space identifier #b space identifier #c space identifier ## space -identifier #d +identifier #d space +macro_id !#d newline \n (later) identifier @efg space identifier @ space identifier @@. space identifier @#@ space +macro_id !@ space newline \n (later) identifier ## space @@ -77,11 +108,11 @@ start_command . identifier x space number 1 identifier y space -unexpected_char \_ +punct \_ identifier z -newline \n (later) - -end +-newline \n (later) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP @@ -107,7 +138,7 @@ wxyz./* unterminated end of line comment WXYZ. /* unterminated end of line comment WxYz./* unterminated end of line comment @&t@ ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl identifier abcd. space identifier abcd end_command . @@ -185,9 +216,9 @@ newline \n (first) identifier WxYz end_command . comment /*_unterminated_end_of_line_comment_ -newline \n (first) - -end +-newline \n (first) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP @@ -200,7 +231,7 @@ AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH andx orx notx eqx gex gtx lex ltx nex allx byx tox withx and. with. ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl reserved_word and space reserved_word or space reserved_word not space @@ -249,9 +280,9 @@ newline \n (later) identifier and. space reserved_word with end_command . -newline \n (first) - -end +-newline \n (first) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP @@ -260,9 +291,10 @@ AT_SETUP([punctuation]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] ** -~&|=>=><=<~=<>(),-+*/[[]]** +~&|=>=><=<~=<>(),-+*/[[]]**!* +% : ; ? _ ` { } ~ !* ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl punct ~ space punct & space punct | space @@ -305,14 +337,27 @@ punct / punct [[ punct ]] punct ** +macro_id !* newline \n (later) -end +punct % space +punct : space +punct ; space +punct ? space +punct \_ space +punct ` space +punct { space +punct } space +punct ~ space +macro_id !* +-newline \n (later) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP -AT_SETUP([numbers]) +AT_SETUP([positive numbers]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl 0 1 01 001. 1. @@ -321,9 +366,9 @@ AT_DATA([input], [dnl 5e1 6E-1 7e+1 6E+01 6e-03 .3E1 .4e-1 .5E+1 .6e+01 .7E-03 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03 -. 1e e1 1e+ 1e- +. 1e e1 1e+ 1e- 1. ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl number 0 space number 1 space number 01 space @@ -371,13 +416,92 @@ start_command . space expected_exponent 1e space identifier e1 space expected_exponent 1e+ space -expected_exponent 1e- +expected_exponent 1e- space +number 1 +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([negative numbers]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl + -0 -1 -01 -001. -1. + -123. /* comment 1 */ /* comment 2 */ + -.1 -0.1 -00.1 -00.10 + -5e1 -6E-1 -7e+1 -6E+01 -6e-03 + -.3E1 -.4e-1 -.5E+1 -.6e+01 -.7E-03 + -1.23e1 -45.6E-1 -78.9e+1 -99.9E+01 -11.2e-03 + -/**/1 + -. -1e -e1 -1e+ -1e- -1. +]) +AT_DATA([expout-base], [dnl +spaces _ +number -0 space +number -1 space +number -01 space +number -001. space +number -1 +end_command . +newline \n (first) + space +number -123 +end_command . space +comment /*_comment_1_*/ space +comment /*_comment_2_*/ +newline \n (first) + space +number -.1 space +number -0.1 space +number -00.1 space +number -00.10 newline \n (later) - -end + space +number -5e1 space +number -6E-1 space +number -7e+1 space +number -6E+01 space +number -6e-03 +newline \n (later) + space +number -.3E1 space +number -.4e-1 space +number -.5E+1 space +number -.6e+01 space +number -.7E-03 +newline \n (later) + space +number -1.23e1 space +number -45.6E-1 space +number -78.9e+1 space +number -99.9E+01 space +number -11.2e-03 +newline \n (later) + space +punct - +comment /**/ +number 1 +newline \n (later) + space +punct - +punct . space +expected_exponent -1e space +punct - +identifier e1 space +expected_exponent -1e+ space +expected_exponent -1e- space +number -1 +end_command . +-newline \n (first) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP + AT_SETUP([strings]) AT_KEYWORDS([segment]) @@ -395,7 +519,7 @@ u'fffd' U"041" + /* also a punctuator on blank line - 'new command' ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl quoted_string 'x' space quoted_string "y" space quoted_string 'abc' @@ -444,9 +568,9 @@ newline \n (later) start_command - space quoted_string 'new_command' -newline \n (later) - -end +-newline \n (later) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP @@ -458,26 +582,27 @@ AT_DATA([input], [dnl title my title. #! /usr/bin/pspp ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl shbang #!_/usr/bin/pspp newline \n (first) identifier title space -unquoted_string my_title +identifier my space +identifier title end_command . newline \n (first) identifier # -unexpected_char ! space +macro_id ! space punct / identifier usr punct / identifier bin punct / identifier pspp -newline \n (later) - -end +-newline \n (later) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP @@ -502,7 +627,7 @@ com is ambiguous with COMPUTE. next command. ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl comment_command *_Comment_commands_"don't newline \n (COMMENT) @@ -540,7 +665,7 @@ identifier COMPUTE end_command . newline \n (first) -separate_commands +separate_commands newline \n (first) spaces ___ @@ -548,7 +673,7 @@ comment_command *_Comment_need_not_start_at_left_margin end_command . newline \n (first) -separate_commands +separate_commands newline \n (first) comment_command *_Comment_ends_with_blank_line @@ -562,10 +687,10 @@ identifier command end_command . newline \n (first) -separate_commands -newline \n (first) - -end +-separate_commands +-newline \n (first) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP @@ -584,10 +709,10 @@ isn't parsed as tokens second paragraph. ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl start_document document DOCUMENT_one_line. -end_command +end_command separate_commands newline \n (first) @@ -602,7 +727,7 @@ document ________one newline \n (DOCUMENT) document ____________line. -end_command +end_command separate_commands newline \n (first) @@ -620,27 +745,18 @@ document newline \n (DOCUMENT) document second_paragraph. -end_command -separate_commands -newline \n (first) - -end +-end_command +-separate_commands +-newline \n (first) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP -AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands]) +AT_SETUP([FILE LABEL command]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl -title/**/'Quoted string title'. -tit /* -"Quoted string on second line". -sub "Quoted string subtitle" - . - -TITL /* Not a */ quoted string title. -SUBT Not a quoted string /* subtitle - FIL label isn't quoted. FILE lab 'is quoted'. @@ -648,43 +764,7 @@ FILE /* /**/ lab not quoted here either ]) -AT_DATA([expout], [dnl -identifier title -comment /**/ -quoted_string 'Quoted_string_title' -end_command . -newline \n (first) - -identifier tit space -comment /* -newline \n (later) - -quoted_string "Quoted_string_on_second_line" -end_command . -newline \n (first) - -identifier sub space -quoted_string "Quoted_string_subtitle" -newline \n (later) - space -end_command . -newline \n (first) - -separate_commands -newline \n (first) - -identifier TITL space -unquoted_string /*_Not_a_*/_quoted_string_title -end_command . -newline \n (first) - -identifier SUBT space -unquoted_string Not_a_quoted_string_/*_subtitle -newline \n (later) - -separate_commands -newline \n (first) - +AT_DATA([expout-base], [dnl identifier FIL space identifier label space unquoted_string isn't_quoted @@ -710,10 +790,10 @@ identifier lab space unquoted_string not_quoted_here_either newline \n (later) -separate_commands -newline \n (first) - -end +-separate_commands +-newline \n (first) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP @@ -745,7 +825,7 @@ begin data "xxx". begin data 123. not data ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl identifier begin space identifier data end_command . @@ -788,7 +868,7 @@ newline \n (data) inline_data 5_6_7_/*_x newline \n (data) -inline_data +inline_data newline \n (data) inline_data end__data @@ -836,9 +916,9 @@ newline \n (first) reserved_word not space identifier data -newline \n (later) - -end +-newline \n (later) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP @@ -860,7 +940,7 @@ do inner command. end repeat. ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl identifier do space identifier repeat space identifier x @@ -918,9 +998,504 @@ newline \n (DO REPEAT) identifier end space identifier repeat end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DO REPEAT command in batch mode]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +do repeat x=a b c + y=d e f +do repeat a=1 thru 5 +another command +second command ++ third command +end /* x */ /* y */ repeat print +end + repeat +do + repeat #a=1 + + inner command +end repeat +]) +AT_DATA([expout-base], [dnl +identifier do space +identifier repeat space +identifier x +punct = +identifier a space +identifier b space +identifier c +newline \n (later) + +spaces __________ +identifier y +punct = +identifier d space +identifier e space +identifier f +newline \n (later) + +start_command +do_repeat_command do_repeat_a=1_thru_5 +newline \n (DO REPEAT) + +do_repeat_command another_command +newline \n (DO REPEAT) + +do_repeat_command second_command +newline \n (DO REPEAT) + +do_repeat_command +_third_command +newline \n (DO REPEAT) + +do_repeat_command end_/*_x_*/_/*_y_*/_repeat_print +newline \n (DO REPEAT) + +identifier end +newline \n (later) + space +identifier repeat +newline \n (later) + +start_command +identifier do +newline \n (later) + +spaces __ +identifier repeat space +identifier #a +punct = +number 1 +newline \n (later) + +separate_commands +newline \n (DO REPEAT) + +do_repeat_command __inner_command +newline \n (DO REPEAT) + +identifier end space +identifier repeat +-newline \n (later) +- +end +]) +PSPP_CHECK_SEGMENT([-b]) +AT_CLEANUP + +AT_SETUP([DEFINE command - simple]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1() +var1 var2 var3 "!enddefine" +!enddefine. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +punct ( +punct ) +spaces +newline \n (DEFINE) + +macro_body var1_var2_var3_"!enddefine" +newline \n (DEFINE) + +macro_id !enddefine +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - no newline after parentheses]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1() var1 var2 var3 /* !enddefine +!enddefine. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +punct ( +punct ) +macro_body _var1_var2_var3_/*_!enddefine +newline \n (DEFINE) + +macro_id !enddefine +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - no newline before !ENDDEFINE]) +AT_KEYWORDS([segment ENDDEFINE]) +AT_DATA([input], [dnl +define !macro1() +var1 var2 var3!enddefine. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +punct ( +punct ) +spaces +newline \n (DEFINE) + +macro_body var1_var2_var3 +macro_id !enddefine +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - all on one line]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1()var1 var2 var3!enddefine. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +punct ( +punct ) +macro_body var1_var2_var3 +macro_id !enddefine +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - empty]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1() +!enddefine. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +punct ( +punct ) +spaces +newline \n (DEFINE) + +macro_id !enddefine +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - blank lines]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1() + + +!enddefine. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +punct ( +punct ) +spaces +newline \n (DEFINE) + +macro_body +newline \n (DEFINE) + +macro_body +newline \n (DEFINE) + +macro_id !enddefine +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - arguments]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1(a(), b(), c()) +!enddefine. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +punct ( +identifier a +punct ( +punct ) +punct , space +identifier b +punct ( +punct ) +punct , space +identifier c +punct ( +punct ) +punct ) +spaces +newline \n (DEFINE) + +macro_id !enddefine +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - multiline arguments]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1( + a(), b( + ), + c() +) +!enddefine. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +punct ( +newline \n (later) + +spaces __ +identifier a +punct ( +punct ) +punct , space +identifier b +punct ( +newline \n (later) + +spaces __ +punct ) +punct , +newline \n (later) + +spaces __ +identifier c +punct ( +punct ) +newline \n (later) + +punct ) +spaces +newline \n (DEFINE) + +macro_id !enddefine +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - arguments start on second line]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1 +(x,y,z +) +content 1 +content 2 +!enddefine. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +newline \n (later) + +punct ( +identifier x +punct , +identifier y +punct , +identifier z +newline \n (later) + +punct ) +spaces +newline \n (DEFINE) + +macro_body content_1 +newline \n (DEFINE) + +macro_body content_2 +newline \n (DEFINE) + +macro_id !enddefine +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - early end of command 1]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1. +data list /x 1. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +end_command . +newline \n (first) + +identifier data space +identifier list space +punct / +identifier x space +number 1 +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - early end of command 2]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1 +x. +data list /x 1. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +newline \n (later) + +identifier x +end_command . +newline \n (first) + +identifier data space +identifier list space +punct / +identifier x space +number 1 +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - early end of command 3]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1(. +x. +data list /x 1. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +punct ( +end_command . +newline \n (first) + +identifier x +end_command . +newline \n (first) + +identifier data space +identifier list space +punct / +identifier x space +number 1 +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - early end of command 4]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +dnl Notice the command terminator at the end of the DEFINE command, +dnl which should not be there and ends it early. +define !macro1. +data list /x 1. +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +end_command . newline \n (first) -end +identifier data space +identifier list space +punct / +identifier x space +number 1 +end_command . +-newline \n (first) +- +end +]) +PSPP_CHECK_SEGMENT([-i]) +AT_CLEANUP + +AT_SETUP([DEFINE command - missing !ENDDEFINE]) +AT_KEYWORDS([segment]) +AT_DATA([input], [dnl +define !macro1() +content line 1 +content line 2 +]) +AT_DATA([expout-base], [dnl +identifier define space +macro_name !macro1 +punct ( +punct ) +spaces +newline \n (DEFINE) + +macro_body content_line_1 +newline \n (DEFINE) + +macro_body content_line_2 +-newline \n (DEFINE) +- +end ]) PSPP_CHECK_SEGMENT([-i]) AT_CLEANUP @@ -936,7 +1511,7 @@ third command fourth command. fifth command. ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl identifier first space identifier command newline \n (later) @@ -972,9 +1547,9 @@ spaces ___ identifier fifth space identifier command end_command . -newline \n (first) - -end +-newline \n (first) +- +end ]) PSPP_CHECK_SEGMENT([-b]) AT_CLEANUP @@ -996,7 +1571,7 @@ twostep cluster fourth command. fifth command. ]) -AT_DATA([expout], [dnl +AT_DATA([expout-base], [dnl identifier command newline \n (later) @@ -1062,9 +1637,22 @@ spaces ___ identifier fifth space identifier command end_command . -newline \n (first) - -end +-newline \n (first) +- +end ]) PSPP_CHECK_SEGMENT([-a]) AT_CLEANUP + +# This checks for regression against bug #61253. To see the read of +# uninitialized data, run with valgrind. The test will pass either +# way. (The bug report has a more complicated crashing case.) +AT_SETUP([input ends in carriage return]) +printf '\r' > input +AT_DATA([expout-base], [dnl +separate_commands +spaces \r +end +]) +PSPP_CHECK_SEGMENT +AT_CLEANUP