pintos-os.org Git - pspp/blob - tests/language/lexer/scan.at

   1 dnl PSPP - a program for statistical analysis.
   2 dnl Copyright (C) 2017 Free Software Foundation, Inc.
   3 dnl
   4 dnl This program is free software: you can redistribute it and/or modify
   5 dnl it under the terms of the GNU General Public License as published by
   6 dnl the Free Software Foundation, either version 3 of the License, or
   7 dnl (at your option) any later version.
   8 dnl
   9 dnl This program is distributed in the hope that it will be useful,
  10 dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 dnl GNU General Public License for more details.
  13 dnl
  14 dnl You should have received a copy of the GNU General Public License
  15 dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 dnl
  17 AT_BANNER([syntax scanning])
  18 m4_define([PSPP_CHECK_SCAN],
  19   [sed 's/^-//' < expout-base > expout
  20    AT_CHECK([scan-test $1 input], [0], [expout])
  21
  22    sed '/^-/d' < expout-base > expout
  23    AT_CHECK([scan-test -s $1 input], [0], [expout])])
  24 \f
  25 AT_SETUP([identifiers])
  26 AT_KEYWORDS([scan])
  27 AT_DATA([input], [dnl
  28 a aB i5 $x @efg @@. !abcd #.# .x _z.
  29 abcd. abcd.
  30 QRSTUV./* end of line comment */
  31 QrStUv./* end of line comment */ @&t@
  32 WXYZ. /* unterminated end of line comment
  33 �. /* U+FFFD is not valid in an identifier
  34 ])
  35 AT_DATA([expout-base], [dnl
  36 ID "a"
  37 SKIP
  38 ID "aB"
  39 SKIP
  40 ID "i5"
  41 SKIP
  42 ID "$x"
  43 SKIP
  44 ID "@efg"
  45 SKIP
  46 ID "@@."
  47 SKIP
  48 MACRO_ID "!abcd"
  49 SKIP
  50 ID "#.#"
  51 SKIP
  52 UNEXPECTED_DOT
  53 ID "x"
  54 SKIP
  55 MACRO_PUNCT "_"
  56 ID "z"
  57 ENDCMD
  58 SKIP
  59 ID "abcd."
  60 SKIP
  61 ID "abcd"
  62 ENDCMD
  63 SKIP
  64 ID "QRSTUV"
  65 ENDCMD
  66 SKIP
  67 SKIP
  68 ID "QrStUv"
  69 ENDCMD
  70 SKIP
  71 SKIP
  72 SKIP
  73 ID "WXYZ"
  74 ENDCMD
  75 SKIP
  76 SKIP
  77 SKIP
  78 UNEXPECTED_CHAR 65533
  79 ENDCMD
  80 SKIP
  81 SKIP
  82 -SKIP
  83 STOP
  84 ])
  85 PSPP_CHECK_SCAN([-i])
  86 AT_CLEANUP
  87 \f
  88 AT_SETUP([reserved words])
  89 AT_KEYWORDS([scan])
  90 AT_DATA([input], [dnl
  91 and or not eq ge gt le lt ne all by to with
  92 AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH
  93 andx orx notx eqx gex gtx lex ltx nex allx byx tox withx
  94 and. with.
  95 ])
  96 AT_DATA([expout-base], [dnl
  97 AND
  98 SKIP
  99 OR
 100 SKIP
 101 NOT
 102 SKIP
 103 EQ
 104 SKIP
 105 GE
 106 SKIP
 107 GT
 108 SKIP
 109 LE
 110 SKIP
 111 LT
 112 SKIP
 113 NE
 114 SKIP
 115 ALL
 116 SKIP
 117 BY
 118 SKIP
 119 TO
 120 SKIP
 121 WITH
 122 SKIP
 123 AND
 124 SKIP
 125 OR
 126 SKIP
 127 NOT
 128 SKIP
 129 EQ
 130 SKIP
 131 GE
 132 SKIP
 133 GT
 134 SKIP
 135 LE
 136 SKIP
 137 LT
 138 SKIP
 139 NE
 140 SKIP
 141 ALL
 142 SKIP
 143 BY
 144 SKIP
 145 TO
 146 SKIP
 147 WITH
 148 SKIP
 149 ID "andx"
 150 SKIP
 151 ID "orx"
 152 SKIP
 153 ID "notx"
 154 SKIP
 155 ID "eqx"
 156 SKIP
 157 ID "gex"
 158 SKIP
 159 ID "gtx"
 160 SKIP
 161 ID "lex"
 162 SKIP
 163 ID "ltx"
 164 SKIP
 165 ID "nex"
 166 SKIP
 167 ID "allx"
 168 SKIP
 169 ID "byx"
 170 SKIP
 171 ID "tox"
 172 SKIP
 173 ID "withx"
 174 SKIP
 175 ID "and."
 176 SKIP
 177 WITH
 178 ENDCMD
 179 -SKIP
 180 STOP
 181 ])
 182 PSPP_CHECK_SCAN([-i])
 183 AT_CLEANUP
 184 \f
 185 AT_SETUP([punctuation])
 186 AT_KEYWORDS([scan])
 187 AT_DATA([input], [dnl
 188 ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] **
 189 ~&|=>=><=<~=<>(),-+*/[[]]**
 190 % : ; ? _ ` { } ~
 191 ])
 192 AT_DATA([expout-base], [dnl
 193 NOT
 194 SKIP
 195 AND
 196 SKIP
 197 OR
 198 SKIP
 199 EQUALS
 200 SKIP
 201 GE
 202 SKIP
 203 GT
 204 SKIP
 205 LE
 206 SKIP
 207 LT
 208 SKIP
 209 NE
 210 SKIP
 211 NE
 212 SKIP
 213 LPAREN
 214 SKIP
 215 RPAREN
 216 SKIP
 217 COMMA
 218 SKIP
 219 DASH
 220 SKIP
 221 PLUS
 222 SKIP
 223 ASTERISK
 224 SKIP
 225 SLASH
 226 SKIP
 227 LBRACK
 228 SKIP
 229 RBRACK
 230 SKIP
 231 EXP
 232 SKIP
 233 NOT
 234 AND
 235 OR
 236 EQUALS
 237 GE
 238 GT
 239 LE
 240 LT
 241 NE
 242 NE
 243 LPAREN
 244 RPAREN
 245 COMMA
 246 DASH
 247 PLUS
 248 ASTERISK
 249 SLASH
 250 LBRACK
 251 RBRACK
 252 EXP
 253 SKIP
 254 MACRO_PUNCT "%"
 255 SKIP
 256 MACRO_PUNCT ":"
 257 SKIP
 258 MACRO_PUNCT ";"
 259 SKIP
 260 MACRO_PUNCT "?"
 261 SKIP
 262 MACRO_PUNCT "_"
 263 SKIP
 264 MACRO_PUNCT "`"
 265 SKIP
 266 MACRO_PUNCT "{"
 267 SKIP
 268 MACRO_PUNCT "}"
 269 SKIP
 270 NOT
 271 -SKIP
 272 STOP
 273 ])
 274 PSPP_CHECK_SCAN([-i])
 275 AT_CLEANUP
 276 \f
 277 AT_SETUP([numbers])
 278 AT_KEYWORDS([scan])
 279 AT_DATA([input], [dnl
 280 0 1 01 001. 1.
 281 123. /* comment 1 */ /* comment 2 */
 282 .1 0.1 00.1 00.10
 283 5e1 6E-1 7e+1 6E+01 6e-03
 284 .3E1 .4e-1 .5E+1 .6e+01 .7E-03
 285 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03
 286 . 1e e1 1e+ 1e-
 287 ])
 288 AT_DATA([expout-base], [dnl
 289 POS_NUM
 290 SKIP
 291 POS_NUM 1
 292 SKIP
 293 POS_NUM 1
 294 SKIP
 295 POS_NUM 1
 296 SKIP
 297 POS_NUM 1
 298 ENDCMD
 299 SKIP
 300 POS_NUM 123
 301 ENDCMD
 302 SKIP
 303 SKIP
 304 SKIP
 305 SKIP
 306 SKIP
 307 ENDCMD
 308 POS_NUM 1
 309 SKIP
 310 POS_NUM 0.1
 311 SKIP
 312 POS_NUM 0.1
 313 SKIP
 314 POS_NUM 0.1
 315 SKIP
 316 POS_NUM 50
 317 SKIP
 318 POS_NUM 0.6
 319 SKIP
 320 POS_NUM 70
 321 SKIP
 322 POS_NUM 60
 323 SKIP
 324 POS_NUM 0.006
 325 SKIP
 326 ENDCMD
 327 POS_NUM 30
 328 SKIP
 329 POS_NUM 0.04
 330 SKIP
 331 POS_NUM 5
 332 SKIP
 333 POS_NUM 6
 334 SKIP
 335 POS_NUM 0.0007
 336 SKIP
 337 POS_NUM 12.3
 338 SKIP
 339 POS_NUM 4.56
 340 SKIP
 341 POS_NUM 789
 342 SKIP
 343 POS_NUM 999
 344 SKIP
 345 POS_NUM 0.0112
 346 SKIP
 347 ENDCMD
 348 SKIP
 349 EXPECTED_EXPONENT "1e"
 350 SKIP
 351 ID "e1"
 352 SKIP
 353 EXPECTED_EXPONENT "1e+"
 354 SKIP
 355 EXPECTED_EXPONENT "1e-"
 356 -SKIP
 357 STOP
 358 ])
 359 PSPP_CHECK_SCAN([-i])
 360 AT_CLEANUP
 361 \f
 362 AT_SETUP([strings])
 363 AT_KEYWORDS([scan])
 364 AT_DATA([input], [dnl
 365 'x' "y" 'abc'
 366 'Don''t' "Can't" 'Won''t'
 367 """quoted""" '"quoted"'
 368 '' "" '''' """"
 369 'missing end quote
 370 "missing double quote
 371 'x' + "y"
 372 + 'z' +
 373 'a' /* abc */ + "b" /*
 374 + 'c' +/* */"d"/* */+'e'
 375 'foo'
 376 +          /* special case: + in column 0 would ordinarily start a new command
 377 'bar'
 378 'foo'
 379  +
 380 'bar'
 381 'foo'
 382 +
 383
 384 'bar'
 385
 386 +
 387 x"4142"+'5152'
 388 "4142"+
 389 x'5152'
 390 x"4142"
 391 +u'304a'
 392 "�あいうえお"
 393 "abc"+U"FFFD"+u'3048'+"xyz"
 394 ])
 395 AT_DATA([expout-base], [dnl
 396 STRING "x"
 397 SKIP
 398 STRING "y"
 399 SKIP
 400 STRING "abc"
 401 SKIP
 402 STRING "Don't"
 403 SKIP
 404 STRING "Can't"
 405 SKIP
 406 STRING "Won't"
 407 SKIP
 408 STRING ""quoted""
 409 SKIP
 410 STRING ""quoted""
 411 SKIP
 412 STRING ""
 413 SKIP
 414 STRING ""
 415 SKIP
 416 STRING "'"
 417 SKIP
 418 STRING """
 419 SKIP
 420 EXPECTED_QUOTE
 421 SKIP
 422 EXPECTED_QUOTE
 423 SKIP
 424 STRING "xyzabcde"
 425 SKIP
 426 STRING "foobar"
 427 SKIP
 428 STRING "foobar"
 429 SKIP
 430 STRING "foo"
 431 SKIP
 432 PLUS
 433 SKIP
 434 ENDCMD
 435 SKIP
 436 STRING "bar"
 437 SKIP
 438 ENDCMD
 439 SKIP
 440 PLUS
 441 SKIP
 442 STRING "AB5152"
 443 SKIP
 444 STRING "4142QR"
 445 SKIP
 446 STRING "ABお"
 447 SKIP
 448 STRING "�あいうえお"
 449 SKIP
 450 STRING "abc�えxyz"
 451 -SKIP
 452 STOP
 453 ])
 454 PSPP_CHECK_SCAN([-i])
 455 AT_CLEANUP
 456 \f
 457 AT_SETUP([@%:@! construct])
 458 AT_KEYWORDS([scan])
 459 AT_DATA([input], [dnl
 460 #! /usr/bin/pspp
 461 #! /usr/bin/pspp
 462 ])
 463 AT_DATA([expout-base], [dnl
 464 SKIP
 465 SKIP
 466 ID "#"
 467 MACRO_ID "!"
 468 SKIP
 469 SLASH
 470 ID "usr"
 471 SLASH
 472 ID "bin"
 473 SLASH
 474 ID "pspp"
 475 -SKIP
 476 STOP
 477 ])
 478 PSPP_CHECK_SCAN([-i])
 479 AT_CLEANUP
 480 \f
 481 AT_SETUP([* and COMMENT commands])
 482 AT_KEYWORDS([scan])
 483 AT_DATA([input], [dnl
 484 * Comment commands "don't
 485 have to contain valid tokens.
 486
 487 ** Check ambiguity with ** token.
 488 ****************.
 489
 490 comment keyword works too.
 491 COMM also.
 492 com is ambiguous with COMPUTE.
 493
 494    * Comment need not start at left margin.
 495
 496 * Comment ends with blank line
 497
 498 next command.
 499
 500 ])
 501 AT_DATA([expout-base], [dnl
 502 SKIP
 503 SKIP
 504 SKIP
 505 ENDCMD
 506 SKIP
 507 ENDCMD
 508 SKIP
 509 SKIP
 510 ENDCMD
 511 SKIP
 512 SKIP
 513 ENDCMD
 514 SKIP
 515 ENDCMD
 516 SKIP
 517 SKIP
 518 ENDCMD
 519 SKIP
 520 SKIP
 521 ENDCMD
 522 SKIP
 523 ID "com"
 524 SKIP
 525 ID "is"
 526 SKIP
 527 ID "ambiguous"
 528 SKIP
 529 WITH
 530 SKIP
 531 ID "COMPUTE"
 532 ENDCMD
 533 SKIP
 534 ENDCMD
 535 SKIP
 536 SKIP
 537 SKIP
 538 ENDCMD
 539 SKIP
 540 ENDCMD
 541 SKIP
 542 SKIP
 543 SKIP
 544 ENDCMD
 545 SKIP
 546 ID "next"
 547 SKIP
 548 ID "command"
 549 ENDCMD
 550 SKIP
 551 -ENDCMD
 552 -SKIP
 553 STOP
 554 ])
 555 PSPP_CHECK_SCAN([-i])
 556 AT_CLEANUP
 557 \f
 558 AT_SETUP([DOCUMENT command])
 559 AT_KEYWORDS([scan])
 560 AT_DATA([input], [dnl
 561 DOCUMENT one line.
 562 DOC more
 563     than
 564         one
 565             line.
 566 docu
 567 first.paragraph
 568 isn't parsed as tokens
 569
 570 second paragraph.
 571 ])
 572 AT_DATA([expout-base], [dnl
 573 ID "DOCUMENT"
 574 STRING "DOCUMENT one line."
 575 ENDCMD
 576 ENDCMD
 577 SKIP
 578 ID "DOCUMENT"
 579 STRING "DOC more"
 580 SKIP
 581 STRING "    than"
 582 SKIP
 583 STRING "        one"
 584 SKIP
 585 STRING "            line."
 586 ENDCMD
 587 ENDCMD
 588 SKIP
 589 ID "DOCUMENT"
 590 STRING "docu"
 591 SKIP
 592 STRING "first.paragraph"
 593 SKIP
 594 STRING "isn't parsed as tokens"
 595 SKIP
 596 STRING ""
 597 SKIP
 598 STRING "second paragraph."
 599 -ENDCMD
 600 -ENDCMD
 601 -SKIP
 602 STOP
 603 ])
 604 PSPP_CHECK_SCAN([-i])
 605 AT_CLEANUP
 606 \f
 607 AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands])
 608 AT_KEYWORDS([scan])
 609 AT_DATA([input], [dnl
 610 title/**/'Quoted string title'.
 611 tit /*
 612 "Quoted string on second line".
 613 sub "Quoted string subtitle"
 614  .
 615
 616 TITL /* Not a */ quoted string title.
 617 SUBT Not a quoted string /* subtitle
 618
 619 FIL label isn't quoted.
 620 FILE
 621   lab 'is quoted'.
 622 FILE /*
 623 /**/  lab not quoted here either
 624
 625 ])
 626 AT_DATA([expout-base], [dnl
 627 ID "title"
 628 SKIP
 629 STRING "Quoted string title"
 630 ENDCMD
 631 SKIP
 632 ID "tit"
 633 SKIP
 634 SKIP
 635 SKIP
 636 STRING "Quoted string on second line"
 637 ENDCMD
 638 SKIP
 639 ID "sub"
 640 SKIP
 641 STRING "Quoted string subtitle"
 642 SKIP
 643 SKIP
 644 ENDCMD
 645 SKIP
 646 ENDCMD
 647 SKIP
 648 ID "TITL"
 649 SKIP
 650 STRING "/* Not a */ quoted string title"
 651 ENDCMD
 652 SKIP
 653 ID "SUBT"
 654 SKIP
 655 STRING "Not a quoted string /* subtitle"
 656 SKIP
 657 ENDCMD
 658 SKIP
 659 ID "FIL"
 660 SKIP
 661 ID "label"
 662 SKIP
 663 STRING "isn't quoted"
 664 ENDCMD
 665 SKIP
 666 ID "FILE"
 667 SKIP
 668 SKIP
 669 ID "lab"
 670 SKIP
 671 STRING "is quoted"
 672 ENDCMD
 673 SKIP
 674 ID "FILE"
 675 SKIP
 676 SKIP
 677 SKIP
 678 SKIP
 679 SKIP
 680 ID "lab"
 681 SKIP
 682 STRING "not quoted here either"
 683 SKIP
 684 -ENDCMD
 685 -SKIP
 686 STOP
 687 ])
 688 PSPP_CHECK_SCAN([-i])
 689 AT_CLEANUP
 690 \f
 691 AT_SETUP([BEGIN DATA command])
 692 AT_KEYWORDS([scan])
 693 AT_DATA([input], [dnl
 694 begin data.
 695 123
 696 xxx
 697 end data.
 698
 699 BEG /**/ DAT /*
 700 5 6 7 /* x
 701
 702 end  data
 703 end data
 704 .
 705 ])
 706 AT_DATA([expout-base], [dnl
 707 ID "begin"
 708 SKIP
 709 ID "data"
 710 ENDCMD
 711 SKIP
 712 STRING "123"
 713 SKIP
 714 STRING "xxx"
 715 SKIP
 716 ID "end"
 717 SKIP
 718 ID "data"
 719 ENDCMD
 720 SKIP
 721 ENDCMD
 722 SKIP
 723 ID "BEG"
 724 SKIP
 725 SKIP
 726 SKIP
 727 ID "DAT"
 728 SKIP
 729 SKIP
 730 SKIP
 731 STRING "5 6 7 /* x"
 732 SKIP
 733 STRING ""
 734 SKIP
 735 STRING "end  data"
 736 SKIP
 737 ID "end"
 738 SKIP
 739 ID "data"
 740 SKIP
 741 ENDCMD
 742 -SKIP
 743 STOP
 744 ])
 745 PSPP_CHECK_SCAN([-i])
 746 AT_CLEANUP
 747 \f
 748 AT_SETUP([DO REPEAT command])
 749 AT_KEYWORDS([scan])
 750 AT_DATA([input], [dnl
 751 do repeat x=a b c
 752           y=d e f.
 753   do repeat a=1 thru 5.
 754 another command.
 755 second command
 756 + third command.
 757 end /* x */ /* y */ repeat print.
 758 end
 759  repeat.
 760 ])
 761 AT_DATA([expout-base], [dnl
 762 ID "do"
 763 SKIP
 764 ID "repeat"
 765 SKIP
 766 ID "x"
 767 EQUALS
 768 ID "a"
 769 SKIP
 770 ID "b"
 771 SKIP
 772 ID "c"
 773 SKIP
 774 SKIP
 775 ID "y"
 776 EQUALS
 777 ID "d"
 778 SKIP
 779 ID "e"
 780 SKIP
 781 ID "f"
 782 ENDCMD
 783 SKIP
 784 STRING "  do repeat a=1 thru 5."
 785 SKIP
 786 STRING "another command."
 787 SKIP
 788 STRING "second command"
 789 SKIP
 790 STRING "+ third command."
 791 SKIP
 792 STRING "end /* x */ /* y */ repeat print."
 793 SKIP
 794 ID "end"
 795 SKIP
 796 SKIP
 797 ID "repeat"
 798 ENDCMD
 799 -SKIP
 800 STOP
 801 ])
 802 PSPP_CHECK_SCAN([-i])
 803 AT_CLEANUP
 804 \f
 805 AT_SETUP([DO REPEAT command in batch mode])
 806 AT_KEYWORDS([scan])
 807 AT_DATA([input], [dnl
 808 do repeat x=a b c
 809           y=d e f
 810 do repeat a=1 thru 5
 811 another command
 812 second command
 813 + third command
 814 end /* x */ /* y */ repeat print
 815 end
 816  repeat
 817 do
 818   repeat #a=1
 819
 820   inner command
 821 end repeat
 822 ])
 823 AT_DATA([expout-base], [dnl
 824 ID "do"
 825 SKIP
 826 ID "repeat"
 827 SKIP
 828 ID "x"
 829 EQUALS
 830 ID "a"
 831 SKIP
 832 ID "b"
 833 SKIP
 834 ID "c"
 835 SKIP
 836 SKIP
 837 ID "y"
 838 EQUALS
 839 ID "d"
 840 SKIP
 841 ID "e"
 842 SKIP
 843 ID "f"
 844 SKIP
 845 ENDCMD
 846 STRING "do repeat a=1 thru 5"
 847 SKIP
 848 STRING "another command"
 849 SKIP
 850 STRING "second command"
 851 SKIP
 852 STRING "+ third command"
 853 SKIP
 854 STRING "end /* x */ /* y */ repeat print"
 855 SKIP
 856 ID "end"
 857 SKIP
 858 SKIP
 859 ID "repeat"
 860 SKIP
 861 ENDCMD
 862 ID "do"
 863 SKIP
 864 SKIP
 865 ID "repeat"
 866 SKIP
 867 ID "#a"
 868 EQUALS
 869 POS_NUM 1
 870 SKIP
 871 ENDCMD
 872 SKIP
 873 STRING "  inner command"
 874 SKIP
 875 ID "end"
 876 SKIP
 877 ID "repeat"
 878 -SKIP
 879 STOP
 880 ])
 881 PSPP_CHECK_SCAN([-b])
 882 AT_CLEANUP
 883 \f
 884 AT_SETUP([batch mode])
 885 AT_KEYWORDS([scan])
 886 AT_DATA([input], [dnl
 887 first command
 888      another line of first command
 889 +  second command
 890 third command
 891
 892 fourth command.
 893    fifth command.
 894 ])
 895 AT_DATA([expout-base], [dnl
 896 ID "first"
 897 SKIP
 898 ID "command"
 899 SKIP
 900 SKIP
 901 ID "another"
 902 SKIP
 903 ID "line"
 904 SKIP
 905 ID "of"
 906 SKIP
 907 ID "first"
 908 SKIP
 909 ID "command"
 910 SKIP
 911 ENDCMD
 912 SKIP
 913 ID "second"
 914 SKIP
 915 ID "command"
 916 SKIP
 917 ENDCMD
 918 ID "third"
 919 SKIP
 920 ID "command"
 921 SKIP
 922 ENDCMD
 923 SKIP
 924 ID "fourth"
 925 SKIP
 926 ID "command"
 927 ENDCMD
 928 SKIP
 929 SKIP
 930 ID "fifth"
 931 SKIP
 932 ID "command"
 933 ENDCMD
 934 -SKIP
 935 STOP
 936 ])
 937 PSPP_CHECK_SCAN([-b])
 938 AT_CLEANUP