pintos-os.org Git - pspp/blob - tests/language/lexer/scan.at

   1 dnl PSPP - a program for statistical analysis.
   2 dnl Copyright (C) 2017 Free Software Foundation, Inc.
   3 dnl
   4 dnl This program is free software: you can redistribute it and/or modify
   5 dnl it under the terms of the GNU General Public License as published by
   6 dnl the Free Software Foundation, either version 3 of the License, or
   7 dnl (at your option) any later version.
   8 dnl
   9 dnl This program is distributed in the hope that it will be useful,
  10 dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 dnl GNU General Public License for more details.
  13 dnl
  14 dnl You should have received a copy of the GNU General Public License
  15 dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 dnl
  17 AT_BANNER([syntax scanning])
  18 m4_define([PSPP_CHECK_SCAN],
  19   [sed 's/^-//' < expout-base > expout
  20    AT_CHECK([scan-test $1 input], [0], [expout])
  21
  22    sed '/^-/d' < expout-base > expout
  23    AT_CHECK([scan-test -s $1 input], [0], [expout])])
  24 \f
  25 AT_SETUP([identifiers])
  26 AT_KEYWORDS([scan])
  27 AT_DATA([input], [dnl
  28 a aB i5 $x @efg @@. !abcd #.# .x _z.
  29 abcd. abcd.
  30 QRSTUV./* end of line comment */
  31 QrStUv./* end of line comment */ @&t@
  32 WXYZ. /* unterminated end of line comment
  33 �. /* U+FFFD is not valid in an identifier
  34 ])
  35 AT_DATA([expout-base], [dnl
  36 ID "a"
  37 SKIP
  38 ID "aB"
  39 SKIP
  40 ID "i5"
  41 SKIP
  42 ID "$x"
  43 SKIP
  44 ID "@efg"
  45 SKIP
  46 ID "@@."
  47 SKIP
  48 MACRO_ID "!abcd"
  49 SKIP
  50 ID "#.#"
  51 SKIP
  52 UNEXPECTED_DOT
  53 ID "x"
  54 SKIP
  55 UNEXPECTED_CHAR 95
  56 ID "z"
  57 ENDCMD
  58 SKIP
  59 ID "abcd."
  60 SKIP
  61 ID "abcd"
  62 ENDCMD
  63 SKIP
  64 ID "QRSTUV"
  65 ENDCMD
  66 SKIP
  67 SKIP
  68 ID "QrStUv"
  69 ENDCMD
  70 SKIP
  71 SKIP
  72 SKIP
  73 ID "WXYZ"
  74 ENDCMD
  75 SKIP
  76 SKIP
  77 SKIP
  78 UNEXPECTED_CHAR 65533
  79 ENDCMD
  80 SKIP
  81 SKIP
  82 -SKIP
  83 STOP
  84 ])
  85 PSPP_CHECK_SCAN([-i])
  86 AT_CLEANUP
  87 \f
  88 AT_SETUP([reserved words])
  89 AT_KEYWORDS([scan])
  90 AT_DATA([input], [dnl
  91 and or not eq ge gt le lt ne all by to with
  92 AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH
  93 andx orx notx eqx gex gtx lex ltx nex allx byx tox withx
  94 and. with.
  95 ])
  96 AT_DATA([expout-base], [dnl
  97 AND
  98 SKIP
  99 OR
 100 SKIP
 101 NOT
 102 SKIP
 103 EQ
 104 SKIP
 105 GE
 106 SKIP
 107 GT
 108 SKIP
 109 LE
 110 SKIP
 111 LT
 112 SKIP
 113 NE
 114 SKIP
 115 ALL
 116 SKIP
 117 BY
 118 SKIP
 119 TO
 120 SKIP
 121 WITH
 122 SKIP
 123 AND
 124 SKIP
 125 OR
 126 SKIP
 127 NOT
 128 SKIP
 129 EQ
 130 SKIP
 131 GE
 132 SKIP
 133 GT
 134 SKIP
 135 LE
 136 SKIP
 137 LT
 138 SKIP
 139 NE
 140 SKIP
 141 ALL
 142 SKIP
 143 BY
 144 SKIP
 145 TO
 146 SKIP
 147 WITH
 148 SKIP
 149 ID "andx"
 150 SKIP
 151 ID "orx"
 152 SKIP
 153 ID "notx"
 154 SKIP
 155 ID "eqx"
 156 SKIP
 157 ID "gex"
 158 SKIP
 159 ID "gtx"
 160 SKIP
 161 ID "lex"
 162 SKIP
 163 ID "ltx"
 164 SKIP
 165 ID "nex"
 166 SKIP
 167 ID "allx"
 168 SKIP
 169 ID "byx"
 170 SKIP
 171 ID "tox"
 172 SKIP
 173 ID "withx"
 174 SKIP
 175 ID "and."
 176 SKIP
 177 WITH
 178 ENDCMD
 179 -SKIP
 180 STOP
 181 ])
 182 PSPP_CHECK_SCAN([-i])
 183 AT_CLEANUP
 184 \f
 185 AT_SETUP([punctuation])
 186 AT_KEYWORDS([scan])
 187 AT_DATA([input], [dnl
 188 ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] **
 189 ~&|=>=><=<~=<>(),-+*/[[]]**
 190 ])
 191 AT_DATA([expout-base], [dnl
 192 NOT
 193 SKIP
 194 AND
 195 SKIP
 196 OR
 197 SKIP
 198 EQUALS
 199 SKIP
 200 GE
 201 SKIP
 202 GT
 203 SKIP
 204 LE
 205 SKIP
 206 LT
 207 SKIP
 208 NE
 209 SKIP
 210 NE
 211 SKIP
 212 LPAREN
 213 SKIP
 214 RPAREN
 215 SKIP
 216 COMMA
 217 SKIP
 218 DASH
 219 SKIP
 220 PLUS
 221 SKIP
 222 ASTERISK
 223 SKIP
 224 SLASH
 225 SKIP
 226 LBRACK
 227 SKIP
 228 RBRACK
 229 SKIP
 230 EXP
 231 SKIP
 232 NOT
 233 AND
 234 OR
 235 EQUALS
 236 GE
 237 GT
 238 LE
 239 LT
 240 NE
 241 NE
 242 LPAREN
 243 RPAREN
 244 COMMA
 245 DASH
 246 PLUS
 247 ASTERISK
 248 SLASH
 249 LBRACK
 250 RBRACK
 251 EXP
 252 -SKIP
 253 STOP
 254 ])
 255 PSPP_CHECK_SCAN([-i])
 256 AT_CLEANUP
 257 \f
 258 AT_SETUP([numbers])
 259 AT_KEYWORDS([scan])
 260 AT_DATA([input], [dnl
 261 0 1 01 001. 1.
 262 123. /* comment 1 */ /* comment 2 */
 263 .1 0.1 00.1 00.10
 264 5e1 6E-1 7e+1 6E+01 6e-03
 265 .3E1 .4e-1 .5E+1 .6e+01 .7E-03
 266 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03
 267 . 1e e1 1e+ 1e-
 268 ])
 269 AT_DATA([expout-base], [dnl
 270 POS_NUM
 271 SKIP
 272 POS_NUM 1
 273 SKIP
 274 POS_NUM 1
 275 SKIP
 276 POS_NUM 1
 277 SKIP
 278 POS_NUM 1
 279 ENDCMD
 280 SKIP
 281 POS_NUM 123
 282 ENDCMD
 283 SKIP
 284 SKIP
 285 SKIP
 286 SKIP
 287 SKIP
 288 ENDCMD
 289 POS_NUM 1
 290 SKIP
 291 POS_NUM 0.1
 292 SKIP
 293 POS_NUM 0.1
 294 SKIP
 295 POS_NUM 0.1
 296 SKIP
 297 POS_NUM 50
 298 SKIP
 299 POS_NUM 0.6
 300 SKIP
 301 POS_NUM 70
 302 SKIP
 303 POS_NUM 60
 304 SKIP
 305 POS_NUM 0.006
 306 SKIP
 307 ENDCMD
 308 POS_NUM 30
 309 SKIP
 310 POS_NUM 0.04
 311 SKIP
 312 POS_NUM 5
 313 SKIP
 314 POS_NUM 6
 315 SKIP
 316 POS_NUM 0.0007
 317 SKIP
 318 POS_NUM 12.3
 319 SKIP
 320 POS_NUM 4.56
 321 SKIP
 322 POS_NUM 789
 323 SKIP
 324 POS_NUM 999
 325 SKIP
 326 POS_NUM 0.0112
 327 SKIP
 328 ENDCMD
 329 SKIP
 330 EXPECTED_EXPONENT "1e"
 331 SKIP
 332 ID "e1"
 333 SKIP
 334 EXPECTED_EXPONENT "1e+"
 335 SKIP
 336 EXPECTED_EXPONENT "1e-"
 337 -SKIP
 338 STOP
 339 ])
 340 PSPP_CHECK_SCAN([-i])
 341 AT_CLEANUP
 342 \f
 343 AT_SETUP([strings])
 344 AT_KEYWORDS([scan])
 345 AT_DATA([input], [dnl
 346 'x' "y" 'abc'
 347 'Don''t' "Can't" 'Won''t'
 348 """quoted""" '"quoted"'
 349 '' "" '''' """"
 350 'missing end quote
 351 "missing double quote
 352 'x' + "y"
 353 + 'z' +
 354 'a' /* abc */ + "b" /*
 355 + 'c' +/* */"d"/* */+'e'
 356 'foo'
 357 +          /* special case: + in column 0 would ordinarily start a new command
 358 'bar'
 359 'foo'
 360  +
 361 'bar'
 362 'foo'
 363 +
 364
 365 'bar'
 366
 367 +
 368 x"4142"+'5152'
 369 "4142"+
 370 x'5152'
 371 x"4142"
 372 +u'304a'
 373 "�あいうえお"
 374 "abc"+U"FFFD"+u'3048'+"xyz"
 375 ])
 376 AT_DATA([expout-base], [dnl
 377 STRING "x"
 378 SKIP
 379 STRING "y"
 380 SKIP
 381 STRING "abc"
 382 SKIP
 383 STRING "Don't"
 384 SKIP
 385 STRING "Can't"
 386 SKIP
 387 STRING "Won't"
 388 SKIP
 389 STRING ""quoted""
 390 SKIP
 391 STRING ""quoted""
 392 SKIP
 393 STRING ""
 394 SKIP
 395 STRING ""
 396 SKIP
 397 STRING "'"
 398 SKIP
 399 STRING """
 400 SKIP
 401 EXPECTED_QUOTE
 402 SKIP
 403 EXPECTED_QUOTE
 404 SKIP
 405 STRING "xyzabcde"
 406 SKIP
 407 STRING "foobar"
 408 SKIP
 409 STRING "foobar"
 410 SKIP
 411 STRING "foo"
 412 SKIP
 413 PLUS
 414 SKIP
 415 ENDCMD
 416 SKIP
 417 STRING "bar"
 418 SKIP
 419 ENDCMD
 420 SKIP
 421 PLUS
 422 SKIP
 423 STRING "AB5152"
 424 SKIP
 425 STRING "4142QR"
 426 SKIP
 427 STRING "ABお"
 428 SKIP
 429 STRING "�あいうえお"
 430 SKIP
 431 STRING "abc�えxyz"
 432 -SKIP
 433 STOP
 434 ])
 435 PSPP_CHECK_SCAN([-i])
 436 AT_CLEANUP
 437 \f
 438 AT_SETUP([@%:@! construct])
 439 AT_KEYWORDS([scan])
 440 AT_DATA([input], [dnl
 441 #! /usr/bin/pspp
 442 #! /usr/bin/pspp
 443 ])
 444 AT_DATA([expout-base], [dnl
 445 SKIP
 446 SKIP
 447 ID "#"
 448 MACRO_ID "!"
 449 SKIP
 450 SLASH
 451 ID "usr"
 452 SLASH
 453 ID "bin"
 454 SLASH
 455 ID "pspp"
 456 -SKIP
 457 STOP
 458 ])
 459 PSPP_CHECK_SCAN([-i])
 460 AT_CLEANUP
 461 \f
 462 AT_SETUP([* and COMMENT commands])
 463 AT_KEYWORDS([scan])
 464 AT_DATA([input], [dnl
 465 * Comment commands "don't
 466 have to contain valid tokens.
 467
 468 ** Check ambiguity with ** token.
 469 ****************.
 470
 471 comment keyword works too.
 472 COMM also.
 473 com is ambiguous with COMPUTE.
 474
 475    * Comment need not start at left margin.
 476
 477 * Comment ends with blank line
 478
 479 next command.
 480
 481 ])
 482 AT_DATA([expout-base], [dnl
 483 SKIP
 484 SKIP
 485 SKIP
 486 ENDCMD
 487 SKIP
 488 ENDCMD
 489 SKIP
 490 SKIP
 491 ENDCMD
 492 SKIP
 493 SKIP
 494 ENDCMD
 495 SKIP
 496 ENDCMD
 497 SKIP
 498 SKIP
 499 ENDCMD
 500 SKIP
 501 SKIP
 502 ENDCMD
 503 SKIP
 504 ID "com"
 505 SKIP
 506 ID "is"
 507 SKIP
 508 ID "ambiguous"
 509 SKIP
 510 WITH
 511 SKIP
 512 ID "COMPUTE"
 513 ENDCMD
 514 SKIP
 515 ENDCMD
 516 SKIP
 517 SKIP
 518 SKIP
 519 ENDCMD
 520 SKIP
 521 ENDCMD
 522 SKIP
 523 SKIP
 524 SKIP
 525 ENDCMD
 526 SKIP
 527 ID "next"
 528 SKIP
 529 ID "command"
 530 ENDCMD
 531 SKIP
 532 -ENDCMD
 533 -SKIP
 534 STOP
 535 ])
 536 PSPP_CHECK_SCAN([-i])
 537 AT_CLEANUP
 538 \f
 539 AT_SETUP([DOCUMENT command])
 540 AT_KEYWORDS([scan])
 541 AT_DATA([input], [dnl
 542 DOCUMENT one line.
 543 DOC more
 544     than
 545         one
 546             line.
 547 docu
 548 first.paragraph
 549 isn't parsed as tokens
 550
 551 second paragraph.
 552 ])
 553 AT_DATA([expout-base], [dnl
 554 ID "DOCUMENT"
 555 STRING "DOCUMENT one line."
 556 ENDCMD
 557 ENDCMD
 558 SKIP
 559 ID "DOCUMENT"
 560 STRING "DOC more"
 561 SKIP
 562 STRING "    than"
 563 SKIP
 564 STRING "        one"
 565 SKIP
 566 STRING "            line."
 567 ENDCMD
 568 ENDCMD
 569 SKIP
 570 ID "DOCUMENT"
 571 STRING "docu"
 572 SKIP
 573 STRING "first.paragraph"
 574 SKIP
 575 STRING "isn't parsed as tokens"
 576 SKIP
 577 STRING ""
 578 SKIP
 579 STRING "second paragraph."
 580 -ENDCMD
 581 -ENDCMD
 582 -SKIP
 583 STOP
 584 ])
 585 PSPP_CHECK_SCAN([-i])
 586 AT_CLEANUP
 587 \f
 588 AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands])
 589 AT_KEYWORDS([scan])
 590 AT_DATA([input], [dnl
 591 title/**/'Quoted string title'.
 592 tit /*
 593 "Quoted string on second line".
 594 sub "Quoted string subtitle"
 595  .
 596
 597 TITL /* Not a */ quoted string title.
 598 SUBT Not a quoted string /* subtitle
 599
 600 FIL label isn't quoted.
 601 FILE
 602   lab 'is quoted'.
 603 FILE /*
 604 /**/  lab not quoted here either
 605
 606 ])
 607 AT_DATA([expout-base], [dnl
 608 ID "title"
 609 SKIP
 610 STRING "Quoted string title"
 611 ENDCMD
 612 SKIP
 613 ID "tit"
 614 SKIP
 615 SKIP
 616 SKIP
 617 STRING "Quoted string on second line"
 618 ENDCMD
 619 SKIP
 620 ID "sub"
 621 SKIP
 622 STRING "Quoted string subtitle"
 623 SKIP
 624 SKIP
 625 ENDCMD
 626 SKIP
 627 ENDCMD
 628 SKIP
 629 ID "TITL"
 630 SKIP
 631 STRING "/* Not a */ quoted string title"
 632 ENDCMD
 633 SKIP
 634 ID "SUBT"
 635 SKIP
 636 STRING "Not a quoted string /* subtitle"
 637 SKIP
 638 ENDCMD
 639 SKIP
 640 ID "FIL"
 641 SKIP
 642 ID "label"
 643 SKIP
 644 STRING "isn't quoted"
 645 ENDCMD
 646 SKIP
 647 ID "FILE"
 648 SKIP
 649 SKIP
 650 ID "lab"
 651 SKIP
 652 STRING "is quoted"
 653 ENDCMD
 654 SKIP
 655 ID "FILE"
 656 SKIP
 657 SKIP
 658 SKIP
 659 SKIP
 660 SKIP
 661 ID "lab"
 662 SKIP
 663 STRING "not quoted here either"
 664 SKIP
 665 -ENDCMD
 666 -SKIP
 667 STOP
 668 ])
 669 PSPP_CHECK_SCAN([-i])
 670 AT_CLEANUP
 671 \f
 672 AT_SETUP([BEGIN DATA command])
 673 AT_KEYWORDS([scan])
 674 AT_DATA([input], [dnl
 675 begin data.
 676 123
 677 xxx
 678 end data.
 679
 680 BEG /**/ DAT /*
 681 5 6 7 /* x
 682
 683 end  data
 684 end data
 685 .
 686 ])
 687 AT_DATA([expout-base], [dnl
 688 ID "begin"
 689 SKIP
 690 ID "data"
 691 ENDCMD
 692 SKIP
 693 STRING "123"
 694 SKIP
 695 STRING "xxx"
 696 SKIP
 697 ID "end"
 698 SKIP
 699 ID "data"
 700 ENDCMD
 701 SKIP
 702 ENDCMD
 703 SKIP
 704 ID "BEG"
 705 SKIP
 706 SKIP
 707 SKIP
 708 ID "DAT"
 709 SKIP
 710 SKIP
 711 SKIP
 712 STRING "5 6 7 /* x"
 713 SKIP
 714 STRING ""
 715 SKIP
 716 STRING "end  data"
 717 SKIP
 718 ID "end"
 719 SKIP
 720 ID "data"
 721 SKIP
 722 ENDCMD
 723 -SKIP
 724 STOP
 725 ])
 726 PSPP_CHECK_SCAN([-i])
 727 AT_CLEANUP
 728 \f
 729 AT_SETUP([DO REPEAT command])
 730 AT_KEYWORDS([scan])
 731 AT_DATA([input], [dnl
 732 do repeat x=a b c
 733           y=d e f.
 734   do repeat a=1 thru 5.
 735 another command.
 736 second command
 737 + third command.
 738 end /* x */ /* y */ repeat print.
 739 end
 740  repeat.
 741 ])
 742 AT_DATA([expout-base], [dnl
 743 ID "do"
 744 SKIP
 745 ID "repeat"
 746 SKIP
 747 ID "x"
 748 EQUALS
 749 ID "a"
 750 SKIP
 751 ID "b"
 752 SKIP
 753 ID "c"
 754 SKIP
 755 SKIP
 756 ID "y"
 757 EQUALS
 758 ID "d"
 759 SKIP
 760 ID "e"
 761 SKIP
 762 ID "f"
 763 ENDCMD
 764 SKIP
 765 STRING "  do repeat a=1 thru 5."
 766 SKIP
 767 STRING "another command."
 768 SKIP
 769 STRING "second command"
 770 SKIP
 771 STRING "+ third command."
 772 SKIP
 773 STRING "end /* x */ /* y */ repeat print."
 774 SKIP
 775 ID "end"
 776 SKIP
 777 SKIP
 778 ID "repeat"
 779 ENDCMD
 780 -SKIP
 781 STOP
 782 ])
 783 PSPP_CHECK_SCAN([-i])
 784 AT_CLEANUP
 785 \f
 786 AT_SETUP([DO REPEAT command in batch mode])
 787 AT_KEYWORDS([scan])
 788 AT_DATA([input], [dnl
 789 do repeat x=a b c
 790           y=d e f
 791 do repeat a=1 thru 5
 792 another command
 793 second command
 794 + third command
 795 end /* x */ /* y */ repeat print
 796 end
 797  repeat
 798 do
 799   repeat #a=1
 800
 801   inner command
 802 end repeat
 803 ])
 804 AT_DATA([expout-base], [dnl
 805 ID "do"
 806 SKIP
 807 ID "repeat"
 808 SKIP
 809 ID "x"
 810 EQUALS
 811 ID "a"
 812 SKIP
 813 ID "b"
 814 SKIP
 815 ID "c"
 816 SKIP
 817 SKIP
 818 ID "y"
 819 EQUALS
 820 ID "d"
 821 SKIP
 822 ID "e"
 823 SKIP
 824 ID "f"
 825 SKIP
 826 ENDCMD
 827 STRING "do repeat a=1 thru 5"
 828 SKIP
 829 STRING "another command"
 830 SKIP
 831 STRING "second command"
 832 SKIP
 833 STRING "+ third command"
 834 SKIP
 835 STRING "end /* x */ /* y */ repeat print"
 836 SKIP
 837 ID "end"
 838 SKIP
 839 SKIP
 840 ID "repeat"
 841 SKIP
 842 ENDCMD
 843 ID "do"
 844 SKIP
 845 SKIP
 846 ID "repeat"
 847 SKIP
 848 ID "#a"
 849 EQUALS
 850 POS_NUM 1
 851 SKIP
 852 ENDCMD
 853 SKIP
 854 STRING "  inner command"
 855 SKIP
 856 ID "end"
 857 SKIP
 858 ID "repeat"
 859 -SKIP
 860 STOP
 861 ])
 862 PSPP_CHECK_SCAN([-b])
 863 AT_CLEANUP
 864 \f
 865 AT_SETUP([batch mode])
 866 AT_KEYWORDS([scan])
 867 AT_DATA([input], [dnl
 868 first command
 869      another line of first command
 870 +  second command
 871 third command
 872
 873 fourth command.
 874    fifth command.
 875 ])
 876 AT_DATA([expout-base], [dnl
 877 ID "first"
 878 SKIP
 879 ID "command"
 880 SKIP
 881 SKIP
 882 ID "another"
 883 SKIP
 884 ID "line"
 885 SKIP
 886 ID "of"
 887 SKIP
 888 ID "first"
 889 SKIP
 890 ID "command"
 891 SKIP
 892 ENDCMD
 893 SKIP
 894 ID "second"
 895 SKIP
 896 ID "command"
 897 SKIP
 898 ENDCMD
 899 ID "third"
 900 SKIP
 901 ID "command"
 902 SKIP
 903 ENDCMD
 904 SKIP
 905 ID "fourth"
 906 SKIP
 907 ID "command"
 908 ENDCMD
 909 SKIP
 910 SKIP
 911 ID "fifth"
 912 SKIP
 913 ID "command"
 914 ENDCMD
 915 -SKIP
 916 STOP
 917 ])
 918 PSPP_CHECK_SCAN([-b])
 919 AT_CLEANUP