pintos-os.org Git - pspp/blob - tests/language/lexer/scan.at

   1 dnl PSPP - a program for statistical analysis.
   2 dnl Copyright (C) 2017 Free Software Foundation, Inc.
   3 dnl
   4 dnl This program is free software: you can redistribute it and/or modify
   5 dnl it under the terms of the GNU General Public License as published by
   6 dnl the Free Software Foundation, either version 3 of the License, or
   7 dnl (at your option) any later version.
   8 dnl
   9 dnl This program is distributed in the hope that it will be useful,
  10 dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 dnl GNU General Public License for more details.
  13 dnl
  14 dnl You should have received a copy of the GNU General Public License
  15 dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 dnl
  17 AT_BANNER([syntax scanning])
  18 m4_define([PSPP_CHECK_SCAN],
  19   [sed 's/^-//' < expout-base > expout
  20    AT_CHECK([scan-test $1 input], [0], [expout])
  21
  22    sed '/^-/d' < expout-base > expout
  23    AT_CHECK([scan-test -s $1 input], [0], [expout])])
  24 \f
  25 AT_SETUP([identifiers])
  26 AT_KEYWORDS([scan])
  27 AT_DATA([input], [dnl
  28 a aB i5 $x @efg @@. #.# .x _z.
  29 abcd. abcd.
  30 QRSTUV./* end of line comment */
  31 QrStUv./* end of line comment */ @&t@
  32 WXYZ. /* unterminated end of line comment
  33 �. /* U+FFFD is not valid in an identifier
  34 ])
  35 AT_DATA([expout-base], [dnl
  36 ID "a"
  37 SKIP
  38 ID "aB"
  39 SKIP
  40 ID "i5"
  41 SKIP
  42 ID "$x"
  43 SKIP
  44 ID "@efg"
  45 SKIP
  46 ID "@@."
  47 SKIP
  48 ID "#.#"
  49 SKIP
  50 UNEXPECTED_DOT
  51 ID "x"
  52 SKIP
  53 UNEXPECTED_CHAR 95
  54 ID "z"
  55 ENDCMD
  56 SKIP
  57 ID "abcd."
  58 SKIP
  59 ID "abcd"
  60 ENDCMD
  61 SKIP
  62 ID "QRSTUV"
  63 ENDCMD
  64 SKIP
  65 SKIP
  66 ID "QrStUv"
  67 ENDCMD
  68 SKIP
  69 SKIP
  70 SKIP
  71 ID "WXYZ"
  72 ENDCMD
  73 SKIP
  74 SKIP
  75 SKIP
  76 UNEXPECTED_CHAR 65533
  77 ENDCMD
  78 SKIP
  79 SKIP
  80 -SKIP
  81 STOP
  82 ])
  83 PSPP_CHECK_SCAN([-i])
  84 AT_CLEANUP
  85 \f
  86 AT_SETUP([reserved words])
  87 AT_KEYWORDS([scan])
  88 AT_DATA([input], [dnl
  89 and or not eq ge gt le lt ne all by to with
  90 AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH
  91 andx orx notx eqx gex gtx lex ltx nex allx byx tox withx
  92 and. with.
  93 ])
  94 AT_DATA([expout-base], [dnl
  95 AND
  96 SKIP
  97 OR
  98 SKIP
  99 NOT
 100 SKIP
 101 EQ
 102 SKIP
 103 GE
 104 SKIP
 105 GT
 106 SKIP
 107 LE
 108 SKIP
 109 LT
 110 SKIP
 111 NE
 112 SKIP
 113 ALL
 114 SKIP
 115 BY
 116 SKIP
 117 TO
 118 SKIP
 119 WITH
 120 SKIP
 121 AND
 122 SKIP
 123 OR
 124 SKIP
 125 NOT
 126 SKIP
 127 EQ
 128 SKIP
 129 GE
 130 SKIP
 131 GT
 132 SKIP
 133 LE
 134 SKIP
 135 LT
 136 SKIP
 137 NE
 138 SKIP
 139 ALL
 140 SKIP
 141 BY
 142 SKIP
 143 TO
 144 SKIP
 145 WITH
 146 SKIP
 147 ID "andx"
 148 SKIP
 149 ID "orx"
 150 SKIP
 151 ID "notx"
 152 SKIP
 153 ID "eqx"
 154 SKIP
 155 ID "gex"
 156 SKIP
 157 ID "gtx"
 158 SKIP
 159 ID "lex"
 160 SKIP
 161 ID "ltx"
 162 SKIP
 163 ID "nex"
 164 SKIP
 165 ID "allx"
 166 SKIP
 167 ID "byx"
 168 SKIP
 169 ID "tox"
 170 SKIP
 171 ID "withx"
 172 SKIP
 173 ID "and."
 174 SKIP
 175 WITH
 176 ENDCMD
 177 -SKIP
 178 STOP
 179 ])
 180 PSPP_CHECK_SCAN([-i])
 181 AT_CLEANUP
 182 \f
 183 AT_SETUP([punctuation])
 184 AT_KEYWORDS([scan])
 185 AT_DATA([input], [dnl
 186 ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] **
 187 ~&|=>=><=<~=<>(),-+*/[[]]**
 188 ])
 189 AT_DATA([expout-base], [dnl
 190 NOT
 191 SKIP
 192 AND
 193 SKIP
 194 OR
 195 SKIP
 196 EQUALS
 197 SKIP
 198 GE
 199 SKIP
 200 GT
 201 SKIP
 202 LE
 203 SKIP
 204 LT
 205 SKIP
 206 NE
 207 SKIP
 208 NE
 209 SKIP
 210 LPAREN
 211 SKIP
 212 RPAREN
 213 SKIP
 214 COMMA
 215 SKIP
 216 DASH
 217 SKIP
 218 PLUS
 219 SKIP
 220 ASTERISK
 221 SKIP
 222 SLASH
 223 SKIP
 224 LBRACK
 225 SKIP
 226 RBRACK
 227 SKIP
 228 EXP
 229 SKIP
 230 NOT
 231 AND
 232 OR
 233 EQUALS
 234 GE
 235 GT
 236 LE
 237 LT
 238 NE
 239 NE
 240 LPAREN
 241 RPAREN
 242 COMMA
 243 DASH
 244 PLUS
 245 ASTERISK
 246 SLASH
 247 LBRACK
 248 RBRACK
 249 EXP
 250 -SKIP
 251 STOP
 252 ])
 253 PSPP_CHECK_SCAN([-i])
 254 AT_CLEANUP
 255 \f
 256 AT_SETUP([numbers])
 257 AT_KEYWORDS([scan])
 258 AT_DATA([input], [dnl
 259 0 1 01 001. 1.
 260 123. /* comment 1 */ /* comment 2 */
 261 .1 0.1 00.1 00.10
 262 5e1 6E-1 7e+1 6E+01 6e-03
 263 .3E1 .4e-1 .5E+1 .6e+01 .7E-03
 264 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03
 265 . 1e e1 1e+ 1e-
 266 ])
 267 AT_DATA([expout-base], [dnl
 268 POS_NUM
 269 SKIP
 270 POS_NUM 1
 271 SKIP
 272 POS_NUM 1
 273 SKIP
 274 POS_NUM 1
 275 SKIP
 276 POS_NUM 1
 277 ENDCMD
 278 SKIP
 279 POS_NUM 123
 280 ENDCMD
 281 SKIP
 282 SKIP
 283 SKIP
 284 SKIP
 285 SKIP
 286 ENDCMD
 287 POS_NUM 1
 288 SKIP
 289 POS_NUM 0.1
 290 SKIP
 291 POS_NUM 0.1
 292 SKIP
 293 POS_NUM 0.1
 294 SKIP
 295 POS_NUM 50
 296 SKIP
 297 POS_NUM 0.6
 298 SKIP
 299 POS_NUM 70
 300 SKIP
 301 POS_NUM 60
 302 SKIP
 303 POS_NUM 0.006
 304 SKIP
 305 ENDCMD
 306 POS_NUM 30
 307 SKIP
 308 POS_NUM 0.04
 309 SKIP
 310 POS_NUM 5
 311 SKIP
 312 POS_NUM 6
 313 SKIP
 314 POS_NUM 0.0007
 315 SKIP
 316 POS_NUM 12.3
 317 SKIP
 318 POS_NUM 4.56
 319 SKIP
 320 POS_NUM 789
 321 SKIP
 322 POS_NUM 999
 323 SKIP
 324 POS_NUM 0.0112
 325 SKIP
 326 ENDCMD
 327 SKIP
 328 EXPECTED_EXPONENT "1e"
 329 SKIP
 330 ID "e1"
 331 SKIP
 332 EXPECTED_EXPONENT "1e+"
 333 SKIP
 334 EXPECTED_EXPONENT "1e-"
 335 -SKIP
 336 STOP
 337 ])
 338 PSPP_CHECK_SCAN([-i])
 339 AT_CLEANUP
 340 \f
 341 AT_SETUP([strings])
 342 AT_KEYWORDS([scan])
 343 AT_DATA([input], [dnl
 344 'x' "y" 'abc'
 345 'Don''t' "Can't" 'Won''t'
 346 """quoted""" '"quoted"'
 347 '' "" '''' """"
 348 'missing end quote
 349 "missing double quote
 350 'x' + "y"
 351 + 'z' +
 352 'a' /* abc */ + "b" /*
 353 + 'c' +/* */"d"/* */+'e'
 354 'foo'
 355 +          /* special case: + in column 0 would ordinarily start a new command
 356 'bar'
 357 'foo'
 358  +
 359 'bar'
 360 'foo'
 361 +
 362
 363 'bar'
 364
 365 +
 366 x"4142"+'5152'
 367 "4142"+
 368 x'5152'
 369 x"4142"
 370 +u'304a'
 371 "�あいうえお"
 372 "abc"+U"FFFD"+u'3048'+"xyz"
 373 ])
 374 AT_DATA([expout-base], [dnl
 375 STRING "x"
 376 SKIP
 377 STRING "y"
 378 SKIP
 379 STRING "abc"
 380 SKIP
 381 STRING "Don't"
 382 SKIP
 383 STRING "Can't"
 384 SKIP
 385 STRING "Won't"
 386 SKIP
 387 STRING ""quoted""
 388 SKIP
 389 STRING ""quoted""
 390 SKIP
 391 STRING ""
 392 SKIP
 393 STRING ""
 394 SKIP
 395 STRING "'"
 396 SKIP
 397 STRING """
 398 SKIP
 399 EXPECTED_QUOTE
 400 SKIP
 401 EXPECTED_QUOTE
 402 SKIP
 403 STRING "xyzabcde"
 404 SKIP
 405 STRING "foobar"
 406 SKIP
 407 STRING "foobar"
 408 SKIP
 409 STRING "foo"
 410 SKIP
 411 PLUS
 412 SKIP
 413 ENDCMD
 414 SKIP
 415 STRING "bar"
 416 SKIP
 417 ENDCMD
 418 SKIP
 419 PLUS
 420 SKIP
 421 STRING "AB5152"
 422 SKIP
 423 STRING "4142QR"
 424 SKIP
 425 STRING "ABお"
 426 SKIP
 427 STRING "�あいうえお"
 428 SKIP
 429 STRING "abc�えxyz"
 430 -SKIP
 431 STOP
 432 ])
 433 PSPP_CHECK_SCAN([-i])
 434 AT_CLEANUP
 435 \f
 436 AT_SETUP([@%:@! construct])
 437 AT_KEYWORDS([scan])
 438 AT_DATA([input], [dnl
 439 #! /usr/bin/pspp
 440 #! /usr/bin/pspp
 441 ])
 442 AT_DATA([expout-base], [dnl
 443 SKIP
 444 SKIP
 445 ID "#"
 446 UNEXPECTED_CHAR 33
 447 SKIP
 448 SLASH
 449 ID "usr"
 450 SLASH
 451 ID "bin"
 452 SLASH
 453 ID "pspp"
 454 -SKIP
 455 STOP
 456 ])
 457 PSPP_CHECK_SCAN([-i])
 458 AT_CLEANUP
 459 \f
 460 AT_SETUP([* and COMMENT commands])
 461 AT_KEYWORDS([scan])
 462 AT_DATA([input], [dnl
 463 * Comment commands "don't
 464 have to contain valid tokens.
 465
 466 ** Check ambiguity with ** token.
 467 ****************.
 468
 469 comment keyword works too.
 470 COMM also.
 471 com is ambiguous with COMPUTE.
 472
 473    * Comment need not start at left margin.
 474
 475 * Comment ends with blank line
 476
 477 next command.
 478
 479 ])
 480 AT_DATA([expout-base], [dnl
 481 SKIP
 482 SKIP
 483 SKIP
 484 ENDCMD
 485 SKIP
 486 ENDCMD
 487 SKIP
 488 SKIP
 489 ENDCMD
 490 SKIP
 491 SKIP
 492 ENDCMD
 493 SKIP
 494 ENDCMD
 495 SKIP
 496 SKIP
 497 ENDCMD
 498 SKIP
 499 SKIP
 500 ENDCMD
 501 SKIP
 502 ID "com"
 503 SKIP
 504 ID "is"
 505 SKIP
 506 ID "ambiguous"
 507 SKIP
 508 WITH
 509 SKIP
 510 ID "COMPUTE"
 511 ENDCMD
 512 SKIP
 513 ENDCMD
 514 SKIP
 515 SKIP
 516 SKIP
 517 ENDCMD
 518 SKIP
 519 ENDCMD
 520 SKIP
 521 SKIP
 522 SKIP
 523 ENDCMD
 524 SKIP
 525 ID "next"
 526 SKIP
 527 ID "command"
 528 ENDCMD
 529 SKIP
 530 -ENDCMD
 531 -SKIP
 532 STOP
 533 ])
 534 PSPP_CHECK_SCAN([-i])
 535 AT_CLEANUP
 536 \f
 537 AT_SETUP([DOCUMENT command])
 538 AT_KEYWORDS([scan])
 539 AT_DATA([input], [dnl
 540 DOCUMENT one line.
 541 DOC more
 542     than
 543         one
 544             line.
 545 docu
 546 first.paragraph
 547 isn't parsed as tokens
 548
 549 second paragraph.
 550 ])
 551 AT_DATA([expout-base], [dnl
 552 ID "DOCUMENT"
 553 STRING "DOCUMENT one line."
 554 ENDCMD
 555 ENDCMD
 556 SKIP
 557 ID "DOCUMENT"
 558 STRING "DOC more"
 559 SKIP
 560 STRING "    than"
 561 SKIP
 562 STRING "        one"
 563 SKIP
 564 STRING "            line."
 565 ENDCMD
 566 ENDCMD
 567 SKIP
 568 ID "DOCUMENT"
 569 STRING "docu"
 570 SKIP
 571 STRING "first.paragraph"
 572 SKIP
 573 STRING "isn't parsed as tokens"
 574 SKIP
 575 STRING ""
 576 SKIP
 577 STRING "second paragraph."
 578 -ENDCMD
 579 -ENDCMD
 580 -SKIP
 581 STOP
 582 ])
 583 PSPP_CHECK_SCAN([-i])
 584 AT_CLEANUP
 585 \f
 586 AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands])
 587 AT_KEYWORDS([scan])
 588 AT_DATA([input], [dnl
 589 title/**/'Quoted string title'.
 590 tit /*
 591 "Quoted string on second line".
 592 sub "Quoted string subtitle"
 593  .
 594
 595 TITL /* Not a */ quoted string title.
 596 SUBT Not a quoted string /* subtitle
 597
 598 FIL label isn't quoted.
 599 FILE
 600   lab 'is quoted'.
 601 FILE /*
 602 /**/  lab not quoted here either
 603
 604 ])
 605 AT_DATA([expout-base], [dnl
 606 ID "title"
 607 SKIP
 608 STRING "Quoted string title"
 609 ENDCMD
 610 SKIP
 611 ID "tit"
 612 SKIP
 613 SKIP
 614 SKIP
 615 STRING "Quoted string on second line"
 616 ENDCMD
 617 SKIP
 618 ID "sub"
 619 SKIP
 620 STRING "Quoted string subtitle"
 621 SKIP
 622 SKIP
 623 ENDCMD
 624 SKIP
 625 ENDCMD
 626 SKIP
 627 ID "TITL"
 628 SKIP
 629 STRING "/* Not a */ quoted string title"
 630 ENDCMD
 631 SKIP
 632 ID "SUBT"
 633 SKIP
 634 STRING "Not a quoted string /* subtitle"
 635 SKIP
 636 ENDCMD
 637 SKIP
 638 ID "FIL"
 639 SKIP
 640 ID "label"
 641 SKIP
 642 STRING "isn't quoted"
 643 ENDCMD
 644 SKIP
 645 ID "FILE"
 646 SKIP
 647 SKIP
 648 ID "lab"
 649 SKIP
 650 STRING "is quoted"
 651 ENDCMD
 652 SKIP
 653 ID "FILE"
 654 SKIP
 655 SKIP
 656 SKIP
 657 SKIP
 658 SKIP
 659 ID "lab"
 660 SKIP
 661 STRING "not quoted here either"
 662 SKIP
 663 -ENDCMD
 664 -SKIP
 665 STOP
 666 ])
 667 PSPP_CHECK_SCAN([-i])
 668 AT_CLEANUP
 669 \f
 670 AT_SETUP([BEGIN DATA command])
 671 AT_KEYWORDS([scan])
 672 AT_DATA([input], [dnl
 673 begin data.
 674 123
 675 xxx
 676 end data.
 677
 678 BEG /**/ DAT /*
 679 5 6 7 /* x
 680
 681 end  data
 682 end data
 683 .
 684 ])
 685 AT_DATA([expout-base], [dnl
 686 ID "begin"
 687 SKIP
 688 ID "data"
 689 ENDCMD
 690 SKIP
 691 STRING "123"
 692 SKIP
 693 STRING "xxx"
 694 SKIP
 695 ID "end"
 696 SKIP
 697 ID "data"
 698 ENDCMD
 699 SKIP
 700 ENDCMD
 701 SKIP
 702 ID "BEG"
 703 SKIP
 704 SKIP
 705 SKIP
 706 ID "DAT"
 707 SKIP
 708 SKIP
 709 SKIP
 710 STRING "5 6 7 /* x"
 711 SKIP
 712 STRING ""
 713 SKIP
 714 STRING "end  data"
 715 SKIP
 716 ID "end"
 717 SKIP
 718 ID "data"
 719 SKIP
 720 ENDCMD
 721 -SKIP
 722 STOP
 723 ])
 724 PSPP_CHECK_SCAN([-i])
 725 AT_CLEANUP
 726 \f
 727 AT_SETUP([DO REPEAT command])
 728 AT_KEYWORDS([scan])
 729 AT_DATA([input], [dnl
 730 do repeat x=a b c
 731           y=d e f.
 732   do repeat a=1 thru 5.
 733 another command.
 734 second command
 735 + third command.
 736 end /* x */ /* y */ repeat print.
 737 end
 738  repeat.
 739 ])
 740 AT_DATA([expout-base], [dnl
 741 ID "do"
 742 SKIP
 743 ID "repeat"
 744 SKIP
 745 ID "x"
 746 EQUALS
 747 ID "a"
 748 SKIP
 749 ID "b"
 750 SKIP
 751 ID "c"
 752 SKIP
 753 SKIP
 754 ID "y"
 755 EQUALS
 756 ID "d"
 757 SKIP
 758 ID "e"
 759 SKIP
 760 ID "f"
 761 ENDCMD
 762 SKIP
 763 STRING "  do repeat a=1 thru 5."
 764 SKIP
 765 STRING "another command."
 766 SKIP
 767 STRING "second command"
 768 SKIP
 769 STRING "+ third command."
 770 SKIP
 771 STRING "end /* x */ /* y */ repeat print."
 772 SKIP
 773 ID "end"
 774 SKIP
 775 SKIP
 776 ID "repeat"
 777 ENDCMD
 778 -SKIP
 779 STOP
 780 ])
 781 PSPP_CHECK_SCAN([-i])
 782 AT_CLEANUP
 783 \f
 784 AT_SETUP([batch mode])
 785 AT_KEYWORDS([scan])
 786 AT_DATA([input], [dnl
 787 first command
 788      another line of first command
 789 +  second command
 790 third command
 791
 792 fourth command.
 793    fifth command.
 794 ])
 795 AT_DATA([expout-base], [dnl
 796 ID "first"
 797 SKIP
 798 ID "command"
 799 SKIP
 800 SKIP
 801 ID "another"
 802 SKIP
 803 ID "line"
 804 SKIP
 805 ID "of"
 806 SKIP
 807 ID "first"
 808 SKIP
 809 ID "command"
 810 SKIP
 811 ENDCMD
 812 SKIP
 813 ID "second"
 814 SKIP
 815 ID "command"
 816 SKIP
 817 ENDCMD
 818 ID "third"
 819 SKIP
 820 ID "command"
 821 SKIP
 822 ENDCMD
 823 SKIP
 824 ID "fourth"
 825 SKIP
 826 ID "command"
 827 ENDCMD
 828 SKIP
 829 SKIP
 830 ID "fifth"
 831 SKIP
 832 ID "command"
 833 ENDCMD
 834 -SKIP
 835 STOP
 836 ])
 837 PSPP_CHECK_SCAN([-b])
 838 AT_CLEANUP