pintos-os.org Git - pspp/blob - tests/language/lexer/scan.at

   1 dnl PSPP - a program for statistical analysis.
   2 dnl Copyright (C) 2017 Free Software Foundation, Inc.
   3 dnl
   4 dnl This program is free software: you can redistribute it and/or modify
   5 dnl it under the terms of the GNU General Public License as published by
   6 dnl the Free Software Foundation, either version 3 of the License, or
   7 dnl (at your option) any later version.
   8 dnl
   9 dnl This program is distributed in the hope that it will be useful,
  10 dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 dnl GNU General Public License for more details.
  13 dnl
  14 dnl You should have received a copy of the GNU General Public License
  15 dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 dnl AT_BANNER([syntax scanning])
  17 m4_define([PSPP_CHECK_SCAN],
  18   [AT_CHECK([scan-test $1 input], [0], [expout])])
  19 \f
  20 AT_SETUP([identifiers])
  21 AT_KEYWORDS([scan])
  22 AT_DATA([input], [dnl
  23 a aB i5 $x @efg @@. #.# .x _z.
  24 abcd. abcd.
  25 QRSTUV./* end of line comment */
  26 QrStUv./* end of line comment */ @&t@
  27 WXYZ. /* unterminated end of line comment
  28 �. /* U+FFFD is not valid in an identifier
  29 ])
  30 AT_DATA([expout], [dnl
  31 ID "a"
  32 SKIP
  33 ID "aB"
  34 SKIP
  35 ID "i5"
  36 SKIP
  37 ID "$x"
  38 SKIP
  39 ID "@efg"
  40 SKIP
  41 ID "@@."
  42 SKIP
  43 ID "#.#"
  44 SKIP
  45 UNEXPECTED_DOT
  46 ID "x"
  47 SKIP
  48 UNEXPECTED_CHAR 95
  49 ID "z"
  50 ENDCMD
  51 SKIP
  52 ID "abcd."
  53 SKIP
  54 ID "abcd"
  55 ENDCMD
  56 SKIP
  57 ID "QRSTUV"
  58 ENDCMD
  59 SKIP
  60 SKIP
  61 ID "QrStUv"
  62 ENDCMD
  63 SKIP
  64 SKIP
  65 SKIP
  66 ID "WXYZ"
  67 ENDCMD
  68 SKIP
  69 SKIP
  70 SKIP
  71 UNEXPECTED_CHAR 65533
  72 ENDCMD
  73 SKIP
  74 SKIP
  75 SKIP
  76 STOP
  77 ])
  78 PSPP_CHECK_SCAN([-i])
  79 AT_CLEANUP
  80 \f
  81 AT_SETUP([reserved words])
  82 AT_KEYWORDS([scan])
  83 AT_DATA([input], [dnl
  84 and or not eq ge gt le lt ne all by to with
  85 AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH
  86 andx orx notx eqx gex gtx lex ltx nex allx byx tox withx
  87 and. with.
  88 ])
  89 AT_DATA([expout], [dnl
  90 AND
  91 SKIP
  92 OR
  93 SKIP
  94 NOT
  95 SKIP
  96 EQ
  97 SKIP
  98 GE
  99 SKIP
 100 GT
 101 SKIP
 102 LE
 103 SKIP
 104 LT
 105 SKIP
 106 NE
 107 SKIP
 108 ALL
 109 SKIP
 110 BY
 111 SKIP
 112 TO
 113 SKIP
 114 WITH
 115 SKIP
 116 AND
 117 SKIP
 118 OR
 119 SKIP
 120 NOT
 121 SKIP
 122 EQ
 123 SKIP
 124 GE
 125 SKIP
 126 GT
 127 SKIP
 128 LE
 129 SKIP
 130 LT
 131 SKIP
 132 NE
 133 SKIP
 134 ALL
 135 SKIP
 136 BY
 137 SKIP
 138 TO
 139 SKIP
 140 WITH
 141 SKIP
 142 ID "andx"
 143 SKIP
 144 ID "orx"
 145 SKIP
 146 ID "notx"
 147 SKIP
 148 ID "eqx"
 149 SKIP
 150 ID "gex"
 151 SKIP
 152 ID "gtx"
 153 SKIP
 154 ID "lex"
 155 SKIP
 156 ID "ltx"
 157 SKIP
 158 ID "nex"
 159 SKIP
 160 ID "allx"
 161 SKIP
 162 ID "byx"
 163 SKIP
 164 ID "tox"
 165 SKIP
 166 ID "withx"
 167 SKIP
 168 ID "and."
 169 SKIP
 170 WITH
 171 ENDCMD
 172 SKIP
 173 STOP
 174 ])
 175 PSPP_CHECK_SCAN([-i])
 176 AT_CLEANUP
 177 \f
 178 AT_SETUP([punctuation])
 179 AT_KEYWORDS([scan])
 180 AT_DATA([input], [dnl
 181 ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] **
 182 ~&|=>=><=<~=<>(),-+*/[[]]**
 183 ])
 184 AT_DATA([expout], [dnl
 185 NOT
 186 SKIP
 187 AND
 188 SKIP
 189 OR
 190 SKIP
 191 EQUALS
 192 SKIP
 193 GE
 194 SKIP
 195 GT
 196 SKIP
 197 LE
 198 SKIP
 199 LT
 200 SKIP
 201 NE
 202 SKIP
 203 NE
 204 SKIP
 205 LPAREN
 206 SKIP
 207 RPAREN
 208 SKIP
 209 COMMA
 210 SKIP
 211 DASH
 212 SKIP
 213 PLUS
 214 SKIP
 215 ASTERISK
 216 SKIP
 217 SLASH
 218 SKIP
 219 LBRACK
 220 SKIP
 221 RBRACK
 222 SKIP
 223 EXP
 224 SKIP
 225 NOT
 226 AND
 227 OR
 228 EQUALS
 229 GE
 230 GT
 231 LE
 232 LT
 233 NE
 234 NE
 235 LPAREN
 236 RPAREN
 237 COMMA
 238 DASH
 239 PLUS
 240 ASTERISK
 241 SLASH
 242 LBRACK
 243 RBRACK
 244 EXP
 245 SKIP
 246 STOP
 247 ])
 248 PSPP_CHECK_SCAN([-i])
 249 AT_CLEANUP
 250 \f
 251 AT_SETUP([numbers])
 252 AT_KEYWORDS([scan])
 253 AT_DATA([input], [dnl
 254 0 1 01 001. 1.
 255 123. /* comment 1 */ /* comment 2 */
 256 .1 0.1 00.1 00.10
 257 5e1 6E-1 7e+1 6E+01 6e-03
 258 .3E1 .4e-1 .5E+1 .6e+01 .7E-03
 259 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03
 260 . 1e e1 1e+ 1e-
 261 ])
 262 AT_DATA([expout], [dnl
 263 POS_NUM
 264 SKIP
 265 POS_NUM 1
 266 SKIP
 267 POS_NUM 1
 268 SKIP
 269 POS_NUM 1
 270 SKIP
 271 POS_NUM 1
 272 ENDCMD
 273 SKIP
 274 POS_NUM 123
 275 ENDCMD
 276 SKIP
 277 SKIP
 278 SKIP
 279 SKIP
 280 SKIP
 281 ENDCMD
 282 POS_NUM 1
 283 SKIP
 284 POS_NUM 0.1
 285 SKIP
 286 POS_NUM 0.1
 287 SKIP
 288 POS_NUM 0.1
 289 SKIP
 290 POS_NUM 50
 291 SKIP
 292 POS_NUM 0.6
 293 SKIP
 294 POS_NUM 70
 295 SKIP
 296 POS_NUM 60
 297 SKIP
 298 POS_NUM 0.006
 299 SKIP
 300 ENDCMD
 301 POS_NUM 30
 302 SKIP
 303 POS_NUM 0.04
 304 SKIP
 305 POS_NUM 5
 306 SKIP
 307 POS_NUM 6
 308 SKIP
 309 POS_NUM 0.0007
 310 SKIP
 311 POS_NUM 12.3
 312 SKIP
 313 POS_NUM 4.56
 314 SKIP
 315 POS_NUM 789
 316 SKIP
 317 POS_NUM 999
 318 SKIP
 319 POS_NUM 0.0112
 320 SKIP
 321 ENDCMD
 322 SKIP
 323 EXPECTED_EXPONENT "1e"
 324 SKIP
 325 ID "e1"
 326 SKIP
 327 EXPECTED_EXPONENT "1e+"
 328 SKIP
 329 EXPECTED_EXPONENT "1e-"
 330 SKIP
 331 STOP
 332 ])
 333 PSPP_CHECK_SCAN([-i])
 334 AT_CLEANUP
 335 \f
 336 AT_SETUP([strings])
 337 AT_KEYWORDS([scan])
 338 AT_DATA([input], [dnl
 339 'x' "y" 'abc'
 340 'Don''t' "Can't" 'Won''t'
 341 """quoted""" '"quoted"'
 342 '' "" '''' """"
 343 'missing end quote
 344 "missing double quote
 345 'x' + "y"
 346 + 'z' +
 347 'a' /* abc */ + "b" /*
 348 + 'c' +/* */"d"/* */+'e'
 349 'foo'
 350 +          /* special case: + in column 0 would ordinarily start a new command
 351 'bar'
 352 'foo'
 353  +
 354 'bar'
 355 'foo'
 356 +
 357
 358 'bar'
 359
 360 +
 361 x"4142"+'5152'
 362 "4142"+
 363 x'5152'
 364 x"4142"
 365 +u'304a'
 366 "�あいうえお"
 367 "abc"+U"FFFD"+u'3048'+"xyz"
 368 ])
 369 AT_DATA([expout], [dnl
 370 STRING "x"
 371 SKIP
 372 STRING "y"
 373 SKIP
 374 STRING "abc"
 375 SKIP
 376 STRING "Don't"
 377 SKIP
 378 STRING "Can't"
 379 SKIP
 380 STRING "Won't"
 381 SKIP
 382 STRING ""quoted""
 383 SKIP
 384 STRING ""quoted""
 385 SKIP
 386 STRING ""
 387 SKIP
 388 STRING ""
 389 SKIP
 390 STRING "'"
 391 SKIP
 392 STRING """
 393 SKIP
 394 EXPECTED_QUOTE
 395 SKIP
 396 EXPECTED_QUOTE
 397 SKIP
 398 STRING "xyzabcde"
 399 SKIP
 400 STRING "foobar"
 401 SKIP
 402 STRING "foobar"
 403 SKIP
 404 STRING "foo"
 405 SKIP
 406 PLUS
 407 SKIP
 408 ENDCMD
 409 SKIP
 410 STRING "bar"
 411 SKIP
 412 ENDCMD
 413 SKIP
 414 PLUS
 415 SKIP
 416 STRING "AB5152"
 417 SKIP
 418 STRING "4142QR"
 419 SKIP
 420 STRING "ABお"
 421 SKIP
 422 STRING "�あいうえお"
 423 SKIP
 424 STRING "abc�えxyz"
 425 SKIP
 426 STOP
 427 ])
 428 PSPP_CHECK_SCAN([-i])
 429 AT_CLEANUP
 430 \f
 431 AT_SETUP([@%:@! construct])
 432 AT_KEYWORDS([scan])
 433 AT_DATA([input], [dnl
 434 #! /usr/bin/pspp
 435 #! /usr/bin/pspp
 436 ])
 437 AT_DATA([expout], [dnl
 438 SKIP
 439 SKIP
 440 ID "#"
 441 UNEXPECTED_CHAR 33
 442 SKIP
 443 SLASH
 444 ID "usr"
 445 SLASH
 446 ID "bin"
 447 SLASH
 448 ID "pspp"
 449 SKIP
 450 STOP
 451 ])
 452 PSPP_CHECK_SCAN([-i])
 453 AT_CLEANUP
 454 \f
 455 AT_SETUP([* and COMMENT commands])
 456 AT_KEYWORDS([scan])
 457 AT_DATA([input], [dnl
 458 * Comment commands "don't
 459 have to contain valid tokens.
 460
 461 ** Check ambiguity with ** token.
 462 ****************.
 463
 464 comment keyword works too.
 465 COMM also.
 466 com is ambiguous with COMPUTE.
 467
 468    * Comment need not start at left margin.
 469
 470 * Comment ends with blank line
 471
 472 next command.
 473
 474 ])
 475 AT_DATA([expout], [dnl
 476 SKIP
 477 SKIP
 478 SKIP
 479 ENDCMD
 480 SKIP
 481 ENDCMD
 482 SKIP
 483 SKIP
 484 ENDCMD
 485 SKIP
 486 SKIP
 487 ENDCMD
 488 SKIP
 489 ENDCMD
 490 SKIP
 491 SKIP
 492 ENDCMD
 493 SKIP
 494 SKIP
 495 ENDCMD
 496 SKIP
 497 ID "com"
 498 SKIP
 499 ID "is"
 500 SKIP
 501 ID "ambiguous"
 502 SKIP
 503 WITH
 504 SKIP
 505 ID "COMPUTE"
 506 ENDCMD
 507 SKIP
 508 ENDCMD
 509 SKIP
 510 SKIP
 511 SKIP
 512 ENDCMD
 513 SKIP
 514 ENDCMD
 515 SKIP
 516 SKIP
 517 SKIP
 518 ENDCMD
 519 SKIP
 520 ID "next"
 521 SKIP
 522 ID "command"
 523 ENDCMD
 524 SKIP
 525 ENDCMD
 526 SKIP
 527 STOP
 528 ])
 529 PSPP_CHECK_SCAN([-i])
 530 AT_CLEANUP
 531 \f
 532 AT_SETUP([DOCUMENT command])
 533 AT_KEYWORDS([scan])
 534 AT_DATA([input], [dnl
 535 DOCUMENT one line.
 536 DOC more
 537     than
 538         one
 539             line.
 540 docu
 541 first.paragraph
 542 isn't parsed as tokens
 543
 544 second paragraph.
 545 ])
 546 AT_DATA([expout], [dnl
 547 ID "DOCUMENT"
 548 STRING "DOCUMENT one line."
 549 ENDCMD
 550 ENDCMD
 551 SKIP
 552 ID "DOCUMENT"
 553 STRING "DOC more"
 554 SKIP
 555 STRING "    than"
 556 SKIP
 557 STRING "        one"
 558 SKIP
 559 STRING "            line."
 560 ENDCMD
 561 ENDCMD
 562 SKIP
 563 ID "DOCUMENT"
 564 STRING "docu"
 565 SKIP
 566 STRING "first.paragraph"
 567 SKIP
 568 STRING "isn't parsed as tokens"
 569 SKIP
 570 STRING ""
 571 SKIP
 572 STRING "second paragraph."
 573 ENDCMD
 574 ENDCMD
 575 SKIP
 576 STOP
 577 ])
 578 PSPP_CHECK_SCAN([-i])
 579 AT_CLEANUP
 580 \f
 581 AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands])
 582 AT_KEYWORDS([scan])
 583 AT_DATA([input], [dnl
 584 title/**/'Quoted string title'.
 585 tit /*
 586 "Quoted string on second line".
 587 sub "Quoted string subtitle"
 588  .
 589
 590 TITL /* Not a */ quoted string title.
 591 SUBT Not a quoted string /* subtitle
 592
 593 FIL label isn't quoted.
 594 FILE
 595   lab 'is quoted'.
 596 FILE /*
 597 /**/  lab not quoted here either
 598
 599 ])
 600 AT_DATA([expout], [dnl
 601 ID "title"
 602 SKIP
 603 STRING "Quoted string title"
 604 ENDCMD
 605 SKIP
 606 ID "tit"
 607 SKIP
 608 SKIP
 609 SKIP
 610 STRING "Quoted string on second line"
 611 ENDCMD
 612 SKIP
 613 ID "sub"
 614 SKIP
 615 STRING "Quoted string subtitle"
 616 SKIP
 617 SKIP
 618 ENDCMD
 619 SKIP
 620 ENDCMD
 621 SKIP
 622 ID "TITL"
 623 SKIP
 624 STRING "/* Not a */ quoted string title"
 625 ENDCMD
 626 SKIP
 627 ID "SUBT"
 628 SKIP
 629 STRING "Not a quoted string /* subtitle"
 630 SKIP
 631 ENDCMD
 632 SKIP
 633 ID "FIL"
 634 SKIP
 635 ID "label"
 636 SKIP
 637 STRING "isn't quoted"
 638 ENDCMD
 639 SKIP
 640 ID "FILE"
 641 SKIP
 642 SKIP
 643 ID "lab"
 644 SKIP
 645 STRING "is quoted"
 646 ENDCMD
 647 SKIP
 648 ID "FILE"
 649 SKIP
 650 SKIP
 651 SKIP
 652 SKIP
 653 SKIP
 654 ID "lab"
 655 SKIP
 656 STRING "not quoted here either"
 657 SKIP
 658 ENDCMD
 659 SKIP
 660 STOP
 661 ])
 662 PSPP_CHECK_SCAN([-i])
 663 AT_CLEANUP
 664 \f
 665 AT_SETUP([BEGIN DATA command])
 666 AT_KEYWORDS([scan])
 667 AT_DATA([input], [dnl
 668 begin data.
 669 123
 670 xxx
 671 end data.
 672
 673 BEG /**/ DAT /*
 674 5 6 7 /* x
 675
 676 end  data
 677 end data
 678 .
 679 ])
 680 AT_DATA([expout], [dnl
 681 ID "begin"
 682 SKIP
 683 ID "data"
 684 ENDCMD
 685 SKIP
 686 STRING "123"
 687 SKIP
 688 STRING "xxx"
 689 SKIP
 690 ID "end"
 691 SKIP
 692 ID "data"
 693 ENDCMD
 694 SKIP
 695 ENDCMD
 696 SKIP
 697 ID "BEG"
 698 SKIP
 699 SKIP
 700 SKIP
 701 ID "DAT"
 702 SKIP
 703 SKIP
 704 SKIP
 705 STRING "5 6 7 /* x"
 706 SKIP
 707 STRING ""
 708 SKIP
 709 STRING "end  data"
 710 SKIP
 711 ID "end"
 712 SKIP
 713 ID "data"
 714 SKIP
 715 ENDCMD
 716 SKIP
 717 STOP
 718 ])
 719 PSPP_CHECK_SCAN([-i])
 720 AT_CLEANUP
 721 \f
 722 AT_SETUP([DO REPEAT command])
 723 AT_KEYWORDS([scan])
 724 AT_DATA([input], [dnl
 725 do repeat x=a b c
 726           y=d e f.
 727   do repeat a=1 thru 5.
 728 another command.
 729 second command
 730 + third command.
 731 end /* x */ /* y */ repeat print.
 732 end
 733  repeat.
 734 ])
 735 AT_DATA([expout], [dnl
 736 ID "do"
 737 SKIP
 738 ID "repeat"
 739 SKIP
 740 ID "x"
 741 EQUALS
 742 ID "a"
 743 SKIP
 744 ID "b"
 745 SKIP
 746 ID "c"
 747 SKIP
 748 SKIP
 749 ID "y"
 750 EQUALS
 751 ID "d"
 752 SKIP
 753 ID "e"
 754 SKIP
 755 ID "f"
 756 ENDCMD
 757 SKIP
 758 STRING "  do repeat a=1 thru 5."
 759 SKIP
 760 STRING "another command."
 761 SKIP
 762 STRING "second command"
 763 SKIP
 764 STRING "+ third command."
 765 SKIP
 766 STRING "end /* x */ /* y */ repeat print."
 767 SKIP
 768 ID "end"
 769 SKIP
 770 SKIP
 771 ID "repeat"
 772 ENDCMD
 773 SKIP
 774 STOP
 775 ])
 776 PSPP_CHECK_SCAN([-i])
 777 AT_CLEANUP
 778 \f
 779 AT_SETUP([batch mode])
 780 AT_KEYWORDS([scan])
 781 AT_DATA([input], [dnl
 782 first command
 783      another line of first command
 784 +  second command
 785 third command
 786
 787 fourth command.
 788    fifth command.
 789 ])
 790 AT_DATA([expout], [dnl
 791 ID "first"
 792 SKIP
 793 ID "command"
 794 SKIP
 795 SKIP
 796 ID "another"
 797 SKIP
 798 ID "line"
 799 SKIP
 800 ID "of"
 801 SKIP
 802 ID "first"
 803 SKIP
 804 ID "command"
 805 SKIP
 806 ENDCMD
 807 SKIP
 808 ID "second"
 809 SKIP
 810 ID "command"
 811 SKIP
 812 ENDCMD
 813 ID "third"
 814 SKIP
 815 ID "command"
 816 SKIP
 817 ENDCMD
 818 SKIP
 819 ID "fourth"
 820 SKIP
 821 ID "command"
 822 ENDCMD
 823 SKIP
 824 SKIP
 825 ID "fifth"
 826 SKIP
 827 ID "command"
 828 ENDCMD
 829 SKIP
 830 STOP
 831 ])
 832 PSPP_CHECK_SCAN([-b])
 833 AT_CLEANUP