pintos-os.org Git - pspp/blob - tests/language/lexer/scan.at

   1 dnl PSPP - a program for statistical analysis.
   2 dnl Copyright (C) 2017 Free Software Foundation, Inc.
   3 dnl
   4 dnl This program is free software: you can redistribute it and/or modify
   5 dnl it under the terms of the GNU General Public License as published by
   6 dnl the Free Software Foundation, either version 3 of the License, or
   7 dnl (at your option) any later version.
   8 dnl
   9 dnl This program is distributed in the hope that it will be useful,
  10 dnl but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 dnl GNU General Public License for more details.
  13 dnl
  14 dnl You should have received a copy of the GNU General Public License
  15 dnl along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 dnl
  17 AT_BANNER([syntax scanning])
  18 m4_define([PSPP_CHECK_SCAN],
  19   [AT_CHECK([scan-test $1 input], [0], [expout])])
  20 \f
  21 AT_SETUP([identifiers])
  22 AT_KEYWORDS([scan])
  23 AT_DATA([input], [dnl
  24 a aB i5 $x @efg @@. #.# .x _z.
  25 abcd. abcd.
  26 QRSTUV./* end of line comment */
  27 QrStUv./* end of line comment */ @&t@
  28 WXYZ. /* unterminated end of line comment
  29 �. /* U+FFFD is not valid in an identifier
  30 ])
  31 AT_DATA([expout], [dnl
  32 ID "a"
  33 SKIP
  34 ID "aB"
  35 SKIP
  36 ID "i5"
  37 SKIP
  38 ID "$x"
  39 SKIP
  40 ID "@efg"
  41 SKIP
  42 ID "@@."
  43 SKIP
  44 ID "#.#"
  45 SKIP
  46 UNEXPECTED_DOT
  47 ID "x"
  48 SKIP
  49 UNEXPECTED_CHAR 95
  50 ID "z"
  51 ENDCMD
  52 SKIP
  53 ID "abcd."
  54 SKIP
  55 ID "abcd"
  56 ENDCMD
  57 SKIP
  58 ID "QRSTUV"
  59 ENDCMD
  60 SKIP
  61 SKIP
  62 ID "QrStUv"
  63 ENDCMD
  64 SKIP
  65 SKIP
  66 SKIP
  67 ID "WXYZ"
  68 ENDCMD
  69 SKIP
  70 SKIP
  71 SKIP
  72 UNEXPECTED_CHAR 65533
  73 ENDCMD
  74 SKIP
  75 SKIP
  76 SKIP
  77 STOP
  78 ])
  79 PSPP_CHECK_SCAN([-i])
  80 AT_CLEANUP
  81 \f
  82 AT_SETUP([reserved words])
  83 AT_KEYWORDS([scan])
  84 AT_DATA([input], [dnl
  85 and or not eq ge gt le lt ne all by to with
  86 AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH
  87 andx orx notx eqx gex gtx lex ltx nex allx byx tox withx
  88 and. with.
  89 ])
  90 AT_DATA([expout], [dnl
  91 AND
  92 SKIP
  93 OR
  94 SKIP
  95 NOT
  96 SKIP
  97 EQ
  98 SKIP
  99 GE
 100 SKIP
 101 GT
 102 SKIP
 103 LE
 104 SKIP
 105 LT
 106 SKIP
 107 NE
 108 SKIP
 109 ALL
 110 SKIP
 111 BY
 112 SKIP
 113 TO
 114 SKIP
 115 WITH
 116 SKIP
 117 AND
 118 SKIP
 119 OR
 120 SKIP
 121 NOT
 122 SKIP
 123 EQ
 124 SKIP
 125 GE
 126 SKIP
 127 GT
 128 SKIP
 129 LE
 130 SKIP
 131 LT
 132 SKIP
 133 NE
 134 SKIP
 135 ALL
 136 SKIP
 137 BY
 138 SKIP
 139 TO
 140 SKIP
 141 WITH
 142 SKIP
 143 ID "andx"
 144 SKIP
 145 ID "orx"
 146 SKIP
 147 ID "notx"
 148 SKIP
 149 ID "eqx"
 150 SKIP
 151 ID "gex"
 152 SKIP
 153 ID "gtx"
 154 SKIP
 155 ID "lex"
 156 SKIP
 157 ID "ltx"
 158 SKIP
 159 ID "nex"
 160 SKIP
 161 ID "allx"
 162 SKIP
 163 ID "byx"
 164 SKIP
 165 ID "tox"
 166 SKIP
 167 ID "withx"
 168 SKIP
 169 ID "and."
 170 SKIP
 171 WITH
 172 ENDCMD
 173 SKIP
 174 STOP
 175 ])
 176 PSPP_CHECK_SCAN([-i])
 177 AT_CLEANUP
 178 \f
 179 AT_SETUP([punctuation])
 180 AT_KEYWORDS([scan])
 181 AT_DATA([input], [dnl
 182 ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] **
 183 ~&|=>=><=<~=<>(),-+*/[[]]**
 184 ])
 185 AT_DATA([expout], [dnl
 186 NOT
 187 SKIP
 188 AND
 189 SKIP
 190 OR
 191 SKIP
 192 EQUALS
 193 SKIP
 194 GE
 195 SKIP
 196 GT
 197 SKIP
 198 LE
 199 SKIP
 200 LT
 201 SKIP
 202 NE
 203 SKIP
 204 NE
 205 SKIP
 206 LPAREN
 207 SKIP
 208 RPAREN
 209 SKIP
 210 COMMA
 211 SKIP
 212 DASH
 213 SKIP
 214 PLUS
 215 SKIP
 216 ASTERISK
 217 SKIP
 218 SLASH
 219 SKIP
 220 LBRACK
 221 SKIP
 222 RBRACK
 223 SKIP
 224 EXP
 225 SKIP
 226 NOT
 227 AND
 228 OR
 229 EQUALS
 230 GE
 231 GT
 232 LE
 233 LT
 234 NE
 235 NE
 236 LPAREN
 237 RPAREN
 238 COMMA
 239 DASH
 240 PLUS
 241 ASTERISK
 242 SLASH
 243 LBRACK
 244 RBRACK
 245 EXP
 246 SKIP
 247 STOP
 248 ])
 249 PSPP_CHECK_SCAN([-i])
 250 AT_CLEANUP
 251 \f
 252 AT_SETUP([numbers])
 253 AT_KEYWORDS([scan])
 254 AT_DATA([input], [dnl
 255 0 1 01 001. 1.
 256 123. /* comment 1 */ /* comment 2 */
 257 .1 0.1 00.1 00.10
 258 5e1 6E-1 7e+1 6E+01 6e-03
 259 .3E1 .4e-1 .5E+1 .6e+01 .7E-03
 260 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03
 261 . 1e e1 1e+ 1e-
 262 ])
 263 AT_DATA([expout], [dnl
 264 POS_NUM
 265 SKIP
 266 POS_NUM 1
 267 SKIP
 268 POS_NUM 1
 269 SKIP
 270 POS_NUM 1
 271 SKIP
 272 POS_NUM 1
 273 ENDCMD
 274 SKIP
 275 POS_NUM 123
 276 ENDCMD
 277 SKIP
 278 SKIP
 279 SKIP
 280 SKIP
 281 SKIP
 282 ENDCMD
 283 POS_NUM 1
 284 SKIP
 285 POS_NUM 0.1
 286 SKIP
 287 POS_NUM 0.1
 288 SKIP
 289 POS_NUM 0.1
 290 SKIP
 291 POS_NUM 50
 292 SKIP
 293 POS_NUM 0.6
 294 SKIP
 295 POS_NUM 70
 296 SKIP
 297 POS_NUM 60
 298 SKIP
 299 POS_NUM 0.006
 300 SKIP
 301 ENDCMD
 302 POS_NUM 30
 303 SKIP
 304 POS_NUM 0.04
 305 SKIP
 306 POS_NUM 5
 307 SKIP
 308 POS_NUM 6
 309 SKIP
 310 POS_NUM 0.0007
 311 SKIP
 312 POS_NUM 12.3
 313 SKIP
 314 POS_NUM 4.56
 315 SKIP
 316 POS_NUM 789
 317 SKIP
 318 POS_NUM 999
 319 SKIP
 320 POS_NUM 0.0112
 321 SKIP
 322 ENDCMD
 323 SKIP
 324 EXPECTED_EXPONENT "1e"
 325 SKIP
 326 ID "e1"
 327 SKIP
 328 EXPECTED_EXPONENT "1e+"
 329 SKIP
 330 EXPECTED_EXPONENT "1e-"
 331 SKIP
 332 STOP
 333 ])
 334 PSPP_CHECK_SCAN([-i])
 335 AT_CLEANUP
 336 \f
 337 AT_SETUP([strings])
 338 AT_KEYWORDS([scan])
 339 AT_DATA([input], [dnl
 340 'x' "y" 'abc'
 341 'Don''t' "Can't" 'Won''t'
 342 """quoted""" '"quoted"'
 343 '' "" '''' """"
 344 'missing end quote
 345 "missing double quote
 346 'x' + "y"
 347 + 'z' +
 348 'a' /* abc */ + "b" /*
 349 + 'c' +/* */"d"/* */+'e'
 350 'foo'
 351 +          /* special case: + in column 0 would ordinarily start a new command
 352 'bar'
 353 'foo'
 354  +
 355 'bar'
 356 'foo'
 357 +
 358
 359 'bar'
 360
 361 +
 362 x"4142"+'5152'
 363 "4142"+
 364 x'5152'
 365 x"4142"
 366 +u'304a'
 367 "�あいうえお"
 368 "abc"+U"FFFD"+u'3048'+"xyz"
 369 ])
 370 AT_DATA([expout], [dnl
 371 STRING "x"
 372 SKIP
 373 STRING "y"
 374 SKIP
 375 STRING "abc"
 376 SKIP
 377 STRING "Don't"
 378 SKIP
 379 STRING "Can't"
 380 SKIP
 381 STRING "Won't"
 382 SKIP
 383 STRING ""quoted""
 384 SKIP
 385 STRING ""quoted""
 386 SKIP
 387 STRING ""
 388 SKIP
 389 STRING ""
 390 SKIP
 391 STRING "'"
 392 SKIP
 393 STRING """
 394 SKIP
 395 EXPECTED_QUOTE
 396 SKIP
 397 EXPECTED_QUOTE
 398 SKIP
 399 STRING "xyzabcde"
 400 SKIP
 401 STRING "foobar"
 402 SKIP
 403 STRING "foobar"
 404 SKIP
 405 STRING "foo"
 406 SKIP
 407 PLUS
 408 SKIP
 409 ENDCMD
 410 SKIP
 411 STRING "bar"
 412 SKIP
 413 ENDCMD
 414 SKIP
 415 PLUS
 416 SKIP
 417 STRING "AB5152"
 418 SKIP
 419 STRING "4142QR"
 420 SKIP
 421 STRING "ABお"
 422 SKIP
 423 STRING "�あいうえお"
 424 SKIP
 425 STRING "abc�えxyz"
 426 SKIP
 427 STOP
 428 ])
 429 PSPP_CHECK_SCAN([-i])
 430 AT_CLEANUP
 431 \f
 432 AT_SETUP([@%:@! construct])
 433 AT_KEYWORDS([scan])
 434 AT_DATA([input], [dnl
 435 #! /usr/bin/pspp
 436 #! /usr/bin/pspp
 437 ])
 438 AT_DATA([expout], [dnl
 439 SKIP
 440 SKIP
 441 ID "#"
 442 UNEXPECTED_CHAR 33
 443 SKIP
 444 SLASH
 445 ID "usr"
 446 SLASH
 447 ID "bin"
 448 SLASH
 449 ID "pspp"
 450 SKIP
 451 STOP
 452 ])
 453 PSPP_CHECK_SCAN([-i])
 454 AT_CLEANUP
 455 \f
 456 AT_SETUP([* and COMMENT commands])
 457 AT_KEYWORDS([scan])
 458 AT_DATA([input], [dnl
 459 * Comment commands "don't
 460 have to contain valid tokens.
 461
 462 ** Check ambiguity with ** token.
 463 ****************.
 464
 465 comment keyword works too.
 466 COMM also.
 467 com is ambiguous with COMPUTE.
 468
 469    * Comment need not start at left margin.
 470
 471 * Comment ends with blank line
 472
 473 next command.
 474
 475 ])
 476 AT_DATA([expout], [dnl
 477 SKIP
 478 SKIP
 479 SKIP
 480 ENDCMD
 481 SKIP
 482 ENDCMD
 483 SKIP
 484 SKIP
 485 ENDCMD
 486 SKIP
 487 SKIP
 488 ENDCMD
 489 SKIP
 490 ENDCMD
 491 SKIP
 492 SKIP
 493 ENDCMD
 494 SKIP
 495 SKIP
 496 ENDCMD
 497 SKIP
 498 ID "com"
 499 SKIP
 500 ID "is"
 501 SKIP
 502 ID "ambiguous"
 503 SKIP
 504 WITH
 505 SKIP
 506 ID "COMPUTE"
 507 ENDCMD
 508 SKIP
 509 ENDCMD
 510 SKIP
 511 SKIP
 512 SKIP
 513 ENDCMD
 514 SKIP
 515 ENDCMD
 516 SKIP
 517 SKIP
 518 SKIP
 519 ENDCMD
 520 SKIP
 521 ID "next"
 522 SKIP
 523 ID "command"
 524 ENDCMD
 525 SKIP
 526 ENDCMD
 527 SKIP
 528 STOP
 529 ])
 530 PSPP_CHECK_SCAN([-i])
 531 AT_CLEANUP
 532 \f
 533 AT_SETUP([DOCUMENT command])
 534 AT_KEYWORDS([scan])
 535 AT_DATA([input], [dnl
 536 DOCUMENT one line.
 537 DOC more
 538     than
 539         one
 540             line.
 541 docu
 542 first.paragraph
 543 isn't parsed as tokens
 544
 545 second paragraph.
 546 ])
 547 AT_DATA([expout], [dnl
 548 ID "DOCUMENT"
 549 STRING "DOCUMENT one line."
 550 ENDCMD
 551 ENDCMD
 552 SKIP
 553 ID "DOCUMENT"
 554 STRING "DOC more"
 555 SKIP
 556 STRING "    than"
 557 SKIP
 558 STRING "        one"
 559 SKIP
 560 STRING "            line."
 561 ENDCMD
 562 ENDCMD
 563 SKIP
 564 ID "DOCUMENT"
 565 STRING "docu"
 566 SKIP
 567 STRING "first.paragraph"
 568 SKIP
 569 STRING "isn't parsed as tokens"
 570 SKIP
 571 STRING ""
 572 SKIP
 573 STRING "second paragraph."
 574 ENDCMD
 575 ENDCMD
 576 SKIP
 577 STOP
 578 ])
 579 PSPP_CHECK_SCAN([-i])
 580 AT_CLEANUP
 581 \f
 582 AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands])
 583 AT_KEYWORDS([scan])
 584 AT_DATA([input], [dnl
 585 title/**/'Quoted string title'.
 586 tit /*
 587 "Quoted string on second line".
 588 sub "Quoted string subtitle"
 589  .
 590
 591 TITL /* Not a */ quoted string title.
 592 SUBT Not a quoted string /* subtitle
 593
 594 FIL label isn't quoted.
 595 FILE
 596   lab 'is quoted'.
 597 FILE /*
 598 /**/  lab not quoted here either
 599
 600 ])
 601 AT_DATA([expout], [dnl
 602 ID "title"
 603 SKIP
 604 STRING "Quoted string title"
 605 ENDCMD
 606 SKIP
 607 ID "tit"
 608 SKIP
 609 SKIP
 610 SKIP
 611 STRING "Quoted string on second line"
 612 ENDCMD
 613 SKIP
 614 ID "sub"
 615 SKIP
 616 STRING "Quoted string subtitle"
 617 SKIP
 618 SKIP
 619 ENDCMD
 620 SKIP
 621 ENDCMD
 622 SKIP
 623 ID "TITL"
 624 SKIP
 625 STRING "/* Not a */ quoted string title"
 626 ENDCMD
 627 SKIP
 628 ID "SUBT"
 629 SKIP
 630 STRING "Not a quoted string /* subtitle"
 631 SKIP
 632 ENDCMD
 633 SKIP
 634 ID "FIL"
 635 SKIP
 636 ID "label"
 637 SKIP
 638 STRING "isn't quoted"
 639 ENDCMD
 640 SKIP
 641 ID "FILE"
 642 SKIP
 643 SKIP
 644 ID "lab"
 645 SKIP
 646 STRING "is quoted"
 647 ENDCMD
 648 SKIP
 649 ID "FILE"
 650 SKIP
 651 SKIP
 652 SKIP
 653 SKIP
 654 SKIP
 655 ID "lab"
 656 SKIP
 657 STRING "not quoted here either"
 658 SKIP
 659 ENDCMD
 660 SKIP
 661 STOP
 662 ])
 663 PSPP_CHECK_SCAN([-i])
 664 AT_CLEANUP
 665 \f
 666 AT_SETUP([BEGIN DATA command])
 667 AT_KEYWORDS([scan])
 668 AT_DATA([input], [dnl
 669 begin data.
 670 123
 671 xxx
 672 end data.
 673
 674 BEG /**/ DAT /*
 675 5 6 7 /* x
 676
 677 end  data
 678 end data
 679 .
 680 ])
 681 AT_DATA([expout], [dnl
 682 ID "begin"
 683 SKIP
 684 ID "data"
 685 ENDCMD
 686 SKIP
 687 STRING "123"
 688 SKIP
 689 STRING "xxx"
 690 SKIP
 691 ID "end"
 692 SKIP
 693 ID "data"
 694 ENDCMD
 695 SKIP
 696 ENDCMD
 697 SKIP
 698 ID "BEG"
 699 SKIP
 700 SKIP
 701 SKIP
 702 ID "DAT"
 703 SKIP
 704 SKIP
 705 SKIP
 706 STRING "5 6 7 /* x"
 707 SKIP
 708 STRING ""
 709 SKIP
 710 STRING "end  data"
 711 SKIP
 712 ID "end"
 713 SKIP
 714 ID "data"
 715 SKIP
 716 ENDCMD
 717 SKIP
 718 STOP
 719 ])
 720 PSPP_CHECK_SCAN([-i])
 721 AT_CLEANUP
 722 \f
 723 AT_SETUP([DO REPEAT command])
 724 AT_KEYWORDS([scan])
 725 AT_DATA([input], [dnl
 726 do repeat x=a b c
 727           y=d e f.
 728   do repeat a=1 thru 5.
 729 another command.
 730 second command
 731 + third command.
 732 end /* x */ /* y */ repeat print.
 733 end
 734  repeat.
 735 ])
 736 AT_DATA([expout], [dnl
 737 ID "do"
 738 SKIP
 739 ID "repeat"
 740 SKIP
 741 ID "x"
 742 EQUALS
 743 ID "a"
 744 SKIP
 745 ID "b"
 746 SKIP
 747 ID "c"
 748 SKIP
 749 SKIP
 750 ID "y"
 751 EQUALS
 752 ID "d"
 753 SKIP
 754 ID "e"
 755 SKIP
 756 ID "f"
 757 ENDCMD
 758 SKIP
 759 STRING "  do repeat a=1 thru 5."
 760 SKIP
 761 STRING "another command."
 762 SKIP
 763 STRING "second command"
 764 SKIP
 765 STRING "+ third command."
 766 SKIP
 767 STRING "end /* x */ /* y */ repeat print."
 768 SKIP
 769 ID "end"
 770 SKIP
 771 SKIP
 772 ID "repeat"
 773 ENDCMD
 774 SKIP
 775 STOP
 776 ])
 777 PSPP_CHECK_SCAN([-i])
 778 AT_CLEANUP
 779 \f
 780 AT_SETUP([batch mode])
 781 AT_KEYWORDS([scan])
 782 AT_DATA([input], [dnl
 783 first command
 784      another line of first command
 785 +  second command
 786 third command
 787
 788 fourth command.
 789    fifth command.
 790 ])
 791 AT_DATA([expout], [dnl
 792 ID "first"
 793 SKIP
 794 ID "command"
 795 SKIP
 796 SKIP
 797 ID "another"
 798 SKIP
 799 ID "line"
 800 SKIP
 801 ID "of"
 802 SKIP
 803 ID "first"
 804 SKIP
 805 ID "command"
 806 SKIP
 807 ENDCMD
 808 SKIP
 809 ID "second"
 810 SKIP
 811 ID "command"
 812 SKIP
 813 ENDCMD
 814 ID "third"
 815 SKIP
 816 ID "command"
 817 SKIP
 818 ENDCMD
 819 SKIP
 820 ID "fourth"
 821 SKIP
 822 ID "command"
 823 ENDCMD
 824 SKIP
 825 SKIP
 826 ID "fifth"
 827 SKIP
 828 ID "command"
 829 ENDCMD
 830 SKIP
 831 STOP
 832 ])
 833 PSPP_CHECK_SCAN([-b])
 834 AT_CLEANUP