tests/language/lexer/scan.at

   1 AT_BANNER([syntax scanning])
   2 m4_define([PSPP_CHECK_SCAN],
   3   [AT_CHECK([scan-test $1 input], [0], [expout])])
   4 \f
   5 AT_SETUP([identifiers])
   6 AT_KEYWORDS([scan])
   7 AT_DATA([input], [dnl
   8 a aB i5 $x @efg @@. #.# .x _z.
   9 abcd. abcd.
  10 QRSTUV./* end of line comment */
  11 QrStUv./* end of line comment */ @&t@
  12 WXYZ. /* unterminated end of line comment
  13 �. /* U+FFFD is not valid in an identifier
  14 ])
  15 AT_DATA([expout], [dnl
  16 ID "a"
  17 SKIP
  18 ID "aB"
  19 SKIP
  20 ID "i5"
  21 SKIP
  22 ID "$x"
  23 SKIP
  24 ID "@efg"
  25 SKIP
  26 ID "@@."
  27 SKIP
  28 ID "#.#"
  29 SKIP
  30 UNEXPECTED_DOT
  31 ID "x"
  32 SKIP
  33 UNEXPECTED_CHAR 95
  34 ID "z"
  35 ENDCMD
  36 SKIP
  37 ID "abcd."
  38 SKIP
  39 ID "abcd"
  40 ENDCMD
  41 SKIP
  42 ID "QRSTUV"
  43 ENDCMD
  44 SKIP
  45 SKIP
  46 ID "QrStUv"
  47 ENDCMD
  48 SKIP
  49 SKIP
  50 SKIP
  51 ID "WXYZ"
  52 ENDCMD
  53 SKIP
  54 SKIP
  55 SKIP
  56 UNEXPECTED_CHAR 65533
  57 ENDCMD
  58 SKIP
  59 SKIP
  60 SKIP
  61 STOP
  62 ])
  63 PSPP_CHECK_SCAN([-i])
  64 AT_CLEANUP
  65 \f
  66 AT_SETUP([reserved words])
  67 AT_KEYWORDS([scan])
  68 AT_DATA([input], [dnl
  69 and or not eq ge gt le lt ne all by to with
  70 AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH
  71 andx orx notx eqx gex gtx lex ltx nex allx byx tox withx
  72 and. with.
  73 ])
  74 AT_DATA([expout], [dnl
  75 AND
  76 SKIP
  77 OR
  78 SKIP
  79 NOT
  80 SKIP
  81 EQ
  82 SKIP
  83 GE
  84 SKIP
  85 GT
  86 SKIP
  87 LE
  88 SKIP
  89 LT
  90 SKIP
  91 NE
  92 SKIP
  93 ALL
  94 SKIP
  95 BY
  96 SKIP
  97 TO
  98 SKIP
  99 WITH
 100 SKIP
 101 AND
 102 SKIP
 103 OR
 104 SKIP
 105 NOT
 106 SKIP
 107 EQ
 108 SKIP
 109 GE
 110 SKIP
 111 GT
 112 SKIP
 113 LE
 114 SKIP
 115 LT
 116 SKIP
 117 NE
 118 SKIP
 119 ALL
 120 SKIP
 121 BY
 122 SKIP
 123 TO
 124 SKIP
 125 WITH
 126 SKIP
 127 ID "andx"
 128 SKIP
 129 ID "orx"
 130 SKIP
 131 ID "notx"
 132 SKIP
 133 ID "eqx"
 134 SKIP
 135 ID "gex"
 136 SKIP
 137 ID "gtx"
 138 SKIP
 139 ID "lex"
 140 SKIP
 141 ID "ltx"
 142 SKIP
 143 ID "nex"
 144 SKIP
 145 ID "allx"
 146 SKIP
 147 ID "byx"
 148 SKIP
 149 ID "tox"
 150 SKIP
 151 ID "withx"
 152 SKIP
 153 ID "and."
 154 SKIP
 155 WITH
 156 ENDCMD
 157 SKIP
 158 STOP
 159 ])
 160 PSPP_CHECK_SCAN([-i])
 161 AT_CLEANUP
 162 \f
 163 AT_SETUP([punctuation])
 164 AT_KEYWORDS([scan])
 165 AT_DATA([input], [dnl
 166 ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] **
 167 ~&|=>=><=<~=<>(),-+*/[[]]**
 168 ])
 169 AT_DATA([expout], [dnl
 170 NOT
 171 SKIP
 172 AND
 173 SKIP
 174 OR
 175 SKIP
 176 EQUALS
 177 SKIP
 178 GE
 179 SKIP
 180 GT
 181 SKIP
 182 LE
 183 SKIP
 184 LT
 185 SKIP
 186 NE
 187 SKIP
 188 NE
 189 SKIP
 190 LPAREN
 191 SKIP
 192 RPAREN
 193 SKIP
 194 COMMA
 195 SKIP
 196 DASH
 197 SKIP
 198 PLUS
 199 SKIP
 200 ASTERISK
 201 SKIP
 202 SLASH
 203 SKIP
 204 LBRACK
 205 SKIP
 206 RBRACK
 207 SKIP
 208 EXP
 209 SKIP
 210 NOT
 211 AND
 212 OR
 213 EQUALS
 214 GE
 215 GT
 216 LE
 217 LT
 218 NE
 219 NE
 220 LPAREN
 221 RPAREN
 222 COMMA
 223 DASH
 224 PLUS
 225 ASTERISK
 226 SLASH
 227 LBRACK
 228 RBRACK
 229 EXP
 230 SKIP
 231 STOP
 232 ])
 233 PSPP_CHECK_SCAN([-i])
 234 AT_CLEANUP
 235 \f
 236 AT_SETUP([numbers])
 237 AT_KEYWORDS([scan])
 238 AT_DATA([input], [dnl
 239 0 1 01 001. 1.
 240 123. /* comment 1 */ /* comment 2 */
 241 .1 0.1 00.1 00.10
 242 5e1 6E-1 7e+1 6E+01 6e-03
 243 .3E1 .4e-1 .5E+1 .6e+01 .7E-03
 244 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03
 245 . 1e e1 1e+ 1e-
 246 ])
 247 AT_DATA([expout], [dnl
 248 POS_NUM
 249 SKIP
 250 POS_NUM 1
 251 SKIP
 252 POS_NUM 1
 253 SKIP
 254 POS_NUM 1
 255 SKIP
 256 POS_NUM 1
 257 ENDCMD
 258 SKIP
 259 POS_NUM 123
 260 ENDCMD
 261 SKIP
 262 SKIP
 263 SKIP
 264 SKIP
 265 SKIP
 266 ENDCMD
 267 POS_NUM 1
 268 SKIP
 269 POS_NUM 0.1
 270 SKIP
 271 POS_NUM 0.1
 272 SKIP
 273 POS_NUM 0.1
 274 SKIP
 275 POS_NUM 50
 276 SKIP
 277 POS_NUM 0.6
 278 SKIP
 279 POS_NUM 70
 280 SKIP
 281 POS_NUM 60
 282 SKIP
 283 POS_NUM 0.006
 284 SKIP
 285 ENDCMD
 286 POS_NUM 30
 287 SKIP
 288 POS_NUM 0.04
 289 SKIP
 290 POS_NUM 5
 291 SKIP
 292 POS_NUM 6
 293 SKIP
 294 POS_NUM 0.0007
 295 SKIP
 296 POS_NUM 12.3
 297 SKIP
 298 POS_NUM 4.56
 299 SKIP
 300 POS_NUM 789
 301 SKIP
 302 POS_NUM 999
 303 SKIP
 304 POS_NUM 0.0112
 305 SKIP
 306 ENDCMD
 307 SKIP
 308 EXPECTED_EXPONENT "1e"
 309 SKIP
 310 ID "e1"
 311 SKIP
 312 EXPECTED_EXPONENT "1e+"
 313 SKIP
 314 EXPECTED_EXPONENT "1e-"
 315 SKIP
 316 STOP
 317 ])
 318 PSPP_CHECK_SCAN([-i])
 319 AT_CLEANUP
 320 \f
 321 AT_SETUP([strings])
 322 AT_KEYWORDS([scan])
 323 AT_DATA([input], [dnl
 324 'x' "y" 'abc'
 325 'Don''t' "Can't" 'Won''t'
 326 """quoted""" '"quoted"'
 327 '' "" '''' """"
 328 'missing end quote
 329 "missing double quote
 330 'x' + "y"
 331 + 'z' +
 332 'a' /* abc */ + "b" /*
 333 + 'c' +/* */"d"/* */+'e'
 334 'foo'
 335 +          /* special case: + in column 0 would ordinarily start a new command
 336 'bar'
 337 'foo'
 338  +
 339 'bar'
 340 'foo'
 341 +
 342
 343 'bar'
 344
 345 +
 346 x"4142"+'5152'
 347 "4142"+
 348 x'5152'
 349 x"4142"
 350 +u'304a'
 351 "�あいうえお"
 352 "abc"+U"FFFD"+u'3048'+"xyz"
 353 ])
 354 AT_DATA([expout], [dnl
 355 STRING "x"
 356 SKIP
 357 STRING "y"
 358 SKIP
 359 STRING "abc"
 360 SKIP
 361 STRING "Don't"
 362 SKIP
 363 STRING "Can't"
 364 SKIP
 365 STRING "Won't"
 366 SKIP
 367 STRING ""quoted""
 368 SKIP
 369 STRING ""quoted""
 370 SKIP
 371 STRING ""
 372 SKIP
 373 STRING ""
 374 SKIP
 375 STRING "'"
 376 SKIP
 377 STRING """
 378 SKIP
 379 EXPECTED_QUOTE
 380 SKIP
 381 EXPECTED_QUOTE
 382 SKIP
 383 STRING "xyzabcde"
 384 SKIP
 385 STRING "foobar"
 386 SKIP
 387 STRING "foobar"
 388 SKIP
 389 STRING "foo"
 390 SKIP
 391 PLUS
 392 SKIP
 393 ENDCMD
 394 SKIP
 395 STRING "bar"
 396 SKIP
 397 ENDCMD
 398 SKIP
 399 PLUS
 400 SKIP
 401 STRING "AB5152"
 402 SKIP
 403 STRING "4142QR"
 404 SKIP
 405 STRING "ABお"
 406 SKIP
 407 STRING "�あいうえお"
 408 SKIP
 409 STRING "abc�えxyz"
 410 SKIP
 411 STOP
 412 ])
 413 PSPP_CHECK_SCAN([-i])
 414 AT_CLEANUP
 415 \f
 416 AT_SETUP([@%:@! construct])
 417 AT_KEYWORDS([scan])
 418 AT_DATA([input], [dnl
 419 #! /usr/bin/pspp
 420 #! /usr/bin/pspp
 421 ])
 422 AT_DATA([expout], [dnl
 423 SKIP
 424 SKIP
 425 ID "#"
 426 UNEXPECTED_CHAR 33
 427 SKIP
 428 SLASH
 429 ID "usr"
 430 SLASH
 431 ID "bin"
 432 SLASH
 433 ID "pspp"
 434 SKIP
 435 STOP
 436 ])
 437 PSPP_CHECK_SCAN([-i])
 438 AT_CLEANUP
 439 \f
 440 AT_SETUP([* and COMMENT commands])
 441 AT_KEYWORDS([scan])
 442 AT_DATA([input], [dnl
 443 * Comment commands "don't
 444 have to contain valid tokens.
 445
 446 ** Check ambiguity with ** token.
 447 ****************.
 448
 449 comment keyword works too.
 450 COMM also.
 451 com is ambiguous with COMPUTE.
 452
 453    * Comment need not start at left margin.
 454
 455 * Comment ends with blank line
 456
 457 next command.
 458
 459 ])
 460 AT_DATA([expout], [dnl
 461 SKIP
 462 SKIP
 463 SKIP
 464 ENDCMD
 465 SKIP
 466 ENDCMD
 467 SKIP
 468 SKIP
 469 ENDCMD
 470 SKIP
 471 SKIP
 472 ENDCMD
 473 SKIP
 474 ENDCMD
 475 SKIP
 476 SKIP
 477 ENDCMD
 478 SKIP
 479 SKIP
 480 ENDCMD
 481 SKIP
 482 ID "com"
 483 SKIP
 484 ID "is"
 485 SKIP
 486 ID "ambiguous"
 487 SKIP
 488 WITH
 489 SKIP
 490 ID "COMPUTE"
 491 ENDCMD
 492 SKIP
 493 ENDCMD
 494 SKIP
 495 SKIP
 496 SKIP
 497 ENDCMD
 498 SKIP
 499 ENDCMD
 500 SKIP
 501 SKIP
 502 SKIP
 503 ENDCMD
 504 SKIP
 505 ID "next"
 506 SKIP
 507 ID "command"
 508 ENDCMD
 509 SKIP
 510 ENDCMD
 511 SKIP
 512 STOP
 513 ])
 514 PSPP_CHECK_SCAN([-i])
 515 AT_CLEANUP
 516 \f
 517 AT_SETUP([DOCUMENT command])
 518 AT_KEYWORDS([scan])
 519 AT_DATA([input], [dnl
 520 DOCUMENT one line.
 521 DOC more
 522     than
 523         one
 524             line.
 525 docu
 526 first.paragraph
 527 isn't parsed as tokens
 528
 529 second paragraph.
 530 ])
 531 AT_DATA([expout], [dnl
 532 ID "DOCUMENT"
 533 STRING "DOCUMENT one line."
 534 ENDCMD
 535 ENDCMD
 536 SKIP
 537 ID "DOCUMENT"
 538 STRING "DOC more"
 539 SKIP
 540 STRING "    than"
 541 SKIP
 542 STRING "        one"
 543 SKIP
 544 STRING "            line."
 545 ENDCMD
 546 ENDCMD
 547 SKIP
 548 ID "DOCUMENT"
 549 STRING "docu"
 550 SKIP
 551 STRING "first.paragraph"
 552 SKIP
 553 STRING "isn't parsed as tokens"
 554 SKIP
 555 STRING ""
 556 SKIP
 557 STRING "second paragraph."
 558 ENDCMD
 559 ENDCMD
 560 SKIP
 561 STOP
 562 ])
 563 PSPP_CHECK_SCAN([-i])
 564 AT_CLEANUP
 565 \f
 566 AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands])
 567 AT_KEYWORDS([scan])
 568 AT_DATA([input], [dnl
 569 title/**/'Quoted string title'.
 570 tit /*
 571 "Quoted string on second line".
 572 sub "Quoted string subtitle"
 573  .
 574
 575 TITL /* Not a */ quoted string title.
 576 SUBT Not a quoted string /* subtitle
 577
 578 FIL label isn't quoted.
 579 FILE
 580   lab 'is quoted'.
 581 FILE /*
 582 /**/  lab not quoted here either
 583
 584 ])
 585 AT_DATA([expout], [dnl
 586 ID "title"
 587 SKIP
 588 STRING "Quoted string title"
 589 ENDCMD
 590 SKIP
 591 ID "tit"
 592 SKIP
 593 SKIP
 594 SKIP
 595 STRING "Quoted string on second line"
 596 ENDCMD
 597 SKIP
 598 ID "sub"
 599 SKIP
 600 STRING "Quoted string subtitle"
 601 SKIP
 602 SKIP
 603 ENDCMD
 604 SKIP
 605 ENDCMD
 606 SKIP
 607 ID "TITL"
 608 SKIP
 609 STRING "/* Not a */ quoted string title"
 610 ENDCMD
 611 SKIP
 612 ID "SUBT"
 613 SKIP
 614 STRING "Not a quoted string /* subtitle"
 615 SKIP
 616 ENDCMD
 617 SKIP
 618 ID "FIL"
 619 SKIP
 620 ID "label"
 621 SKIP
 622 STRING "isn't quoted"
 623 ENDCMD
 624 SKIP
 625 ID "FILE"
 626 SKIP
 627 SKIP
 628 ID "lab"
 629 SKIP
 630 STRING "is quoted"
 631 ENDCMD
 632 SKIP
 633 ID "FILE"
 634 SKIP
 635 SKIP
 636 SKIP
 637 SKIP
 638 SKIP
 639 ID "lab"
 640 SKIP
 641 STRING "not quoted here either"
 642 SKIP
 643 ENDCMD
 644 SKIP
 645 STOP
 646 ])
 647 PSPP_CHECK_SCAN([-i])
 648 AT_CLEANUP
 649 \f
 650 AT_SETUP([BEGIN DATA command])
 651 AT_KEYWORDS([scan])
 652 AT_DATA([input], [dnl
 653 begin data.
 654 123
 655 xxx
 656 end data.
 657
 658 BEG /**/ DAT /*
 659 5 6 7 /* x
 660
 661 end  data
 662 end data
 663 .
 664 ])
 665 AT_DATA([expout], [dnl
 666 ID "begin"
 667 SKIP
 668 ID "data"
 669 ENDCMD
 670 SKIP
 671 STRING "123"
 672 SKIP
 673 STRING "xxx"
 674 SKIP
 675 ID "end"
 676 SKIP
 677 ID "data"
 678 ENDCMD
 679 SKIP
 680 ENDCMD
 681 SKIP
 682 ID "BEG"
 683 SKIP
 684 SKIP
 685 SKIP
 686 ID "DAT"
 687 SKIP
 688 SKIP
 689 SKIP
 690 STRING "5 6 7 /* x"
 691 SKIP
 692 STRING ""
 693 SKIP
 694 STRING "end  data"
 695 SKIP
 696 ID "end"
 697 SKIP
 698 ID "data"
 699 SKIP
 700 ENDCMD
 701 SKIP
 702 STOP
 703 ])
 704 PSPP_CHECK_SCAN([-i])
 705 AT_CLEANUP
 706 \f
 707 AT_SETUP([DO REPEAT command])
 708 AT_KEYWORDS([scan])
 709 AT_DATA([input], [dnl
 710 do repeat x=a b c
 711           y=d e f.
 712   do repeat a=1 thru 5.
 713 another command.
 714 second command
 715 + third command.
 716 end /* x */ /* y */ repeat print.
 717 end
 718  repeat.
 719 ])
 720 AT_DATA([expout], [dnl
 721 ID "do"
 722 SKIP
 723 ID "repeat"
 724 SKIP
 725 ID "x"
 726 EQUALS
 727 ID "a"
 728 SKIP
 729 ID "b"
 730 SKIP
 731 ID "c"
 732 SKIP
 733 SKIP
 734 ID "y"
 735 EQUALS
 736 ID "d"
 737 SKIP
 738 ID "e"
 739 SKIP
 740 ID "f"
 741 ENDCMD
 742 SKIP
 743 STRING "  do repeat a=1 thru 5."
 744 SKIP
 745 STRING "another command."
 746 SKIP
 747 STRING "second command"
 748 SKIP
 749 STRING "+ third command."
 750 SKIP
 751 STRING "end /* x */ /* y */ repeat print."
 752 SKIP
 753 ID "end"
 754 SKIP
 755 SKIP
 756 ID "repeat"
 757 ENDCMD
 758 SKIP
 759 STOP
 760 ])
 761 PSPP_CHECK_SCAN([-i])
 762 AT_CLEANUP
 763 \f
 764 AT_SETUP([batch mode])
 765 AT_KEYWORDS([scan])
 766 AT_DATA([input], [dnl
 767 first command
 768      another line of first command
 769 +  second command
 770 third command
 771
 772 fourth command.
 773    fifth command.
 774 ])
 775 AT_DATA([expout], [dnl
 776 ID "first"
 777 SKIP
 778 ID "command"
 779 SKIP
 780 SKIP
 781 ID "another"
 782 SKIP
 783 ID "line"
 784 SKIP
 785 ID "of"
 786 SKIP
 787 ID "first"
 788 SKIP
 789 ID "command"
 790 SKIP
 791 ENDCMD
 792 SKIP
 793 ID "second"
 794 SKIP
 795 ID "command"
 796 SKIP
 797 ENDCMD
 798 ID "third"
 799 SKIP
 800 ID "command"
 801 SKIP
 802 ENDCMD
 803 SKIP
 804 ID "fourth"
 805 SKIP
 806 ID "command"
 807 ENDCMD
 808 SKIP
 809 SKIP
 810 ID "fifth"
 811 SKIP
 812 ID "command"
 813 ENDCMD
 814 SKIP
 815 STOP
 816 ])
 817 PSPP_CHECK_SCAN([-b])
 818 AT_CLEANUP