lexer: New functions lex_spec_missing(), lex_spec_only_once().
[pspp-builds.git] / tests / language / lexer / scan.at
1 AT_BANNER([syntax scanning])
2 m4_define([PSPP_CHECK_SCAN],
3   [AT_CHECK([scan-test $1 input], [0], [expout])])
4 \f
5 AT_SETUP([identifiers])
6 AT_KEYWORDS([scan])
7 AT_DATA([input], [dnl
8 a aB i5 $x @efg @@. #.# .x _z.
9 abcd. abcd.
10 QRSTUV./* end of line comment */
11 QrStUv./* end of line comment */ @&t@
12 WXYZ. /* unterminated end of line comment
13 �. /* U+FFFD is not valid in an identifier
14 ])
15 AT_DATA([expout], [dnl
16 ID "a"
17 SKIP
18 ID "aB"
19 SKIP
20 ID "i5"
21 SKIP
22 ID "$x"
23 SKIP
24 ID "@efg"
25 SKIP
26 ID "@@."
27 SKIP
28 ID "#.#"
29 SKIP
30 UNEXPECTED_DOT
31 ID "x"
32 SKIP
33 UNEXPECTED_CHAR 95
34 ID "z"
35 ENDCMD
36 SKIP
37 ID "abcd."
38 SKIP
39 ID "abcd"
40 ENDCMD
41 SKIP
42 ID "QRSTUV"
43 ENDCMD
44 SKIP
45 SKIP
46 ID "QrStUv"
47 ENDCMD
48 SKIP
49 SKIP
50 SKIP
51 ID "WXYZ"
52 ENDCMD
53 SKIP
54 SKIP
55 SKIP
56 UNEXPECTED_CHAR 65533
57 ENDCMD
58 SKIP
59 SKIP
60 SKIP
61 STOP
62 ])
63 PSPP_CHECK_SCAN([-i])
64 AT_CLEANUP
65 \f
66 AT_SETUP([reserved words])
67 AT_KEYWORDS([scan])
68 AT_DATA([input], [dnl
69 and or not eq ge gt le lt ne all by to with
70 AND OR NOT EQ GE GT LE LT NE ALL BY TO WITH
71 andx orx notx eqx gex gtx lex ltx nex allx byx tox withx
72 and. with.
73 ])
74 AT_DATA([expout], [dnl
75 AND
76 SKIP
77 OR
78 SKIP
79 NOT
80 SKIP
81 EQ
82 SKIP
83 GE
84 SKIP
85 GT
86 SKIP
87 LE
88 SKIP
89 LT
90 SKIP
91 NE
92 SKIP
93 ALL
94 SKIP
95 BY
96 SKIP
97 TO
98 SKIP
99 WITH
100 SKIP
101 AND
102 SKIP
103 OR
104 SKIP
105 NOT
106 SKIP
107 EQ
108 SKIP
109 GE
110 SKIP
111 GT
112 SKIP
113 LE
114 SKIP
115 LT
116 SKIP
117 NE
118 SKIP
119 ALL
120 SKIP
121 BY
122 SKIP
123 TO
124 SKIP
125 WITH
126 SKIP
127 ID "andx"
128 SKIP
129 ID "orx"
130 SKIP
131 ID "notx"
132 SKIP
133 ID "eqx"
134 SKIP
135 ID "gex"
136 SKIP
137 ID "gtx"
138 SKIP
139 ID "lex"
140 SKIP
141 ID "ltx"
142 SKIP
143 ID "nex"
144 SKIP
145 ID "allx"
146 SKIP
147 ID "byx"
148 SKIP
149 ID "tox"
150 SKIP
151 ID "withx"
152 SKIP
153 ID "and."
154 SKIP
155 WITH
156 ENDCMD
157 SKIP
158 STOP
159 ])
160 PSPP_CHECK_SCAN([-i])
161 AT_CLEANUP
162 \f
163 AT_SETUP([punctuation])
164 AT_KEYWORDS([scan])
165 AT_DATA([input], [dnl
166 ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] **
167 ~&|=>=><=<~=<>(),-+*/[[]]**
168 ])
169 AT_DATA([expout], [dnl
170 NOT
171 SKIP
172 AND
173 SKIP
174 OR
175 SKIP
176 EQUALS
177 SKIP
178 GE
179 SKIP
180 GT
181 SKIP
182 LE
183 SKIP
184 LT
185 SKIP
186 NE
187 SKIP
188 NE
189 SKIP
190 LPAREN
191 SKIP
192 RPAREN
193 SKIP
194 COMMA
195 SKIP
196 DASH
197 SKIP
198 PLUS
199 SKIP
200 ASTERISK
201 SKIP
202 SLASH
203 SKIP
204 LBRACK
205 SKIP
206 RBRACK
207 SKIP
208 EXP
209 SKIP
210 NOT
211 AND
212 OR
213 EQUALS
214 GE
215 GT
216 LE
217 LT
218 NE
219 NE
220 LPAREN
221 RPAREN
222 COMMA
223 DASH
224 PLUS
225 ASTERISK
226 SLASH
227 LBRACK
228 RBRACK
229 EXP
230 SKIP
231 STOP
232 ])
233 PSPP_CHECK_SCAN([-i])
234 AT_CLEANUP
235 \f
236 AT_SETUP([numbers])
237 AT_KEYWORDS([scan])
238 AT_DATA([input], [dnl
239 0 1 01 001. 1.
240 123. /* comment 1 */ /* comment 2 */
241 .1 0.1 00.1 00.10
242 5e1 6E-1 7e+1 6E+01 6e-03
243 .3E1 .4e-1 .5E+1 .6e+01 .7E-03
244 1.23e1 45.6E-1 78.9e+1 99.9E+01 11.2e-03
245 . 1e e1 1e+ 1e-
246 ])
247 AT_DATA([expout], [dnl
248 POS_NUM
249 SKIP
250 POS_NUM 1
251 SKIP
252 POS_NUM 1
253 SKIP
254 POS_NUM 1
255 SKIP
256 POS_NUM 1
257 ENDCMD
258 SKIP
259 POS_NUM 123
260 ENDCMD
261 SKIP
262 SKIP
263 SKIP
264 SKIP
265 SKIP
266 ENDCMD
267 POS_NUM 1
268 SKIP
269 POS_NUM 0.1
270 SKIP
271 POS_NUM 0.1
272 SKIP
273 POS_NUM 0.1
274 SKIP
275 POS_NUM 50
276 SKIP
277 POS_NUM 0.6
278 SKIP
279 POS_NUM 70
280 SKIP
281 POS_NUM 60
282 SKIP
283 POS_NUM 0.006
284 SKIP
285 ENDCMD
286 POS_NUM 30
287 SKIP
288 POS_NUM 0.04
289 SKIP
290 POS_NUM 5
291 SKIP
292 POS_NUM 6
293 SKIP
294 POS_NUM 0.0007
295 SKIP
296 POS_NUM 12.3
297 SKIP
298 POS_NUM 4.56
299 SKIP
300 POS_NUM 789
301 SKIP
302 POS_NUM 999
303 SKIP
304 POS_NUM 0.0112
305 SKIP
306 ENDCMD
307 SKIP
308 EXPECTED_EXPONENT "1e"
309 SKIP
310 ID "e1"
311 SKIP
312 EXPECTED_EXPONENT "1e+"
313 SKIP
314 EXPECTED_EXPONENT "1e-"
315 SKIP
316 STOP
317 ])
318 PSPP_CHECK_SCAN([-i])
319 AT_CLEANUP
320 \f
321 AT_SETUP([strings])
322 AT_KEYWORDS([scan])
323 AT_DATA([input], [dnl
324 'x' "y" 'abc'
325 'Don''t' "Can't" 'Won''t'
326 """quoted""" '"quoted"'
327 '' "" '''' """"
328 'missing end quote
329 "missing double quote
330 'x' + "y"
331 + 'z' +
332 'a' /* abc */ + "b" /*
333 + 'c' +/* */"d"/* */+'e'
334 'foo'
335 +          /* special case: + in column 0 would ordinarily start a new command
336 'bar'
337 'foo'
338  +
339 'bar'
340 'foo'
341 +
342
343 'bar'
344
345 +
346 x"4142"+'5152'
347 "4142"+
348 x'5152'
349 x"4142"
350 +u'304a'
351 "�あいうえお"
352 "abc"+U"FFFD"+u'3048'+"xyz"
353 ])
354 AT_DATA([expout], [dnl
355 STRING "x"
356 SKIP
357 STRING "y"
358 SKIP
359 STRING "abc"
360 SKIP
361 STRING "Don't"
362 SKIP
363 STRING "Can't"
364 SKIP
365 STRING "Won't"
366 SKIP
367 STRING ""quoted""
368 SKIP
369 STRING ""quoted""
370 SKIP
371 STRING ""
372 SKIP
373 STRING ""
374 SKIP
375 STRING "'"
376 SKIP
377 STRING """
378 SKIP
379 EXPECTED_QUOTE
380 SKIP
381 EXPECTED_QUOTE
382 SKIP
383 STRING "xyzabcde"
384 SKIP
385 STRING "foobar"
386 SKIP
387 STRING "foobar"
388 SKIP
389 STRING "foo"
390 SKIP
391 PLUS
392 SKIP
393 ENDCMD
394 SKIP
395 STRING "bar"
396 SKIP
397 ENDCMD
398 SKIP
399 PLUS
400 SKIP
401 STRING "AB5152"
402 SKIP
403 STRING "4142QR"
404 SKIP
405 STRING "ABお"
406 SKIP
407 STRING "�あいうえお"
408 SKIP
409 STRING "abc�えxyz"
410 SKIP
411 STOP
412 ])
413 PSPP_CHECK_SCAN([-i])
414 AT_CLEANUP
415 \f
416 AT_SETUP([@%:@! construct])
417 AT_KEYWORDS([scan])
418 AT_DATA([input], [dnl
419 #! /usr/bin/pspp
420 #! /usr/bin/pspp
421 ])
422 AT_DATA([expout], [dnl
423 SKIP
424 SKIP
425 ID "#"
426 UNEXPECTED_CHAR 33
427 SKIP
428 SLASH
429 ID "usr"
430 SLASH
431 ID "bin"
432 SLASH
433 ID "pspp"
434 SKIP
435 STOP
436 ])
437 PSPP_CHECK_SCAN([-i])
438 AT_CLEANUP
439 \f
440 AT_SETUP([* and COMMENT commands])
441 AT_KEYWORDS([scan])
442 AT_DATA([input], [dnl
443 * Comment commands "don't
444 have to contain valid tokens.
445
446 ** Check ambiguity with ** token.
447 ****************.
448
449 comment keyword works too.
450 COMM also.
451 com is ambiguous with COMPUTE.
452
453    * Comment need not start at left margin.
454
455 * Comment ends with blank line
456
457 next command.
458
459 ])
460 AT_DATA([expout], [dnl
461 SKIP
462 SKIP
463 SKIP
464 ENDCMD
465 SKIP
466 ENDCMD
467 SKIP
468 SKIP
469 ENDCMD
470 SKIP
471 SKIP
472 ENDCMD
473 SKIP
474 ENDCMD
475 SKIP
476 SKIP
477 ENDCMD
478 SKIP
479 SKIP
480 ENDCMD
481 SKIP
482 ID "com"
483 SKIP
484 ID "is"
485 SKIP
486 ID "ambiguous"
487 SKIP
488 WITH
489 SKIP
490 ID "COMPUTE"
491 ENDCMD
492 SKIP
493 ENDCMD
494 SKIP
495 SKIP
496 SKIP
497 ENDCMD
498 SKIP
499 ENDCMD
500 SKIP
501 SKIP
502 SKIP
503 ENDCMD
504 SKIP
505 ID "next"
506 SKIP
507 ID "command"
508 ENDCMD
509 SKIP
510 ENDCMD
511 SKIP
512 STOP
513 ])
514 PSPP_CHECK_SCAN([-i])
515 AT_CLEANUP
516 \f
517 AT_SETUP([DOCUMENT command])
518 AT_KEYWORDS([scan])
519 AT_DATA([input], [dnl
520 DOCUMENT one line.
521 DOC more
522     than
523         one
524             line.
525 docu
526 first.paragraph
527 isn't parsed as tokens
528
529 second paragraph.
530 ])
531 AT_DATA([expout], [dnl
532 ID "DOCUMENT"
533 STRING "DOCUMENT one line."
534 ENDCMD
535 ENDCMD
536 SKIP
537 ID "DOCUMENT"
538 STRING "DOC more"
539 SKIP
540 STRING "    than"
541 SKIP
542 STRING "        one"
543 SKIP
544 STRING "            line."
545 ENDCMD
546 ENDCMD
547 SKIP
548 ID "DOCUMENT"
549 STRING "docu"
550 SKIP
551 STRING "first.paragraph"
552 SKIP
553 STRING "isn't parsed as tokens"
554 SKIP
555 STRING ""
556 SKIP
557 STRING "second paragraph."
558 ENDCMD
559 ENDCMD
560 SKIP
561 STOP
562 ])
563 PSPP_CHECK_SCAN([-i])
564 AT_CLEANUP
565 \f
566 AT_SETUP([TITLE, SUBTITLE, FILE LABEL commands])
567 AT_KEYWORDS([scan])
568 AT_DATA([input], [dnl
569 title/**/'Quoted string title'.
570 tit /*
571 "Quoted string on second line".
572 sub "Quoted string subtitle"
573  .
574
575 TITL /* Not a */ quoted string title.
576 SUBT Not a quoted string /* subtitle
577
578 FIL label isn't quoted.
579 FILE
580   lab 'is quoted'.
581 FILE /*
582 /**/  lab not quoted here either
583
584 ])
585 AT_DATA([expout], [dnl
586 ID "title"
587 SKIP
588 STRING "Quoted string title"
589 ENDCMD
590 SKIP
591 ID "tit"
592 SKIP
593 SKIP
594 SKIP
595 STRING "Quoted string on second line"
596 ENDCMD
597 SKIP
598 ID "sub"
599 SKIP
600 STRING "Quoted string subtitle"
601 SKIP
602 SKIP
603 ENDCMD
604 SKIP
605 ENDCMD
606 SKIP
607 ID "TITL"
608 SKIP
609 STRING "/* Not a */ quoted string title"
610 ENDCMD
611 SKIP
612 ID "SUBT"
613 SKIP
614 STRING "Not a quoted string /* subtitle"
615 SKIP
616 ENDCMD
617 SKIP
618 ID "FIL"
619 SKIP
620 ID "label"
621 SKIP
622 STRING "isn't quoted"
623 ENDCMD
624 SKIP
625 ID "FILE"
626 SKIP
627 SKIP
628 ID "lab"
629 SKIP
630 STRING "is quoted"
631 ENDCMD
632 SKIP
633 ID "FILE"
634 SKIP
635 SKIP
636 SKIP
637 SKIP
638 SKIP
639 ID "lab"
640 SKIP
641 STRING "not quoted here either"
642 SKIP
643 ENDCMD
644 SKIP
645 STOP
646 ])
647 PSPP_CHECK_SCAN([-i])
648 AT_CLEANUP
649 \f
650 AT_SETUP([BEGIN DATA command])
651 AT_KEYWORDS([scan])
652 AT_DATA([input], [dnl
653 begin data.
654 123
655 xxx
656 end data.
657
658 BEG /**/ DAT /*
659 5 6 7 /* x
660
661 end  data
662 end data
663 .
664 ])
665 AT_DATA([expout], [dnl
666 ID "begin"
667 SKIP
668 ID "data"
669 ENDCMD
670 SKIP
671 STRING "123"
672 SKIP
673 STRING "xxx"
674 SKIP
675 ID "end"
676 SKIP
677 ID "data"
678 ENDCMD
679 SKIP
680 ENDCMD
681 SKIP
682 ID "BEG"
683 SKIP
684 SKIP
685 SKIP
686 ID "DAT"
687 SKIP
688 SKIP
689 SKIP
690 STRING "5 6 7 /* x"
691 SKIP
692 STRING ""
693 SKIP
694 STRING "end  data"
695 SKIP
696 ID "end"
697 SKIP
698 ID "data"
699 SKIP
700 ENDCMD
701 SKIP
702 STOP
703 ])
704 PSPP_CHECK_SCAN([-i])
705 AT_CLEANUP
706 \f
707 AT_SETUP([DO REPEAT command])
708 AT_KEYWORDS([scan])
709 AT_DATA([input], [dnl
710 do repeat x=a b c
711           y=d e f.
712   do repeat a=1 thru 5.
713 another command.
714 second command
715 + third command.
716 end /* x */ /* y */ repeat print.
717 end
718  repeat.
719 ])
720 AT_DATA([expout], [dnl
721 ID "do"
722 SKIP
723 ID "repeat"
724 SKIP
725 ID "x"
726 EQUALS
727 ID "a"
728 SKIP
729 ID "b"
730 SKIP
731 ID "c"
732 SKIP
733 SKIP
734 ID "y"
735 EQUALS
736 ID "d"
737 SKIP
738 ID "e"
739 SKIP
740 ID "f"
741 ENDCMD
742 SKIP
743 STRING "  do repeat a=1 thru 5."
744 SKIP
745 STRING "another command."
746 SKIP
747 STRING "second command"
748 SKIP
749 STRING "+ third command."
750 SKIP
751 STRING "end /* x */ /* y */ repeat print."
752 SKIP
753 ID "end"
754 SKIP
755 SKIP
756 ID "repeat"
757 ENDCMD
758 SKIP
759 STOP
760 ])
761 PSPP_CHECK_SCAN([-i])
762 AT_CLEANUP
763 \f
764 AT_SETUP([batch mode])
765 AT_KEYWORDS([scan])
766 AT_DATA([input], [dnl
767 first command
768      another line of first command
769 +  second command
770 third command
771
772 fourth command.
773    fifth command.
774 ])
775 AT_DATA([expout], [dnl
776 ID "first"
777 SKIP
778 ID "command"
779 SKIP
780 SKIP
781 ID "another"
782 SKIP
783 ID "line"
784 SKIP
785 ID "of"
786 SKIP
787 ID "first"
788 SKIP
789 ID "command"
790 SKIP
791 ENDCMD
792 SKIP
793 ID "second"
794 SKIP
795 ID "command"
796 SKIP
797 ENDCMD
798 ID "third"
799 SKIP
800 ID "command"
801 SKIP
802 ENDCMD
803 SKIP
804 ID "fourth"
805 SKIP
806 ID "command"
807 ENDCMD
808 SKIP
809 SKIP
810 ID "fifth"
811 SKIP
812 ID "command"
813 ENDCMD
814 SKIP
815 STOP
816 ])
817 PSPP_CHECK_SCAN([-b])
818 AT_CLEANUP