X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=tests%2Flanguage%2Flexer%2Fscan-test.c;h=95c52a42be48cc204f9bd79ef0ef0a692213d1ce;hb=6417b81fb2d5665471ba9fadd180ed1ddeb29246;hp=313fa16398590589e0eeced47b584516eb237354;hpb=9e28bd4b67bfed8806899042231c4ea1761c949b;p=pspp diff --git a/tests/language/lexer/scan-test.c b/tests/language/lexer/scan-test.c index 313fa16398..95c52a42be 100644 --- a/tests/language/lexer/scan-test.c +++ b/tests/language/lexer/scan-test.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 2010, 2011 Free Software Foundation, Inc. + Copyright (C) 2010, 2011, 2013 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -39,108 +39,100 @@ /* -a/--auto, -b/--batch, -i/--interactive: syntax mode. */ static enum segmenter_mode mode = SEG_MODE_AUTO; +/* -s, --strip-trailing-newline: Strip trailing newline from last line of + input. */ +static bool strip_trailing_newline; + static const char *parse_options (int argc, char **argv); static void usage (void) NO_RETURN; int main (int argc, char *argv[]) { - struct segment - { - enum segment_type type; - struct substring string; - }; - - size_t offset; const char *file_name; - char *input; - struct segmenter s; - struct segment *segs; - size_t n_segs, allocated_segs; size_t length; - size_t i; - int n; + char *input; + + struct string_lexer slex; set_program_name (argv[0]); file_name = parse_options (argc, argv); - /* Read from stdin into 'input'. Ensure that 'input' ends in a new-line - followed by a null byte. */ + /* Read from stdin into 'input'. */ input = (!strcmp (file_name, "-") - ? fread_file (stdin, &length) - : read_file (file_name, &length)); + ? fread_file (stdin, 0, &length) + : read_file (file_name, 0, &length)); if (input == NULL) error (EXIT_FAILURE, errno, "reading %s failed", file_name); - input = xrealloc (input, length + 3); - if (length == 0 || input[length - 1] != '\n') - input[length++] = '\n'; - input[length++] = '\0'; - - segs = NULL; - n_segs = allocated_segs = 0; - segmenter_init (&s, mode); - for (offset = 0; offset < length; offset += n) + if (strip_trailing_newline && length && input[length - 1] == '\n') { - enum segment_type type; + length--; + if (length && input[length - 1] == '\r') + length--; + } - n = segmenter_push (&s, input + offset, length - offset, &type); - assert (n >= 0); - assert (offset + n <= length); + struct token *tokens = NULL; + size_t n_tokens = 0; + size_t allocated_tokens = 0; + string_lexer_init (&slex, input, length, mode, false); + for (;;) + { + if (n_tokens >= allocated_tokens) + tokens = x2nrealloc (tokens, &allocated_tokens, sizeof *tokens); + enum string_lexer_result result + = string_lexer_next (&slex, &tokens[n_tokens]); - if (n_segs >= allocated_segs) - segs = x2nrealloc (segs, &allocated_segs, sizeof *segs); + if (result == SLR_ERROR) + tokens[n_tokens].type = T_STOP; + n_tokens++; - segs[n_segs].type = type; - segs[n_segs].string.string = input + offset; - segs[n_segs].string.length = n; - n_segs++; + if (result == SLR_END) + break; } - for (i = 0; i < n_segs; ) + for (size_t i = 0; i < n_tokens; ) { - enum scan_result result; - struct scanner scanner; - struct token token; - int saved = -1; - - scanner_init (&scanner, &token); - do + struct merger m = MERGER_INIT; + int retval; + struct token out; + for (size_t j = i; ; j++) { - struct segment *seg; - - assert (i < n_segs); - - seg = &segs[i++]; - result = scanner_push (&scanner, seg->type, seg->string, &token); - if (result == SCAN_SAVE) - saved = i; + assert (j < n_tokens); + retval = merger_add (&m, &tokens[j], &out); + if (retval != -1) + break; } - while (result == SCAN_MORE || result == SCAN_SAVE); - if (result == SCAN_BACK) - { - assert (saved >= 0); - i = saved; - } + const struct token *t = retval ? &out : &tokens[i]; - printf ("%s", scan_type_to_string (token.type)); - if (token.number != 0.0) + fputs (token_type_to_name (t->type), stdout); + if (t->number != 0.0) { - char s[DBL_BUFSIZE_BOUND]; + double x = t->number; - dtoastr (s, sizeof s, 0, 0, token.number); - printf (" %s", s); + if (x > LONG_MIN && x <= LONG_MAX && floor (x) == x) + printf (" %ld", (long int) x); + else + printf (" %.3g", x); } - if (token.string.string != NULL || token.string.length > 0) - printf (" \"%.*s\"", (int) token.string.length, token.string.string); + if (t->string.string != NULL || t->string.length > 0) + printf (" \"%.*s\"", (int) t->string.length, t->string.string); printf ("\n"); - token_destroy (&token); + if (retval) + { + i += retval; + token_uninit (&out); + } + else + i++; } + for (size_t i = 0; i < n_tokens; i++) + token_uninit (&tokens[i]); + free (tokens); free (input); - free (segs); return 0; } @@ -155,11 +147,12 @@ parse_options (int argc, char **argv) {"auto", no_argument, NULL, 'a'}, {"batch", no_argument, NULL, 'b'}, {"interactive", no_argument, NULL, 'i'}, + {"strip-trailing-newline", no_argument, NULL, 's'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0}, }; - int c = getopt_long (argc, argv, "abih", options, NULL); + int c = getopt_long (argc, argv, "sabih", options, NULL); if (c == -1) break; @@ -177,6 +170,10 @@ parse_options (int argc, char **argv) mode = SEG_MODE_INTERACTIVE; break; + case 's': + strip_trailing_newline = true; + break; + case 'h': usage (); @@ -207,10 +204,10 @@ usage (void) usage: %s [OPTIONS] INPUT\n\ \n\ Options:\n\ - -1, --one-segment feed one segment at a time\n\ -a, --auto use \"auto\" syntax mode\n\ -b, --batch use \"batch\" syntax mode\n\ -i, --interactive use \"interactive\" syntax mode (default)\n\ + -s, --strip-trailing-newline remove newline from end of input\n\ -v, --verbose include rows and column numbers in output\n\ -h, --help print this help message\n", program_name, program_name);