X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Flexer%2Flexer.c;h=aa681c80306b65f0cfb7f2c72f3edfae479ccead;hb=3da49359c52cb783db907cc197847bbd5e721c97;hp=5b24522f5c9097172b2852c567d21a120dfe5517;hpb=ff46735d4c81b49fa17c8bd26f3f5dd7bde9a15d;p=pspp diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c index 5b24522f5c..aa681c8030 100644 --- a/src/language/lexer/lexer.c +++ b/src/language/lexer/lexer.c @@ -1,5 +1,5 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc. + Copyright (C) 1997-9, 2000, 2006, 2009, 2010 Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -17,7 +17,8 @@ #include #include "lexer.h" #include -#include +#include +#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include #include "xalloc.h" @@ -38,11 +40,6 @@ #define _(msgid) gettext (msgid) #define N_(msgid) msgid - -#define DUMP_TOKENS 0 - - - struct lexer { struct string line_buffer; @@ -81,10 +78,6 @@ enum string_type }; static int parse_string (struct lexer *, enum string_type); - -#if DUMP_TOKENS -static void dump_token (struct lexer *); -#endif /* Initialization. */ @@ -174,20 +167,17 @@ lex_get (struct lexer *lexer) return; } - /* If a token was pushed ahead, return it. */ - if (lexer->put_token) - { - restore_token (lexer); -#if DUMP_TOKENS - dump_token (lexer); -#endif - return; - } + /* If a token was pushed ahead, return it. */ + if (lexer->put_token) + { + restore_token (lexer); + return; + } - for (;;) - { - /* Skip whitespace. */ - while (isspace ((unsigned char) *lexer->prog)) + for (;;) + { + /* Skip whitespace. */ + while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; if (*lexer->prog) @@ -197,27 +187,18 @@ lex_get (struct lexer *lexer) { lexer->dot = 0; lexer->token = '.'; -#if DUMP_TOKENS - dump_token (lexer); -#endif return; } else if (!lex_get_line (lexer)) { lexer->prog = NULL; lexer->token = T_STOP; -#if DUMP_TOKENS - dump_token (lexer); -#endif return; } if (lexer->put_token) { restore_token (lexer); -#if DUMP_TOKENS - dump_token (lexer); -#endif return; } } @@ -244,10 +225,10 @@ lex_get (struct lexer *lexer) if (*lexer->prog == '-') { ds_put_char (&lexer->tokstr, *lexer->prog++); - while (isspace ((unsigned char) *lexer->prog)) + while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; - if (!isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.') + if (!c_isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.') { lexer->token = '-'; break; @@ -258,12 +239,12 @@ lex_get (struct lexer *lexer) lexer->token = T_POS_NUM; /* Parse the number, copying it into tokstr. */ - while (isdigit ((unsigned char) *lexer->prog)) + while (c_isdigit ((unsigned char) *lexer->prog)) ds_put_char (&lexer->tokstr, *lexer->prog++); if (*lexer->prog == '.') { ds_put_char (&lexer->tokstr, *lexer->prog++); - while (isdigit ((unsigned char) *lexer->prog)) + while (c_isdigit ((unsigned char) *lexer->prog)) ds_put_char (&lexer->tokstr, *lexer->prog++); } if (*lexer->prog == 'e' || *lexer->prog == 'E') @@ -271,12 +252,12 @@ lex_get (struct lexer *lexer) ds_put_char (&lexer->tokstr, *lexer->prog++); if (*lexer->prog == '+' || *lexer->prog == '-') ds_put_char (&lexer->tokstr, *lexer->prog++); - while (isdigit ((unsigned char) *lexer->prog)) + while (c_isdigit ((unsigned char) *lexer->prog)) ds_put_char (&lexer->tokstr, *lexer->prog++); } /* Parse as floating point. */ - lexer->tokval = strtod (ds_cstr (&lexer->tokstr), &tail); + lexer->tokval = c_strtod (ds_cstr (&lexer->tokstr), &tail); if (*tail) { msg (SE, _("%s does not form a valid number."), @@ -295,6 +276,7 @@ lex_get (struct lexer *lexer) break; case '(': case ')': case ',': case '=': case '+': case '/': + case '[': case ']': lexer->token = *lexer->prog++; break; @@ -382,19 +364,15 @@ lex_get (struct lexer *lexer) } else { - if (isgraph ((unsigned char) *lexer->prog)) - msg (SE, _("Bad character in input: `%c'."), *lexer->prog++); - else - msg (SE, _("Bad character in input: `\\%o'."), *lexer->prog++); + unsigned char c = *lexer->prog++; + char *c_name = xasprintf (c_isgraph (c) ? "%c" : "\\%o", c); + msg (SE, _("Bad character in input: `%s'."), c_name); + free (c_name); continue; } } break; } - -#if DUMP_TOKENS - dump_token (lexer); -#endif } /* Parses an identifier at the current position into tokid and @@ -437,31 +415,34 @@ lex_sbc_missing (struct lexer *lexer, const char *sbc) void lex_error (struct lexer *lexer, const char *message, ...) { - char *token_rep; - char where[128]; + struct string s; + + ds_init_empty (&s); - token_rep = lex_token_representation (lexer); if (lexer->token == T_STOP) - strcpy (where, "end of file"); + ds_put_cstr (&s, _("Syntax error at end of file")); else if (lexer->token == '.') - strcpy (where, "end of command"); + ds_put_cstr (&s, _("Syntax error at end of command")); else - snprintf (where, sizeof where, "`%s'", token_rep); - free (token_rep); + { + char *token_rep = lex_token_representation (lexer); + ds_put_format (&s, _("Syntax error at `%s'"), token_rep); + free (token_rep); + } if (message) { - char buf[1024]; va_list args; + ds_put_cstr (&s, ": "); + va_start (args, message); - vsnprintf (buf, 1024, message, args); + ds_put_vformat (&s, message, args); va_end (args); - - msg (SE, _("Syntax error %s at %s."), buf, where); } - else - msg (SE, _("Syntax error at %s."), where); + + msg (SE, "%s.", ds_cstr (&s)); + ds_destroy (&s); } /* Checks that we're at end of command. @@ -622,7 +603,7 @@ lex_force_match (struct lexer *lexer, int t) bool lex_force_string (struct lexer *lexer) { - if (lexer->token == T_STRING) + if (lex_is_string (lexer)) return true; else { @@ -690,7 +671,7 @@ lex_look_ahead (struct lexer *lexer) for (;;) { - while (isspace ((unsigned char) *lexer->prog)) + while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; if (*lexer->prog) break; @@ -865,22 +846,22 @@ lex_preprocess_line (struct string *line, if (syntax == GETL_BATCH) { int first = ds_first (line); - *line_starts_command = !isspace (first); + *line_starts_command = !c_isspace (first); if (first == '+' || first == '-') *ds_data (line) = ' '; } } -/* Reads a line, without performing any preprocessing. - Sets *SYNTAX, if SYNTAX is non-null, to the line's syntax - mode. */ +/* Reads a line, without performing any preprocessing. */ bool lex_get_line_raw (struct lexer *lexer) { bool ok = getl_read_line (lexer->ss, &lexer->line_buffer); - enum syntax_mode mode = lex_current_syntax_mode (lexer); - journal_write (mode == GETL_BATCH, ds_cstr (&lexer->line_buffer)); - + if (ok) + { + const char *line = ds_cstr (&lexer->line_buffer); + text_item_submit (text_item_create (TEXT_ITEM_SYNTAX, line)); + } return ok; } @@ -950,7 +931,7 @@ lex_token_representation (struct lexer *lexer) char *sp, *dp; for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++) - if (!isprint ((unsigned char) *sp)) + if (!c_isprint ((unsigned char) *sp)) { hexstring = 1; break; @@ -1164,7 +1145,7 @@ parse_string (struct lexer *lexer, enum string_type type) break; for (;;) { - while (isspace ((unsigned char) *lexer->prog)) + while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; if (*lexer->prog) break; @@ -1186,7 +1167,7 @@ parse_string (struct lexer *lexer, enum string_type type) break; for (;;) { - while (isspace ((unsigned char) *lexer->prog)) + while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; if (*lexer->prog) break; @@ -1215,70 +1196,9 @@ finish: if (type != CHARACTER_STRING) convert_numeric_string_to_char_string (lexer, type); - if (ds_length (&lexer->tokstr) > 255) - { - msg (SE, _("String exceeds 255 characters in length (%zu characters)."), - ds_length (&lexer->tokstr)); - ds_truncate (&lexer->tokstr, 255); - } - return T_STRING; } -#if DUMP_TOKENS -/* Reads one token from the lexer and writes a textual representation - on stdout for debugging purposes. */ -static void -dump_token (struct lexer *lexer) -{ - { - const char *curfn; - int curln; - - curln = getl_source_location (lexer->ss); - curfn = getl_source_name (lexer->ss); - if (curfn) - fprintf (stderr, "%s:%d\t", curfn, curln); - } - - switch (lexer->token) - { - case T_ID: - fprintf (stderr, "ID\t%s\n", lexer->tokid); - break; - - case T_POS_NUM: - case T_NEG_NUM: - fprintf (stderr, "NUM\t%f\n", lexer->tokval); - break; - - case T_STRING: - fprintf (stderr, "STRING\t\"%s\"\n", ds_cstr (&lexer->tokstr)); - break; - - case T_STOP: - fprintf (stderr, "STOP\n"); - break; - - case T_EXP: - fprintf (stderr, "MISC\tEXP\""); - break; - - case 0: - fprintf (stderr, "MISC\tEOF\n"); - break; - - default: - if (lex_is_keyword (lexer->token)) - fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (lexer->token)); - else - fprintf (stderr, "PUNCT\t%c\n", lexer->token); - break; - } -} -#endif /* DUMP_TOKENS */ - - /* Token Accessor Functions */ int