X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Flexer%2Flexer.c;h=68dd1c8638f190aea05bedd096912f3b07e34c11;hb=eb72e2843fa902b185680a756e79e91b68caea49;hp=907a77cee6490f72a72ca9dc84585ea5856acb57;hpb=8381768f3394a907c621cb9acbb77b83f5cd4875;p=pspp diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c index 907a77cee6..68dd1c8638 100644 --- a/src/language/lexer/lexer.c +++ b/src/language/lexer/lexer.c @@ -1,6 +1,5 @@ /* PSPP - computes sample statistics. - Copyright (C) 1997-9, 2000 Free Software Foundation, Inc. - Written by Ben Pfaff . + Copyright (C) 1997-9, 2000, 2006 Free Software Foundation, Inc. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -41,15 +40,15 @@ #define _(msgid) gettext (msgid) #define N_(msgid) msgid -/* -#define DUMP_TOKENS 1 -*/ + +#define DUMP_TOKENS 0 + struct lexer { struct string line_buffer; - bool (*read_line) (struct string *, enum getl_syntax *); + struct source_stream *ss; int token; /* Current token. */ double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */ @@ -62,7 +61,6 @@ struct lexer char *prog; /* Pointer to next token in line_buffer. */ bool dot; /* True only if this line ends with a terminal dot. */ - bool eof; /* True only if the last token returned was T_STOP. */ int put_token ; /* If nonzero, next token returned by lex_get(). Used only in exceptional circumstances. */ @@ -86,28 +84,32 @@ enum string_type static int parse_string (struct lexer *, enum string_type); #if DUMP_TOKENS -static void dump_token (void); +static void dump_token (struct lexer *); #endif /* Initialization. */ /* Initializes the lexer. */ struct lexer * -lex_create (bool (*read_line_func) (struct string *, enum getl_syntax *)) +lex_create (struct source_stream *ss) { struct lexer *lexer = xzalloc (sizeof (*lexer)); ds_init_empty (&lexer->tokstr); ds_init_empty (&lexer->put_tokstr); ds_init_empty (&lexer->line_buffer); - lexer->read_line = read_line_func; - - if (!lex_get_line (lexer)) - lexer->eof = true; + lexer->ss = ss; return lexer; } +struct source_stream * +lex_get_source_stream (const struct lexer *lex) +{ + return lex->ss; +} + + void lex_destroy (struct lexer *lexer) { @@ -152,30 +154,31 @@ save_token (struct lexer *lexer) void lex_get (struct lexer *lexer) { + /* Find a token. */ + for (;;) + { + if (NULL == lexer->prog && ! lex_get_line (lexer) ) + { + lexer->token = T_STOP; + return; + } + /* If a token was pushed ahead, return it. */ if (lexer->put_token) { restore_token (lexer); #if DUMP_TOKENS - dump_token (); + dump_token (lexer); #endif return; } - /* Find a token. */ for (;;) { /* Skip whitespace. */ - if (lexer->eof) - { - lexer->token = T_STOP; - return; - } - - for (;;) - { while (isspace ((unsigned char) *lexer->prog)) lexer->prog++; + if (*lexer->prog) break; @@ -184,16 +187,16 @@ lex_get (struct lexer *lexer) lexer->dot = 0; lexer->token = '.'; #if DUMP_TOKENS - dump_token (); + dump_token (lexer); #endif return; } else if (!lex_get_line (lexer)) { - lexer->eof = true; + lexer->prog = NULL; lexer->token = T_STOP; #if DUMP_TOKENS - dump_token (); + dump_token (lexer); #endif return; } @@ -202,7 +205,7 @@ lex_get (struct lexer *lexer) { restore_token (lexer); #if DUMP_TOKENS - dump_token (); + dump_token (lexer); #endif return; } @@ -379,7 +382,7 @@ lex_get (struct lexer *lexer) } #if DUMP_TOKENS - dump_token (); + dump_token (lexer); #endif } @@ -653,7 +656,7 @@ lex_look_ahead (struct lexer *lexer) for (;;) { - if (lexer->eof) + if (NULL == lexer->prog && ! lex_get_line (lexer) ) return 0; for (;;) @@ -752,7 +755,7 @@ lex_discard_line (struct lexer *lexer) void lex_discard_rest_of_command (struct lexer *lexer) { - if (!getl_is_interactive ()) + if (!getl_is_interactive (lexer->ss)) { while (lexer->token != T_STOP && lexer->token != '.') lex_get (lexer); @@ -842,7 +845,7 @@ bool lex_get_line_raw (struct lexer *lexer, enum getl_syntax *syntax) { enum getl_syntax dummy; - bool ok = lexer->read_line (&lexer->line_buffer, + bool ok = getl_read_line (lexer->ss, &lexer->line_buffer, syntax != NULL ? syntax : &dummy); return ok; } @@ -857,10 +860,14 @@ lex_get_line (struct lexer *lexer) enum getl_syntax syntax; if (!lex_get_line_raw (lexer, &syntax)) + { + lexer->prog = NULL; return false; + } lex_preprocess_line (&lexer->line_buffer, syntax, &line_starts_command, &lexer->dot); + if (line_starts_command) lexer->put_token = '.'; @@ -976,13 +983,6 @@ lex_negative_to_dash (struct lexer *lexer) } } -/* We're not at eof any more. */ -void -lex_reset_eof (struct lexer *lexer) -{ - lexer->eof = false; -} - /* Skip a COMMENT command. */ void lex_skip_comment (struct lexer *lexer) @@ -992,7 +992,7 @@ lex_skip_comment (struct lexer *lexer) if (!lex_get_line (lexer)) { lexer->put_token = T_STOP; - lexer->eof = true; + lexer->prog = NULL; return; } @@ -1047,7 +1047,7 @@ convert_numeric_string_to_char_string (struct lexer *lexer, if (ds_length (&lexer->tokstr) % chars_per_byte) msg (SE, _("String of %s digits has %d characters, which is not a " "multiple of %d."), - base_name, ds_length (&lexer->tokstr), chars_per_byte); + base_name, (int) ds_length (&lexer->tokstr), chars_per_byte); p = ds_cstr (&lexer->tokstr); for (i = 0; i < byte_cnt; i++) @@ -1126,7 +1126,7 @@ parse_string (struct lexer *lexer, enum string_type type) lexer->prog++; /* Skip whitespace after final quote mark. */ - if (lexer->eof) + if (lexer->prog == NULL) break; for (;;) { @@ -1148,7 +1148,7 @@ parse_string (struct lexer *lexer, enum string_type type) lexer->prog++; /* Skip whitespace after plus sign. */ - if (lexer->eof) + if (lexer->prog == NULL) break; for (;;) { @@ -1184,7 +1184,7 @@ finish: if (ds_length (&lexer->tokstr) > 255) { msg (SE, _("String exceeds 255 characters in length (%d characters)."), - ds_length (&lexer->tokstr)); + (int) ds_length (&lexer->tokstr)); ds_truncate (&lexer->tokstr, 255); } @@ -1201,7 +1201,8 @@ dump_token (struct lexer *lexer) const char *curfn; int curln; - getl_location (&curfn, &curln); + curln = getl_source_location (lexer->ss); + curfn = getl_source_name (lexer->ss); if (curfn) fprintf (stderr, "%s:%d\t", curfn, curln); } @@ -1234,8 +1235,8 @@ dump_token (struct lexer *lexer) break; default: - if (lex_is_keyword (token)) - fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (token)); + if (lex_is_keyword (lexer->token)) + fprintf (stderr, "KEYWORD\t%s\n", lex_token_name (lexer->token)); else fprintf (stderr, "PUNCT\t%c\n", lexer->token); break;