X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flexer.c;h=568a87e316654c45b3b979460d8805bf0c9f8261;hb=ed7bce25787929340a3f264f00dde7c979e571a9;hp=dd20c8b9524db89a474dc1bad03c2e0eac34223a;hpb=97d6c6f6b1922621ca013668eba9a9a9f71d60fe;p=pspp diff --git a/src/lexer.c b/src/lexer.c index dd20c8b952..568a87e316 100644 --- a/src/lexer.c +++ b/src/lexer.c @@ -44,14 +44,14 @@ /* Current token. */ int token; -/* T_NUM: the token's value. */ +/* T_POS_NUM, T_NEG_NUM: the token's value. */ double tokval; /* T_ID: the identifier. */ -char tokid[9]; +char tokid[LONG_NAME_LEN + 1]; /* T_ID, T_STRING: token string value. - For T_ID, this is not truncated to 8 characters as is tokid. */ + For T_ID, this is not truncated to SHORT_NAME_LEN characters as is tokid. */ struct string tokstr; /* Static variables. */ @@ -71,7 +71,7 @@ static char *prog; /* Nonzero only if this line ends with a terminal dot. */ static int dot; -/* Nonzero only if the last token returned was T_EOF. */ +/* Nonzero only if the last token returned was T_STOP. */ static int eof; /* If nonzero, next token returned by lex_get(). @@ -81,7 +81,6 @@ static struct string put_tokstr; static double put_tokval; static void unexpected_eof (void); -static inline int check_id (const char *id, size_t len); static void convert_numeric_string_to_char_string (int type); static int parse_string (int type); @@ -95,10 +94,17 @@ static void dump_token (void); void lex_init (void) { - ds_init (NULL, &put_tokstr, 64); + ds_init (&put_tokstr, 64); if (!lex_get_line ()) unexpected_eof (); } + +void +lex_done (void) +{ + ds_destroy(&put_tokstr); +} + /* Common functions. */ @@ -109,9 +115,9 @@ restore_token (void) { assert (put_token != 0); token = put_token; - ds_replace (&tokstr, ds_value (&put_tokstr)); - strncpy (tokid, ds_value (&put_tokstr), 8); - tokid[8] = 0; + ds_replace (&tokstr, ds_c_str (&put_tokstr)); + strncpy (tokid, ds_c_str (&put_tokstr), SHORT_NAME_LEN); + tokid[SHORT_NAME_LEN] = 0; tokval = put_tokval; put_token = 0; } @@ -122,7 +128,7 @@ static void save_token (void) { put_token = token; - ds_replace (&put_tokstr, ds_value (&tokstr)); + ds_replace (&put_tokstr, ds_c_str (&tokstr)); put_tokval = tokval; } @@ -131,6 +137,8 @@ save_token (void) void lex_get (void) { + int i; + /* If a token was pushed ahead, return it. */ if (put_token) { @@ -208,7 +216,7 @@ lex_get (void) negative numbers into two tokens. */ if (*cp == '-') { - ds_putchar (&tokstr, *prog++); + ds_putc (&tokstr, *prog++); while (isspace ((unsigned char) *prog)) prog++; @@ -217,39 +225,41 @@ lex_get (void) token = '-'; break; } + token = T_NEG_NUM; } - + else + token = T_POS_NUM; + /* Parse the number, copying it into tokstr. */ while (isdigit ((unsigned char) *prog)) - ds_putchar (&tokstr, *prog++); + ds_putc (&tokstr, *prog++); if (*prog == '.') { - ds_putchar (&tokstr, *prog++); + ds_putc (&tokstr, *prog++); while (isdigit ((unsigned char) *prog)) - ds_putchar (&tokstr, *prog++); + ds_putc (&tokstr, *prog++); } if (*prog == 'e' || *prog == 'E') { - ds_putchar (&tokstr, *prog++); + ds_putc (&tokstr, *prog++); if (*prog == '+' || *prog == '-') - ds_putchar (&tokstr, *prog++); + ds_putc (&tokstr, *prog++); while (isdigit ((unsigned char) *prog)) - ds_putchar (&tokstr, *prog++); + ds_putc (&tokstr, *prog++); } /* Parse as floating point. */ - tokval = strtod (ds_value (&tokstr), &tail); + tokval = strtod (ds_c_str (&tokstr), &tail); if (*tail) { msg (SE, _("%s does not form a valid number."), - ds_value (&tokstr)); + ds_c_str (&tokstr)); tokval = 0.0; ds_clear (&tokstr); - ds_putchar (&tokstr, '0'); + ds_putc (&tokstr, '0'); } - token = T_NUM; break; } @@ -346,15 +356,19 @@ lex_get (void) } /* Copy id to tokstr. */ - ds_putchar (&tokstr, toupper ((unsigned char) *prog++)); + ds_putc (&tokstr, *prog++); while (CHAR_IS_IDN (*prog)) - ds_putchar (&tokstr, toupper ((unsigned char) *prog++)); + ds_putc (&tokstr, *prog++); - /* Copy tokstr to tokid, truncating it to 8 characters. */ - strncpy (tokid, ds_value (&tokstr), 8); - tokid[8] = 0; + /* Copy tokstr to tokid, truncating it to LONG_NAME_LEN characters.*/ + strncpy (tokid, ds_c_str (&tokstr), LONG_NAME_LEN); + tokid[LONG_NAME_LEN] = 0; - token = check_id (ds_value (&tokstr), ds_length (&tokstr)); + /* Convert to upper case */ + for ( i = 0 ; i < ds_length(&tokstr) ; ++i ) + tokstr.string[i] = toupper(tokstr.string[i]); + + token = lex_id_to_token (ds_c_str (&tokstr), ds_length (&tokstr)); break; default: @@ -418,11 +432,27 @@ lex_end_of_command (void) /* Token testing functions. */ -/* Returns nonzero if the current token is an integer. */ -int -lex_integer_p (void) +/* Returns true if the current token is a number. */ +bool +lex_is_number (void) +{ + return token == T_POS_NUM || token == T_NEG_NUM; +} + +/* Returns the value of the current token, which must be a + floating point number. */ +double +lex_number (void) { - return (token == T_NUM + assert (lex_is_number ()); + return tokval; +} + +/* Returns true iff the current token is an integer. */ +bool +lex_is_integer (void) +{ + return (lex_is_number () && tokval != NOT_LONG && tokval >= LONG_MIN && tokval <= LONG_MAX @@ -434,26 +464,9 @@ lex_integer_p (void) long lex_integer (void) { - assert (lex_integer_p ()); + assert (lex_is_integer ()); return tokval; } -/* Returns nonzero if the current token is an floating point. */ -int -lex_double_p (void) -{ - return ( token == T_NUM - && tokval != NOT_DOUBLE ); -} - -/* Returns the value of the current token, which must be a - floating point number. */ -double -lex_double (void) -{ - assert (lex_double_p ()); - return tokval; -} - /* Token matching functions. */ @@ -491,7 +504,7 @@ lex_match_id (const char *s) int lex_match_int (int x) { - if (lex_integer_p () && lex_integer () == x) + if (lex_is_integer () && lex_integer () == x) { lex_get (); return 1; @@ -556,7 +569,7 @@ lex_force_string (void) int lex_force_int (void) { - if (lex_integer_p ()) + if (lex_is_integer ()) return 1; else { @@ -570,7 +583,7 @@ lex_force_int (void) int lex_force_num (void) { - if (token == T_NUM) + if (lex_is_number ()) return 1; else { @@ -628,6 +641,23 @@ lex_id_match (const char *kw, const char *tok) { return lex_id_match_len (kw, strlen (kw), tok, strlen (tok)); } + +/* Returns the proper token type, either T_ID or a reserved keyword + enum, for ID[], which must contain LEN characters. */ +int +lex_id_to_token (const char *id, size_t len) +{ + const char **kwp; + + if (len < 2 || len > 4) + return T_ID; + + for (kwp = keywords; *kwp; kwp++) + if (!strcasecmp (*kwp, id)) + return T_FIRST_KEYWORD + (kwp - keywords); + + return T_ID; +} /* Weird token functions. */ @@ -690,8 +720,8 @@ lex_put_back_id (const char *id) save_token (); token = T_ID; ds_replace (&tokstr, id); - strncpy (tokid, ds_value (&tokstr), 8); - tokid[8] = 0; + strncpy (tokid, ds_c_str (&tokstr), SHORT_NAME_LEN); + tokid[SHORT_NAME_LEN] = 0; } /* Weird line processing functions. */ @@ -700,7 +730,7 @@ lex_put_back_id (const char *id) const char * lex_entire_line (void) { - return ds_value (&getl_buf); + return ds_c_str (&getl_buf); } /* As lex_entire_line(), but only returns the part of the current line @@ -764,7 +794,7 @@ lex_preprocess_line (void) /* Remove C-style comments begun by slash-star and terminated by star-slash or newline. */ quote = comment = 0; - for (cp = ds_value (&getl_buf); *cp; ) + for (cp = ds_c_str (&getl_buf); *cp; ) { /* If we're not commented out, toggle quoting. */ if (!comment) @@ -805,7 +835,7 @@ lex_preprocess_line (void) /* Strip trailing whitespace and terminal dot. */ { size_t len = ds_length (&getl_buf); - char *s = ds_value (&getl_buf); + char *s = ds_c_str (&getl_buf); /* Strip trailing whitespace. */ while (len > 0 && isspace ((unsigned char) s[len - 1])) @@ -830,7 +860,7 @@ lex_preprocess_line (void) as necessary. */ if (getl_interactive != 2 && getl_mode == GETL_MODE_BATCH) { - char *s = ds_value (&getl_buf); + char *s = ds_c_str (&getl_buf); if (s[0] == '+' || s[0] == '-' || s[0] == '.') s[0] = ' '; @@ -838,7 +868,7 @@ lex_preprocess_line (void) put_token = '.'; } - prog = ds_value (&getl_buf); + prog = ds_c_str (&getl_buf); } /* Token names. */ @@ -870,8 +900,9 @@ lex_token_representation (void) switch (token) { case T_ID: - case T_NUM: - return xstrdup (ds_value (&tokstr)); + case T_POS_NUM: + case T_NEG_NUM: + return xstrdup (ds_c_str (&tokstr)); break; case T_STRING: @@ -879,7 +910,7 @@ lex_token_representation (void) int hexstring = 0; char *sp, *dp; - for (sp = ds_value (&tokstr); sp < ds_end (&tokstr); sp++) + for (sp = ds_c_str (&tokstr); sp < ds_end (&tokstr); sp++) if (!isprint ((unsigned char) *sp)) { hexstring = 1; @@ -894,14 +925,14 @@ lex_token_representation (void) *dp++ = '\''; if (!hexstring) - for (sp = ds_value (&tokstr); *sp; ) + for (sp = ds_c_str (&tokstr); *sp; ) { if (*sp == '\'') *dp++ = '\''; *dp++ = (unsigned char) *sp++; } else - for (sp = ds_value (&tokstr); sp < ds_end (&tokstr); sp++) + for (sp = ds_c_str (&tokstr); sp < ds_end (&tokstr); sp++) { *dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"]; *dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"]; @@ -945,11 +976,11 @@ lex_token_representation (void) void lex_negative_to_dash (void) { - if (token == T_NUM && tokval < 0.0) + if (token == T_NEG_NUM) { - token = T_NUM; + token = T_POS_NUM; tokval = -tokval; - ds_replace (&tokstr, ds_value (&tokstr) + 1); + ds_replace (&tokstr, ds_c_str (&tokstr) + 1); save_token (); token = '-'; } @@ -968,7 +999,13 @@ lex_skip_comment (void) { for (;;) { - lex_get_line (); + if (!lex_get_line ()) + { + put_token = T_STOP; + eof = 1; + return; + } + if (put_token == '.') break; @@ -987,23 +1024,6 @@ unexpected_eof (void) msg (FE, _("Unexpected end of file.")); } -/* Returns the proper token type, either T_ID or a reserved keyword - enum, for ID[], which must contain LEN characters. */ -static inline int -check_id (const char *id, size_t len) -{ - const char **kwp; - - if (len < 2 || len > 4) - return T_ID; - - for (kwp = keywords; *kwp; kwp++) - if (!strcmp (*kwp, id)) - return T_FIRST_KEYWORD + (kwp - keywords); - - return T_ID; -} - /* When invoked, tokstr contains a string of binary, octal, or hex digits, for values of TYPE of 0, 1, or 2, respectively. The string is converted to characters having the specified values. */ @@ -1028,7 +1048,7 @@ convert_numeric_string_to_char_string (int type) "multiple of %d."), gettext (base_name), ds_length (&tokstr), cpb); - p = ds_value (&tokstr); + p = ds_c_str (&tokstr); for (i = 0; i < nb; i++) { int value; @@ -1058,7 +1078,7 @@ convert_numeric_string_to_char_string (int type) value = value * base + v; } - ds_value (&tokstr)[i] = (unsigned char) value; + ds_c_str (&tokstr)[i] = (unsigned char) value; } ds_truncate (&tokstr, nb); @@ -1097,7 +1117,7 @@ parse_string (int type) break; } - ds_putchar (&tokstr, *prog++); + ds_putc (&tokstr, *prog++); } prog++; @@ -1167,7 +1187,7 @@ finish: int warned = 0; for (i = 0; i < ds_length (&tokstr); i++) - if (ds_value (&tokstr)[i] == 0) + if (ds_c_str (&tokstr)[i] == 0) { if (!warned) { @@ -1175,7 +1195,7 @@ finish: "characters. Replacing with spaces.")); warned = 1; } - ds_value (&tokstr)[i] = ' '; + ds_c_str (&tokstr)[i] = ' '; } } @@ -1203,12 +1223,13 @@ dump_token (void) fprintf (stderr, "ID\t%s\n", tokid); break; - case T_NUM: + case T_POS_NUM: + case T_NEG_NUM: fprintf (stderr, "NUM\t%f\n", tokval); break; case T_STRING: - fprintf (stderr, "STRING\t\"%s\"\n", ds_value (&tokstr)); + fprintf (stderr, "STRING\t\"%s\"\n", ds_c_str (&tokstr)); break; case T_STOP: @@ -1231,4 +1252,4 @@ dump_token (void) break; } } -#endif /* DEBUGGING */ +#endif /* DUMP_TOKENS */