X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Flexer%2Flexer.c;h=a409a83ee957647a7b5d9b5f8195732ac8e0a61c;hb=17753a2ef0a1e33b5478a970c747a2a57454ee07;hp=8d0352ae8ab88ad0c3cf309d765c2b56fe01ae7e;hpb=52c400a0ac67e4a43c5cae5a0d40e1452326240c;p=pspp-builds.git diff --git a/src/language/lexer/lexer.c b/src/language/lexer/lexer.c index 8d0352ae..a409a83e 100644 --- a/src/language/lexer/lexer.c +++ b/src/language/lexer/lexer.c @@ -49,11 +49,7 @@ struct lexer int token; /* Current token. */ double tokval; /* T_POS_NUM, T_NEG_NUM: the token's value. */ - char tokid [VAR_NAME_LEN + 1]; /* T_ID: the identifier. */ - - struct string tokstr; /* T_ID, T_STRING: token string value. - For T_ID, this is not truncated as is - tokid. */ + struct string tokstr; /* T_ID, T_STRING: token string value. */ char *prog; /* Pointer to next token in line_buffer. */ bool dot; /* True only if this line ends with a terminal dot. */ @@ -138,7 +134,6 @@ restore_token (struct lexer *lexer) assert (lexer->put_token != 0); lexer->token = lexer->put_token; ds_assign_string (&lexer->tokstr, &lexer->put_tokstr); - str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr)); lexer->tokval = lexer->put_tokval; lexer->put_token = 0; } @@ -186,7 +181,7 @@ lex_get (struct lexer *lexer) if (lexer->dot) { lexer->dot = 0; - lexer->token = '.'; + lexer->token = T_ENDCMD; return; } else if (!lex_get_line (lexer)) @@ -215,22 +210,17 @@ lex_get (struct lexer *lexer) { char *tail; - /* `-' can introduce a negative number, or it can be a - token by itself. If it is not followed by a digit or a - decimal point, it is definitely not a number. - Otherwise, it might be either, but most of the time we - want it as a number. When the syntax calls for a `-' - token, lex_negative_to_dash() must be used to break - negative numbers into two tokens. */ + /* `-' can introduce a negative number, or it can be a token by + itself. */ if (*lexer->prog == '-') { - ds_put_char (&lexer->tokstr, *lexer->prog++); + ds_put_byte (&lexer->tokstr, *lexer->prog++); while (c_isspace ((unsigned char) *lexer->prog)) lexer->prog++; if (!c_isdigit ((unsigned char) *lexer->prog) && *lexer->prog != '.') { - lexer->token = '-'; + lexer->token = T_DASH; break; } lexer->token = T_NEG_NUM; @@ -240,20 +230,20 @@ lex_get (struct lexer *lexer) /* Parse the number, copying it into tokstr. */ while (c_isdigit ((unsigned char) *lexer->prog)) - ds_put_char (&lexer->tokstr, *lexer->prog++); + ds_put_byte (&lexer->tokstr, *lexer->prog++); if (*lexer->prog == '.') { - ds_put_char (&lexer->tokstr, *lexer->prog++); + ds_put_byte (&lexer->tokstr, *lexer->prog++); while (c_isdigit ((unsigned char) *lexer->prog)) - ds_put_char (&lexer->tokstr, *lexer->prog++); + ds_put_byte (&lexer->tokstr, *lexer->prog++); } if (*lexer->prog == 'e' || *lexer->prog == 'E') { - ds_put_char (&lexer->tokstr, *lexer->prog++); + ds_put_byte (&lexer->tokstr, *lexer->prog++); if (*lexer->prog == '+' || *lexer->prog == '-') - ds_put_char (&lexer->tokstr, *lexer->prog++); + ds_put_byte (&lexer->tokstr, *lexer->prog++); while (c_isdigit ((unsigned char) *lexer->prog)) - ds_put_char (&lexer->tokstr, *lexer->prog++); + ds_put_byte (&lexer->tokstr, *lexer->prog++); } /* Parse as floating point. */ @@ -265,7 +255,7 @@ lex_get (struct lexer *lexer) lexer->tokval = 0.0; ds_clear (&lexer->tokstr); - ds_put_char (&lexer->tokstr, '0'); + ds_put_byte (&lexer->tokstr, '0'); } break; @@ -275,10 +265,45 @@ lex_get (struct lexer *lexer) lexer->token = parse_string (lexer, CHARACTER_STRING); break; - case '(': case ')': case ',': case '=': case '+': case '/': - case '[': case ']': - lexer->token = *lexer->prog++; - break; + case '+': + lexer->token = T_PLUS; + lexer->prog++; + break; + + case '/': + lexer->token = T_SLASH; + lexer->prog++; + break; + + case '=': + lexer->token = T_EQUALS; + lexer->prog++; + break; + + case '(': + lexer->token = T_LPAREN; + lexer->prog++; + break; + + case ')': + lexer->token = T_RPAREN; + lexer->prog++; + break; + + case '[': + lexer->token = T_LBRACK; + lexer->prog++; + break; + + case ']': + lexer->token = T_RBRACK; + lexer->prog++; + break; + + case ',': + lexer->token = T_COMMA; + lexer->prog++; + break; case '*': if (*++lexer->prog == '*') @@ -287,7 +312,7 @@ lex_get (struct lexer *lexer) lexer->token = T_EXP; } else - lexer->token = '*'; + lexer->token = T_ASTERISK; break; case '<': @@ -375,8 +400,7 @@ lex_get (struct lexer *lexer) } } -/* Parses an identifier at the current position into tokid and - tokstr. +/* Parses an identifier at the current position into tokstr. Returns the correct token type. */ static int parse_id (struct lexer *lexer) @@ -390,7 +414,6 @@ parse_id (struct lexer *lexer) lexer->prog += ss_length (id); ds_assign_substring (&lexer->tokstr, id); - str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr)); return lex_id_to_token (id); } @@ -421,7 +444,7 @@ lex_error (struct lexer *lexer, const char *message, ...) if (lexer->token == T_STOP) ds_put_cstr (&s, _("Syntax error at end of file")); - else if (lexer->token == '.') + else if (lexer->token == T_ENDCMD) ds_put_cstr (&s, _("Syntax error at end of command")); else { @@ -452,7 +475,7 @@ lex_error (struct lexer *lexer, const char *message, ...) int lex_end_of_command (struct lexer *lexer) { - if (lexer->token != '.') + if (lexer->token != T_ENDCMD) { lex_error (lexer, _("expecting end of command")); return CMD_FAILURE; @@ -512,7 +535,7 @@ lex_integer (struct lexer *lexer) /* If TOK is the current token, skips it and returns true Otherwise, returns false. */ bool -lex_match (struct lexer *lexer, int t) +lex_match (struct lexer *lexer, enum token_type t) { if (lexer->token == t) { @@ -541,7 +564,7 @@ bool lex_match_id_n (struct lexer *lexer, const char *s, size_t n) { if (lexer->token == T_ID - && lex_id_match_n (ss_cstr (s), ss_cstr (lexer->tokid), n)) + && lex_id_match_n (ss_cstr (s), lex_tokss (lexer), n)) { lex_get (lexer); return true; @@ -584,7 +607,7 @@ lex_force_match_id (struct lexer *lexer, const char *s) /* If the current token is T, skips the token. Otherwise, reports an error and returns from the current function with return value false. */ bool -lex_force_match (struct lexer *lexer, int t) +lex_force_match (struct lexer *lexer, enum token_type t) { if (lexer->token == t) { @@ -652,13 +675,8 @@ lex_force_id (struct lexer *lexer) /* Weird token functions. */ -/* Returns the first character of the next token, except that if the - next token is not an identifier, the character returned will not be - a character that can begin an identifier. Specifically, the - hexstring lead-in X' causes lookahead() to return '. Note that an - alphanumeric return value doesn't guarantee an ID token, it could - also be a reserved-word token. */ -int +/* Returns the likely type of the next token, or 0 if it's hard to tell. */ +enum token_type lex_look_ahead (struct lexer *lexer) { if (lexer->put_token) @@ -677,7 +695,7 @@ lex_look_ahead (struct lexer *lexer) break; if (lexer->dot) - return '.'; + return T_ENDCMD; else if (!lex_get_line (lexer)) return 0; @@ -685,36 +703,84 @@ lex_look_ahead (struct lexer *lexer) return lexer->put_token; } - if ((toupper ((unsigned char) *lexer->prog) == 'X' - || toupper ((unsigned char) *lexer->prog) == 'B' - || toupper ((unsigned char) *lexer->prog) == 'O') - && (lexer->prog[1] == '\'' || lexer->prog[1] == '"')) - return '\''; + switch (toupper ((unsigned char) *lexer->prog)) + { + case 'X': case 'B': case 'O': + if (lexer->prog[1] == '\'' || lexer->prog[1] == '"') + return T_STRING; + /* Fall through */ + + case '-': + return T_DASH; + + case '.': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return T_POS_NUM; + + case '\'': case '"': + return T_STRING; + + case '+': + return T_PLUS; - return *lexer->prog; + case '/': + return T_SLASH; + + case '=': + return T_EQUALS; + + case '(': + return T_LPAREN; + + case ')': + return T_RPAREN; + + case '[': + return T_LBRACK; + + case ']': + return T_RBRACK; + + case ',': + return T_COMMA; + + case '*': + return lexer->prog[1] == '*' ? T_EXP : T_ASTERISK; + + case '<': + return (lexer->prog[1] == '=' ? T_LE + : lexer->prog[1] == '>' ? T_NE + : T_LT); + + case '>': + return lexer->prog[1] == '=' ? T_GE : T_GT; + + case '~': + return lexer->prog[1] == '=' ? T_NE : T_NOT; + + case '&': + return T_AND; + + case '|': + return T_OR; + + default: + if (lex_is_id1 (*lexer->prog)) + return T_ID; + return 0; + } } } /* Makes the current token become the next token to be read; the current token is set to T. */ void -lex_put_back (struct lexer *lexer, int t) +lex_put_back (struct lexer *lexer, enum token_type t) { save_token (lexer); lexer->token = t; } - -/* Makes the current token become the next token to be read; the - current token is set to the identifier ID. */ -void -lex_put_back_id (struct lexer *lexer, const char *id) -{ - assert (lex_id_to_token (ss_cstr (id)) == T_ID); - save_token (lexer); - lexer->token = T_ID; - ds_assign_cstr (&lexer->tokstr, id); - str_copy_trunc (lexer->tokid, sizeof lexer->tokid, ds_cstr (&lexer->tokstr)); -} /* Weird line processing functions. */ @@ -771,7 +837,7 @@ lex_discard_rest_of_command (struct lexer *lexer) { if (!getl_is_interactive (lexer->ss)) { - while (lexer->token != T_STOP && lexer->token != '.') + while (lexer->token != T_STOP && lexer->token != T_ENDCMD) lex_get (lexer); } else @@ -840,8 +906,7 @@ lex_preprocess_line (struct string *line, { strip_comments (line); ds_rtrim (line, ss_cstr (CC_SPACES)); - *line_ends_command = (ds_chomp (line, settings_get_endcmd ()) - || (ds_is_empty (line) && settings_get_nulline ())); + *line_ends_command = ds_chomp (line, '.') || ds_is_empty (line); *line_starts_command = false; if (syntax == GETL_BATCH) { @@ -883,7 +948,7 @@ lex_get_line (struct lexer *lexer) &line_starts_command, &lexer->dot); if (line_starts_command) - lexer->put_token = '.'; + lexer->put_token = T_ENDCMD; lexer->prog = ds_cstr (&lexer->line_buffer); return true; @@ -893,20 +958,97 @@ lex_get_line (struct lexer *lexer) /* Returns the name of a token. */ const char * -lex_token_name (int token) +lex_token_name (enum token_type token) { - if (lex_is_keyword (token)) - return lex_id_name (token); - else if (token < 256) + switch (token) { - static char t[256][2]; - char *s = t[token]; - s[0] = token; - s[1] = '\0'; - return s; + case T_ID: + case T_POS_NUM: + case T_NEG_NUM: + case T_STRING: + case TOKEN_N_TYPES: + NOT_REACHED (); + + case T_STOP: + return ""; + + case T_ENDCMD: + return "."; + + case T_PLUS: + return "+"; + + case T_DASH: + return "-"; + + case T_ASTERISK: + return "*"; + + case T_SLASH: + return "/"; + + case T_EQUALS: + return "="; + + case T_LPAREN: + return "("; + + case T_RPAREN: + return ")"; + + case T_LBRACK: + return "["; + + case T_RBRACK: + return "]"; + + case T_COMMA: + return ","; + + case T_AND: + return "AND"; + + case T_OR: + return "OR"; + + case T_NOT: + return "NOT"; + + case T_EQ: + return "EQ"; + + case T_GE: + return ">="; + + case T_GT: + return ">"; + + case T_LE: + return "<="; + + case T_LT: + return "<"; + + case T_NE: + return "~="; + + case T_ALL: + return "ALL"; + + case T_BY: + return "BY"; + + case T_TO: + return "TO"; + + case T_WITH: + return "WITH"; + + case T_EXP: + return "**"; } - else - NOT_REACHED (); + + NOT_REACHED (); } /* Returns an ASCII representation of the current token as a @@ -921,37 +1063,37 @@ lex_token_representation (struct lexer *lexer) case T_ID: case T_POS_NUM: case T_NEG_NUM: - return ds_xstrdup (&lexer->tokstr); - break; + return ss_xstrdup (lex_tokss (lexer)); case T_STRING: { + struct substring ss; int hexstring = 0; char *sp, *dp; - for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++) + ss = lex_tokss (lexer); + for (sp = ss_data (ss); sp < ss_end (ss); sp++) if (!c_isprint ((unsigned char) *sp)) { hexstring = 1; break; } - token_rep = xmalloc (2 + ds_length (&lexer->tokstr) * 2 + 1 + 1); + token_rep = xmalloc (2 + ss_length (ss) * 2 + 1 + 1); dp = token_rep; if (hexstring) *dp++ = 'X'; *dp++ = '\''; - if (!hexstring) - for (sp = ds_cstr (&lexer->tokstr); *sp; ) + for (sp = ss_data (ss); sp < ss_end (ss); sp++) + if (!hexstring) { if (*sp == '\'') *dp++ = '\''; - *dp++ = (unsigned char) *sp++; + *dp++ = (unsigned char) *sp; } - else - for (sp = ds_cstr (&lexer->tokstr); sp < ds_end (&lexer->tokstr); sp++) + else { *dp++ = (((unsigned char) *sp) >> 4)["0123456789ABCDEF"]; *dp++ = (((unsigned char) *sp) & 15)["0123456789ABCDEF"]; @@ -961,42 +1103,14 @@ lex_token_representation (struct lexer *lexer) return token_rep; } - break; - - case T_STOP: - token_rep = xmalloc (1); - *token_rep = '\0'; - return token_rep; - - case T_EXP: - return xstrdup ("**"); default: return xstrdup (lex_token_name (lexer->token)); } - - NOT_REACHED (); } /* Really weird functions. */ -/* Most of the time, a `-' is a lead-in to a negative number. But - sometimes it's actually part of the syntax. If a dash can be part - of syntax then this function is called to rip it off of a - number. */ -void -lex_negative_to_dash (struct lexer *lexer) -{ - if (lexer->token == T_NEG_NUM) - { - lexer->token = T_POS_NUM; - lexer->tokval = -lexer->tokval; - ds_assign_substring (&lexer->tokstr, ds_substr (&lexer->tokstr, 1, SIZE_MAX)); - save_token (lexer); - lexer->token = '-'; - } -} - /* Skip a COMMENT command. */ void lex_skip_comment (struct lexer *lexer) @@ -1010,7 +1124,7 @@ lex_skip_comment (struct lexer *lexer) return; } - if (lexer->put_token == '.') + if (lexer->put_token == T_ENDCMD) break; ds_cstr (&lexer->line_buffer); /* Ensures ds_end will point to a valid char */ @@ -1135,7 +1249,7 @@ parse_string (struct lexer *lexer, enum string_type type) break; } - ds_put_char (&lexer->tokstr, *lexer->prog++); + ds_put_byte (&lexer->tokstr, *lexer->prog++); } lexer->prog++; @@ -1200,7 +1314,7 @@ finish: /* Token Accessor Functions */ -int +enum token_type lex_token (const struct lexer *lexer) { return lexer->token; @@ -1212,16 +1326,22 @@ lex_tokval (const struct lexer *lexer) return lexer->tokval; } +/* Returns the null-terminated string value associated with LEXER's current + token. For a T_ID token, this is the identifier, and for a T_STRING token, + this is the string. For other tokens the value is undefined. */ const char * -lex_tokid (const struct lexer *lexer) +lex_tokcstr (const struct lexer *lexer) { - return lexer->tokid; + return ds_cstr (&lexer->tokstr); } -const struct string * -lex_tokstr (const struct lexer *lexer) +/* Returns the string value associated with LEXER's current token. For a T_ID + token, this is the identifier, and for a T_STRING token, this is the string. + For other tokens the value is undefined. */ +struct substring +lex_tokss (const struct lexer *lexer) { - return &lexer->tokstr; + return ds_ss (&lexer->tokstr); } /* If the lexer is positioned at the (pseudo)identifier S, which @@ -1236,13 +1356,13 @@ lex_match_hyphenated_word (struct lexer *lexer, const char *s) if (hyphen == NULL) return lex_match_id (lexer, s); else if (lexer->token != T_ID - || !lex_id_match (ss_buffer (s, hyphen - s), ss_cstr (lexer->tokid)) - || lex_look_ahead (lexer) != '-') + || !lex_id_match (ss_buffer (s, hyphen - s), lex_tokss (lexer)) + || lex_look_ahead (lexer) != T_DASH) return false; else { lex_get (lexer); - lex_force_match (lexer, '-'); + lex_force_match (lexer, T_DASH); lex_force_match_id (lexer, hyphen + 1); return true; }