From f626663385d57d16eff7e1d06e07b558cf669385 Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Wed, 21 Jul 2021 22:48:07 -0700 Subject: [PATCH] Turn !* into a single token, for macro expansion purposes. --- src/language/lexer/macro.c | 18 ++++-------------- src/language/lexer/segment.c | 15 ++++++++++++++- tests/language/lexer/scan.at | 5 ++++- tests/language/lexer/segment.at | 8 +++++--- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/language/lexer/macro.c b/src/language/lexer/macro.c index 22e92ca05e..ab1e78efcf 100644 --- a/src/language/lexer/macro.c +++ b/src/language/lexer/macro.c @@ -897,16 +897,6 @@ expand_macro_function (const struct macro_expander *me, const struct macro_token *input, size_t n_input, struct string *output); -/* Returns true if the N tokens within MTS start with !*, false otherwise. */ -static bool -is_bang_star (const struct macro_token *mts, size_t n) -{ - return (n > 1 - && mts[0].token.type == T_MACRO_ID - && ss_equals (mts[0].token.string, ss_cstr ("!")) - && mts[1].token.type == T_ASTERISK); -} - /* Parses one function argument from the N_INPUT tokens in INPUT Each argument to a macro function is one of: @@ -941,7 +931,7 @@ parse_function_arg (const struct macro_expander *me, return 1; } - if (is_bang_star (input, n_input)) + if (ss_equals (token->string, ss_cstr ("!*"))) { for (size_t i = 0; i < me->macro->n_params; i++) { @@ -951,7 +941,7 @@ parse_function_arg (const struct macro_expander *me, ds_put_byte (farg, ' '); macro_tokens_to_syntax (me->args[i], farg, NULL, NULL); } - return 2; + return 1; } const char *var = stringi_map_find__ (me->vars, @@ -1998,11 +1988,11 @@ macro_expand__ (const struct macro_token *mts, size_t n, macro_expand_arg (me, param - me->macro->params, exp); return 1; } - else if (is_bang_star (mts, n)) + else if (ss_equals (token->string, ss_cstr ("!*"))) { for (size_t j = 0; j < me->macro->n_params; j++) macro_expand_arg (me, j, exp); - return 2; + return 1; } } diff --git a/src/language/lexer/segment.c b/src/language/lexer/segment.c index 5c298a781d..1cf7ac6f6d 100644 --- a/src/language/lexer/segment.c +++ b/src/language/lexer/segment.c @@ -1018,7 +1018,20 @@ segmenter_parse_mid_command__ (struct segmenter *s, s, input, n, eof, type); case '!': - return segmenter_parse_id__ (s, input, n, eof, type); + if (n < 2) + { + if (!eof) + return -1; + *type = SEG_PUNCT; + return 1; + } + else if (input[1] == '*') + { + *type = SEG_MACRO_ID; + return 2; + } + else + return segmenter_parse_id__ (s, input, n, eof, type); default: if (lex_uc_is_space (uc)) diff --git a/tests/language/lexer/scan.at b/tests/language/lexer/scan.at index 53e96921f3..187c22efee 100644 --- a/tests/language/lexer/scan.at +++ b/tests/language/lexer/scan.at @@ -25,7 +25,7 @@ m4_define([PSPP_CHECK_SCAN], AT_SETUP([identifiers]) AT_KEYWORDS([scan]) AT_DATA([input], [dnl -a aB i5 $x @efg @@. !abcd #.# .x _z. +a aB i5 $x @efg @@. !abcd !* !*a #.# .x _z. abcd. abcd. QRSTUV./* end of line comment */ QrStUv./* end of line comment */ @&t@ @@ -40,6 +40,9 @@ ID "$x" ID "@efg" ID "@@." MACRO_ID "!abcd" +MACRO_ID "!*" +MACRO_ID "!*" +ID "a" ID "#.#" MACRO_PUNCT "." ID "x" diff --git a/tests/language/lexer/segment.at b/tests/language/lexer/segment.at index 5d9440fe21..8936ae98df 100644 --- a/tests/language/lexer/segment.at +++ b/tests/language/lexer/segment.at @@ -291,8 +291,8 @@ AT_SETUP([punctuation]) AT_KEYWORDS([segment]) AT_DATA([input], [dnl ~ & | = >= > <= < ~= <> ( ) , - + * / [[ ]] ** -~&|=>=><=<~=<>(),-+*/[[]]** -% : ; ? _ ` { } ~ +~&|=>=><=<~=<>(),-+*/[[]]**!* +% : ; ? _ ` { } ~ !* ]) AT_DATA([expout-base], [dnl punct ~ space @@ -337,6 +337,7 @@ punct / punct [[ punct ]] punct ** +macro_id !* newline \n (later) punct % space @@ -347,7 +348,8 @@ punct \_ space punct ` space punct { space punct } space -punct ~ +punct ~ space +macro_id !* -newline \n (later) - end -- 2.30.2