1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2021 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "language/lexer/macro.h"
25 #include "data/settings.h"
26 #include "language/lexer/lexer.h"
27 #include "language/lexer/segment.h"
28 #include "language/lexer/scan.h"
29 #include "libpspp/assertion.h"
30 #include "libpspp/cast.h"
31 #include "libpspp/i18n.h"
32 #include "libpspp/message.h"
33 #include "libpspp/str.h"
34 #include "libpspp/string-array.h"
35 #include "libpspp/string-map.h"
36 #include "libpspp/stringi-set.h"
38 #include "gl/c-ctype.h"
39 #include "gl/ftoastr.h"
42 #define _(msgid) gettext (msgid)
45 macro_token_copy (struct macro_token *dst, const struct macro_token *src)
47 token_copy (&dst->token, &src->token);
48 ss_alloc_substring (&dst->representation, src->representation);
52 macro_token_uninit (struct macro_token *mt)
54 token_uninit (&mt->token);
55 ss_dealloc (&mt->representation);
59 macro_token_to_representation (struct macro_token *mt, struct string *s)
61 ds_put_substring (s, mt->representation);
65 is_macro_keyword (struct substring s)
67 static struct stringi_set keywords = STRINGI_SET_INITIALIZER (keywords);
68 if (stringi_set_is_empty (&keywords))
70 static const char *kws[] = {
91 for (size_t i = 0; i < sizeof kws / sizeof *kws; i++)
92 stringi_set_insert (&keywords, kws[i]);
95 ss_ltrim (&s, ss_cstr ("!"));
96 return stringi_set_contains_len (&keywords, s.string, s.length);
100 macro_tokens_copy (struct macro_tokens *dst, const struct macro_tokens *src)
102 *dst = (struct macro_tokens) {
103 .mts = xmalloc (src->n * sizeof *dst->mts),
107 for (size_t i = 0; i < src->n; i++)
108 macro_token_copy (&dst->mts[i], &src->mts[i]);
112 macro_tokens_uninit (struct macro_tokens *mts)
114 for (size_t i = 0; i < mts->n; i++)
115 macro_token_uninit (&mts->mts[i]);
120 macro_tokens_add_uninit (struct macro_tokens *mts)
122 if (mts->n >= mts->allocated)
123 mts->mts = x2nrealloc (mts->mts, &mts->allocated, sizeof *mts->mts);
124 return &mts->mts[mts->n++];
128 macro_tokens_add (struct macro_tokens *mts, const struct macro_token *mt)
130 macro_token_copy (macro_tokens_add_uninit (mts), mt);
134 macro_tokens_from_string (struct macro_tokens *mts, const struct substring src,
135 enum segmenter_mode mode)
139 struct segmenter segmenter;
140 struct substring body;
143 struct state state = {
144 .segmenter = segmenter_init (mode, true),
147 struct state saved = state;
149 while (state.body.length > 0)
151 struct macro_token mt = {
152 .token = { .type = T_STOP },
153 .representation = { .string = state.body.string },
155 struct token *token = &mt.token;
157 struct scanner scanner;
158 scanner_init (&scanner, token);
162 enum segment_type type;
163 int seg_len = segmenter_push (&state.segmenter, state.body.string,
164 state.body.length, true, &type);
165 assert (seg_len >= 0);
167 struct substring segment = ss_head (state.body, seg_len);
168 ss_advance (&state.body, seg_len);
170 enum scan_result result = scanner_push (&scanner, type, segment, token);
171 if (result == SCAN_SAVE)
173 else if (result == SCAN_BACK)
178 else if (result == SCAN_DONE)
182 /* We have a token in 'token'. */
183 if (is_scan_type (token->type))
185 if (token->type != SCAN_SKIP)
188 /* XXX report error */
193 mt.representation.length = state.body.string - mt.representation.string;
194 macro_tokens_add (mts, &mt);
196 token_uninit (token);
201 macro_tokens_print (const struct macro_tokens *mts, FILE *stream)
203 for (size_t i = 0; i < mts->n; i++)
204 token_print (&mts->mts[i].token, stream);
209 TC_ENDCMD, /* No space before or after (new-line after). */
210 TC_BINOP, /* Space on both sides. */
211 TC_COMMA, /* Space afterward. */
212 TC_ID, /* Don't need spaces except sequentially. */
213 TC_PUNCT, /* Don't need spaces except sequentially. */
217 needs_space (enum token_class prev, enum token_class next)
219 /* Don't need a space before or after the end of a command.
220 (A new-line is needed afterward as a special case.) */
221 if (prev == TC_ENDCMD || next == TC_ENDCMD)
224 /* Binary operators always have a space on both sides. */
225 if (prev == TC_BINOP || next == TC_BINOP)
228 /* A comma always has a space afterward. */
229 if (prev == TC_COMMA)
232 /* Otherwise, PREV is TC_ID or TC_PUNCT, which only need a space if there are
233 two or them in a row. */
237 static enum token_class
238 classify_token (enum token_type type)
291 macro_tokens_to_representation (struct macro_tokens *mts, struct string *s,
292 size_t *ofs, size_t *len)
294 assert ((ofs != NULL) == (len != NULL));
299 for (size_t i = 0; i < mts->n; i++)
303 enum token_type prev = mts->mts[i - 1].token.type;
304 enum token_type next = mts->mts[i].token.type;
306 if (prev == T_ENDCMD)
307 ds_put_byte (s, '\n');
310 enum token_class pc = classify_token (prev);
311 enum token_class nc = classify_token (next);
312 if (needs_space (pc, nc))
313 ds_put_byte (s, ' ');
318 ofs[i] = s->ss.length;
319 macro_token_to_representation (&mts->mts[i], s);
321 len[i] = s->ss.length - ofs[i];
326 macro_destroy (struct macro *m)
333 for (size_t i = 0; i < m->n_params; i++)
335 struct macro_param *p = &m->params[i];
338 macro_tokens_uninit (&p->def);
346 token_uninit (&p->charend);
350 token_uninit (&p->enclose[0]);
351 token_uninit (&p->enclose[1]);
359 macro_tokens_uninit (&m->body);
364 macro_set_create (void)
366 struct macro_set *set = xmalloc (sizeof *set);
367 *set = (struct macro_set) {
368 .macros = HMAP_INITIALIZER (set->macros),
374 macro_set_destroy (struct macro_set *set)
379 struct macro *macro, *next;
380 HMAP_FOR_EACH_SAFE (macro, next, struct macro, hmap_node, &set->macros)
382 hmap_delete (&set->macros, ¯o->hmap_node);
383 macro_destroy (macro);
385 hmap_destroy (&set->macros);
390 hash_macro_name (const char *name)
392 return utf8_hash_case_string (name, 0);
395 static struct macro *
396 macro_set_find__ (struct macro_set *set, const char *name)
399 HMAP_FOR_EACH_WITH_HASH (macro, struct macro, hmap_node,
400 hash_macro_name (name), &set->macros)
401 if (!utf8_strcasecmp (macro->name, name))
408 macro_set_find (const struct macro_set *set, const char *name)
410 return macro_set_find__ (CONST_CAST (struct macro_set *, set), name);
413 /* Adds M to SET. M replaces any existing macro with the same name. Takes
416 macro_set_add (struct macro_set *set, struct macro *m)
418 struct macro *victim = macro_set_find__ (set, m->name);
421 hmap_delete (&set->macros, &victim->hmap_node);
422 macro_destroy (victim);
425 hmap_insert (&set->macros, &m->hmap_node, hash_macro_name (m->name));
428 struct macro_expansion_stack
430 const struct macro_expansion_stack *next;
432 const char *file_name;
437 static void PRINTF_FORMAT (3, 4)
438 macro_error (const struct macro_expansion_stack *stack,
439 const struct macro_token *mt,
440 const char *format, ...)
442 struct msg_stack **ms = NULL;
443 size_t allocated_ms = 0;
446 for (const struct macro_expansion_stack *p = stack; p; p = p->next)
448 if (n_ms >= allocated_ms)
449 ms = x2nrealloc (ms, &allocated_ms, sizeof *ms);
451 /* TRANSLATORS: These strings are used for explaining the context of an
452 error. The "While expanding" message appears first, followed by zero
453 or more of the "inside expansion" messages. `innermost',
454 `next_inner`, etc., are names of macros, and `foobar' is a piece of
457 foo.sps:12: At `foobar' in the expansion of 'innermost',
458 foo.sps:23: inside the expansion of 'next_inner',
459 foo.sps:34: inside the expansion of 'next_inner2',
460 foo.sps:45: inside the expansion of 'outermost',
461 foo.sps:76: This is the actual error message. */
465 if (mt && mt->representation.length)
468 lex_ellipsize (mt->representation, syntax, sizeof syntax);
469 description = xasprintf (_("At `%s' in the expansion of `%s',"),
473 description = xasprintf (_("In the expansion of `%s',"), p->name);
476 description = xasprintf (_("inside the expansion of `%s',"), p->name);
478 ms[n_ms] = xmalloc (sizeof *ms[n_ms]);
479 *ms[n_ms] = (struct msg_stack) {
481 .file_name = xstrdup_if_nonnull (p->file_name),
482 .first_line = p->first_line,
483 .last_line = p->last_line,
485 .description = description,
491 va_start (args, format);
492 char *s = xvasprintf (format, args);
495 struct msg *m = xmalloc (sizeof *m);
497 .category = MSG_C_SYNTAX,
498 .severity = MSG_S_ERROR,
511 /* Accumulating tokens in me->params toward the end of any type of
515 /* Expecting the opening delimiter of an ARG_ENCLOSE argument. */
518 /* Expecting a keyword for a keyword argument. */
521 /* Expecting an equal sign for a keyword argument. */
526 struct macro_expander
528 const struct macro_set *macros;
533 const struct macro *macro;
534 struct macro_tokens **args;
535 const struct macro_param *param;
539 me_finished (struct macro_expander *me)
541 for (size_t i = 0; i < me->macro->n_params; i++)
544 me->args[i] = xmalloc (sizeof *me->args[i]);
545 macro_tokens_copy (me->args[i], &me->macro->params[i].def);
551 me_next_arg (struct macro_expander *me)
555 assert (!me->macro->n_params);
556 return me_finished (me);
558 else if (me->param->positional)
561 if (me->param >= &me->macro->params[me->macro->n_params])
562 return me_finished (me);
565 me->state = (!me->param->positional ? ME_KEYWORD
566 : me->param->arg_type == ARG_ENCLOSE ? ME_ENCLOSE
573 for (size_t i = 0; i < me->macro->n_params; i++)
576 me->state = ME_KEYWORD;
579 return me_finished (me);
584 me_error (struct macro_expander *me)
586 me->state = ME_ERROR;
591 me_add_arg (struct macro_expander *me, const struct macro_token *mt)
593 const struct macro_param *p = me->param;
595 const struct token *token = &mt->token;
596 if ((token->type == T_ENDCMD || token->type == T_STOP)
597 && p->arg_type != ARG_CMDEND)
599 msg (SE, _("Unexpected end of command reading argument %s "
600 "to macro %s."), me->param->name, me->macro->name);
602 return me_error (me);
607 struct macro_tokens **argp = &me->args[p - me->macro->params];
609 *argp = xzalloc (sizeof **argp);
610 struct macro_tokens *arg = *argp;
611 if (p->arg_type == ARG_N_TOKENS)
613 macro_tokens_add (arg, mt);
614 if (arg->n >= p->n_tokens)
615 return me_next_arg (me);
618 else if (p->arg_type == ARG_CMDEND)
620 if (token->type == T_ENDCMD || token->type == T_STOP)
621 return me_next_arg (me);
622 macro_tokens_add (arg, mt);
627 const struct token *end
628 = p->arg_type == ARG_CHAREND ? &p->charend : &p->enclose[1];
629 if (token_equal (token, end))
630 return me_next_arg (me);
631 macro_tokens_add (arg, mt);
637 me_expected (struct macro_expander *me, const struct macro_token *actual,
638 const struct token *expected)
640 const struct substring actual_s
641 = (actual->representation.length ? actual->representation
642 : ss_cstr (_("<end of input>")));
643 char *expected_s = token_to_string (expected);
644 msg (SE, _("Found `%.*s' while expecting `%s' reading argument %s "
646 (int) actual_s.length, actual_s.string, expected_s,
647 me->param->name, me->macro->name);
650 return me_error (me);
654 me_enclose (struct macro_expander *me, const struct macro_token *mt)
656 const struct token *token = &mt->token;
659 if (token_equal (&me->param->enclose[0], token))
665 return me_expected (me, mt, &me->param->enclose[0]);
668 static const struct macro_param *
669 macro_find_parameter_by_name (const struct macro *m, struct substring name)
671 ss_ltrim (&name, ss_cstr ("!"));
673 for (size_t i = 0; i < m->n_params; i++)
675 const struct macro_param *p = &m->params[i];
676 struct substring p_name = ss_cstr (p->name + 1);
677 if (!utf8_strncasecmp (p_name.string, p_name.length,
678 name.string, name.length))
685 me_keyword (struct macro_expander *me, const struct macro_token *mt)
687 const struct token *token = &mt->token;
688 if (token->type != T_ID)
689 return me_finished (me);
691 const struct macro_param *p = macro_find_parameter_by_name (me->macro,
695 size_t arg_index = p - me->macro->params;
697 if (me->args[arg_index])
700 _("Argument %s multiply specified in call to macro %s."),
701 p->name, me->macro->name);
702 return me_error (me);
706 me->state = ME_EQUALS;
710 return me_finished (me);
714 me_equals (struct macro_expander *me, const struct macro_token *mt)
716 const struct token *token = &mt->token;
719 if (token->type == T_EQUALS)
725 return me_expected (me, mt, &(struct token) { .type = T_EQUALS });
729 macro_expander_create (const struct macro_set *macros,
730 const struct token *token,
731 struct macro_expander **mep)
734 if (macro_set_is_empty (macros))
736 if (token->type != T_ID && token->type != T_MACRO_ID)
739 const struct macro *macro = macro_set_find (macros, token->string.string);
743 struct macro_expander *me = xmalloc (sizeof *me);
744 *me = (struct macro_expander) {
751 if (!macro->n_params)
755 me->state = (!macro->params[0].positional ? ME_KEYWORD
756 : macro->params[0].arg_type == ARG_ENCLOSE ? ME_ENCLOSE
758 me->args = xcalloc (macro->n_params, sizeof *me->args);
759 me->param = macro->params;
765 macro_expander_destroy (struct macro_expander *me)
770 for (size_t i = 0; i < me->macro->n_params; i++)
773 macro_tokens_uninit (me->args[i]);
780 /* Adds TOKEN to the collection of tokens in ME that potentially need to be
783 Returns -1 if the tokens added do not actually invoke a macro. The caller
784 should consume the first token without expanding it.
786 Returns 0 if the macro expander needs more tokens, for macro arguments or to
787 decide whether this is actually a macro invocation. The caller should call
788 macro_expander_add() again with the next token.
790 Returns a positive number to indicate that the returned number of tokens
791 invoke a macro. The number returned might be less than the number of tokens
792 added because it can take a few tokens of lookahead to determine whether the
793 macro invocation is finished. The caller should call
794 macro_expander_get_expansion() to obtain the expansion. */
796 macro_expander_add (struct macro_expander *me, const struct macro_token *mt)
804 return me_add_arg (me, mt);
807 return me_enclose (me, mt);
810 return me_keyword (me, mt);
813 return me_equals (me, mt);
820 /* Each argument to a macro function is one of:
822 - A quoted string or other single literal token.
824 - An argument to the macro being expanded, e.g. !1 or a named argument.
828 - A function invocation.
830 Each function invocation yields a character sequence to be turned into a
831 sequence of tokens. The case where that character sequence is a single
832 quoted string is an important special case.
834 struct parse_macro_function_ctx
836 const struct macro_token *input;
838 int nesting_countdown;
839 const struct macro_set *macros;
840 const struct macro_expander *me;
841 const struct macro_expansion_stack *stack;
842 struct string_map *vars;
847 macro_expand (const struct macro_tokens *,
848 int nesting_countdown, const struct macro_set *,
849 const struct macro_expander *, struct string_map *vars,
850 const struct macro_expansion_stack *stack,
851 bool *expand, bool *break_,
852 struct macro_tokens *exp);
855 expand_macro_function (struct parse_macro_function_ctx *ctx,
856 struct string *output, size_t *input_consumed);
858 /* Returns true if the pair of tokens starting at offset OFS within MTS are !*,
861 is_bang_star (const struct macro_token *mts, size_t n, size_t ofs)
864 && mts[ofs].token.type == T_MACRO_ID
865 && ss_equals (mts[ofs].token.string, ss_cstr ("!"))
866 && mts[ofs + 1].token.type == T_ASTERISK);
870 parse_function_arg (struct parse_macro_function_ctx *ctx,
871 size_t i, struct string *farg)
873 const struct macro_token *tokens = ctx->input;
874 const struct token *token = &tokens[i].token;
875 if (token->type == T_MACRO_ID)
877 const struct macro_param *param = macro_find_parameter_by_name (
878 ctx->me->macro, token->string);
881 size_t param_idx = param - ctx->me->macro->params;
882 const struct macro_tokens *marg = ctx->me->args[param_idx];
883 for (size_t i = 0; i < marg->n; i++)
886 ds_put_byte (farg, ' ');
887 ds_put_substring (farg, marg->mts[i].representation);
892 if (is_bang_star (ctx->input, ctx->n_input, i))
894 for (size_t i = 0; i < ctx->me->macro->n_params; i++)
896 if (!ctx->me->macro->params[i].positional)
899 const struct macro_tokens *marg = ctx->me->args[i];
900 for (size_t j = 0; j < marg->n; j++)
903 ds_put_byte (farg, ' ');
904 ds_put_substring (farg, marg->mts[j].representation);
912 const char *value = string_map_find__ (ctx->vars,
913 token->string.string,
914 token->string.length);
917 ds_put_cstr (farg, value);
922 struct parse_macro_function_ctx subctx = {
923 .input = &ctx->input[i],
924 .n_input = ctx->n_input - i,
925 .nesting_countdown = ctx->nesting_countdown,
926 .macros = ctx->macros,
930 .expand = ctx->expand,
932 size_t subinput_consumed;
933 if (expand_macro_function (&subctx, farg, &subinput_consumed))
934 return subinput_consumed;
937 ds_put_substring (farg, tokens[i].representation);
942 parse_macro_function (struct parse_macro_function_ctx *ctx,
943 struct string_array *args,
944 struct substring function,
945 int min_args, int max_args,
946 size_t *input_consumed)
948 const struct macro_token *tokens = ctx->input;
949 size_t n_tokens = ctx->n_input;
952 || tokens[0].token.type != T_MACRO_ID
953 || !ss_equals_case (tokens[0].token.string, function)) /* XXX abbrevs allowed */
956 if (n_tokens < 2 || tokens[1].token.type != T_LPAREN)
958 macro_error (ctx->stack, n_tokens > 1 ? &tokens[1] : NULL,
959 _("`(' expected following %s"), function.string);
963 string_array_init (args);
965 for (size_t i = 2;; )
969 if (tokens[i].token.type == T_RPAREN)
971 *input_consumed = i + 1;
972 if (args->n < min_args || args->n > max_args)
974 macro_error (ctx->stack, &tokens[i],
975 _("Wrong number of arguments to macro function %s."),
982 struct string s = DS_EMPTY_INITIALIZER;
983 i += parse_function_arg (ctx, i, &s);
989 string_array_append_nocopy (args, ds_steal_cstr (&s));
991 if (tokens[i].token.type == T_COMMA)
993 else if (tokens[i].token.type != T_RPAREN)
995 macro_error (ctx->stack, &tokens[i],
996 _("`,' or `)' expected in call to macro function %s."),
1003 macro_error (ctx->stack, NULL, _("Missing `)' in call to macro function %s."),
1007 string_array_destroy (args);
1012 unquote_string (const char *s, struct string *content)
1014 struct string_lexer slex;
1015 string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE /* XXX */,
1018 struct token token1;
1019 if (!string_lexer_next (&slex, &token1))
1022 if (token1.type != T_STRING)
1024 token_uninit (&token1);
1028 struct token token2;
1029 if (string_lexer_next (&slex, &token2))
1031 token_uninit (&token1);
1032 token_uninit (&token2);
1036 ds_put_substring (content, token1.string);
1037 token_uninit (&token1);
1042 unquote_string_in_place (const char *s, struct string *tmp)
1044 ds_init_empty (tmp);
1045 return unquote_string (s, tmp) ? ds_cstr (tmp) : s;
1049 parse_integer (const char *s, int *np)
1054 long int n = strtol (s, &tail, 10);
1055 *np = n < INT_MIN ? INT_MIN : n > INT_MAX ? INT_MAX : n;
1056 tail += strspn (tail, CC_SPACES);
1057 return *tail == '\0' && errno != ERANGE && n == *np;
1061 expand_macro_function (struct parse_macro_function_ctx *ctx,
1062 struct string *output,
1063 size_t *input_consumed)
1065 struct string_array args;
1067 if (parse_macro_function (ctx, &args, ss_cstr ("!length"), 1, 1,
1069 ds_put_format (output, "%zu", strlen (args.strings[0]));
1070 else if (parse_macro_function (ctx, &args, ss_cstr ("!blanks"), 1, 1,
1074 if (!parse_integer (args.strings[0], &n))
1076 macro_error (ctx->stack, NULL,
1077 _("Argument to !BLANKS must be non-negative integer "
1078 "(not \"%s\")"), args.strings[0]);
1079 string_array_destroy (&args);
1083 ds_put_byte_multiple (output, ' ', n);
1085 else if (parse_macro_function (ctx, &args, ss_cstr ("!concat"), 1, INT_MAX,
1088 for (size_t i = 0; i < args.n; i++)
1089 if (!unquote_string (args.strings[i], output))
1090 ds_put_cstr (output, args.strings[i]);
1092 else if (parse_macro_function (ctx, &args, ss_cstr ("!head"), 1, 1,
1096 const char *s = unquote_string_in_place (args.strings[0], &tmp);
1098 struct macro_tokens mts = { .n = 0 };
1099 macro_tokens_from_string (&mts, ss_cstr (s), SEG_MODE_INTERACTIVE /* XXX */);
1101 ds_put_substring (output, mts.mts[0].representation);
1102 macro_tokens_uninit (&mts);
1105 else if (parse_macro_function (ctx, &args, ss_cstr ("!index"), 2, 2,
1108 const char *haystack = args.strings[0];
1109 const char *needle = strstr (haystack, args.strings[1]);
1110 ds_put_format (output, "%zu", needle ? needle - haystack + 1 : 0);
1112 else if (parse_macro_function (ctx, &args, ss_cstr ("!quote"), 1, 1,
1115 if (unquote_string (args.strings[0], NULL))
1116 ds_put_cstr (output, args.strings[0]);
1119 ds_extend (output, strlen (args.strings[0]) + 2);
1120 ds_put_byte (output, '\'');
1121 for (const char *p = args.strings[0]; *p; p++)
1124 ds_put_byte (output, '\'');
1125 ds_put_byte (output, *p);
1127 ds_put_byte (output, '\'');
1130 else if (parse_macro_function (ctx, &args, ss_cstr ("!substr"), 2, 3,
1134 if (!parse_integer (args.strings[1], &start) || start < 1)
1136 macro_error (ctx->stack, NULL,
1137 _("Second argument of !SUBSTR must be "
1138 "positive integer (not \"%s\")"),
1140 string_array_destroy (&args);
1144 int count = INT_MAX;
1145 if (args.n > 2 && (!parse_integer (args.strings[2], &count) || count < 0))
1147 macro_error (ctx->stack, NULL,
1148 _("Third argument of !SUBSTR must be "
1149 "non-negative integer (not \"%s\")"),
1151 string_array_destroy (&args);
1155 struct substring s = ss_cstr (args.strings[0]);
1156 ds_put_substring (output, ss_substr (s, start - 1, count));
1158 else if (parse_macro_function (ctx, &args, ss_cstr ("!tail"), 1, 1,
1162 const char *s = unquote_string_in_place (args.strings[0], &tmp);
1164 struct macro_tokens mts = { .n = 0 };
1165 macro_tokens_from_string (&mts, ss_cstr (s), SEG_MODE_INTERACTIVE /* XXX */);
1168 struct macro_tokens tail = { .mts = mts.mts + 1, .n = mts.n - 1 };
1169 macro_tokens_to_representation (&tail, output, NULL, NULL);
1171 macro_tokens_uninit (&mts);
1174 else if (parse_macro_function (ctx, &args, ss_cstr ("!unquote"), 1, 1,
1177 if (!unquote_string (args.strings[0], output))
1178 ds_put_cstr (output, args.strings[0]);
1180 else if (parse_macro_function (ctx, &args, ss_cstr ("!upcase"), 1, 1,
1184 const char *s = unquote_string_in_place (args.strings[0], &tmp);
1185 char *upper = utf8_to_upper (s);
1186 ds_put_cstr (output, upper);
1190 else if (parse_macro_function (ctx, &args, ss_cstr ("!eval"), 1, 1,
1193 struct macro_tokens mts = { .n = 0 };
1194 macro_tokens_from_string (&mts, ss_cstr (args.strings[0]),
1195 SEG_MODE_INTERACTIVE /* XXX */);
1196 struct macro_tokens exp = { .n = 0 };
1197 macro_expand (&mts, ctx->nesting_countdown - 1, ctx->macros, ctx->me,
1199 &(struct macro_expansion_stack) {
1202 }, ctx->expand, NULL, &exp);
1203 macro_tokens_to_representation (&exp, output, NULL, NULL);
1204 macro_tokens_uninit (&exp);
1205 macro_tokens_uninit (&mts);
1207 else if (ctx->n_input > 0
1208 && ctx->input[0].token.type == T_MACRO_ID
1209 && ss_equals_case (ctx->input[0].token.string, ss_cstr ("!null")))
1211 *input_consumed = 1;
1217 string_array_destroy (&args);
1223 int nesting_countdown;
1224 const struct macro_set *macros;
1225 const struct macro_expander *me;
1226 const struct macro_expansion_stack *stack;
1227 struct string_map *vars;
1231 static char *macro_evaluate_or (const struct expr_context *ctx,
1232 const struct macro_token **tokens,
1233 const struct macro_token *end);
1236 macro_evaluate_literal (const struct expr_context *ctx,
1237 const struct macro_token **tokens,
1238 const struct macro_token *end)
1240 const struct macro_token *p = *tokens;
1243 if (p->token.type == T_LPAREN)
1246 char *value = macro_evaluate_or (ctx, &p, end);
1249 if (p >= end || p->token.type != T_RPAREN)
1252 macro_error (ctx->stack, p < end ? p : NULL,
1253 _("Expecting ')' in macro expression."));
1261 struct parse_macro_function_ctx fctx = {
1264 .nesting_countdown = ctx->nesting_countdown,
1265 .macros = ctx->macros,
1267 .stack = ctx->stack,
1269 .expand = ctx->expand,
1271 struct string function_output = DS_EMPTY_INITIALIZER;
1272 size_t function_consumed = parse_function_arg (&fctx, 0, &function_output);
1273 struct string unquoted = DS_EMPTY_INITIALIZER;
1274 if (unquote_string (ds_cstr (&function_output), &unquoted))
1276 ds_swap (&function_output, &unquoted);
1277 ds_destroy (&unquoted);
1279 *tokens = p + function_consumed;
1280 return ds_steal_cstr (&function_output);
1283 /* Returns true if MT is valid as a macro operator. Only operators written as
1284 symbols (e.g. <>) are usable in macro expressions, not operator written as
1285 letters (e.g. EQ). */
1287 is_macro_operator (const struct macro_token *mt)
1289 return (mt->representation.length > 0
1290 && !c_isalpha (mt->representation.string[0]));
1293 static enum token_type
1294 parse_relational_op (const struct macro_token *mt)
1296 switch (mt->token.type)
1306 return is_macro_operator (mt) ? mt->token.type : T_STOP;
1309 return (ss_equals_case (mt->token.string, ss_cstr ("!EQ")) ? T_EQ
1310 : ss_equals_case (mt->token.string, ss_cstr ("!NE")) ? T_NE
1311 : ss_equals_case (mt->token.string, ss_cstr ("!LT")) ? T_LT
1312 : ss_equals_case (mt->token.string, ss_cstr ("!GT")) ? T_GT
1313 : ss_equals_case (mt->token.string, ss_cstr ("!LE")) ? T_LE
1314 : ss_equals_case (mt->token.string, ss_cstr ("!GE")) ? T_GE
1323 macro_evaluate_relational (const struct expr_context *ctx,
1324 const struct macro_token **tokens,
1325 const struct macro_token *end)
1327 const struct macro_token *p = *tokens;
1328 char *lhs = macro_evaluate_literal (ctx, &p, end);
1332 enum token_type op = p >= end ? T_STOP : parse_relational_op (p);
1340 char *rhs = macro_evaluate_literal (ctx, &p, end);
1347 struct string lhs_tmp, rhs_tmp;
1348 int cmp = strcmp/*XXX*/ (unquote_string_in_place (lhs, &lhs_tmp),
1349 unquote_string_in_place (rhs, &rhs_tmp));
1350 ds_destroy (&lhs_tmp);
1351 ds_destroy (&rhs_tmp);
1356 bool b = (op == T_EQUALS || op == T_EQ ? !cmp
1358 : op == T_LT ? cmp < 0
1359 : op == T_GT ? cmp > 0
1360 : op == T_LE ? cmp <= 0
1361 :/*op == T_GE*/cmp >= 0);
1364 return xstrdup (b ? "1" : "0");
1368 macro_evaluate_not (const struct expr_context *ctx,
1369 const struct macro_token **tokens,
1370 const struct macro_token *end)
1372 const struct macro_token *p = *tokens;
1374 unsigned int negations = 0;
1376 && (ss_equals_case (p->representation, ss_cstr ("!NOT"))
1377 || ss_equals (p->representation, ss_cstr ("~"))))
1383 char *operand = macro_evaluate_relational (ctx, &p, end);
1384 if (!operand || !negations)
1390 bool b = strcmp (operand, "0") ^ (negations & 1);
1393 return xstrdup (b ? "1" : "0");
1397 macro_evaluate_and (const struct expr_context *ctx,
1398 const struct macro_token **tokens,
1399 const struct macro_token *end)
1401 const struct macro_token *p = *tokens;
1402 char *lhs = macro_evaluate_not (ctx, &p, end);
1407 && (ss_equals_case (p->representation, ss_cstr ("!AND"))
1408 || ss_equals (p->representation, ss_cstr ("&"))))
1411 char *rhs = macro_evaluate_not (ctx, &p, end);
1418 bool b = strcmp (lhs, "0") && strcmp (rhs, "0");
1421 lhs = xstrdup (b ? "1" : "0");
1428 macro_evaluate_or (const struct expr_context *ctx,
1429 const struct macro_token **tokens,
1430 const struct macro_token *end)
1432 const struct macro_token *p = *tokens;
1433 char *lhs = macro_evaluate_and (ctx, &p, end);
1438 && (ss_equals_case (p->representation, ss_cstr ("!OR"))
1439 || ss_equals (p->representation, ss_cstr ("|"))))
1442 char *rhs = macro_evaluate_and (ctx, &p, end);
1449 bool b = strcmp (lhs, "0") || strcmp (rhs, "0");
1452 lhs = xstrdup (b ? "1" : "0");
1459 macro_evaluate_expression (const struct macro_token **tokens, size_t n_tokens,
1460 int nesting_countdown, const struct macro_set *macros,
1461 const struct macro_expander *me,
1462 const struct macro_expansion_stack *stack,
1463 struct string_map *vars, bool *expand)
1465 const struct expr_context ctx = {
1466 .nesting_countdown = nesting_countdown,
1473 return macro_evaluate_or (&ctx, tokens, *tokens + n_tokens);
1477 macro_evaluate_number (const struct macro_token **tokens, size_t n_tokens,
1478 int nesting_countdown, const struct macro_set *macros,
1479 const struct macro_expander *me,
1480 const struct macro_expansion_stack *stack,
1481 struct string_map *vars,
1482 bool *expand, double *number)
1484 char *s = macro_evaluate_expression (tokens, n_tokens, nesting_countdown,
1485 macros, me, stack, vars, expand);
1489 struct macro_tokens mts = { .n = 0 };
1490 macro_tokens_from_string (&mts, ss_cstr (s), SEG_MODE_INTERACTIVE /* XXX */);
1491 if (mts.n != 1 || !token_is_number (&mts.mts[0].token))
1493 macro_tokens_print (&mts, stdout);
1494 macro_error (stack, mts.n > 0 ? &mts.mts[0] : NULL,
1495 _("Macro expression must evaluate to "
1496 "a number (not \"%s\")"), s);
1498 macro_tokens_uninit (&mts);
1502 *number = token_number (&mts.mts[0].token);
1504 macro_tokens_uninit (&mts);
1508 static const struct macro_token *
1509 find_ifend_clause (const struct macro_token *p, const struct macro_token *end)
1512 for (; p < end; p++)
1514 if (p->token.type != T_MACRO_ID)
1517 if (ss_equals_case (p->token.string, ss_cstr ("!IF")))
1519 else if (ss_equals_case (p->token.string, ss_cstr ("!IFEND")))
1525 else if (ss_equals_case (p->token.string, ss_cstr ("!ELSE")) && !nesting)
1532 macro_expand_if (const struct macro_token *tokens, size_t n_tokens,
1533 int nesting_countdown, const struct macro_set *macros,
1534 const struct macro_expander *me,
1535 const struct macro_expansion_stack *stack,
1536 struct string_map *vars,
1537 bool *expand, bool *break_, struct macro_tokens *exp)
1539 const struct macro_token *p = tokens;
1540 const struct macro_token *end = tokens + n_tokens;
1542 if (p >= end || !ss_equals_case (p->token.string, ss_cstr ("!IF")))
1546 char *result = macro_evaluate_expression (&p, end - p,
1547 nesting_countdown, macros, me,
1548 stack, vars, expand);
1551 bool b = strcmp (result, "0");
1555 || p->token.type != T_MACRO_ID
1556 || !ss_equals_case (p->token.string, ss_cstr ("!THEN")))
1558 macro_error (stack, p < end ? p : NULL,
1559 _("!THEN expected in macro !IF construct."));
1563 const struct macro_token *start_then = p + 1;
1564 const struct macro_token *end_then = find_ifend_clause (start_then, end);
1567 macro_error (stack, NULL,
1568 _("!ELSE or !IFEND expected in macro !IF construct."));
1572 const struct macro_token *start_else, *end_if;
1573 if (ss_equals_case (end_then->token.string, ss_cstr ("!ELSE")))
1575 start_else = end_then + 1;
1576 end_if = find_ifend_clause (start_else, end);
1578 || !ss_equals_case (end_if->token.string, ss_cstr ("!IFEND")))
1580 macro_error (stack, end_if ? end_if : NULL,
1581 _("!IFEND expected in macro !IF construct."));
1591 const struct macro_token *start;
1596 n = end_then - start_then;
1598 else if (start_else)
1601 n = end_if - start_else;
1611 struct macro_tokens mts = {
1612 .mts = CONST_CAST (struct macro_token *, start),
1615 macro_expand (&mts, nesting_countdown, macros, me, vars,
1616 &(struct macro_expansion_stack) {
1620 expand, break_, exp);
1622 return (end_if + 1) - tokens;
1626 macro_parse_let (const struct macro_token *tokens, size_t n_tokens,
1627 int nesting_countdown, const struct macro_set *macros,
1628 const struct macro_expander *me,
1629 const struct macro_expansion_stack *stack,
1630 struct string_map *vars, bool *expand)
1632 const struct macro_token *p = tokens;
1633 const struct macro_token *end = tokens + n_tokens;
1635 if (p >= end || !ss_equals_case (p->token.string, ss_cstr ("!LET")))
1639 if (p >= end || p->token.type != T_MACRO_ID)
1641 macro_error (stack, p < end ? p : NULL,
1642 _("Expected macro variable name following !LET."));
1645 const struct substring var_name = p->token.string;
1646 if (is_macro_keyword (var_name)
1647 || macro_find_parameter_by_name (me->macro, var_name))
1649 macro_error (stack, p < end ? p : NULL,
1650 _("Cannot use argument name or macro keyword "
1651 "\"%.*s\" as !LET variable"),
1652 (int) var_name.length, var_name.string);
1657 if (p >= end || p->token.type != T_EQUALS)
1659 macro_error (stack, p < end ? p : NULL,
1660 _("Expected `=' following !LET"));
1665 char *value = macro_evaluate_expression (&p, end - p,
1666 nesting_countdown, macros, me, stack,
1671 string_map_replace_nocopy (vars, ss_xstrdup (var_name), value);
1675 static const struct macro_token *
1676 find_doend (const struct macro_expansion_stack *stack,
1677 const struct macro_token *p, const struct macro_token *end)
1680 for (; p < end; p++)
1682 if (p->token.type != T_MACRO_ID)
1685 if (ss_equals_case (p->token.string, ss_cstr ("!DO")))
1687 else if (ss_equals_case (p->token.string, ss_cstr ("!DOEND")))
1694 macro_error (stack, NULL, _("Missing !DOEND."));
1699 macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
1700 int nesting_countdown, const struct macro_set *macros,
1701 const struct macro_expander *me,
1702 const struct macro_expansion_stack *stack,
1703 struct string_map *vars,
1704 bool *expand, struct macro_tokens *exp)
1706 const struct macro_token *p = tokens;
1707 const struct macro_token *end = tokens + n_tokens;
1709 if (p >= end || !ss_equals_case (p->token.string, ss_cstr ("!DO")))
1713 if (p >= end || p->token.type != T_MACRO_ID)
1715 macro_error (stack, p < end ? p : NULL,
1716 _("Expected macro variable name following !DO"));
1719 const struct substring var_name = p->token.string;
1720 if (is_macro_keyword (var_name)
1721 || macro_find_parameter_by_name (me->macro, var_name))
1723 macro_error (stack, p, _("Cannot use argument name or macro "
1724 "keyword as !DO variable"));
1729 struct macro_expansion_stack next_stack = {
1730 .name = "!DO", .next = stack,
1732 int miterate = settings_get_miterate ();
1733 if (p < end && p->token.type == T_MACRO_ID
1734 && ss_equals_case (p->token.string, ss_cstr ("!IN")))
1737 char *list = macro_evaluate_expression (&p, end - p,
1738 nesting_countdown, macros, me,
1739 &next_stack, vars, expand);
1743 struct macro_tokens items = { .n = 0 };
1744 macro_tokens_from_string (&items, ss_cstr (list),
1745 SEG_MODE_INTERACTIVE /* XXX */);
1748 const struct macro_token *do_end = find_doend (stack, p, end);
1751 macro_tokens_uninit (&items);
1755 const struct macro_tokens inner = {
1756 .mts = CONST_CAST (struct macro_token *, p),
1759 for (size_t i = 0; i < items.n; i++)
1763 macro_error (stack, NULL,
1764 _("!DO loop over list exceeded "
1765 "maximum number of iterations %d. "
1766 "(Use SET MITERATE to change the limit.)"),
1770 string_map_replace_nocopy (vars, ss_xstrdup (var_name),
1771 ss_xstrdup (items.mts[i].representation));
1773 bool break_ = false;
1774 macro_expand (&inner, nesting_countdown, macros,
1775 me, vars, &next_stack, expand, &break_, exp);
1779 return do_end - tokens + 1;
1781 else if (p < end && p->token.type == T_EQUALS)
1785 if (!macro_evaluate_number (&p, end - p, nesting_countdown, macros, me,
1786 &next_stack, vars, expand, &first))
1789 if (p >= end || p->token.type != T_MACRO_ID
1790 || !ss_equals_case (p->token.string, ss_cstr ("!TO")))
1792 macro_error (stack, p < end ? p : NULL,
1793 _("Expected !TO in numerical !DO loop"));
1799 if (!macro_evaluate_number (&p, end - p, nesting_countdown, macros, me,
1800 &next_stack, vars, expand, &last))
1804 if (p < end && p->token.type == T_MACRO_ID
1805 && ss_equals_case (p->token.string, ss_cstr ("!BY")))
1808 if (!macro_evaluate_number (&p, end - p, nesting_countdown, macros, me,
1809 &next_stack, vars, expand, &by))
1814 macro_error (stack, NULL, _("!BY value cannot be zero."));
1819 const struct macro_token *do_end = find_doend (stack, p, end);
1822 const struct macro_tokens inner = {
1823 .mts = CONST_CAST (struct macro_token *, p),
1827 if ((by > 0 && first <= last) || (by < 0 && first >= last))
1830 for (double index = first;
1831 by > 0 ? (index <= last) : (index >= last);
1836 macro_error (stack, NULL,
1837 _("Numerical !DO loop exceeded "
1838 "maximum number of iterations %d. "
1839 "(Use SET MITERATE to change the limit.)"),
1844 char index_s[DBL_BUFSIZE_BOUND];
1845 c_dtoastr (index_s, sizeof index_s, 0, 0, index);
1846 string_map_replace_nocopy (vars, ss_xstrdup (var_name),
1849 bool break_ = false;
1850 macro_expand (&inner, nesting_countdown, macros,
1851 me, vars, &next_stack, expand, &break_, exp);
1857 return do_end - tokens + 1;
1861 macro_error (stack, p < end ? p : NULL,
1862 _("Expected `=' or !IN in !DO loop."));
1868 macro_expand (const struct macro_tokens *mts,
1869 int nesting_countdown, const struct macro_set *macros,
1870 const struct macro_expander *me, struct string_map *vars,
1871 const struct macro_expansion_stack *stack,
1872 bool *expand, bool *break_, struct macro_tokens *exp)
1874 if (nesting_countdown <= 0)
1876 macro_error (stack, NULL, _("Maximum nesting level %d exceeded."),
1877 settings_get_mnest ());
1878 for (size_t i = 0; i < mts->n; i++)
1879 macro_tokens_add (exp, &mts->mts[i]);
1883 struct string_map own_vars = STRING_MAP_INITIALIZER (own_vars);
1887 for (size_t i = 0; i < mts->n && (!break_ || !*break_); i++)
1889 const struct macro_token *mt = &mts->mts[i];
1890 const struct token *token = &mt->token;
1891 if (token->type == T_MACRO_ID && me)
1893 const struct macro_param *param = macro_find_parameter_by_name (
1894 me->macro, token->string);
1897 const struct macro_tokens *arg = me->args[param - me->macro->params];
1898 if (*expand && param->expand_arg)
1899 macro_expand (arg, nesting_countdown, macros, NULL, NULL,
1900 &(struct macro_expansion_stack) {
1901 .name = param->name,
1903 }, expand, break_, exp);
1905 for (size_t i = 0; i < arg->n; i++)
1906 macro_tokens_add (exp, &arg->mts[i]);
1910 if (is_bang_star (mts->mts, mts->n, i))
1912 for (size_t j = 0; j < me->macro->n_params; j++)
1914 const struct macro_param *param = &me->macro->params[j];
1915 if (!param->positional)
1918 const struct macro_tokens *arg = me->args[j];
1919 if (*expand && param->expand_arg)
1920 macro_expand (arg, nesting_countdown, macros, NULL, NULL,
1921 &(struct macro_expansion_stack) {
1924 }, expand, break_, exp);
1926 for (size_t k = 0; k < arg->n; k++)
1927 macro_tokens_add (exp, &arg->mts[k]);
1933 size_t n = macro_expand_if (&mts->mts[i], mts->n - i,
1934 nesting_countdown, macros, me, stack,
1935 vars, expand, break_, exp);
1943 if (token->type == T_MACRO_ID && vars)
1945 const char *value = string_map_find__ (vars, token->string.string,
1946 token->string.length);
1949 macro_tokens_from_string (exp, ss_cstr (value),
1950 SEG_MODE_INTERACTIVE /* XXX */);
1957 struct macro_expander *subme;
1958 int retval = macro_expander_create (macros, token, &subme);
1959 for (size_t j = 1; !retval; j++)
1961 const struct macro_token endcmd = { .token = { .type = T_ENDCMD } };
1962 retval = macro_expander_add (
1963 subme, i + j < mts->n ? &mts->mts[i + j] : &endcmd);
1968 macro_expand (&subme->macro->body, nesting_countdown - 1, macros,
1970 &(struct macro_expansion_stack) {
1971 .name = subme->macro->name,
1972 .file_name = subme->macro->file_name,
1973 .first_line = subme->macro->first_line,
1974 .last_line = subme->macro->last_line,
1976 }, expand, break_, exp);
1977 macro_expander_destroy (subme);
1981 macro_expander_destroy (subme);
1984 if (token->type != T_MACRO_ID)
1986 macro_tokens_add (exp, mt);
1990 if (ss_equals_case (token->string, ss_cstr ("!break")))
1993 macro_error (stack, mt, _("!BREAK outside !DO."));
2001 struct parse_macro_function_ctx ctx = {
2002 .input = &mts->mts[i],
2003 .n_input = mts->n - i,
2004 .nesting_countdown = nesting_countdown,
2011 struct string function_output = DS_EMPTY_INITIALIZER;
2012 size_t function_consumed;
2013 if (expand_macro_function (&ctx, &function_output, &function_consumed))
2015 i += function_consumed - 1;
2017 macro_tokens_from_string (exp, function_output.ss,
2018 SEG_MODE_INTERACTIVE /* XXX */);
2019 ds_destroy (&function_output);
2024 size_t n = macro_parse_let (&mts->mts[i], mts->n - i,
2025 nesting_countdown, macros, me, stack,
2033 n = macro_expand_do (&mts->mts[i], mts->n - i,
2034 nesting_countdown, macros, me, stack, vars,
2042 if (ss_equals_case (token->string, ss_cstr ("!onexpand")))
2044 else if (ss_equals_case (token->string, ss_cstr ("!offexpand")))
2047 macro_tokens_add (exp, mt);
2049 if (vars == &own_vars)
2050 string_map_destroy (&own_vars);
2054 macro_expander_get_expansion (struct macro_expander *me, struct macro_tokens *exp)
2057 struct macro_expansion_stack stack = {
2058 .name = me->macro->name,
2059 .file_name = me->macro->file_name,
2060 .first_line = me->macro->first_line,
2061 .last_line = me->macro->last_line,
2063 macro_expand (&me->macro->body, settings_get_mnest (),
2064 me->macros, me, NULL, &stack, &expand, NULL, exp);