1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2021 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "language/lexer/macro.h"
25 #include "data/settings.h"
26 #include "language/lexer/segment.h"
27 #include "language/lexer/scan.h"
28 #include "libpspp/assertion.h"
29 #include "libpspp/cast.h"
30 #include "libpspp/i18n.h"
31 #include "libpspp/message.h"
32 #include "libpspp/str.h"
33 #include "libpspp/string-array.h"
34 #include "libpspp/string-map.h"
35 #include "libpspp/stringi-set.h"
37 #include "gl/c-ctype.h"
38 #include "gl/ftoastr.h"
41 #define _(msgid) gettext (msgid)
44 macro_token_copy (struct macro_token *dst, const struct macro_token *src)
46 token_copy (&dst->token, &src->token);
47 ss_alloc_substring (&dst->representation, src->representation);
51 macro_token_uninit (struct macro_token *mt)
53 token_uninit (&mt->token);
54 ss_dealloc (&mt->representation);
58 macro_token_to_representation (struct macro_token *mt, struct string *s)
60 ds_put_substring (s, mt->representation);
64 is_macro_keyword (struct substring s)
66 static struct stringi_set keywords = STRINGI_SET_INITIALIZER (keywords);
67 if (stringi_set_is_empty (&keywords))
69 static const char *kws[] = {
90 for (size_t i = 0; i < sizeof kws / sizeof *kws; i++)
91 stringi_set_insert (&keywords, kws[i]);
94 ss_ltrim (&s, ss_cstr ("!"));
95 return stringi_set_contains_len (&keywords, s.string, s.length);
99 macro_tokens_copy (struct macro_tokens *dst, const struct macro_tokens *src)
101 *dst = (struct macro_tokens) {
102 .mts = xmalloc (src->n * sizeof *dst->mts),
106 for (size_t i = 0; i < src->n; i++)
107 macro_token_copy (&dst->mts[i], &src->mts[i]);
111 macro_tokens_uninit (struct macro_tokens *mts)
113 for (size_t i = 0; i < mts->n; i++)
114 macro_token_uninit (&mts->mts[i]);
119 macro_tokens_add_uninit (struct macro_tokens *mts)
121 if (mts->n >= mts->allocated)
122 mts->mts = x2nrealloc (mts->mts, &mts->allocated, sizeof *mts->mts);
123 return &mts->mts[mts->n++];
127 macro_tokens_add (struct macro_tokens *mts, const struct macro_token *mt)
129 macro_token_copy (macro_tokens_add_uninit (mts), mt);
133 macro_tokens_from_string (struct macro_tokens *mts, const struct substring src,
134 enum segmenter_mode mode)
138 struct segmenter segmenter;
139 struct substring body;
142 struct state state = {
143 .segmenter = segmenter_init (mode, true),
146 struct state saved = state;
148 while (state.body.length > 0)
150 struct macro_token mt = {
151 .token = { .type = T_STOP },
152 .representation = { .string = state.body.string },
154 struct token *token = &mt.token;
156 struct scanner scanner;
157 scanner_init (&scanner, token);
161 enum segment_type type;
162 int seg_len = segmenter_push (&state.segmenter, state.body.string,
163 state.body.length, true, &type);
164 assert (seg_len >= 0);
166 struct substring segment = ss_head (state.body, seg_len);
167 ss_advance (&state.body, seg_len);
169 enum scan_result result = scanner_push (&scanner, type, segment, token);
170 if (result == SCAN_SAVE)
172 else if (result == SCAN_BACK)
177 else if (result == SCAN_DONE)
181 /* We have a token in 'token'. */
182 if (is_scan_type (token->type))
184 if (token->type != SCAN_SKIP)
187 /* XXX report error */
192 mt.representation.length = state.body.string - mt.representation.string;
193 macro_tokens_add (mts, &mt);
195 token_uninit (token);
200 macro_tokens_print (const struct macro_tokens *mts, FILE *stream)
202 for (size_t i = 0; i < mts->n; i++)
203 token_print (&mts->mts[i].token, stream);
208 TC_ENDCMD, /* No space before or after (new-line after). */
209 TC_BINOP, /* Space on both sides. */
210 TC_COMMA, /* Space afterward. */
211 TC_ID, /* Don't need spaces except sequentially. */
212 TC_PUNCT, /* Don't need spaces except sequentially. */
216 needs_space (enum token_class prev, enum token_class next)
218 /* Don't need a space before or after the end of a command.
219 (A new-line is needed afterward as a special case.) */
220 if (prev == TC_ENDCMD || next == TC_ENDCMD)
223 /* Binary operators always have a space on both sides. */
224 if (prev == TC_BINOP || next == TC_BINOP)
227 /* A comma always has a space afterward. */
228 if (prev == TC_COMMA)
231 /* Otherwise, PREV is TC_ID or TC_PUNCT, which only need a space if there are
232 two or them in a row. */
236 static enum token_class
237 classify_token (enum token_type type)
290 macro_tokens_to_representation (struct macro_tokens *mts, struct string *s,
291 size_t *ofs, size_t *len)
293 assert ((ofs != NULL) == (len != NULL));
298 for (size_t i = 0; i < mts->n; i++)
302 enum token_type prev = mts->mts[i - 1].token.type;
303 enum token_type next = mts->mts[i].token.type;
305 if (prev == T_ENDCMD)
306 ds_put_byte (s, '\n');
309 enum token_class pc = classify_token (prev);
310 enum token_class nc = classify_token (next);
311 if (needs_space (pc, nc))
312 ds_put_byte (s, ' ');
317 ofs[i] = s->ss.length;
318 macro_token_to_representation (&mts->mts[i], s);
320 len[i] = s->ss.length - ofs[i];
325 macro_destroy (struct macro *m)
332 for (size_t i = 0; i < m->n_params; i++)
334 struct macro_param *p = &m->params[i];
337 macro_tokens_uninit (&p->def);
345 token_uninit (&p->charend);
349 token_uninit (&p->enclose[0]);
350 token_uninit (&p->enclose[1]);
358 macro_tokens_uninit (&m->body);
363 macro_set_create (void)
365 struct macro_set *set = xmalloc (sizeof *set);
366 *set = (struct macro_set) {
367 .macros = HMAP_INITIALIZER (set->macros),
373 macro_set_destroy (struct macro_set *set)
378 struct macro *macro, *next;
379 HMAP_FOR_EACH_SAFE (macro, next, struct macro, hmap_node, &set->macros)
381 hmap_delete (&set->macros, ¯o->hmap_node);
382 macro_destroy (macro);
384 hmap_destroy (&set->macros);
389 hash_macro_name (const char *name)
391 return utf8_hash_case_string (name, 0);
394 static struct macro *
395 macro_set_find__ (struct macro_set *set, const char *name)
398 HMAP_FOR_EACH_WITH_HASH (macro, struct macro, hmap_node,
399 hash_macro_name (name), &set->macros)
400 if (!utf8_strcasecmp (macro->name, name))
407 macro_set_find (const struct macro_set *set, const char *name)
409 return macro_set_find__ (CONST_CAST (struct macro_set *, set), name);
412 /* Adds M to SET. M replaces any existing macro with the same name. Takes
415 macro_set_add (struct macro_set *set, struct macro *m)
417 struct macro *victim = macro_set_find__ (set, m->name);
420 hmap_delete (&set->macros, &victim->hmap_node);
421 macro_destroy (victim);
424 hmap_insert (&set->macros, &m->hmap_node, hash_macro_name (m->name));
427 struct macro_expansion_stack
429 const struct macro_expansion_stack *next;
431 const char *file_name;
436 static void PRINTF_FORMAT (2, 3)
437 macro_error (const struct macro_expansion_stack *stack,
438 const char *format, ...)
440 struct msg_stack **ms = NULL;
441 size_t allocated_ms = 0;
444 for (const struct macro_expansion_stack *p = stack; p; p = p->next)
446 if (n_ms >= allocated_ms)
447 ms = x2nrealloc (ms, &allocated_ms, sizeof *ms);
449 ms[n_ms] = xmalloc (sizeof *ms[n_ms]);
450 *ms[n_ms] = (struct msg_stack) {
452 .file_name = xstrdup_if_nonnull (p->file_name),
453 .first_line = p->first_line,
454 .last_line = p->last_line,
458 /* TRANSLATORS: These strings are used for explaining the context of
459 an error. The "While expanding" message appears first, followed
460 by zero or more of the "inside expansion" messages, like this:
462 foo.sps:12: While expanding 'innermost',
463 foo.sps:23: inside expansion of 'next_inner',
464 foo.sps:34: inside expansion of 'next_inner2',
465 foo.sps:45: inside expansion of 'outermost',
466 foo.sps:76: This is the actual error message. */
467 ? xasprintf (_("While expanding \"%s\","), p->name)
468 : xasprintf (_("inside expansion of \"%s\","), p->name))
474 va_start (args, format);
476 struct msg *m = xmalloc (sizeof *m);
478 .category = MSG_C_SYNTAX,
479 .severity = MSG_S_ERROR,
482 .text = xvasprintf (format, args),
494 /* Accumulating tokens in me->params toward the end of any type of
498 /* Expecting the opening delimiter of an ARG_ENCLOSE argument. */
501 /* Expecting a keyword for a keyword argument. */
504 /* Expecting an equal sign for a keyword argument. */
509 struct macro_expander
511 const struct macro_set *macros;
516 const struct macro *macro;
517 struct macro_tokens **args;
518 const struct macro_param *param;
522 me_finished (struct macro_expander *me)
524 for (size_t i = 0; i < me->macro->n_params; i++)
527 me->args[i] = xmalloc (sizeof *me->args[i]);
528 macro_tokens_copy (me->args[i], &me->macro->params[i].def);
534 me_next_arg (struct macro_expander *me)
538 assert (!me->macro->n_params);
539 return me_finished (me);
541 else if (me->param->positional)
544 if (me->param >= &me->macro->params[me->macro->n_params])
545 return me_finished (me);
548 me->state = (!me->param->positional ? ME_KEYWORD
549 : me->param->arg_type == ARG_ENCLOSE ? ME_ENCLOSE
556 for (size_t i = 0; i < me->macro->n_params; i++)
559 me->state = ME_KEYWORD;
562 return me_finished (me);
567 me_error (struct macro_expander *me)
569 me->state = ME_ERROR;
574 me_add_arg (struct macro_expander *me, const struct macro_token *mt)
576 const struct macro_param *p = me->param;
578 const struct token *token = &mt->token;
579 if ((token->type == T_ENDCMD || token->type == T_STOP)
580 && p->arg_type != ARG_CMDEND)
582 msg (SE, _("Unexpected end of command reading argument %s "
583 "to macro %s."), me->param->name, me->macro->name);
585 return me_error (me);
590 struct macro_tokens **argp = &me->args[p - me->macro->params];
592 *argp = xzalloc (sizeof **argp);
593 struct macro_tokens *arg = *argp;
594 if (p->arg_type == ARG_N_TOKENS)
596 macro_tokens_add (arg, mt);
597 if (arg->n >= p->n_tokens)
598 return me_next_arg (me);
601 else if (p->arg_type == ARG_CMDEND)
603 if (token->type == T_ENDCMD || token->type == T_STOP)
604 return me_next_arg (me);
605 macro_tokens_add (arg, mt);
610 const struct token *end
611 = p->arg_type == ARG_CHAREND ? &p->charend : &p->enclose[1];
612 if (token_equal (token, end))
613 return me_next_arg (me);
614 macro_tokens_add (arg, mt);
620 me_expected (struct macro_expander *me, const struct macro_token *actual,
621 const struct token *expected)
623 const struct substring actual_s
624 = (actual->representation.length ? actual->representation
625 : ss_cstr (_("<end of input>")));
626 char *expected_s = token_to_string (expected);
627 msg (SE, _("Found `%.*s' while expecting `%s' reading argument %s "
629 (int) actual_s.length, actual_s.string, expected_s,
630 me->param->name, me->macro->name);
633 return me_error (me);
637 me_enclose (struct macro_expander *me, const struct macro_token *mt)
639 const struct token *token = &mt->token;
642 if (token_equal (&me->param->enclose[0], token))
648 return me_expected (me, mt, &me->param->enclose[0]);
651 static const struct macro_param *
652 macro_find_parameter_by_name (const struct macro *m, struct substring name)
654 ss_ltrim (&name, ss_cstr ("!"));
656 for (size_t i = 0; i < m->n_params; i++)
658 const struct macro_param *p = &m->params[i];
659 struct substring p_name = ss_cstr (p->name + 1);
660 if (!utf8_strncasecmp (p_name.string, p_name.length,
661 name.string, name.length))
668 me_keyword (struct macro_expander *me, const struct macro_token *mt)
670 const struct token *token = &mt->token;
671 if (token->type != T_ID)
672 return me_finished (me);
674 const struct macro_param *p = macro_find_parameter_by_name (me->macro,
678 size_t arg_index = p - me->macro->params;
680 if (me->args[arg_index])
683 _("Argument %s multiply specified in call to macro %s."),
684 p->name, me->macro->name);
685 return me_error (me);
689 me->state = ME_EQUALS;
693 return me_finished (me);
697 me_equals (struct macro_expander *me, const struct macro_token *mt)
699 const struct token *token = &mt->token;
702 if (token->type == T_EQUALS)
708 return me_expected (me, mt, &(struct token) { .type = T_EQUALS });
712 macro_expander_create (const struct macro_set *macros,
713 const struct token *token,
714 struct macro_expander **mep)
717 if (macro_set_is_empty (macros))
719 if (token->type != T_ID && token->type != T_MACRO_ID)
722 const struct macro *macro = macro_set_find (macros, token->string.string);
726 struct macro_expander *me = xmalloc (sizeof *me);
727 *me = (struct macro_expander) {
734 if (!macro->n_params)
738 me->state = (!macro->params[0].positional ? ME_KEYWORD
739 : macro->params[0].arg_type == ARG_ENCLOSE ? ME_ENCLOSE
741 me->args = xcalloc (macro->n_params, sizeof *me->args);
742 me->param = macro->params;
748 macro_expander_destroy (struct macro_expander *me)
753 for (size_t i = 0; i < me->macro->n_params; i++)
756 macro_tokens_uninit (me->args[i]);
763 /* Adds TOKEN to the collection of tokens in ME that potentially need to be
766 Returns -1 if the tokens added do not actually invoke a macro. The caller
767 should consume the first token without expanding it.
769 Returns 0 if the macro expander needs more tokens, for macro arguments or to
770 decide whether this is actually a macro invocation. The caller should call
771 macro_expander_add() again with the next token.
773 Returns a positive number to indicate that the returned number of tokens
774 invoke a macro. The number returned might be less than the number of tokens
775 added because it can take a few tokens of lookahead to determine whether the
776 macro invocation is finished. The caller should call
777 macro_expander_get_expansion() to obtain the expansion. */
779 macro_expander_add (struct macro_expander *me, const struct macro_token *mt)
787 return me_add_arg (me, mt);
790 return me_enclose (me, mt);
793 return me_keyword (me, mt);
796 return me_equals (me, mt);
803 /* Each argument to a macro function is one of:
805 - A quoted string or other single literal token.
807 - An argument to the macro being expanded, e.g. !1 or a named argument.
811 - A function invocation.
813 Each function invocation yields a character sequence to be turned into a
814 sequence of tokens. The case where that character sequence is a single
815 quoted string is an important special case.
817 struct parse_macro_function_ctx
819 const struct macro_token *input;
821 int nesting_countdown;
822 const struct macro_set *macros;
823 const struct macro_expander *me;
824 const struct macro_expansion_stack *stack;
825 struct string_map *vars;
830 macro_expand (const struct macro_tokens *,
831 int nesting_countdown, const struct macro_set *,
832 const struct macro_expander *, struct string_map *vars,
833 const struct macro_expansion_stack *stack,
834 bool *expand, bool *break_,
835 struct macro_tokens *exp);
838 expand_macro_function (struct parse_macro_function_ctx *ctx,
839 struct string *output, size_t *input_consumed);
841 /* Returns true if the pair of tokens starting at offset OFS within MTS are !*,
844 is_bang_star (const struct macro_token *mts, size_t n, size_t ofs)
847 && mts[ofs].token.type == T_MACRO_ID
848 && ss_equals (mts[ofs].token.string, ss_cstr ("!"))
849 && mts[ofs + 1].token.type == T_ASTERISK);
853 parse_function_arg (struct parse_macro_function_ctx *ctx,
854 size_t i, struct string *farg)
856 const struct macro_token *tokens = ctx->input;
857 const struct token *token = &tokens[i].token;
858 if (token->type == T_MACRO_ID)
860 const struct macro_param *param = macro_find_parameter_by_name (
861 ctx->me->macro, token->string);
864 size_t param_idx = param - ctx->me->macro->params;
865 const struct macro_tokens *marg = ctx->me->args[param_idx];
866 for (size_t i = 0; i < marg->n; i++)
869 ds_put_byte (farg, ' ');
870 ds_put_substring (farg, marg->mts[i].representation);
875 if (is_bang_star (ctx->input, ctx->n_input, i))
877 for (size_t i = 0; i < ctx->me->macro->n_params; i++)
879 if (!ctx->me->macro->params[i].positional)
882 const struct macro_tokens *marg = ctx->me->args[i];
883 for (size_t j = 0; j < marg->n; j++)
886 ds_put_byte (farg, ' ');
887 ds_put_substring (farg, marg->mts[j].representation);
895 const char *value = string_map_find__ (ctx->vars,
896 token->string.string,
897 token->string.length);
900 ds_put_cstr (farg, value);
905 struct parse_macro_function_ctx subctx = {
906 .input = &ctx->input[i],
907 .n_input = ctx->n_input - i,
908 .nesting_countdown = ctx->nesting_countdown,
909 .macros = ctx->macros,
913 .expand = ctx->expand,
915 size_t subinput_consumed;
916 if (expand_macro_function (&subctx, farg, &subinput_consumed))
917 return subinput_consumed;
920 ds_put_substring (farg, tokens[i].representation);
925 parse_macro_function (struct parse_macro_function_ctx *ctx,
926 struct string_array *args,
927 struct substring function,
928 int min_args, int max_args,
929 size_t *input_consumed)
931 const struct macro_token *tokens = ctx->input;
932 size_t n_tokens = ctx->n_input;
935 || tokens[0].token.type != T_MACRO_ID
936 || !ss_equals_case (tokens[0].token.string, function)) /* XXX abbrevs allowed */
939 if (n_tokens < 2 || tokens[1].token.type != T_LPAREN)
941 macro_error (ctx->stack, _("`(' expected following %s"), function.string);
945 string_array_init (args);
947 for (size_t i = 2;; )
951 if (tokens[i].token.type == T_RPAREN)
953 *input_consumed = i + 1;
954 if (args->n < min_args || args->n > max_args)
956 macro_error (ctx->stack,
957 _("Wrong number of arguments to macro function %s."),
964 struct string s = DS_EMPTY_INITIALIZER;
965 i += parse_function_arg (ctx, i, &s);
971 string_array_append_nocopy (args, ds_steal_cstr (&s));
973 if (tokens[i].token.type == T_COMMA)
975 else if (tokens[i].token.type != T_RPAREN)
977 macro_error (ctx->stack,
978 _("`,' or `)' expected in call to macro function %s."),
985 macro_error (ctx->stack, _("Missing `)' in call to macro function %s."),
989 string_array_destroy (args);
994 unquote_string (const char *s, struct string *content)
996 struct string_lexer slex;
997 string_lexer_init (&slex, s, strlen (s), SEG_MODE_INTERACTIVE /* XXX */,
1000 struct token token1;
1001 if (!string_lexer_next (&slex, &token1))
1004 if (token1.type != T_STRING)
1006 token_uninit (&token1);
1010 struct token token2;
1011 if (string_lexer_next (&slex, &token2))
1013 token_uninit (&token1);
1014 token_uninit (&token2);
1018 ds_put_substring (content, token1.string);
1019 token_uninit (&token1);
1024 unquote_string_in_place (const char *s, struct string *tmp)
1026 ds_init_empty (tmp);
1027 return unquote_string (s, tmp) ? ds_cstr (tmp) : s;
1031 parse_integer (const char *s, int *np)
1036 long int n = strtol (s, &tail, 10);
1037 *np = n < INT_MIN ? INT_MIN : n > INT_MAX ? INT_MAX : n;
1038 tail += strspn (tail, CC_SPACES);
1039 return *tail == '\0' && errno != ERANGE && n == *np;
1043 expand_macro_function (struct parse_macro_function_ctx *ctx,
1044 struct string *output,
1045 size_t *input_consumed)
1047 struct string_array args;
1049 if (parse_macro_function (ctx, &args, ss_cstr ("!length"), 1, 1,
1051 ds_put_format (output, "%zu", strlen (args.strings[0]));
1052 else if (parse_macro_function (ctx, &args, ss_cstr ("!blanks"), 1, 1,
1056 if (!parse_integer (args.strings[0], &n))
1058 macro_error (ctx->stack,
1059 _("Argument to !BLANKS must be non-negative integer "
1060 "(not \"%s\")"), args.strings[0]);
1061 string_array_destroy (&args);
1065 ds_put_byte_multiple (output, ' ', n);
1067 else if (parse_macro_function (ctx, &args, ss_cstr ("!concat"), 1, INT_MAX,
1070 for (size_t i = 0; i < args.n; i++)
1071 if (!unquote_string (args.strings[i], output))
1072 ds_put_cstr (output, args.strings[i]);
1074 else if (parse_macro_function (ctx, &args, ss_cstr ("!head"), 1, 1,
1078 const char *s = unquote_string_in_place (args.strings[0], &tmp);
1080 struct macro_tokens mts = { .n = 0 };
1081 macro_tokens_from_string (&mts, ss_cstr (s), SEG_MODE_INTERACTIVE /* XXX */);
1083 ds_put_substring (output, mts.mts[0].representation);
1084 macro_tokens_uninit (&mts);
1087 else if (parse_macro_function (ctx, &args, ss_cstr ("!index"), 2, 2,
1090 const char *haystack = args.strings[0];
1091 const char *needle = strstr (haystack, args.strings[1]);
1092 ds_put_format (output, "%zu", needle ? needle - haystack + 1 : 0);
1094 else if (parse_macro_function (ctx, &args, ss_cstr ("!quote"), 1, 1,
1097 if (unquote_string (args.strings[0], NULL))
1098 ds_put_cstr (output, args.strings[0]);
1101 ds_extend (output, strlen (args.strings[0]) + 2);
1102 ds_put_byte (output, '\'');
1103 for (const char *p = args.strings[0]; *p; p++)
1106 ds_put_byte (output, '\'');
1107 ds_put_byte (output, *p);
1109 ds_put_byte (output, '\'');
1112 else if (parse_macro_function (ctx, &args, ss_cstr ("!substr"), 2, 3,
1116 if (!parse_integer (args.strings[1], &start) || start < 1)
1118 macro_error (ctx->stack, _("Second argument of !SUBSTR must be "
1119 "positive integer (not \"%s\")"),
1121 string_array_destroy (&args);
1125 int count = INT_MAX;
1126 if (args.n > 2 && (!parse_integer (args.strings[2], &count) || count < 0))
1128 macro_error (ctx->stack, _("Third argument of !SUBSTR must be "
1129 "non-negative integer (not \"%s\")"),
1131 string_array_destroy (&args);
1135 struct substring s = ss_cstr (args.strings[0]);
1136 ds_put_substring (output, ss_substr (s, start - 1, count));
1138 else if (parse_macro_function (ctx, &args, ss_cstr ("!tail"), 1, 1,
1142 const char *s = unquote_string_in_place (args.strings[0], &tmp);
1144 struct macro_tokens mts = { .n = 0 };
1145 macro_tokens_from_string (&mts, ss_cstr (s), SEG_MODE_INTERACTIVE /* XXX */);
1148 struct macro_tokens tail = { .mts = mts.mts + 1, .n = mts.n - 1 };
1149 macro_tokens_to_representation (&tail, output, NULL, NULL);
1151 macro_tokens_uninit (&mts);
1154 else if (parse_macro_function (ctx, &args, ss_cstr ("!unquote"), 1, 1,
1157 if (!unquote_string (args.strings[0], output))
1158 ds_put_cstr (output, args.strings[0]);
1160 else if (parse_macro_function (ctx, &args, ss_cstr ("!upcase"), 1, 1,
1164 const char *s = unquote_string_in_place (args.strings[0], &tmp);
1165 char *upper = utf8_to_upper (s);
1166 ds_put_cstr (output, upper);
1170 else if (parse_macro_function (ctx, &args, ss_cstr ("!eval"), 1, 1,
1173 struct macro_tokens mts = { .n = 0 };
1174 macro_tokens_from_string (&mts, ss_cstr (args.strings[0]),
1175 SEG_MODE_INTERACTIVE /* XXX */);
1176 struct macro_tokens exp = { .n = 0 };
1177 macro_expand (&mts, ctx->nesting_countdown - 1, ctx->macros, ctx->me,
1179 &(struct macro_expansion_stack) {
1182 }, ctx->expand, NULL, &exp);
1183 macro_tokens_to_representation (&exp, output, NULL, NULL);
1184 macro_tokens_uninit (&exp);
1185 macro_tokens_uninit (&mts);
1187 else if (ctx->n_input > 0
1188 && ctx->input[0].token.type == T_MACRO_ID
1189 && ss_equals_case (ctx->input[0].token.string, ss_cstr ("!null")))
1191 *input_consumed = 1;
1197 string_array_destroy (&args);
1203 int nesting_countdown;
1204 const struct macro_set *macros;
1205 const struct macro_expander *me;
1206 const struct macro_expansion_stack *stack;
1207 struct string_map *vars;
1211 static char *macro_evaluate_or (const struct expr_context *ctx,
1212 const struct macro_token **tokens,
1213 const struct macro_token *end);
1216 macro_evaluate_literal (const struct expr_context *ctx,
1217 const struct macro_token **tokens,
1218 const struct macro_token *end)
1220 const struct macro_token *p = *tokens;
1223 if (p->token.type == T_LPAREN)
1226 char *value = macro_evaluate_or (ctx, &p, end);
1229 if (p >= end || p->token.type != T_RPAREN)
1232 macro_error (ctx->stack, _("Expecting ')' in macro expression."));
1240 struct parse_macro_function_ctx fctx = {
1243 .nesting_countdown = ctx->nesting_countdown,
1244 .macros = ctx->macros,
1246 .stack = ctx->stack,
1248 .expand = ctx->expand,
1250 struct string function_output = DS_EMPTY_INITIALIZER;
1251 size_t function_consumed = parse_function_arg (&fctx, 0, &function_output);
1252 struct string unquoted = DS_EMPTY_INITIALIZER;
1253 if (unquote_string (ds_cstr (&function_output), &unquoted))
1255 ds_swap (&function_output, &unquoted);
1256 ds_destroy (&unquoted);
1258 *tokens = p + function_consumed;
1259 return ds_steal_cstr (&function_output);
1262 /* Returns true if MT is valid as a macro operator. Only operators written as
1263 symbols (e.g. <>) are usable in macro expressions, not operator written as
1264 letters (e.g. EQ). */
1266 is_macro_operator (const struct macro_token *mt)
1268 return (mt->representation.length > 0
1269 && !c_isalpha (mt->representation.string[0]));
1272 static enum token_type
1273 parse_relational_op (const struct macro_token *mt)
1275 switch (mt->token.type)
1285 return is_macro_operator (mt) ? mt->token.type : T_STOP;
1288 return (ss_equals_case (mt->token.string, ss_cstr ("!EQ")) ? T_EQ
1289 : ss_equals_case (mt->token.string, ss_cstr ("!NE")) ? T_NE
1290 : ss_equals_case (mt->token.string, ss_cstr ("!LT")) ? T_LT
1291 : ss_equals_case (mt->token.string, ss_cstr ("!GT")) ? T_GT
1292 : ss_equals_case (mt->token.string, ss_cstr ("!LE")) ? T_LE
1293 : ss_equals_case (mt->token.string, ss_cstr ("!GE")) ? T_GE
1302 macro_evaluate_relational (const struct expr_context *ctx,
1303 const struct macro_token **tokens,
1304 const struct macro_token *end)
1306 const struct macro_token *p = *tokens;
1307 char *lhs = macro_evaluate_literal (ctx, &p, end);
1311 enum token_type op = p >= end ? T_STOP : parse_relational_op (p);
1319 char *rhs = macro_evaluate_literal (ctx, &p, end);
1326 struct string lhs_tmp, rhs_tmp;
1327 int cmp = strcmp/*XXX*/ (unquote_string_in_place (lhs, &lhs_tmp),
1328 unquote_string_in_place (rhs, &rhs_tmp));
1329 ds_destroy (&lhs_tmp);
1330 ds_destroy (&rhs_tmp);
1335 bool b = (op == T_EQUALS || op == T_EQ ? !cmp
1337 : op == T_LT ? cmp < 0
1338 : op == T_GT ? cmp > 0
1339 : op == T_LE ? cmp <= 0
1340 :/*op == T_GE*/cmp >= 0);
1343 return xstrdup (b ? "1" : "0");
1347 macro_evaluate_not (const struct expr_context *ctx,
1348 const struct macro_token **tokens,
1349 const struct macro_token *end)
1351 const struct macro_token *p = *tokens;
1353 unsigned int negations = 0;
1355 && (ss_equals_case (p->representation, ss_cstr ("!NOT"))
1356 || ss_equals (p->representation, ss_cstr ("~"))))
1362 char *operand = macro_evaluate_relational (ctx, &p, end);
1363 if (!operand || !negations)
1369 bool b = strcmp (operand, "0") ^ (negations & 1);
1372 return xstrdup (b ? "1" : "0");
1376 macro_evaluate_and (const struct expr_context *ctx,
1377 const struct macro_token **tokens,
1378 const struct macro_token *end)
1380 const struct macro_token *p = *tokens;
1381 char *lhs = macro_evaluate_not (ctx, &p, end);
1386 && (ss_equals_case (p->representation, ss_cstr ("!AND"))
1387 || ss_equals (p->representation, ss_cstr ("&"))))
1390 char *rhs = macro_evaluate_not (ctx, &p, end);
1397 bool b = strcmp (lhs, "0") && strcmp (rhs, "0");
1400 lhs = xstrdup (b ? "1" : "0");
1407 macro_evaluate_or (const struct expr_context *ctx,
1408 const struct macro_token **tokens,
1409 const struct macro_token *end)
1411 const struct macro_token *p = *tokens;
1412 char *lhs = macro_evaluate_and (ctx, &p, end);
1417 && (ss_equals_case (p->representation, ss_cstr ("!OR"))
1418 || ss_equals (p->representation, ss_cstr ("|"))))
1421 char *rhs = macro_evaluate_and (ctx, &p, end);
1428 bool b = strcmp (lhs, "0") || strcmp (rhs, "0");
1431 lhs = xstrdup (b ? "1" : "0");
1438 macro_evaluate_expression (const struct macro_token **tokens, size_t n_tokens,
1439 int nesting_countdown, const struct macro_set *macros,
1440 const struct macro_expander *me,
1441 const struct macro_expansion_stack *stack,
1442 struct string_map *vars, bool *expand)
1444 const struct expr_context ctx = {
1445 .nesting_countdown = nesting_countdown,
1452 return macro_evaluate_or (&ctx, tokens, *tokens + n_tokens);
1456 macro_evaluate_number (const struct macro_token **tokens, size_t n_tokens,
1457 int nesting_countdown, const struct macro_set *macros,
1458 const struct macro_expander *me,
1459 const struct macro_expansion_stack *stack,
1460 struct string_map *vars,
1461 bool *expand, double *number)
1463 char *s = macro_evaluate_expression (tokens, n_tokens, nesting_countdown,
1464 macros, me, stack, vars, expand);
1468 struct macro_tokens mts = { .n = 0 };
1469 macro_tokens_from_string (&mts, ss_cstr (s), SEG_MODE_INTERACTIVE /* XXX */);
1470 if (mts.n != 1 || !token_is_number (&mts.mts[0].token))
1472 macro_tokens_print (&mts, stdout);
1473 macro_error (stack, _("Macro expression must evaluate to "
1474 "a number (not \"%s\")"), s);
1476 macro_tokens_uninit (&mts);
1480 *number = token_number (&mts.mts[0].token);
1482 macro_tokens_uninit (&mts);
1486 static const struct macro_token *
1487 find_ifend_clause (const struct macro_token *p, const struct macro_token *end)
1490 for (; p < end; p++)
1492 if (p->token.type != T_MACRO_ID)
1495 if (ss_equals_case (p->token.string, ss_cstr ("!IF")))
1497 else if (ss_equals_case (p->token.string, ss_cstr ("!IFEND")))
1503 else if (ss_equals_case (p->token.string, ss_cstr ("!ELSE")) && !nesting)
1510 macro_expand_if (const struct macro_token *tokens, size_t n_tokens,
1511 int nesting_countdown, const struct macro_set *macros,
1512 const struct macro_expander *me,
1513 const struct macro_expansion_stack *stack,
1514 struct string_map *vars,
1515 bool *expand, bool *break_, struct macro_tokens *exp)
1517 const struct macro_token *p = tokens;
1518 const struct macro_token *end = tokens + n_tokens;
1520 if (p >= end || !ss_equals_case (p->token.string, ss_cstr ("!IF")))
1524 char *result = macro_evaluate_expression (&p, end - p,
1525 nesting_countdown, macros, me,
1526 stack, vars, expand);
1529 bool b = strcmp (result, "0");
1533 || p->token.type != T_MACRO_ID
1534 || !ss_equals_case (p->token.string, ss_cstr ("!THEN")))
1536 macro_error (stack, _("!THEN expected in macro !IF construct."));
1540 const struct macro_token *start_then = p + 1;
1541 const struct macro_token *end_then = find_ifend_clause (start_then, end);
1545 _("!ELSE or !IFEND expected in macro !IF construct."));
1549 const struct macro_token *start_else, *end_if;
1550 if (ss_equals_case (end_then->token.string, ss_cstr ("!ELSE")))
1552 start_else = end_then + 1;
1553 end_if = find_ifend_clause (start_else, end);
1555 || !ss_equals_case (end_if->token.string, ss_cstr ("!IFEND")))
1557 macro_error (stack, _("!IFEND expected in macro !IF construct."));
1567 const struct macro_token *start;
1572 n = end_then - start_then;
1574 else if (start_else)
1577 n = end_if - start_else;
1587 struct macro_tokens mts = {
1588 .mts = CONST_CAST (struct macro_token *, start),
1591 macro_expand (&mts, nesting_countdown, macros, me, vars,
1592 &(struct macro_expansion_stack) {
1596 expand, break_, exp);
1598 return (end_if + 1) - tokens;
1602 macro_parse_let (const struct macro_token *tokens, size_t n_tokens,
1603 int nesting_countdown, const struct macro_set *macros,
1604 const struct macro_expander *me,
1605 const struct macro_expansion_stack *stack,
1606 struct string_map *vars, bool *expand)
1608 const struct macro_token *p = tokens;
1609 const struct macro_token *end = tokens + n_tokens;
1611 if (p >= end || !ss_equals_case (p->token.string, ss_cstr ("!LET")))
1615 if (p >= end || p->token.type != T_MACRO_ID)
1617 macro_error (stack, _("Expected macro variable name following !LET."));
1620 const struct substring var_name = p->token.string;
1621 if (is_macro_keyword (var_name)
1622 || macro_find_parameter_by_name (me->macro, var_name))
1624 macro_error (stack, _("Cannot use argument name or macro keyword "
1625 "\"%.*s\" as !LET variable"),
1626 (int) var_name.length, var_name.string);
1631 if (p >= end || p->token.type != T_EQUALS)
1633 macro_error (stack, _("Expected `=' following !LET"));
1638 char *value = macro_evaluate_expression (&p, end - p,
1639 nesting_countdown, macros, me, stack,
1644 string_map_replace_nocopy (vars, ss_xstrdup (var_name), value);
1648 static const struct macro_token *
1649 find_doend (const struct macro_expansion_stack *stack,
1650 const struct macro_token *p, const struct macro_token *end)
1653 for (; p < end; p++)
1655 if (p->token.type != T_MACRO_ID)
1658 if (ss_equals_case (p->token.string, ss_cstr ("!DO")))
1660 else if (ss_equals_case (p->token.string, ss_cstr ("!DOEND")))
1667 macro_error (stack, _("Missing !DOEND."));
1672 macro_expand_do (const struct macro_token *tokens, size_t n_tokens,
1673 int nesting_countdown, const struct macro_set *macros,
1674 const struct macro_expander *me,
1675 const struct macro_expansion_stack *stack,
1676 struct string_map *vars,
1677 bool *expand, struct macro_tokens *exp)
1679 const struct macro_token *p = tokens;
1680 const struct macro_token *end = tokens + n_tokens;
1682 if (p >= end || !ss_equals_case (p->token.string, ss_cstr ("!DO")))
1686 if (p >= end || p->token.type != T_MACRO_ID)
1688 macro_error (stack, _("Expected macro variable name following !DO"));
1691 const struct substring var_name = p->token.string;
1692 if (is_macro_keyword (var_name)
1693 || macro_find_parameter_by_name (me->macro, var_name))
1695 macro_error (stack, _("Cannot use argument name or macro "
1696 "keyword \"%.*s\" as !DO variable"),
1697 (int) var_name.length, var_name.string);
1702 struct macro_expansion_stack next_stack = {
1703 .name = "!DO", .next = stack,
1705 int miterate = settings_get_miterate ();
1706 if (p < end && p->token.type == T_MACRO_ID
1707 && ss_equals_case (p->token.string, ss_cstr ("!IN")))
1710 char *list = macro_evaluate_expression (&p, end - p,
1711 nesting_countdown, macros, me,
1712 &next_stack, vars, expand);
1716 struct macro_tokens items = { .n = 0 };
1717 macro_tokens_from_string (&items, ss_cstr (list),
1718 SEG_MODE_INTERACTIVE /* XXX */);
1721 const struct macro_token *do_end = find_doend (stack, p, end);
1724 macro_tokens_uninit (&items);
1728 const struct macro_tokens inner = {
1729 .mts = CONST_CAST (struct macro_token *, p),
1732 for (size_t i = 0; i < items.n; i++)
1736 macro_error (stack, _("!DO loop over list exceeded "
1737 "maximum number of iterations %d. "
1738 "(Use SET MITERATE to change the limit.)"),
1742 string_map_replace_nocopy (vars, ss_xstrdup (var_name),
1743 ss_xstrdup (items.mts[i].representation));
1745 bool break_ = false;
1746 macro_expand (&inner, nesting_countdown, macros,
1747 me, vars, &next_stack, expand, &break_, exp);
1751 return do_end - tokens + 1;
1753 else if (p < end && p->token.type == T_EQUALS)
1757 if (!macro_evaluate_number (&p, end - p, nesting_countdown, macros, me,
1758 &next_stack, vars, expand, &first))
1761 if (p >= end || p->token.type != T_MACRO_ID
1762 || !ss_equals_case (p->token.string, ss_cstr ("!TO")))
1764 macro_error (stack, _("Expected !TO in numerical !DO loop"));
1770 if (!macro_evaluate_number (&p, end - p, nesting_countdown, macros, me,
1771 &next_stack, vars, expand, &last))
1775 if (p < end && p->token.type == T_MACRO_ID
1776 && ss_equals_case (p->token.string, ss_cstr ("!BY")))
1779 if (!macro_evaluate_number (&p, end - p, nesting_countdown, macros, me,
1780 &next_stack, vars, expand, &by))
1785 macro_error (stack, _("!BY value cannot be zero."));
1790 const struct macro_token *do_end = find_doend (stack, p, end);
1793 const struct macro_tokens inner = {
1794 .mts = CONST_CAST (struct macro_token *, p),
1798 if ((by > 0 && first <= last) || (by < 0 && first >= last))
1801 for (double index = first;
1802 by > 0 ? (index <= last) : (index >= last);
1808 _("Numerical !DO loop exceeded "
1809 "maximum number of iterations %d. "
1810 "(Use SET MITERATE to change the limit.)"),
1815 char index_s[DBL_BUFSIZE_BOUND];
1816 c_dtoastr (index_s, sizeof index_s, 0, 0, index);
1817 string_map_replace_nocopy (vars, ss_xstrdup (var_name),
1820 bool break_ = false;
1821 macro_expand (&inner, nesting_countdown, macros,
1822 me, vars, &next_stack, expand, &break_, exp);
1828 return do_end - tokens + 1;
1832 macro_error (stack, _("Expected `=' or !IN in !DO loop."));
1838 macro_expand (const struct macro_tokens *mts,
1839 int nesting_countdown, const struct macro_set *macros,
1840 const struct macro_expander *me, struct string_map *vars,
1841 const struct macro_expansion_stack *stack,
1842 bool *expand, bool *break_, struct macro_tokens *exp)
1844 if (nesting_countdown <= 0)
1846 macro_error (stack, _("Maximum nesting level %d exceeded."),
1847 settings_get_mnest ());
1848 for (size_t i = 0; i < mts->n; i++)
1849 macro_tokens_add (exp, &mts->mts[i]);
1853 struct string_map own_vars = STRING_MAP_INITIALIZER (own_vars);
1857 for (size_t i = 0; i < mts->n && (!break_ || !*break_); i++)
1859 const struct macro_token *mt = &mts->mts[i];
1860 const struct token *token = &mt->token;
1861 if (token->type == T_MACRO_ID && me)
1863 const struct macro_param *param = macro_find_parameter_by_name (
1864 me->macro, token->string);
1867 const struct macro_tokens *arg = me->args[param - me->macro->params];
1868 if (*expand && param->expand_arg)
1869 macro_expand (arg, nesting_countdown, macros, NULL, NULL,
1870 &(struct macro_expansion_stack) {
1871 .name = param->name,
1873 }, expand, break_, exp);
1875 for (size_t i = 0; i < arg->n; i++)
1876 macro_tokens_add (exp, &arg->mts[i]);
1880 if (is_bang_star (mts->mts, mts->n, i))
1882 for (size_t j = 0; j < me->macro->n_params; j++)
1884 const struct macro_param *param = &me->macro->params[j];
1885 if (!param->positional)
1888 const struct macro_tokens *arg = me->args[j];
1889 if (*expand && param->expand_arg)
1890 macro_expand (arg, nesting_countdown, macros, NULL, NULL,
1891 &(struct macro_expansion_stack) {
1894 }, expand, break_, exp);
1896 for (size_t k = 0; k < arg->n; k++)
1897 macro_tokens_add (exp, &arg->mts[k]);
1903 size_t n = macro_expand_if (&mts->mts[i], mts->n - i,
1904 nesting_countdown, macros, me, stack,
1905 vars, expand, break_, exp);
1913 if (token->type == T_MACRO_ID && vars)
1915 const char *value = string_map_find__ (vars, token->string.string,
1916 token->string.length);
1919 macro_tokens_from_string (exp, ss_cstr (value),
1920 SEG_MODE_INTERACTIVE /* XXX */);
1927 struct macro_expander *subme;
1928 int retval = macro_expander_create (macros, token, &subme);
1929 for (size_t j = 1; !retval; j++)
1931 const struct macro_token endcmd = { .token = { .type = T_ENDCMD } };
1932 retval = macro_expander_add (
1933 subme, i + j < mts->n ? &mts->mts[i + j] : &endcmd);
1938 macro_expand (&subme->macro->body, nesting_countdown - 1, macros,
1940 &(struct macro_expansion_stack) {
1941 .name = subme->macro->name,
1942 .file_name = subme->macro->file_name,
1943 .first_line = subme->macro->first_line,
1944 .last_line = subme->macro->last_line,
1946 }, expand, break_, exp);
1947 macro_expander_destroy (subme);
1951 macro_expander_destroy (subme);
1954 if (token->type != T_MACRO_ID)
1956 macro_tokens_add (exp, mt);
1960 if (ss_equals_case (token->string, ss_cstr ("!break")))
1963 macro_error (stack, _("!BREAK outside !DO."));
1971 struct parse_macro_function_ctx ctx = {
1972 .input = &mts->mts[i],
1973 .n_input = mts->n - i,
1974 .nesting_countdown = nesting_countdown,
1981 struct string function_output = DS_EMPTY_INITIALIZER;
1982 size_t function_consumed;
1983 if (expand_macro_function (&ctx, &function_output, &function_consumed))
1985 i += function_consumed - 1;
1987 macro_tokens_from_string (exp, function_output.ss,
1988 SEG_MODE_INTERACTIVE /* XXX */);
1989 ds_destroy (&function_output);
1994 size_t n = macro_parse_let (&mts->mts[i], mts->n - i,
1995 nesting_countdown, macros, me, stack,
2003 n = macro_expand_do (&mts->mts[i], mts->n - i,
2004 nesting_countdown, macros, me, stack, vars,
2012 if (ss_equals_case (token->string, ss_cstr ("!onexpand")))
2014 else if (ss_equals_case (token->string, ss_cstr ("!offexpand")))
2017 macro_tokens_add (exp, mt);
2019 if (vars == &own_vars)
2020 string_map_destroy (&own_vars);
2024 macro_expander_get_expansion (struct macro_expander *me, struct macro_tokens *exp)
2027 struct macro_expansion_stack stack = {
2028 .name = me->macro->name,
2029 .file_name = me->macro->file_name,
2030 .first_line = me->macro->first_line,
2031 .last_line = me->macro->last_line,
2033 macro_expand (&me->macro->body, settings_get_mnest (),
2034 me->macros, me, NULL, &stack, &expand, NULL, exp);