From: Ben Pfaff Date: Mon, 10 May 2021 05:37:17 +0000 (-0700) Subject: !quote, !unquote X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c8553a5f7b9974755bf6384060db3cb6d7e6f165;p=pspp !quote, !unquote --- diff --git a/src/language/control/define.c b/src/language/control/define.c index b0699a56b1..686685fc9c 100644 --- a/src/language/control/define.c +++ b/src/language/control/define.c @@ -72,71 +72,6 @@ parse_quoted_token (struct lexer *lexer, struct token *token) return true; } -static void -macro_tokenize (struct macro *m, const struct substring body, struct lexer *lexer) -{ - struct state - { - struct segmenter segmenter; - struct substring body; - }; - - struct state state = { - .segmenter = SEGMENTER_INIT (lex_get_syntax_mode (lexer)), - .body = body, - }; - struct state saved = state; - - while (state.body.length > 0) - { - struct macro_token mt = { - .token = { .type = T_STOP }, - .representation = { .string = state.body.string }, - }; - struct token *token = &mt.token; - - struct scanner scanner; - scanner_init (&scanner, token); - - for (;;) - { - enum segment_type type; - int seg_len = segmenter_push (&state.segmenter, state.body.string, - state.body.length, true, &type); - assert (seg_len >= 0); - - struct substring segment = ss_head (state.body, seg_len); - ss_advance (&state.body, seg_len); - - enum scan_result result = scanner_push (&scanner, type, segment, token); - if (result == SCAN_SAVE) - saved = state; - else if (result == SCAN_BACK) - { - state = saved; - break; - } - else if (result == SCAN_DONE) - break; - } - - /* We have a token in 'token'. */ - if (is_scan_type (token->type)) - { - if (token->type != SCAN_SKIP) - { - /* XXX report error */ - } - } - else - { - mt.representation.length = state.body.string - mt.representation.string; - macro_tokens_add (&m->body, &mt); - } - token_uninit (token); - } -} - int cmd_define (struct lexer *lexer, struct dataset *ds UNUSED) { @@ -275,7 +210,7 @@ cmd_define (struct lexer *lexer, struct dataset *ds UNUSED) lex_get (lexer); } - macro_tokenize (m, body.ss, lexer); + macro_tokens_from_string (&m->body, body.ss, lex_get_syntax_mode (lexer)); ds_destroy (&body); lex_define_macro (lexer, m); diff --git a/src/language/lexer/macro.c b/src/language/lexer/macro.c index 05e3b45b83..a15b7064a5 100644 --- a/src/language/lexer/macro.c +++ b/src/language/lexer/macro.c @@ -80,6 +80,72 @@ macro_tokens_add (struct macro_tokens *mts, const struct macro_token *mt) macro_token_copy (macro_tokens_add_uninit (mts), mt); } +void +macro_tokens_from_string (struct macro_tokens *mts, const struct substring src, + enum segmenter_mode mode) +{ + struct state + { + struct segmenter segmenter; + struct substring body; + }; + + struct state state = { + .segmenter = SEGMENTER_INIT (mode), + .body = src, + }; + struct state saved = state; + + while (state.body.length > 0) + { + struct macro_token mt = { + .token = { .type = T_STOP }, + .representation = { .string = state.body.string }, + }; + struct token *token = &mt.token; + + struct scanner scanner; + scanner_init (&scanner, token); + + for (;;) + { + enum segment_type type; + int seg_len = segmenter_push (&state.segmenter, state.body.string, + state.body.length, true, &type); + assert (seg_len >= 0); + + struct substring segment = ss_head (state.body, seg_len); + ss_advance (&state.body, seg_len); + + enum scan_result result = scanner_push (&scanner, type, segment, token); + if (result == SCAN_SAVE) + saved = state; + else if (result == SCAN_BACK) + { + state = saved; + break; + } + else if (result == SCAN_DONE) + break; + } + + /* We have a token in 'token'. */ + if (is_scan_type (token->type)) + { + if (token->type != SCAN_SKIP) + { + /* XXX report error */ + } + } + else + { + mt.representation.length = state.body.string - mt.representation.string; + macro_tokens_add (mts, &mt); + } + token_uninit (token); + } +} + void macro_tokens_print (const struct macro_tokens *mts, FILE *stream) { @@ -667,16 +733,19 @@ expand_macro_function (struct parse_macro_function_ctx *ctx, if (args.mts[0].token.type != T_POS_NUM) { printf ("argument to !BLANKS must be positive integer\n"); - macro_token_uninit (output); + macro_tokens_uninit (&args); return false; } struct string s = DS_EMPTY_INITIALIZER; ds_put_byte_multiple (&s, ' ', args.mts[0].token.number); + struct substring s_copy; + ss_alloc_substring (&s_copy, s.ss); + *output = (struct macro_token) { .token = { .type = T_ID, .string = s.ss }, - .representation = s.ss, + .representation = s_copy, }; } else if (parse_macro_function (ctx, &args, ss_cstr ("!concat"), 1, INT_MAX, @@ -710,6 +779,30 @@ expand_macro_function (struct parse_macro_function_ctx *ctx, ss_alloc_substring (&output->representation, s.ss); } } + else if (parse_macro_function (ctx, &args, ss_cstr ("!quote"), 1, 1, + input_consumed)) + { + if (args.mts[0].token.type == T_STRING) + macro_token_copy (output, &args.mts[0]); + else + { + *output = (struct macro_token) { .token = { .type = T_STRING } }; + ss_alloc_substring (&output->token.string, args.mts[0].representation); + output->representation = ss_cstr (token_to_string (&output->token)); + } + } + else if (parse_macro_function (ctx, &args, ss_cstr ("!unquote"), 1, 1, + input_consumed)) + { + if (args.mts[0].token.type == T_STRING) + { + *output = (struct macro_token) { .token = { .type = T_MACRO_ID } }; + ss_alloc_substring (&output->token.string, args.mts[0].token.string); + output->representation = ss_cstr (token_to_string (&output->token)); + } + else + macro_token_copy (output, &args.mts[0]); + } else return false; @@ -796,7 +889,12 @@ macro_expand (const struct macro_tokens *mts, { i += function_consumed - 1; - + if (function_output.token.type == T_MACRO_ID) + macro_tokens_from_string (exp, function_output.token.string, + SEG_MODE_INTERACTIVE /* XXX */); + else + macro_tokens_add (exp, &function_output); + macro_token_uninit (&function_output); continue; } diff --git a/src/language/lexer/macro.h b/src/language/lexer/macro.h index 6eb239240a..23ae1d9a18 100644 --- a/src/language/lexer/macro.h +++ b/src/language/lexer/macro.h @@ -22,6 +22,7 @@ #include "libpspp/hmap.h" #include "libpspp/str.h" +#include "language/lexer/segment.h" #include "language/lexer/token.h" struct macro_expander; @@ -47,6 +48,9 @@ void macro_tokens_uninit (struct macro_tokens *); struct macro_token *macro_tokens_add_uninit (struct macro_tokens *); void macro_tokens_add (struct macro_tokens *, const struct macro_token *); +void macro_tokens_from_string (struct macro_tokens *, const struct substring, + enum segmenter_mode); + void macro_tokens_print (const struct macro_tokens *, FILE *); struct macro_param