+void
+macro_tokens_from_string (struct macro_tokens *mts, const struct substring src,
+ enum segmenter_mode mode)
+{
+ struct state
+ {
+ struct segmenter segmenter;
+ struct substring body;
+ };
+
+ struct state state = {
+ .segmenter = SEGMENTER_INIT (mode),
+ .body = src,
+ };
+ struct state saved = state;
+
+ while (state.body.length > 0)
+ {
+ struct macro_token mt = {
+ .token = { .type = T_STOP },
+ .representation = { .string = state.body.string },
+ };
+ struct token *token = &mt.token;
+
+ struct scanner scanner;
+ scanner_init (&scanner, token);
+
+ for (;;)
+ {
+ enum segment_type type;
+ int seg_len = segmenter_push (&state.segmenter, state.body.string,
+ state.body.length, true, &type);
+ assert (seg_len >= 0);
+
+ struct substring segment = ss_head (state.body, seg_len);
+ ss_advance (&state.body, seg_len);
+
+ enum scan_result result = scanner_push (&scanner, type, segment, token);
+ if (result == SCAN_SAVE)
+ saved = state;
+ else if (result == SCAN_BACK)
+ {
+ state = saved;
+ break;
+ }
+ else if (result == SCAN_DONE)
+ break;
+ }
+
+ /* We have a token in 'token'. */
+ if (is_scan_type (token->type))
+ {
+ if (token->type != SCAN_SKIP)
+ {
+ /* XXX report error */
+ }
+ }
+ else
+ {
+ mt.representation.length = state.body.string - mt.representation.string;
+ macro_tokens_add (mts, &mt);
+ }
+ token_uninit (token);
+ }
+}
+