case '<': return T_LT;
case '>': return T_GT;
case '~': return T_NOT;
+ default: return T_MACRO_PUNCT;
}
NOT_REACHED ();
case SEG_DO_REPEAT_COMMAND:
case SEG_INLINE_DATA:
case SEG_DOCUMENT:
+ case SEG_MACRO_BODY:
token->type = T_STRING;
ss_alloc_substring (&token->string, s);
return SCAN_DONE;
else
{
token->type = scan_punct__ (s);
+ if (token->type == T_MACRO_PUNCT)
+ ss_alloc_substring (&token->string, s);
return SCAN_DONE;
}
ss_alloc_substring (&token->string, s);
return SCAN_DONE;
- case SEG_UNEXPECTED_DOT:
- token->type = SCAN_UNEXPECTED_DOT;
- return SCAN_DONE;
-
case SEG_UNEXPECTED_CHAR:
return scan_unexpected_char (&s, token);
}
scanner_init (struct scanner *scanner, struct token *token)
{
scanner->state = S_START;
- token_init (token);
+ *token = (struct token) { .type = T_STOP };
}
/* Adds the segment with type TYPE and UTF-8 text S to SCANNER. TOKEN must be
string_lexer_init (struct string_lexer *slex, const char *input, size_t length,
enum segmenter_mode mode)
{
- slex->input = input;
- slex->length = length;
- slex->offset = 0;
- segmenter_init (&slex->segmenter, mode);
+ *slex = (struct string_lexer) {
+ .input = input,
+ .length = length,
+ .offset = 0,
+ .segmenter = segmenter_init (mode, true),
+ };
}
/* */