S_DOCUMENT_1,
S_DOCUMENT_2,
S_DOCUMENT_3,
- S_FILE_LABEL,
+ S_FILE_LABEL_1,
+ S_FILE_LABEL_2,
+ S_FILE_LABEL_3,
S_DO_REPEAT_1,
S_DO_REPEAT_2,
S_DO_REPEAT_3,
S_BEGIN_DATA_2,
S_BEGIN_DATA_3,
S_BEGIN_DATA_4,
- S_TITLE_1,
- S_TITLE_2
};
#define SS_START_OF_LINE (1u << 0)
return is_end_of_line (input, n, eof, ofs);
}
+static bool
+is_all_spaces (const char *input_, size_t n)
+{
+ const uint8_t *input = CHAR_CAST (const uint8_t *, input_);
+
+ int mblen;
+ for (int ofs = 0; ofs < n; ofs += mblen)
+ {
+ ucs4_t uc;
+ mblen = u8_mbtouc (&uc, input + ofs, n - ofs);
+ if (!lex_uc_is_space (uc))
+ return false;
+ }
+ return true;
+}
+
static int
segmenter_parse_newline__ (const char *input, size_t n, bool eof,
enum segment_type *type)
static int
segmenter_parse_number__ (struct segmenter *s, const char *input, size_t n,
- bool eof, enum segment_type *type)
+ bool eof, enum segment_type *type, int ofs)
{
- int ofs;
-
assert (s->state == S_GENERAL);
- ofs = skip_digits (input, n, eof, 0);
+ ofs = skip_digits (input, n, eof, ofs);
if (ofs < 0)
return -1;
case SEG_DO_REPEAT_COMMAND:
case SEG_INLINE_DATA:
case SEG_MACRO_ID:
+ case SEG_MACRO_NAME:
case SEG_MACRO_BODY:
case SEG_START_DOCUMENT:
case SEG_DOCUMENT:
case SEG_END:
case SEG_EXPECTED_QUOTE:
case SEG_EXPECTED_EXPONENT:
- case SEG_UNEXPECTED_DOT:
case SEG_UNEXPECTED_CHAR:
id[0] = '\0';
return ofs + retval;
*type = SEG_START_DOCUMENT;
return 0;
}
- else if (lex_id_match (ss_cstr ("TITLE"), word)
- || lex_id_match (ss_cstr ("SUBTITLE"), word))
- {
- int result = segmenter_unquoted (input, n, eof, ofs);
- if (result < 0)
- return -1;
- else if (result)
- {
- s->state = S_TITLE_1;
- return ofs;
- }
- }
else if (lex_id_match_n (ss_cstr ("DEFINE"), word, 6))
{
s->state = S_DEFINE_1;
return -1;
else if (lex_id_match (ss_cstr ("LABEL"), ss_cstr (id)))
{
- s->state = S_FILE_LABEL;
+ s->state = S_FILE_LABEL_1;
s->substate = 0;
return ofs;
}
*type = SEG_PUNCT;
return 1;
- case '(': case ')': case ',': case '=': case '-':
+ case '-':
+ ofs = skip_spaces (input, n, eof, 1);
+ if (ofs < 0)
+ return -1;
+ else if (ofs < n && c_isdigit (input[ofs]))
+ return segmenter_parse_number__ (s, input, n, eof, type, ofs);
+ else if (ofs < n && input[ofs] == '.')
+ {
+ if (ofs + 1 >= n)
+ {
+ if (!eof)
+ return -1;
+ }
+ else if (c_isdigit (input[ofs + 1]))
+ return segmenter_parse_number__ (s, input, n, eof, type, ofs);
+ }
+ /* Fall through. */
+ case '(': case ')': case ',': case '=':
case '[': case ']': case '&': case '|': case '+':
*type = SEG_PUNCT;
s->substate = 0;
return -1;
}
else if (c_isdigit (input[1]))
- return segmenter_parse_number__ (s, input, n, eof, type);
+ return segmenter_parse_number__ (s, input, n, eof, type, 0);
int eol = at_end_of_line (input, n, eof, 1);
if (eol < 0)
s->substate = SS_START_OF_COMMAND;
}
else
- *type = SEG_UNEXPECTED_DOT;
+ *type = SEG_PUNCT;
return 1;
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
- return segmenter_parse_number__ (s, input, n, eof, type);
+ return segmenter_parse_number__ (s, input, n, eof, type, 0);
case 'u': case 'U':
return segmenter_maybe_parse_string__ (SEG_UNICODE_STRING,
s, input, n, eof, type);
case '!':
- return segmenter_parse_id__ (s, input, n, eof, type);
+ if (n < 2)
+ {
+ if (!eof)
+ return -1;
+ *type = SEG_PUNCT;
+ return 1;
+ }
+ else if (input[1] == '*')
+ {
+ *type = SEG_MACRO_ID;
+ return 2;
+ }
+ else
+ return segmenter_parse_id__ (s, input, n, eof, type);
default:
if (lex_uc_is_space (uc))
if (ofs < 0)
return -1;
- if (input[ofs - 1] == '\r' && input[ofs] == '\n')
+ if (ofs < n && input[ofs - 1] == '\r' && input[ofs] == '\n')
{
if (ofs == 1)
{
}
static int
-segmenter_parse_file_label__ (struct segmenter *s,
- const char *input, size_t n, bool eof,
- enum segment_type *type)
+segmenter_parse_file_label_1__ (struct segmenter *s,
+ const char *input, size_t n, bool eof,
+ enum segment_type *type)
{
struct segmenter sub;
int ofs;
else
{
if (result)
- s->state = S_TITLE_1;
+ s->state = S_FILE_LABEL_2;
else
*s = sub;
return ofs;
}
}
+static int
+segmenter_parse_file_label_2__ (struct segmenter *s,
+ const char *input, size_t n, bool eof,
+ enum segment_type *type)
+{
+ int ofs;
+
+ ofs = skip_spaces (input, n, eof, 0);
+ if (ofs < 0)
+ return -1;
+ s->state = S_FILE_LABEL_3;
+ *type = SEG_SPACES;
+ return ofs;
+}
+
+static int
+segmenter_parse_file_label_3__ (struct segmenter *s,
+ const char *input, size_t n, bool eof,
+ enum segment_type *type)
+{
+ int endcmd;
+ int ofs;
+
+ endcmd = -1;
+ ofs = 0;
+ while (ofs < n)
+ {
+ ucs4_t uc;
+ int mblen;
+
+ mblen = segmenter_u8_to_uc__ (&uc, input, n, eof, ofs);
+ if (mblen < 0)
+ return -1;
+
+ switch (uc)
+ {
+ case '\n':
+ goto end_of_line;
+
+ case '.':
+ endcmd = ofs;
+ break;
+
+ default:
+ if (!lex_uc_is_space (uc))
+ endcmd = -1;
+ break;
+ }
+
+ ofs += mblen;
+ }
+
+ if (eof)
+ {
+ end_of_line:
+ s->state = S_GENERAL;
+ s->substate = 0;
+ *type = SEG_UNQUOTED_STRING;
+ return endcmd >= 0 ? endcmd : ofs;
+ }
+
+ return -1;
+}
+
static int
segmenter_subparse (struct segmenter *s,
const char *input, size_t n, bool eof,
- The DEFINE keyword.
+ - An identifier. We transform this into SEG_MACRO_NAME instead of
+ SEG_IDENTIFIER or SEG_MACRO_NAME because this identifier must never be
+ macro-expanded.
+
- Anything but "(".
- "(" followed by a sequence of tokens possibly including balanced parentheses
up to a final ")".
- - A newline.
-
- - A sequence of lines that don't start with "!ENDDEFINE", one string per line,
- each ending in a newline.
-
- - "!ENDDEFINE".
-
+ - A sequence of any number of lines, one string per line, ending with
+ "!ENDDEFINE". The first line is usually blank (that is, a newline follows
+ the "("). The last line usually just has "!ENDDEFINE." on it, but it can
+ start with other tokens. The whole DEFINE...!ENDDEFINE can be on a single
+ line, even.
*/
static int
-segmenter_parse_define_1__ (struct segmenter *s,
- const char *input, size_t n, bool eof,
- enum segment_type *type)
+segmenter_parse_define_1_2__ (struct segmenter *s,
+ const char *input, size_t n, bool eof,
+ enum segment_type *type)
{
int ofs = segmenter_subparse (s, input, n, eof, type);
if (ofs < 0)
return -1;
- if (*type == SEG_SEPARATE_COMMANDS
+ if (s->state == S_DEFINE_1
+ && (*type == SEG_IDENTIFIER || *type == SEG_MACRO_ID))
+ {
+ *type = SEG_MACRO_NAME;
+ s->state = S_DEFINE_2;
+ }
+ else if (*type == SEG_SEPARATE_COMMANDS
|| *type == SEG_END_COMMAND
|| *type == SEG_START_COMMAND)
{
}
else if (*type == SEG_PUNCT && input[0] == '(')
{
- s->state = S_DEFINE_2;
+ s->state = S_DEFINE_3;
s->nest = 1;
return ofs;
}
}
static int
-segmenter_parse_define_2__ (struct segmenter *s,
+segmenter_parse_define_3__ (struct segmenter *s,
const char *input, size_t n, bool eof,
enum segment_type *type)
{
{
s->nest--;
if (!s->nest)
- s->state = S_DEFINE_3;
+ {
+ s->state = S_DEFINE_4;
+ s->substate = 0;
+ }
return ofs;
}
return ofs;
}
-static int
-segmenter_parse_define_3__ (struct segmenter *s,
- const char *input, size_t n, bool eof,
- enum segment_type *type)
+static size_t
+find_enddefine (struct substring input)
{
- int ofs = segmenter_subparse (s, input, n, eof, type);
- if (ofs < 0)
- return -1;
-
- if (*type == SEG_NEWLINE)
- s->state = S_DEFINE_4;
-
- return ofs;
-}
-
-static bool
-is_enddefine (const char *input, size_t n)
-{
- int ofs = skip_spaces_and_comments (input, n, true, 0);
- assert (ofs >= 0);
-
+ size_t n = input.length;
const struct substring enddefine = ss_cstr ("!ENDDEFINE");
- if (n - ofs < enddefine.length)
- return false;
-
- if (!ss_equals_case (ss_buffer (input + ofs, enddefine.length), enddefine))
- return false;
-
- if (ofs + enddefine.length >= n)
- return true;
-
- const uint8_t *u_input = CHAR_CAST (const uint8_t *, input);
- ucs4_t uc;
- u8_mbtouc (&uc, u_input + ofs, n - ofs);
- return uc == '.' || !lex_uc_is_idn (uc);
+ for (int ofs = 0;;)
+ {
+ /* Skip !ENDDEFINE in comment. */
+ ofs = skip_spaces_and_comments (input.string, n, true, ofs);
+ if (ofs + enddefine.length > n)
+ return SIZE_MAX;
+
+ char c = input.string[ofs];
+ if (c == '!'
+ && ss_equals_case (ss_substr (input, ofs, enddefine.length),
+ enddefine))
+ return ofs;
+ else if (c == '\'' || c == '"')
+ {
+ /* Skip quoted !ENDDEFINE. */
+ ofs++;
+ for (;;)
+ {
+ if (ofs >= n)
+ return SIZE_MAX;
+ else if (input.string[ofs++] == c)
+ break;
+ }
+ }
+ else
+ ofs++;
+ }
}
+/* We are in the body of a macro definition, looking for additional lines of
+ the body or !ENDDEFINE. */
static int
segmenter_parse_define_4__ (struct segmenter *s,
const char *input, size_t n, bool eof,
enum segment_type *type)
{
- int ofs;
-
- ofs = segmenter_parse_full_line__ (input, n, eof, type);
+ /* Gather a whole line. */
+ const char *newline = memchr (input, '\n', n);
+ int ofs = (newline ? newline - input - (newline > input && newline[-1] == '\r')
+ : eof ? n
+ : -1);
if (ofs < 0)
return -1;
- else if (is_enddefine (input, ofs))
+
+ /* Does the line contain !ENDDEFINE? */
+ size_t end = find_enddefine (ss_buffer (input, ofs));
+ if (end == SIZE_MAX)
{
- s->state = S_GENERAL;
- s->substate = SS_START_OF_COMMAND | SS_START_OF_LINE;
- return segmenter_push (s, input, n, eof, type);
+ /* No !ENDDEFINE. We have a full line of macro body.
+
+ The line might be blank, whether completely empty or just spaces and
+ comments. That's OK: we need to report blank lines because they can
+ have significance.
+
+ However, if the first line of the macro body (the same line as the
+ closing parenthesis in the argument definition) is blank, we just
+ report it as spaces because it's not significant. */
+ *type = (s->substate == 0 && is_all_spaces (input, ofs)
+ ? SEG_SPACES : SEG_MACRO_BODY);
+ s->state = S_DEFINE_5;
+ s->substate = 1;
+ return ofs;
}
else
{
- *type = SEG_MACRO_BODY;
- s->state = S_DEFINE_5;
- return input[ofs - 1] == '\n' ? 0 : ofs;
+ /* Macro ends at the !ENDDEFINE on this line. */
+ s->state = S_GENERAL;
+ s->substate = 0;
+ if (!end)
+ {
+ /* Line starts with !ENDDEFINE. */
+ return segmenter_push (s, input, n, eof, type);
+ }
+ else
+ {
+ if (is_all_spaces (input, end))
+ {
+ /* Line starts with spaces followed by !ENDDEFINE. */
+ *type = SEG_SPACES;
+ }
+ else
+ {
+ /* Line starts with some content followed by !ENDDEFINE. */
+ *type = SEG_MACRO_BODY;
+ }
+ return end;
+ }
}
}
const char *input, size_t n, bool eof,
enum segment_type *type)
{
- int ofs;
-
- ofs = segmenter_parse_newline__ (input, n, eof, type);
+ int ofs = segmenter_parse_newline__ (input, n, eof, type);
if (ofs < 0)
return -1;
return ofs;
}
-static int
-segmenter_parse_title_1__ (struct segmenter *s,
- const char *input, size_t n, bool eof,
- enum segment_type *type)
-{
- int ofs;
-
- ofs = skip_spaces (input, n, eof, 0);
- if (ofs < 0)
- return -1;
- s->state = S_TITLE_2;
- *type = SEG_SPACES;
- return ofs;
-}
-
-static int
-segmenter_parse_title_2__ (struct segmenter *s,
- const char *input, size_t n, bool eof,
- enum segment_type *type)
-{
- int endcmd;
- int ofs;
-
- endcmd = -1;
- ofs = 0;
- while (ofs < n)
- {
- ucs4_t uc;
- int mblen;
-
- mblen = segmenter_u8_to_uc__ (&uc, input, n, eof, ofs);
- if (mblen < 0)
- return -1;
-
- switch (uc)
- {
- case '\n':
- goto end_of_line;
-
- case '.':
- endcmd = ofs;
- break;
-
- default:
- if (!lex_uc_is_space (uc))
- endcmd = -1;
- break;
- }
-
- ofs += mblen;
- }
-
- if (eof)
- {
- end_of_line:
- s->state = S_GENERAL;
- s->substate = 0;
- *type = SEG_UNQUOTED_STRING;
- return endcmd >= 0 ? endcmd : ofs;
- }
-
- return -1;
-}
-
/* Returns the name of segment TYPE as a string. The caller must not modify
or free the returned string.
}
}
-/* Initializes S as a segmenter with the given syntax MODE.
+/* Returns a segmenter with the given syntax MODE.
+
+ If IS_SNIPPET is false, then the segmenter will parse as if it's being given
+ a whole file. This means, for example, that it will interpret - or + at the
+ beginning of the syntax as a separator between commands (since - or + at the
+ beginning of a line has this meaning).
+
+ If IS_SNIPPET is true, then the segmenter will parse as if it's being given
+ an isolated piece of syntax. This means that, for example, that it will
+ interpret - or + at the beginning of the syntax as an operator token or (if
+ followed by a digit) as part of a number.
A segmenter does not contain any external references, so nothing needs to be
done to destroy one. For the same reason, segmenters may be copied with
plain struct assignment (or memcpy). */
-void
-segmenter_init (struct segmenter *s, enum segmenter_mode mode)
+struct segmenter
+segmenter_init (enum segmenter_mode mode, bool is_snippet)
{
- s->state = S_SHBANG;
- s->substate = 0;
- s->mode = mode;
+ return (struct segmenter) {
+ .state = is_snippet ? S_GENERAL : S_SHBANG,
+ .mode = mode,
+ };
}
/* Returns the mode passed to segmenter_init() for S. */
case S_DOCUMENT_3:
return segmenter_parse_document_3__ (s, type);
- case S_FILE_LABEL:
- return segmenter_parse_file_label__ (s, input, n, eof, type);
+ case S_FILE_LABEL_1:
+ return segmenter_parse_file_label_1__ (s, input, n, eof, type);
+ case S_FILE_LABEL_2:
+ return segmenter_parse_file_label_2__ (s, input, n, eof, type);
+ case S_FILE_LABEL_3:
+ return segmenter_parse_file_label_3__ (s, input, n, eof, type);
case S_DO_REPEAT_1:
return segmenter_parse_do_repeat_1__ (s, input, n, eof, type);
return segmenter_parse_do_repeat_3__ (s, input, n, eof, type);
case S_DEFINE_1:
- return segmenter_parse_define_1__ (s, input, n, eof, type);
case S_DEFINE_2:
- return segmenter_parse_define_2__ (s, input, n, eof, type);
+ return segmenter_parse_define_1_2__ (s, input, n, eof, type);
case S_DEFINE_3:
return segmenter_parse_define_3__ (s, input, n, eof, type);
case S_DEFINE_4:
return segmenter_parse_begin_data_3__ (s, input, n, eof, type);
case S_BEGIN_DATA_4:
return segmenter_parse_begin_data_4__ (s, input, n, eof, type);
-
- case S_TITLE_1:
- return segmenter_parse_title_1__ (s, input, n, eof, type);
- case S_TITLE_2:
- return segmenter_parse_title_2__ (s, input, n, eof, type);
}
NOT_REACHED ();
case S_DOCUMENT_3:
return PROMPT_FIRST;
- case S_FILE_LABEL:
+ case S_FILE_LABEL_1:
return PROMPT_LATER;
+ case S_FILE_LABEL_2:
+ case S_FILE_LABEL_3:
+ return PROMPT_FIRST;
case S_DO_REPEAT_1:
case S_DO_REPEAT_2:
case S_BEGIN_DATA_4:
return PROMPT_DATA;
- case S_TITLE_1:
- case S_TITLE_2:
- return PROMPT_FIRST;
}
NOT_REACHED ();
projects / pspp / blobdiff