-/* When invoked, tokstr contains a string of binary, octal, or
- hex digits, according to TYPE. The string is converted to
- characters having the specified values. */
-static void
-convert_numeric_string_to_char_string (struct lexer *lexer,
- enum string_type type)
-{
- const char *base_name;
- int base;
- int chars_per_byte;
- size_t byte_cnt;
- size_t i;
- char *p;
+ /* If we've reached the end of a line, or the end of a command, then pass
+ the line to the output engine as a syntax text item. */
+ int n_lines = state.newlines;
+ if (state.last_segment == SEG_END_COMMAND && !src->suppress_next_newline)
+ {
+ n_lines++;
+ src->suppress_next_newline = true;
+ }
+ else if (n_lines > 0 && src->suppress_next_newline)
+ {
+ n_lines--;
+ src->suppress_next_newline = false;
+ }
+ for (int i = 0; i < n_lines; i++)
+ {
+ /* Beginning of line. */
+ const char *line = &src->buffer[src->journal_pos - src->tail];
+
+ /* Calculate line length, including \n or \r\n end-of-line if present.
+
+ We use src->head even though that may be beyond what we've actually
+ converted to tokens (which is only through state.line_pos). That's
+ because, if we're emitting the line due to SEG_END_COMMAND, we want to
+ take the whole line through the newline, not just through the '.'. */
+ size_t max_len = src->head - src->journal_pos;
+ const char *newline = memchr (line, '\n', max_len);
+ size_t line_len = newline ? newline - line + 1 : max_len;
+
+ /* Calculate line length excluding end-of-line. */
+ size_t copy_len = line_len;
+ if (copy_len > 0 && line[copy_len - 1] == '\n')
+ copy_len--;
+ if (copy_len > 0 && line[copy_len - 1] == '\r')
+ copy_len--;
+
+ /* Submit the line as syntax. */
+ text_item_submit (text_item_create_nocopy (TEXT_ITEM_SYNTAX,
+ xmemdup0 (line, copy_len)));
+
+ src->journal_pos += line_len;
+ }
+
+ token->token_len = state.seg_pos - src->seg_pos;
+
+ src->segmenter = state.segmenter;
+ src->seg_pos = state.seg_pos;
+ src->line_pos = state.line_pos;
+ src->n_newlines += state.newlines;