+\f
+static void
+lex_source_push_endcmd__ (struct lex_source *src)
+{
+ struct lex_token *token = lex_push_token__ (src);
+ token->token.type = T_ENDCMD;
+ token->token_pos = 0;
+ token->token_len = 0;
+ token->line_pos = 0;
+ token->first_line = 0;
+}
+
+static struct lex_source *
+lex_source_create (struct lex_reader *reader)
+{
+ struct lex_source *src;
+ enum segmenter_mode mode;
+
+ src = xzalloc (sizeof *src);
+ src->reader = reader;
+
+ if (reader->syntax == LEX_SYNTAX_AUTO)
+ mode = SEG_MODE_AUTO;
+ else if (reader->syntax == LEX_SYNTAX_INTERACTIVE)
+ mode = SEG_MODE_INTERACTIVE;
+ else if (reader->syntax == LEX_SYNTAX_BATCH)
+ mode = SEG_MODE_BATCH;
+ else
+ NOT_REACHED ();
+ segmenter_init (&src->segmenter, mode);
+
+ src->tokens = deque_init (&src->deque, 4, sizeof *src->tokens);
+
+ lex_source_push_endcmd__ (src);
+
+ return src;
+}
+
+static void
+lex_source_destroy (struct lex_source *src)
+{
+ char *file_name = src->reader->file_name;
+ char *encoding = src->reader->encoding;
+ if (src->reader->class->destroy != NULL)
+ src->reader->class->destroy (src->reader);
+ free (file_name);
+ free (encoding);
+ free (src->buffer);
+ while (!deque_is_empty (&src->deque))
+ lex_source_pop__ (src);
+ free (src->tokens);
+ ll_remove (&src->ll);
+ free (src);
+}
+\f
+struct lex_file_reader
+ {
+ struct lex_reader reader;
+ struct u8_istream *istream;
+ };
+
+static struct lex_reader_class lex_file_reader_class;
+
+/* Creates and returns a new lex_reader that will read from file FILE_NAME (or
+ from stdin if FILE_NAME is "-"). The file is expected to be encoded with
+ ENCODING, which should take one of the forms accepted by
+ u8_istream_for_file(). SYNTAX and ERROR become the syntax mode and error
+ mode of the new reader, respectively.
+
+ Returns a null pointer if FILE_NAME cannot be opened. */
+struct lex_reader *
+lex_reader_for_file (const char *file_name, const char *encoding,
+ enum lex_syntax_mode syntax,
+ enum lex_error_mode error)
+{
+ struct lex_file_reader *r;
+ struct u8_istream *istream;
+
+ istream = (!strcmp(file_name, "-")
+ ? u8_istream_for_fd (encoding, STDIN_FILENO)
+ : u8_istream_for_file (encoding, file_name, O_RDONLY));
+ if (istream == NULL)
+ {
+ msg (ME, _("Opening `%s': %s."), file_name, strerror (errno));
+ return NULL;
+ }
+
+ r = xmalloc (sizeof *r);
+ lex_reader_init (&r->reader, &lex_file_reader_class);
+ r->reader.syntax = syntax;
+ r->reader.error = error;
+ r->reader.file_name = xstrdup (file_name);
+ r->reader.encoding = encoding ? xstrdup (encoding) : NULL;
+ r->reader.line_number = 1;
+ r->istream = istream;
+
+ return &r->reader;
+}
+
+static struct lex_file_reader *
+lex_file_reader_cast (struct lex_reader *r)
+{
+ return UP_CAST (r, struct lex_file_reader, reader);
+}
+
+static size_t
+lex_file_read (struct lex_reader *r_, char *buf, size_t n,
+ enum prompt_style prompt_style UNUSED)
+{
+ struct lex_file_reader *r = lex_file_reader_cast (r_);
+ ssize_t n_read = u8_istream_read (r->istream, buf, n);
+ if (n_read < 0)
+ {
+ msg (ME, _("Error reading `%s': %s."), r_->file_name, strerror (errno));
+ return 0;
+ }
+ return n_read;
+}
+
+static void
+lex_file_close (struct lex_reader *r_)
+{
+ struct lex_file_reader *r = lex_file_reader_cast (r_);
+
+ if (u8_istream_fileno (r->istream) != STDIN_FILENO)
+ {
+ if (u8_istream_close (r->istream) != 0)
+ msg (ME, _("Error closing `%s': %s."), r_->file_name, strerror (errno));
+ }
+ else
+ u8_istream_free (r->istream);
+
+ free (r);
+}
+
+static struct lex_reader_class lex_file_reader_class =
+ {
+ lex_file_read,
+ lex_file_close
+ };
+\f
+struct lex_string_reader
+ {
+ struct lex_reader reader;
+ struct substring s;
+ size_t offset;
+ };
+
+static struct lex_reader_class lex_string_reader_class;
+
+/* Creates and returns a new lex_reader for the contents of S, which must be
+ encoded in the given ENCODING. The new reader takes ownership of S and will free it
+ with ss_dealloc() when it is closed. */
+struct lex_reader *
+lex_reader_for_substring_nocopy (struct substring s, const char *encoding)
+{
+ struct lex_string_reader *r;
+
+ r = xmalloc (sizeof *r);
+ lex_reader_init (&r->reader, &lex_string_reader_class);
+ r->reader.syntax = LEX_SYNTAX_AUTO;
+ r->reader.encoding = encoding ? xstrdup (encoding) : NULL;
+ r->s = s;
+ r->offset = 0;
+
+ return &r->reader;
+}
+
+/* Creates and returns a new lex_reader for a copy of null-terminated string S,
+ which must be encoded in ENCODING. The caller retains ownership of S. */
+struct lex_reader *
+lex_reader_for_string (const char *s, const char *encoding)
+{
+ struct substring ss;
+ ss_alloc_substring (&ss, ss_cstr (s));
+ return lex_reader_for_substring_nocopy (ss, encoding);
+}
+
+/* Formats FORMAT as a printf()-like format string and creates and returns a
+ new lex_reader for the formatted result. */
+struct lex_reader *
+lex_reader_for_format (const char *format, const char *encoding, ...)
+{
+ struct lex_reader *r;
+ va_list args;
+
+ va_start (args, encoding);
+ r = lex_reader_for_substring_nocopy (ss_cstr (xvasprintf (format, args)), encoding);
+ va_end (args);
+
+ return r;
+}
+
+static struct lex_string_reader *
+lex_string_reader_cast (struct lex_reader *r)
+{
+ return UP_CAST (r, struct lex_string_reader, reader);
+}
+
+static size_t
+lex_string_read (struct lex_reader *r_, char *buf, size_t n,
+ enum prompt_style prompt_style UNUSED)
+{
+ struct lex_string_reader *r = lex_string_reader_cast (r_);
+ size_t chunk;
+
+ chunk = MIN (n, r->s.length - r->offset);
+ memcpy (buf, r->s.string + r->offset, chunk);
+ r->offset += chunk;
+
+ return chunk;
+}
+
+static void
+lex_string_close (struct lex_reader *r_)
+{
+ struct lex_string_reader *r = lex_string_reader_cast (r_);
+
+ ss_dealloc (&r->s);
+ free (r);
+}
+
+static struct lex_reader_class lex_string_reader_class =
+ {
+ lex_string_read,
+ lex_string_close
+ };