/* PSPP - a program for statistical analysis.
- Copyright (C) 1997-2004, 2006 Free Software Foundation, Inc.
+ Copyright (C) 1997-2004, 2006, 2010, 2011, 2012, 2016 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#include <config.h>
-#include <language/data-io/data-reader.h>
+#include "language/data-io/data-reader.h"
#include <ctype.h>
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
-#include <sys/stat.h>
-
-#include <data/casereader.h>
-#include <data/file-handle-def.h>
-#include <data/file-name.h>
-#include <data/procedure.h>
-#include <language/command.h>
-#include <language/data-io/file-handle.h>
-#include <language/lexer/lexer.h>
-#include <language/prompt.h>
-#include <libpspp/assertion.h>
-#include <libpspp/cast.h>
-#include <libpspp/integer-format.h>
-#include <libpspp/message.h>
-#include <libpspp/str.h>
-
-#include "minmax.h"
-#include "xalloc.h"
+
+#include "data/casereader.h"
+#include "data/dataset.h"
+#include "data/file-handle-def.h"
+#include "data/file-name.h"
+#include "language/command.h"
+#include "language/data-io/file-handle.h"
+#include "language/lexer/lexer.h"
+#include "libpspp/assertion.h"
+#include "libpspp/cast.h"
+#include "libpspp/encoding-guesser.h"
+#include "libpspp/integer-format.h"
+#include "libpspp/line-reader.h"
+#include "libpspp/message.h"
+#include "libpspp/str.h"
+
+#include "gl/minmax.h"
+#include "gl/xalloc.h"
#include "gettext.h"
#define _(msgid) gettext (msgid)
DFM_SAW_BEGIN_DATA = 004, /* For inline_file only, whether we've
already read a BEGIN DATA line. */
DFM_TABS_EXPANDED = 010, /* Tabs have been expanded. */
+ DFM_CONSUME = 020 /* read_inline_record() should get a token? */
};
/* Data file reader. */
{
struct file_handle *fh; /* File handle. */
struct fh_lock *lock; /* Mutual exclusion lock for file. */
- struct msg_locator where; /* Current location in data file. */
+ int line_number; /* Current line or record number. */
struct string line; /* Current line. */
struct string scratch; /* Extra line buffer. */
enum dfm_reader_flags flags; /* Zero or more of DFM_*. */
FILE *file; /* Associated file. */
- off_t file_size; /* File size, or -1 if unavailable. */
size_t pos; /* Offset in line of current character. */
unsigned eof_cnt; /* # of attempts to advance past EOF. */
struct lexer *lexer; /* The lexer reading the file */
+ char *encoding; /* Current encoding. */
+
+ /* For FH_MODE_TEXT only. */
+ struct line_reader *line_reader;
/* For FH_MODE_360_VARIABLE and FH_MODE_360_SPANNED files only. */
size_t block_left; /* Bytes left in current block. */
/* This was the last client, so close the underlying file. */
if (fh_get_referent (r->fh) != FH_REF_INLINE)
- fn_close (fh_get_file_name (r->fh), r->file);
+ fn_close (r->fh, r->file);
else
{
/* Skip any remaining data on the inline file. */
}
}
+ line_reader_free (r->line_reader);
+ free (r->encoding);
fh_unref (r->fh);
ds_destroy (&r->line);
ds_destroy (&r->scratch);
free (r);
}
-/* Opens the file designated by file handle FH for reading as a
- data file. Providing fh_inline_file() for FH designates the
- "inline file", that is, data included inline in the command
- file between BEGIN FILE and END FILE. Returns a reader if
- successful, or a null pointer otherwise. */
+/* Opens the file designated by file handle FH for reading as a data file.
+ Returns a reader if successful, or a null pointer otherwise.
+
+ If FH is fh_inline_file() then the new reader reads data included inline in
+ the command file between BEGIN FILE and END FILE, obtaining data from LEXER.
+ LEXER must remain valid as long as the new reader is in use. ENCODING is
+ ignored.
+
+ If FH is not fh_inline_file(), then the encoding of the file read is by
+ default that of FH itself. If ENCODING is nonnull, then it overrides the
+ default encoding. LEXER is ignored. */
struct dfm_reader *
-dfm_open_reader (struct file_handle *fh, struct lexer *lexer)
+dfm_open_reader (struct file_handle *fh, struct lexer *lexer,
+ const char *encoding)
{
struct dfm_reader *r;
struct fh_lock *lock;
r->block_left = 0;
if (fh_get_referent (fh) != FH_REF_INLINE)
{
- struct stat s;
- r->where.file_name = CONST_CAST (char *, fh_get_file_name (fh));
- r->where.line_number = 0;
- r->file = fn_open (fh_get_file_name (fh),
- fh_get_mode (fh) == FH_MODE_TEXT ? "r" : "rb");
+ r->line_number = 0;
+ r->file = fn_open (fh, "rb");
if (r->file == NULL)
{
- msg (ME, _("Could not open \"%s\" for reading as a data file: %s."),
+ msg (ME, _("Could not open `%s' for reading as a data file: %s."),
fh_get_file_name (r->fh), strerror (errno));
- fh_unlock (r->lock);
- fh_unref (fh);
- free (r);
- return NULL;
+ goto error;
}
- r->file_size = fstat (fileno (r->file), &s) == 0 ? s.st_size : -1;
}
- else
- r->file_size = -1;
fh_lock_set_aux (lock, r);
+ if (encoding == NULL)
+ encoding = fh_get_encoding (fh);
+ if (fh_get_referent (fh) == FH_REF_FILE && fh_get_mode (fh) == FH_MODE_TEXT)
+ {
+ r->line_reader = line_reader_for_fd (encoding, fileno (r->file));
+ if (r->line_reader == NULL)
+ {
+ msg (ME, _("Could not read `%s' as a text file with encoding `%s': "
+ "%s."),
+ fh_get_file_name (r->fh), encoding, strerror (errno));
+ goto error;
+ }
+ r->encoding = xstrdup (line_reader_get_encoding (r->line_reader));
+ }
+ else
+ {
+ r->line_reader = NULL;
+ r->encoding = xstrdup (encoding_guess_parse_encoding (encoding));
+ }
+
return r;
+
+error:
+ fh_unlock (r->lock);
+ fh_unref (fh);
+ free (r);
+ return NULL;
}
/* Returns true if an I/O error occurred on READER, false otherwise. */
bool
dfm_reader_error (const struct dfm_reader *r)
{
- return fh_get_referent (r->fh) == FH_REF_FILE && ferror (r->file);
+ return (fh_get_referent (r->fh) == FH_REF_FILE
+ && (r->line_reader != NULL
+ ? line_reader_error (r->line_reader) != 0
+ : ferror (r->file)));
}
/* Reads a record from the inline file into R.
if ((r->flags & DFM_SAW_BEGIN_DATA) == 0)
{
r->flags |= DFM_SAW_BEGIN_DATA;
+ r->flags &= ~DFM_CONSUME;
- while (lex_token (r->lexer) == '.')
+ while (lex_token (r->lexer) == T_ENDCMD)
lex_get (r->lexer);
- if (!lex_force_match_id (r->lexer, "BEGIN") || !lex_force_match_id (r->lexer, "DATA"))
+
+ if (!lex_force_match_id (r->lexer, "BEGIN")
+ || !lex_force_match_id (r->lexer, "DATA"))
return false;
- prompt_set_style (PROMPT_DATA);
- }
- if (!lex_get_line_raw (r->lexer))
- {
- lex_discard_line (r->lexer);
- msg (SE, _("Unexpected end-of-file while reading data in BEGIN "
- "DATA. This probably indicates "
- "a missing or misformatted END DATA command. "
- "END DATA must appear by itself on a single line "
- "with exactly one space between words."));
- return false;
+ lex_match (r->lexer, T_ENDCMD);
}
- if (ds_length (lex_entire_line_ds (r->lexer) ) >= 8
- && !strncasecmp (lex_entire_line (r->lexer), "end data", 8))
+ if (r->flags & DFM_CONSUME)
+ lex_get (r->lexer);
+
+ if (!lex_is_string (r->lexer))
{
- lex_discard_line (r->lexer);
+ if (!lex_match_id (r->lexer, "END") || !lex_match_id (r->lexer, "DATA"))
+ {
+ msg (SE, _("Missing %s while reading inline data. "
+ "This probably indicates a missing or incorrectly "
+ "formatted %s command. %s must appear "
+ "by itself on a single line with exactly one space "
+ "between words."), "END DATA", "END DATA", "END DATA");
+ lex_discard_rest_of_command (r->lexer);
+ }
return false;
}
- ds_assign_string (&r->line, lex_entire_line_ds (r->lexer) );
+ ds_assign_substring (&r->line, lex_tokss (r->lexer));
+ r->flags |= DFM_CONSUME;
return true;
}
-/* Report a read error or unexpected end-of-file condition on R. */
+/* Report a read error on R. */
static void
read_error (struct dfm_reader *r)
{
- if (ferror (r->file))
- msg (ME, _("Error reading file %s: %s."),
- fh_get_name (r->fh), strerror (errno));
- else if (feof (r->file))
- msg (ME, _("Unexpected end of file reading %s."), fh_get_name (r->fh));
- else
- NOT_REACHED ();
+ msg (ME, _("Error reading file %s: %s."),
+ fh_get_name (r->fh), strerror (errno));
}
/* Report a partial read at end of file reading R. */
return 1;
}
+static bool
+read_text_record (struct dfm_reader *r)
+{
+ bool is_auto;
+ bool ok;
+
+ /* Read a line. If the line reader's encoding changes, update r->encoding to
+ match. */
+ is_auto = line_reader_is_auto (r->line_reader);
+ ok = line_reader_read (r->line_reader, &r->line, SIZE_MAX);
+ if (is_auto && !line_reader_is_auto (r->line_reader))
+ {
+ free (r->encoding);
+ r->encoding = xstrdup (line_reader_get_encoding (r->line_reader));
+ }
+
+ /* Detect and report read error. */
+ if (!ok)
+ {
+ int error = line_reader_error (r->line_reader);
+ if (error != 0)
+ msg (ME, _("Error reading file %s: %s."),
+ fh_get_name (r->fh), strerror (error));
+ }
+
+ return ok;
+}
+
/* Reads a record from a disk file into R.
Returns true if successful, false on error or at end of file. */
static bool
switch (fh_get_mode (r->fh))
{
case FH_MODE_TEXT:
- if (ds_read_line (&r->line, r->file, SIZE_MAX))
- {
- ds_chomp (&r->line, '\n');
- return true;
- }
- else
- {
- if (ferror (r->file))
- read_error (r);
- return false;
- }
- return true;
+ return read_text_record (r);
case FH_MODE_FIXED:
if (ds_read_stream (&r->line, 1, fh_get_record_width (r->fh), r->file))
partial_record (r);
return false;
}
- return true;
case FH_MODE_VARIABLE:
{
{
bool ok = read_file_record (r);
if (ok)
- r->where.line_number++;
+ r->line_number++;
return ok;
}
else
msg (ME, _("Attempt to read beyond end-of-file on file %s."),
fh_get_name (r->fh));
else
- msg (ME, _("Attempt to read beyond END DATA."));
+ msg (ME, _("Attempt to read beyond %s."), "END DATA");
}
}
if (r->fh != fh_inline_file ()
&& (fh_get_mode (r->fh) != FH_MODE_TEXT
|| fh_get_tab_width (r->fh) == 0
- || ds_find_char (&r->line, '\t') == SIZE_MAX))
+ || ds_find_byte (&r->line, '\t') == SIZE_MAX))
return;
/* Expand tabs from r->line into r->scratch, and figure out
c = ds_data (&r->line)[ofs];
if (c != '\t')
- ds_put_char (&r->scratch, c);
+ ds_put_byte (&r->scratch, c);
else
{
do
- ds_put_char (&r->scratch, ' ');
+ ds_put_byte (&r->scratch, ' ');
while (ds_length (&r->scratch) % tab_width != 0);
}
}
r->pos = new_pos;
}
-/* Returns the legacy character encoding of data read from READER. */
+/* Returns the character encoding of data read from READER. */
const char *
-dfm_reader_get_legacy_encoding (const struct dfm_reader *reader)
+dfm_reader_get_encoding (const struct dfm_reader *reader)
{
- return fh_get_legacy_encoding (reader->fh);
-}
-
-/* Returns a number between 0 and 100 that approximates the
- percentage of the data in READER that has already been read,
- or -1 if this value cannot be estimated.
-
- ftello is slow in glibc (it flushes the read buffer), so don't
- call this function unless you need to. */
-int
-dfm_get_percent_read (const struct dfm_reader *reader)
-{
- if (reader->file_size >= 0)
- {
- off_t position = ftello (reader->file);
- if (position >= 0)
- {
- double p = 100.0 * position / reader->file_size;
- return p < 0 ? 0 : p > 100 ? 100 : p;
- }
- }
- return -1;
+ return reader->encoding;
}
/* Causes dfm_get_record() or dfm_get_whole_record() to read in
return ds_pointer_to_position (&r->line, p) + 1;
}
-/* Pushes the file name and line number on the fn/ln stack. */
-void
-dfm_push (struct dfm_reader *r)
+const char *
+dfm_get_file_name (const struct dfm_reader *r)
{
- if (r->fh != fh_inline_file ())
- msg_push_msg_locator (&r->where);
+ return (fh_get_referent (r->fh) == FH_REF_FILE
+ ? fh_get_file_name (r->fh)
+ : NULL);
}
-/* Pops the file name and line number from the fn/ln stack. */
-void
-dfm_pop (struct dfm_reader *r)
+int
+dfm_get_line_number (const struct dfm_reader *r)
{
- if (r->fh != fh_inline_file ())
- msg_pop_msg_locator (&r->where);
+ return fh_get_referent (r->fh) == FH_REF_FILE ? r->line_number : -1;
}
\f
/* BEGIN DATA...END DATA procedure. */
"input program does not access the inline file."));
return CMD_CASCADING_FAILURE;
}
+ lex_match (lexer, T_ENDCMD);
/* Open inline file. */
- r = dfm_open_reader (fh_inline_file (), lexer);
+ r = dfm_open_reader (fh_inline_file (), lexer, NULL);
r->flags |= DFM_SAW_BEGIN_DATA;
+ r->flags &= ~DFM_CONSUME;
/* Input procedure reads from inline file. */
- prompt_set_style (PROMPT_DATA);
casereader_destroy (proc_open (ds));
ok = proc_commit (ds);
dfm_close_reader (r);
projects / pspp / blobdiff