1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-2004, 2006, 2010, 2011, 2012 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "language/data-io/data-reader.h"
28 #include "data/casereader.h"
29 #include "data/dataset.h"
30 #include "data/file-handle-def.h"
31 #include "data/file-name.h"
32 #include "language/command.h"
33 #include "language/data-io/file-handle.h"
34 #include "language/lexer/lexer.h"
35 #include "libpspp/assertion.h"
36 #include "libpspp/cast.h"
37 #include "libpspp/encoding-guesser.h"
38 #include "libpspp/integer-format.h"
39 #include "libpspp/line-reader.h"
40 #include "libpspp/message.h"
41 #include "libpspp/str.h"
43 #include "gl/minmax.h"
44 #include "gl/xalloc.h"
47 #define _(msgid) gettext (msgid)
48 #define N_(msgid) (msgid)
50 /* Flags for DFM readers. */
53 DFM_ADVANCE = 002, /* Read next line on dfm_get_record() call? */
54 DFM_SAW_BEGIN_DATA = 004, /* For inline_file only, whether we've
55 already read a BEGIN DATA line. */
56 DFM_TABS_EXPANDED = 010, /* Tabs have been expanded. */
57 DFM_CONSUME = 020 /* read_inline_record() should get a token? */
60 /* Data file reader. */
63 struct file_handle *fh; /* File handle. */
64 struct fh_lock *lock; /* Mutual exclusion lock for file. */
65 int line_number; /* Current line or record number. */
66 struct string line; /* Current line. */
67 struct string scratch; /* Extra line buffer. */
68 enum dfm_reader_flags flags; /* Zero or more of DFM_*. */
69 FILE *file; /* Associated file. */
70 off_t file_size; /* File size, or -1 if unavailable. */
71 size_t pos; /* Offset in line of current character. */
72 unsigned eof_cnt; /* # of attempts to advance past EOF. */
73 struct lexer *lexer; /* The lexer reading the file */
74 char *encoding; /* Current encoding. */
76 /* For FH_MODE_TEXT only. */
77 struct line_reader *line_reader;
79 /* For FH_MODE_360_VARIABLE and FH_MODE_360_SPANNED files only. */
80 size_t block_left; /* Bytes left in current block. */
83 /* Closes reader R opened by dfm_open_reader(). */
85 dfm_close_reader (struct dfm_reader *r)
90 if (fh_unlock (r->lock))
92 /* File is still locked by another client. */
96 /* This was the last client, so close the underlying file. */
97 if (fh_get_referent (r->fh) != FH_REF_INLINE)
98 fn_close (fh_get_file_name (r->fh), r->file);
101 /* Skip any remaining data on the inline file. */
102 if (r->flags & DFM_SAW_BEGIN_DATA)
104 dfm_reread_record (r, 0);
106 dfm_forward_record (r);
110 line_reader_free (r->line_reader);
113 ds_destroy (&r->line);
114 ds_destroy (&r->scratch);
118 /* Opens the file designated by file handle FH for reading as a data file.
119 Returns a reader if successful, or a null pointer otherwise.
121 If FH is fh_inline_file() then the new reader reads data included inline in
122 the command file between BEGIN FILE and END FILE, obtaining data from LEXER.
123 LEXER must remain valid as long as the new reader is in use. ENCODING is
126 If FH is not fh_inline_file(), then the encoding of the file read is by
127 default that of FH itself. If ENCODING is nonnull, then it overrides the
128 default encoding. LEXER is ignored. */
130 dfm_open_reader (struct file_handle *fh, struct lexer *lexer,
131 const char *encoding)
133 struct dfm_reader *r;
134 struct fh_lock *lock;
136 /* TRANSLATORS: this fragment will be interpolated into
137 messages in fh_lock() that identify types of files. */
138 lock = fh_lock (fh, FH_REF_FILE | FH_REF_INLINE, N_("data file"),
143 r = fh_lock_get_aux (lock);
147 r = xmalloc (sizeof *r);
151 ds_init_empty (&r->line);
152 ds_init_empty (&r->scratch);
153 r->flags = DFM_ADVANCE;
156 if (fh_get_referent (fh) != FH_REF_INLINE)
160 r->file = fn_open (fh_get_file_name (fh), "rb");
163 msg (ME, _("Could not open `%s' for reading as a data file: %s."),
164 fh_get_file_name (r->fh), strerror (errno));
166 r->file_size = fstat (fileno (r->file), &s) == 0 ? s.st_size : -1;
170 fh_lock_set_aux (lock, r);
172 if (encoding == NULL)
173 encoding = fh_get_encoding (fh);
174 if (fh_get_referent (fh) == FH_REF_FILE && fh_get_mode (fh) == FH_MODE_TEXT)
176 r->line_reader = line_reader_for_fd (encoding, fileno (r->file));
177 if (r->line_reader == NULL)
179 msg (ME, _("Could not read `%s' as a text file with encoding `%s': "
181 fh_get_file_name (r->fh), encoding, strerror (errno));
184 r->encoding = xstrdup (line_reader_get_encoding (r->line_reader));
188 r->line_reader = NULL;
189 r->encoding = xstrdup (encoding_guess_parse_encoding (encoding));
201 /* Returns true if an I/O error occurred on READER, false otherwise. */
203 dfm_reader_error (const struct dfm_reader *r)
205 return (fh_get_referent (r->fh) == FH_REF_FILE
206 && (r->line_reader != NULL
207 ? line_reader_error (r->line_reader) != 0
208 : ferror (r->file)));
211 /* Reads a record from the inline file into R.
212 Returns true if successful, false on failure. */
214 read_inline_record (struct dfm_reader *r)
216 if ((r->flags & DFM_SAW_BEGIN_DATA) == 0)
218 r->flags |= DFM_SAW_BEGIN_DATA;
219 r->flags &= ~DFM_CONSUME;
221 while (lex_token (r->lexer) == T_ENDCMD)
224 if (!lex_force_match_id (r->lexer, "BEGIN")
225 || !lex_force_match_id (r->lexer, "DATA"))
228 lex_match (r->lexer, T_ENDCMD);
231 if (r->flags & DFM_CONSUME)
234 if (!lex_is_string (r->lexer))
236 if (!lex_match_id (r->lexer, "END") || !lex_match_id (r->lexer, "DATA"))
238 msg (SE, _("Missing END DATA while reading inline data. "
239 "This probably indicates a missing or incorrectly "
240 "formatted END DATA command. END DATA must appear "
241 "by itself on a single line with exactly one space "
243 lex_discard_rest_of_command (r->lexer);
248 ds_assign_substring (&r->line, lex_tokss (r->lexer));
249 r->flags |= DFM_CONSUME;
254 /* Report a read error on R. */
256 read_error (struct dfm_reader *r)
258 msg (ME, _("Error reading file %s: %s."),
259 fh_get_name (r->fh), strerror (errno));
262 /* Report a partial read at end of file reading R. */
264 partial_record (struct dfm_reader *r)
266 msg (ME, _("Unexpected end of file in partial record reading %s."),
267 fh_get_name (r->fh));
270 /* Tries to read SIZE bytes from R into BUFFER. Returns 1 if
271 successful, 0 if end of file was reached before any bytes
272 could be read, and -1 if some bytes were read but fewer than
273 SIZE due to end of file or an error mid-read. In the latter
274 case, reports an error. */
276 try_to_read_fully (struct dfm_reader *r, void *buffer, size_t size)
278 size_t bytes_read = fread (buffer, 1, size, r->file);
279 if (bytes_read == size)
281 else if (bytes_read == 0)
290 /* Type of a descriptor word. */
297 /* Reads a block descriptor word or record descriptor word
298 (according to TYPE) from R. Returns 1 if successful, 0 if
299 end of file was reached before any bytes could be read, -1 if
300 an error occurred. Reports an error in the latter case.
302 If successful, stores the number of remaining bytes in the
303 block or record (that is, the block or record length, minus
304 the 4 bytes in the BDW or RDW itself) into *REMAINING_SIZE.
305 If SEGMENT is nonnull, also stores the segment control
306 character (SCC) into *SEGMENT. */
308 read_descriptor_word (struct dfm_reader *r, enum descriptor_type type,
309 size_t *remaining_size, int *segment)
311 uint8_t raw_descriptor[4];
314 status = try_to_read_fully (r, raw_descriptor, sizeof raw_descriptor);
318 *remaining_size = (raw_descriptor[0] << 8) | raw_descriptor[1];
320 *segment = raw_descriptor[2];
322 if (*remaining_size < 4)
326 ? _("Corrupt block descriptor word at offset 0x%lx in %s.")
327 : _("Corrupt record descriptor word at offset 0x%lx in %s.")),
328 (long) ftello (r->file) - 4, fh_get_name (r->fh));
332 *remaining_size -= 4;
336 /* Reports that reader R has read a corrupt record size. */
338 corrupt_size (struct dfm_reader *r)
340 msg (ME, _("Corrupt record size at offset 0x%lx in %s."),
341 (long) ftello (r->file) - 4, fh_get_name (r->fh));
344 /* Reads a 32-byte little-endian signed number from R and stores
345 its value into *SIZE_OUT. Returns 1 if successful, 0 if end
346 of file was reached before any bytes could be read, -1 if an
347 error occurred. Reports an error in the latter case. Numbers
348 less than 0 are considered errors. */
350 read_size (struct dfm_reader *r, size_t *size_out)
355 status = try_to_read_fully (r, &size, sizeof size);
359 integer_convert (INTEGER_LSB_FIRST, &size, INTEGER_NATIVE, &size,
372 read_text_record (struct dfm_reader *r)
377 /* Read a line. If the line reader's encoding changes, update r->encoding to
379 is_auto = line_reader_is_auto (r->line_reader);
380 ok = line_reader_read (r->line_reader, &r->line, SIZE_MAX);
381 if (is_auto && !line_reader_is_auto (r->line_reader))
384 r->encoding = xstrdup (line_reader_get_encoding (r->line_reader));
387 /* Detect and report read error. */
390 int error = line_reader_error (r->line_reader);
392 msg (ME, _("Error reading file %s: %s."),
393 fh_get_name (r->fh), strerror (error));
399 /* Reads a record from a disk file into R.
400 Returns true if successful, false on error or at end of file. */
402 read_file_record (struct dfm_reader *r)
404 assert (r->fh != fh_inline_file ());
407 switch (fh_get_mode (r->fh))
410 return read_text_record (r);
413 if (ds_read_stream (&r->line, 1, fh_get_record_width (r->fh), r->file))
417 if (ferror (r->file))
419 else if (!ds_is_empty (&r->line))
424 case FH_MODE_VARIABLE:
427 size_t trailing_size;
430 /* Read leading record size. */
431 status = read_size (r, &leading_size);
435 /* Read record data. */
436 if (!ds_read_stream (&r->line, leading_size, 1, r->file))
438 if (ferror (r->file))
445 /* Read trailing record size and check that it's the same
446 as the leading record size. */
447 status = read_size (r, &trailing_size);
454 if (leading_size != trailing_size)
463 case FH_MODE_360_VARIABLE:
464 case FH_MODE_360_SPANNED:
471 /* If we've exhausted our current block, start another
472 one by reading the new block descriptor word. */
473 if (r->block_left == 0)
475 status = read_descriptor_word (r, BLOCK, &r->block_left, NULL);
478 else if (status == 0)
479 return !ds_is_empty (&r->line);
482 /* Read record descriptor. */
483 if (r->block_left < 4)
489 status = read_descriptor_word (r, RECORD, &record_size, &segment);
496 if (record_size > r->block_left)
498 msg (ME, _("Record exceeds remaining block length."));
502 /* Read record data. */
503 if (!ds_read_stream (&r->line, record_size, 1, r->file))
505 if (ferror (r->file))
511 r->block_left -= record_size;
513 /* In variable mode, read only a single record.
514 In spanned mode, a segment value of 0 should
515 designate a whole record without spanning, 1 the
516 first segment in a record, 2 the last segment in a
517 record, and 3 an intermediate segment in a record.
518 For compatibility, though, we actually pay attention
519 only to whether the segment value is even or odd. */
520 if (fh_get_mode (r->fh) == FH_MODE_360_VARIABLE
521 || (segment & 1) == 0)
529 /* Reads a record from R, setting the current position to the
530 start of the line. If an error occurs or end-of-file is
531 encountered, the current line is set to null. */
533 read_record (struct dfm_reader *r)
535 if (fh_get_referent (r->fh) == FH_REF_FILE)
537 bool ok = read_file_record (r);
543 return read_inline_record (r);
546 /* Returns the number of attempts, thus far, to advance past
547 end-of-file in reader R. Reads forward in HANDLE's file, if
548 necessary, to find out.
550 Normally, the user stops attempting to read from the file the
551 first time EOF is reached (a return value of 1). If the user
552 tries to read past EOF again (a return value of 2 or more),
553 an error message is issued, and the caller should more
554 forcibly abort to avoid an infinite loop. */
556 dfm_eof (struct dfm_reader *r)
558 if (r->flags & DFM_ADVANCE)
560 r->flags &= ~DFM_ADVANCE;
562 if (r->eof_cnt == 0 && read_record (r) )
571 if (r->fh != fh_inline_file ())
572 msg (ME, _("Attempt to read beyond end-of-file on file %s."),
573 fh_get_name (r->fh));
575 msg (ME, _("Attempt to read beyond END DATA."));
582 /* Returns the current record in the file corresponding to
583 HANDLE. Aborts if reading from the file is necessary or at
584 end of file, so call dfm_eof() first. */
586 dfm_get_record (struct dfm_reader *r)
588 assert ((r->flags & DFM_ADVANCE) == 0);
589 assert (r->eof_cnt == 0);
591 return ds_substr (&r->line, r->pos, SIZE_MAX);
594 /* Expands tabs in the current line into the equivalent number of
595 spaces, if appropriate for this kind of file. Aborts if
596 reading from the file is necessary or at end of file, so call
599 dfm_expand_tabs (struct dfm_reader *r)
601 size_t ofs, new_pos, tab_width;
603 assert ((r->flags & DFM_ADVANCE) == 0);
604 assert (r->eof_cnt == 0);
606 if (r->flags & DFM_TABS_EXPANDED)
608 r->flags |= DFM_TABS_EXPANDED;
610 if (r->fh != fh_inline_file ()
611 && (fh_get_mode (r->fh) != FH_MODE_TEXT
612 || fh_get_tab_width (r->fh) == 0
613 || ds_find_byte (&r->line, '\t') == SIZE_MAX))
616 /* Expand tabs from r->line into r->scratch, and figure out
617 new value for r->pos. */
618 tab_width = fh_get_tab_width (r->fh);
619 ds_clear (&r->scratch);
621 for (ofs = 0; ofs < ds_length (&r->line); ofs++)
626 new_pos = ds_length (&r->scratch);
628 c = ds_data (&r->line)[ofs];
630 ds_put_byte (&r->scratch, c);
634 ds_put_byte (&r->scratch, ' ');
635 while (ds_length (&r->scratch) % tab_width != 0);
638 if (new_pos == SIZE_MAX)
640 /* Maintain the same relationship between position and line
641 length that we had before. DATA LIST uses a
642 beyond-the-end position to deal with an empty field at
643 the end of the line. */
644 assert (r->pos >= ds_length (&r->line));
645 new_pos = (r->pos - ds_length (&r->line)) + ds_length (&r->scratch);
648 /* Swap r->line and r->scratch and set new r->pos. */
649 ds_swap (&r->line, &r->scratch);
653 /* Returns the character encoding of data read from READER. */
655 dfm_reader_get_encoding (const struct dfm_reader *reader)
657 return reader->encoding;
660 /* Returns a number between 0 and 100 that approximates the
661 percentage of the data in READER that has already been read,
662 or -1 if this value cannot be estimated.
664 ftello is slow in glibc (it flushes the read buffer), so don't
665 call this function unless you need to. */
667 dfm_get_percent_read (const struct dfm_reader *reader)
669 if (reader->file_size >= 0)
673 position = (reader->line_reader != NULL
674 ? line_reader_tell (reader->line_reader)
675 : ftello (reader->file));
678 double p = 100.0 * position / reader->file_size;
679 return p < 0 ? 0 : p > 100 ? 100 : p;
685 /* Causes dfm_get_record() or dfm_get_whole_record() to read in
686 the next record the next time it is executed on file
689 dfm_forward_record (struct dfm_reader *r)
691 r->flags |= DFM_ADVANCE;
694 /* Cancels the effect of any previous dfm_fwd_record() executed
695 on file HANDLE. Sets the current line to begin in the 1-based
698 dfm_reread_record (struct dfm_reader *r, size_t column)
700 r->flags &= ~DFM_ADVANCE;
701 r->pos = MAX (column, 1) - 1;
704 /* Sets the current line to begin COLUMNS characters following
705 the current start. */
707 dfm_forward_columns (struct dfm_reader *r, size_t columns)
709 dfm_reread_record (r, (r->pos + 1) + columns);
712 /* Returns the 1-based column to which the line pointer in HANDLE
713 is set. Unless dfm_reread_record() or dfm_forward_columns()
714 have been called, this is 1. */
716 dfm_column_start (const struct dfm_reader *r)
721 /* Returns the number of columns we are currently beyond the end
722 of the line. At or before end-of-line, this is 0; one column
723 after end-of-line, this is 1; and so on. */
725 dfm_columns_past_end (const struct dfm_reader *r)
727 return r->pos < ds_length (&r->line) ? 0 : ds_length (&r->line) - r->pos;
730 /* Returns the 1-based column within the current line that P
733 dfm_get_column (const struct dfm_reader *r, const char *p)
735 return ds_pointer_to_position (&r->line, p) + 1;
739 dfm_get_file_name (const struct dfm_reader *r)
741 return (fh_get_referent (r->fh) == FH_REF_FILE
742 ? fh_get_file_name (r->fh)
747 dfm_get_line_number (const struct dfm_reader *r)
749 return fh_get_referent (r->fh) == FH_REF_FILE ? r->line_number : -1;
752 /* BEGIN DATA...END DATA procedure. */
754 /* Perform BEGIN DATA...END DATA as a procedure in itself. */
756 cmd_begin_data (struct lexer *lexer, struct dataset *ds)
758 struct dfm_reader *r;
761 if (!fh_is_locked (fh_inline_file (), FH_ACC_READ))
763 msg (SE, _("This command is not valid here since the current "
764 "input program does not access the inline file."));
765 return CMD_CASCADING_FAILURE;
767 lex_match (lexer, T_ENDCMD);
769 /* Open inline file. */
770 r = dfm_open_reader (fh_inline_file (), lexer, NULL);
771 r->flags |= DFM_SAW_BEGIN_DATA;
772 r->flags &= ~DFM_CONSUME;
774 /* Input procedure reads from inline file. */
775 casereader_destroy (proc_open (ds));
776 ok = proc_commit (ds);
777 dfm_close_reader (r);
779 return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;