1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 1997-2004, 2006, 2010, 2011 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "language/data-io/data-reader.h"
28 #include "data/casereader.h"
29 #include "data/file-handle-def.h"
30 #include "data/file-name.h"
31 #include "data/procedure.h"
32 #include "language/command.h"
33 #include "language/data-io/file-handle.h"
34 #include "language/lexer/lexer.h"
35 #include "language/prompt.h"
36 #include "libpspp/assertion.h"
37 #include "libpspp/cast.h"
38 #include "libpspp/integer-format.h"
39 #include "libpspp/message.h"
40 #include "libpspp/str.h"
42 #include "gl/minmax.h"
43 #include "gl/xalloc.h"
46 #define _(msgid) gettext (msgid)
47 #define N_(msgid) (msgid)
49 /* Flags for DFM readers. */
52 DFM_ADVANCE = 002, /* Read next line on dfm_get_record() call? */
53 DFM_SAW_BEGIN_DATA = 004, /* For inline_file only, whether we've
54 already read a BEGIN DATA line. */
55 DFM_TABS_EXPANDED = 010, /* Tabs have been expanded. */
58 /* Data file reader. */
61 struct file_handle *fh; /* File handle. */
62 struct fh_lock *lock; /* Mutual exclusion lock for file. */
63 struct msg_locator where; /* Current location in data file. */
64 struct string line; /* Current line. */
65 struct string scratch; /* Extra line buffer. */
66 enum dfm_reader_flags flags; /* Zero or more of DFM_*. */
67 FILE *file; /* Associated file. */
68 off_t file_size; /* File size, or -1 if unavailable. */
69 size_t pos; /* Offset in line of current character. */
70 unsigned eof_cnt; /* # of attempts to advance past EOF. */
71 struct lexer *lexer; /* The lexer reading the file */
73 /* For FH_MODE_360_VARIABLE and FH_MODE_360_SPANNED files only. */
74 size_t block_left; /* Bytes left in current block. */
77 /* Closes reader R opened by dfm_open_reader(). */
79 dfm_close_reader (struct dfm_reader *r)
84 if (fh_unlock (r->lock))
86 /* File is still locked by another client. */
90 /* This was the last client, so close the underlying file. */
91 if (fh_get_referent (r->fh) != FH_REF_INLINE)
92 fn_close (fh_get_file_name (r->fh), r->file);
95 /* Skip any remaining data on the inline file. */
96 if (r->flags & DFM_SAW_BEGIN_DATA)
98 dfm_reread_record (r, 0);
100 dfm_forward_record (r);
105 ds_destroy (&r->line);
106 ds_destroy (&r->scratch);
110 /* Opens the file designated by file handle FH for reading as a
111 data file. Providing fh_inline_file() for FH designates the
112 "inline file", that is, data included inline in the command
113 file between BEGIN FILE and END FILE. Returns a reader if
114 successful, or a null pointer otherwise. */
116 dfm_open_reader (struct file_handle *fh, struct lexer *lexer)
118 struct dfm_reader *r;
119 struct fh_lock *lock;
121 /* TRANSLATORS: this fragment will be interpolated into
122 messages in fh_lock() that identify types of files. */
123 lock = fh_lock (fh, FH_REF_FILE | FH_REF_INLINE, N_("data file"),
128 r = fh_lock_get_aux (lock);
132 r = xmalloc (sizeof *r);
136 ds_init_empty (&r->line);
137 ds_init_empty (&r->scratch);
138 r->flags = DFM_ADVANCE;
141 if (fh_get_referent (fh) != FH_REF_INLINE)
144 r->where.file_name = CONST_CAST (char *, fh_get_file_name (fh));
145 r->where.line_number = 0;
146 r->file = fn_open (fh_get_file_name (fh), "rb");
149 msg (ME, _("Could not open `%s' for reading as a data file: %s."),
150 fh_get_file_name (r->fh), strerror (errno));
156 r->file_size = fstat (fileno (r->file), &s) == 0 ? s.st_size : -1;
160 fh_lock_set_aux (lock, r);
165 /* Returns true if an I/O error occurred on READER, false otherwise. */
167 dfm_reader_error (const struct dfm_reader *r)
169 return fh_get_referent (r->fh) == FH_REF_FILE && ferror (r->file);
172 /* Reads a record from the inline file into R.
173 Returns true if successful, false on failure. */
175 read_inline_record (struct dfm_reader *r)
177 if ((r->flags & DFM_SAW_BEGIN_DATA) == 0)
179 r->flags |= DFM_SAW_BEGIN_DATA;
181 while (lex_token (r->lexer) == T_ENDCMD)
183 if (!lex_force_match_id (r->lexer, "BEGIN") || !lex_force_match_id (r->lexer, "DATA"))
185 prompt_set_style (PROMPT_DATA);
188 if (!lex_get_line_raw (r->lexer))
190 lex_discard_line (r->lexer);
191 msg (SE, _("Unexpected end-of-file while reading data in BEGIN "
192 "DATA. This probably indicates "
193 "a missing or incorrectly formatted END DATA command. "
194 "END DATA must appear by itself on a single line "
195 "with exactly one space between words."));
199 if (ds_length (lex_entire_line_ds (r->lexer) ) >= 8
200 && !strncasecmp (lex_entire_line (r->lexer), "end data", 8))
202 lex_discard_line (r->lexer);
206 ds_assign_string (&r->line, lex_entire_line_ds (r->lexer) );
211 /* Report a read error or unexpected end-of-file condition on R. */
213 read_error (struct dfm_reader *r)
215 if (ferror (r->file))
216 msg (ME, _("Error reading file %s: %s."),
217 fh_get_name (r->fh), strerror (errno));
218 else if (feof (r->file))
219 msg (ME, _("Unexpected end of file reading %s."), fh_get_name (r->fh));
224 /* Report a partial read at end of file reading R. */
226 partial_record (struct dfm_reader *r)
228 msg (ME, _("Unexpected end of file in partial record reading %s."),
229 fh_get_name (r->fh));
232 /* Tries to read SIZE bytes from R into BUFFER. Returns 1 if
233 successful, 0 if end of file was reached before any bytes
234 could be read, and -1 if some bytes were read but fewer than
235 SIZE due to end of file or an error mid-read. In the latter
236 case, reports an error. */
238 try_to_read_fully (struct dfm_reader *r, void *buffer, size_t size)
240 size_t bytes_read = fread (buffer, 1, size, r->file);
241 if (bytes_read == size)
243 else if (bytes_read == 0)
252 /* Type of a descriptor word. */
259 /* Reads a block descriptor word or record descriptor word
260 (according to TYPE) from R. Returns 1 if successful, 0 if
261 end of file was reached before any bytes could be read, -1 if
262 an error occurred. Reports an error in the latter case.
264 If successful, stores the number of remaining bytes in the
265 block or record (that is, the block or record length, minus
266 the 4 bytes in the BDW or RDW itself) into *REMAINING_SIZE.
267 If SEGMENT is nonnull, also stores the segment control
268 character (SCC) into *SEGMENT. */
270 read_descriptor_word (struct dfm_reader *r, enum descriptor_type type,
271 size_t *remaining_size, int *segment)
273 uint8_t raw_descriptor[4];
276 status = try_to_read_fully (r, raw_descriptor, sizeof raw_descriptor);
280 *remaining_size = (raw_descriptor[0] << 8) | raw_descriptor[1];
282 *segment = raw_descriptor[2];
284 if (*remaining_size < 4)
288 ? _("Corrupt block descriptor word at offset 0x%lx in %s.")
289 : _("Corrupt record descriptor word at offset 0x%lx in %s.")),
290 (long) ftello (r->file) - 4, fh_get_name (r->fh));
294 *remaining_size -= 4;
298 /* Reports that reader R has read a corrupt record size. */
300 corrupt_size (struct dfm_reader *r)
302 msg (ME, _("Corrupt record size at offset 0x%lx in %s."),
303 (long) ftello (r->file) - 4, fh_get_name (r->fh));
306 /* Reads a 32-byte little-endian signed number from R and stores
307 its value into *SIZE_OUT. Returns 1 if successful, 0 if end
308 of file was reached before any bytes could be read, -1 if an
309 error occurred. Reports an error in the latter case. Numbers
310 less than 0 are considered errors. */
312 read_size (struct dfm_reader *r, size_t *size_out)
317 status = try_to_read_fully (r, &size, sizeof size);
321 integer_convert (INTEGER_LSB_FIRST, &size, INTEGER_NATIVE, &size,
333 /* Reads a record from a disk file into R.
334 Returns true if successful, false on error or at end of file. */
336 read_file_record (struct dfm_reader *r)
338 assert (r->fh != fh_inline_file ());
341 switch (fh_get_mode (r->fh))
344 if (ds_read_line (&r->line, r->file, SIZE_MAX))
346 ds_chomp (&r->line, '\n');
351 if (ferror (r->file))
357 if (ds_read_stream (&r->line, 1, fh_get_record_width (r->fh), r->file))
361 if (ferror (r->file))
363 else if (!ds_is_empty (&r->line))
368 case FH_MODE_VARIABLE:
371 size_t trailing_size;
374 /* Read leading record size. */
375 status = read_size (r, &leading_size);
379 /* Read record data. */
380 if (!ds_read_stream (&r->line, leading_size, 1, r->file))
382 if (ferror (r->file))
389 /* Read trailing record size and check that it's the same
390 as the leading record size. */
391 status = read_size (r, &trailing_size);
398 if (leading_size != trailing_size)
407 case FH_MODE_360_VARIABLE:
408 case FH_MODE_360_SPANNED:
415 /* If we've exhausted our current block, start another
416 one by reading the new block descriptor word. */
417 if (r->block_left == 0)
419 status = read_descriptor_word (r, BLOCK, &r->block_left, NULL);
422 else if (status == 0)
423 return !ds_is_empty (&r->line);
426 /* Read record descriptor. */
427 if (r->block_left < 4)
433 status = read_descriptor_word (r, RECORD, &record_size, &segment);
440 if (record_size > r->block_left)
442 msg (ME, _("Record exceeds remaining block length."));
446 /* Read record data. */
447 if (!ds_read_stream (&r->line, record_size, 1, r->file))
449 if (ferror (r->file))
455 r->block_left -= record_size;
457 /* In variable mode, read only a single record.
458 In spanned mode, a segment value of 0 should
459 designate a whole record without spanning, 1 the
460 first segment in a record, 2 the last segment in a
461 record, and 3 an intermediate segment in a record.
462 For compatibility, though, we actually pay attention
463 only to whether the segment value is even or odd. */
464 if (fh_get_mode (r->fh) == FH_MODE_360_VARIABLE
465 || (segment & 1) == 0)
473 /* Reads a record from R, setting the current position to the
474 start of the line. If an error occurs or end-of-file is
475 encountered, the current line is set to null. */
477 read_record (struct dfm_reader *r)
479 if (fh_get_referent (r->fh) == FH_REF_FILE)
481 bool ok = read_file_record (r);
483 r->where.line_number++;
487 return read_inline_record (r);
490 /* Returns the number of attempts, thus far, to advance past
491 end-of-file in reader R. Reads forward in HANDLE's file, if
492 necessary, to find out.
494 Normally, the user stops attempting to read from the file the
495 first time EOF is reached (a return value of 1). If the user
496 tries to read past EOF again (a return value of 2 or more),
497 an error message is issued, and the caller should more
498 forcibly abort to avoid an infinite loop. */
500 dfm_eof (struct dfm_reader *r)
502 if (r->flags & DFM_ADVANCE)
504 r->flags &= ~DFM_ADVANCE;
506 if (r->eof_cnt == 0 && read_record (r) )
515 if (r->fh != fh_inline_file ())
516 msg (ME, _("Attempt to read beyond end-of-file on file %s."),
517 fh_get_name (r->fh));
519 msg (ME, _("Attempt to read beyond END DATA."));
526 /* Returns the current record in the file corresponding to
527 HANDLE. Aborts if reading from the file is necessary or at
528 end of file, so call dfm_eof() first. */
530 dfm_get_record (struct dfm_reader *r)
532 assert ((r->flags & DFM_ADVANCE) == 0);
533 assert (r->eof_cnt == 0);
535 return ds_substr (&r->line, r->pos, SIZE_MAX);
538 /* Expands tabs in the current line into the equivalent number of
539 spaces, if appropriate for this kind of file. Aborts if
540 reading from the file is necessary or at end of file, so call
543 dfm_expand_tabs (struct dfm_reader *r)
545 size_t ofs, new_pos, tab_width;
547 assert ((r->flags & DFM_ADVANCE) == 0);
548 assert (r->eof_cnt == 0);
550 if (r->flags & DFM_TABS_EXPANDED)
552 r->flags |= DFM_TABS_EXPANDED;
554 if (r->fh != fh_inline_file ()
555 && (fh_get_mode (r->fh) != FH_MODE_TEXT
556 || fh_get_tab_width (r->fh) == 0
557 || ds_find_byte (&r->line, '\t') == SIZE_MAX))
560 /* Expand tabs from r->line into r->scratch, and figure out
561 new value for r->pos. */
562 tab_width = fh_get_tab_width (r->fh);
563 ds_clear (&r->scratch);
565 for (ofs = 0; ofs < ds_length (&r->line); ofs++)
570 new_pos = ds_length (&r->scratch);
572 c = ds_data (&r->line)[ofs];
574 ds_put_byte (&r->scratch, c);
578 ds_put_byte (&r->scratch, ' ');
579 while (ds_length (&r->scratch) % tab_width != 0);
582 if (new_pos == SIZE_MAX)
584 /* Maintain the same relationship between position and line
585 length that we had before. DATA LIST uses a
586 beyond-the-end position to deal with an empty field at
587 the end of the line. */
588 assert (r->pos >= ds_length (&r->line));
589 new_pos = (r->pos - ds_length (&r->line)) + ds_length (&r->scratch);
592 /* Swap r->line and r->scratch and set new r->pos. */
593 ds_swap (&r->line, &r->scratch);
597 /* Returns the legacy character encoding of data read from READER. */
599 dfm_reader_get_legacy_encoding (const struct dfm_reader *reader)
601 return fh_get_legacy_encoding (reader->fh);
604 /* Returns a number between 0 and 100 that approximates the
605 percentage of the data in READER that has already been read,
606 or -1 if this value cannot be estimated.
608 ftello is slow in glibc (it flushes the read buffer), so don't
609 call this function unless you need to. */
611 dfm_get_percent_read (const struct dfm_reader *reader)
613 if (reader->file_size >= 0)
615 off_t position = ftello (reader->file);
618 double p = 100.0 * position / reader->file_size;
619 return p < 0 ? 0 : p > 100 ? 100 : p;
625 /* Causes dfm_get_record() or dfm_get_whole_record() to read in
626 the next record the next time it is executed on file
629 dfm_forward_record (struct dfm_reader *r)
631 r->flags |= DFM_ADVANCE;
634 /* Cancels the effect of any previous dfm_fwd_record() executed
635 on file HANDLE. Sets the current line to begin in the 1-based
638 dfm_reread_record (struct dfm_reader *r, size_t column)
640 r->flags &= ~DFM_ADVANCE;
641 r->pos = MAX (column, 1) - 1;
644 /* Sets the current line to begin COLUMNS characters following
645 the current start. */
647 dfm_forward_columns (struct dfm_reader *r, size_t columns)
649 dfm_reread_record (r, (r->pos + 1) + columns);
652 /* Returns the 1-based column to which the line pointer in HANDLE
653 is set. Unless dfm_reread_record() or dfm_forward_columns()
654 have been called, this is 1. */
656 dfm_column_start (const struct dfm_reader *r)
661 /* Returns the number of columns we are currently beyond the end
662 of the line. At or before end-of-line, this is 0; one column
663 after end-of-line, this is 1; and so on. */
665 dfm_columns_past_end (const struct dfm_reader *r)
667 return r->pos < ds_length (&r->line) ? 0 : ds_length (&r->line) - r->pos;
670 /* Returns the 1-based column within the current line that P
673 dfm_get_column (const struct dfm_reader *r, const char *p)
675 return ds_pointer_to_position (&r->line, p) + 1;
679 dfm_get_file_name (const struct dfm_reader *r)
681 return fh_get_referent (r->fh) == FH_REF_FILE ? r->where.file_name : NULL;
685 dfm_get_line_number (const struct dfm_reader *r)
687 return fh_get_referent (r->fh) == FH_REF_FILE ? r->where.line_number : -1;
690 /* BEGIN DATA...END DATA procedure. */
692 /* Perform BEGIN DATA...END DATA as a procedure in itself. */
694 cmd_begin_data (struct lexer *lexer, struct dataset *ds)
696 struct dfm_reader *r;
699 if (!fh_is_locked (fh_inline_file (), FH_ACC_READ))
701 msg (SE, _("This command is not valid here since the current "
702 "input program does not access the inline file."));
703 return CMD_CASCADING_FAILURE;
706 /* Open inline file. */
707 r = dfm_open_reader (fh_inline_file (), lexer);
708 r->flags |= DFM_SAW_BEGIN_DATA;
710 /* Input procedure reads from inline file. */
711 prompt_set_style (PROMPT_DATA);
712 casereader_destroy (proc_open (ds));
713 ok = proc_commit (ds);
714 dfm_close_reader (r);
716 return ok ? CMD_SUCCESS : CMD_CASCADING_FAILURE;