1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2009, 2010, 2011, 2012, 2013 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
19 #include "libpspp/message.h"
20 #include "libpspp/misc.h"
22 #include "gl/minmax.h"
23 #include "gl/c-strtod.h"
26 #define _(msgid) gettext (msgid)
27 #define N_(msgid) (msgid)
29 #include "spreadsheet-reader.h"
34 gnumeric_open_reader (const struct spreadsheet_read_options *opts, struct dictionary **dict)
36 msg (ME, _("Support for %s files was not compiled into this installation of PSPP"), "Gnumeric");
43 #include "data/gnumeric-reader.h"
48 #include <libxml/xmlreader.h>
51 #include "data/case.h"
52 #include "data/casereader-provider.h"
53 #include "data/dictionary.h"
54 #include "data/identifier.h"
55 #include "data/value.h"
56 #include "data/variable.h"
57 #include "libpspp/i18n.h"
58 #include "libpspp/str.h"
60 #include "gl/xalloc.h"
62 static void gnm_file_casereader_destroy (struct casereader *, void *);
64 static struct ccase *gnm_file_casereader_read (struct casereader *, void *);
67 static const struct casereader_class gnm_file_casereader_class =
69 gnm_file_casereader_read,
70 gnm_file_casereader_destroy,
77 STATE_PRE_INIT = 0, /* Initial state */
78 STATE_SHEET_COUNT, /* Found the sheet index */
79 STATE_INIT , /* Other Initial state */
80 STATE_SHEET_START, /* Found the start of a sheet */
81 STATE_SHEET_NAME, /* Found the sheet name */
84 STATE_SHEET_FOUND, /* Found the sheet that we actually want */
85 STATE_CELLS_START, /* Found the start of the cell array */
86 STATE_CELL /* Found a cell */
91 /* The name of the sheet (utf8 encoding) */
105 /* The libxml reader for this instance */
106 xmlTextReaderPtr xtr;
108 /* An internal state variable */
109 enum reader_state state;
122 state_data_destroy (struct state_data *sd)
124 xmlFreeTextReader (sd->xtr);
128 struct gnumeric_reader
130 struct spreadsheet spreadsheet;
133 struct state_data rsd;
134 struct state_data msd;
141 struct sheet_detail *sheets;
143 const xmlChar *target_sheet;
144 int target_sheet_index;
146 struct caseproto *proto;
147 struct dictionary *dict;
148 struct ccase *first_case;
149 bool used_first_case;
154 gnumeric_destroy (struct spreadsheet *s)
156 struct gnumeric_reader *r = (struct gnumeric_reader *) s;
158 if (0 == --r->ref_cnt)
162 for (i = 0; i < s->n_sheets; ++i)
164 xmlFree (r->sheets[i].name);
168 state_data_destroy (&r->msd);
176 gnumeric_get_sheet_name (struct spreadsheet *s, int n)
178 struct gnumeric_reader *gr = (struct gnumeric_reader *) s;
179 assert (n < s->n_sheets);
181 return gr->sheets[n].name;
185 static void process_node (struct gnumeric_reader *r, struct state_data *sd);
190 gnumeric_get_sheet_range (struct spreadsheet *s, int n)
193 struct gnumeric_reader *gr = (struct gnumeric_reader *) s;
195 assert (n < s->n_sheets);
198 (gr->sheets[n].stop_col == -1)
200 (1 == (ret = xmlTextReaderRead (gr->msd.xtr)))
203 process_node (gr, &gr->msd);
206 return create_cell_ref (
207 gr->sheets[n].start_col,
208 gr->sheets[n].start_row,
209 gr->sheets[n].stop_col,
210 gr->sheets[n].stop_row);
215 gnm_file_casereader_destroy (struct casereader *reader UNUSED, void *r_)
217 struct gnumeric_reader *r = r_;
222 state_data_destroy (&r->rsd);
226 xmlFreeTextReader (r->rsd.xtr);
229 if ( ! r->used_first_case )
230 case_unref (r->first_case);
232 caseproto_unref (r->proto);
235 gnumeric_destroy (&r->spreadsheet);
240 process_node (struct gnumeric_reader *r, struct state_data *sd)
242 xmlChar *name = xmlTextReaderName (sd->xtr);
244 name = xmlStrdup (_xml ("--"));
246 sd->node_type = xmlTextReaderNodeType (sd->xtr);
251 sd->current_sheet = -1;
252 if (0 == xmlStrcasecmp (name, _xml("gnm:SheetNameIndex")) &&
253 XML_READER_TYPE_ELEMENT == sd->node_type)
255 sd->state = STATE_SHEET_COUNT;
259 case STATE_SHEET_COUNT:
260 if (0 == xmlStrcasecmp (name, _xml("gnm:SheetName")) &&
261 XML_READER_TYPE_ELEMENT == sd->node_type)
264 if (sd->current_sheet + 1 > r->spreadsheet.n_sheets)
266 struct sheet_detail *detail ;
267 r->sheets = xrealloc (r->sheets, (sd->current_sheet + 1) * sizeof *r->sheets);
268 detail = &r->sheets[sd->current_sheet];
269 detail->start_col = detail->stop_col = detail->start_row = detail->stop_row = -1;
271 r->spreadsheet.n_sheets = sd->current_sheet + 1;
274 else if (0 == xmlStrcasecmp (name, _xml("gnm:SheetNameIndex")) &&
275 XML_READER_TYPE_END_ELEMENT == sd->node_type)
277 sd->state = STATE_INIT;
278 sd->current_sheet = -1;
280 else if (XML_READER_TYPE_TEXT == sd->node_type)
282 if ( r->sheets [r->spreadsheet.n_sheets - 1].name == NULL)
283 r->sheets [r->spreadsheet.n_sheets - 1].name = CHAR_CAST (char *, xmlTextReaderValue (sd->xtr));
288 if (0 == xmlStrcasecmp (name, _xml("gnm:Sheet")) &&
289 XML_READER_TYPE_ELEMENT == sd->node_type)
292 sd->state = STATE_SHEET_START;
295 case STATE_SHEET_START:
296 if (0 == xmlStrcasecmp (name, _xml("gnm:Name")) &&
297 XML_READER_TYPE_ELEMENT == sd->node_type)
299 sd->state = STATE_SHEET_NAME;
302 case STATE_SHEET_NAME:
303 if (0 == xmlStrcasecmp (name, _xml("gnm:Name")) &&
304 XML_READER_TYPE_END_ELEMENT == sd->node_type)
306 sd->state = STATE_INIT;
308 else if (0 == xmlStrcasecmp (name, _xml("gnm:Sheet")) &&
309 XML_READER_TYPE_END_ELEMENT == sd->node_type)
311 sd->state = STATE_INIT;
313 else if (XML_READER_TYPE_TEXT == sd->node_type)
315 if ( r->target_sheet != NULL)
317 xmlChar *value = xmlTextReaderValue (sd->xtr);
318 if ( 0 == xmlStrcmp (value, r->target_sheet))
319 sd->state = STATE_SHEET_FOUND;
322 else if (r->target_sheet_index == sd->current_sheet + 1)
324 sd->state = STATE_SHEET_FOUND;
326 else if (r->target_sheet_index == -1)
328 sd->state = STATE_SHEET_FOUND;
332 case STATE_SHEET_FOUND:
333 if (0 == xmlStrcasecmp (name, _xml("gnm:Cells")) &&
334 XML_READER_TYPE_ELEMENT == sd->node_type)
336 sd->min_col = INT_MAX;
337 if (! xmlTextReaderIsEmptyElement (sd->xtr))
338 sd->state = STATE_CELLS_START;
340 else if (0 == xmlStrcasecmp (name, _xml("gnm:MaxRow")) &&
341 XML_READER_TYPE_ELEMENT == sd->node_type)
343 sd->state = STATE_MAXROW;
345 else if (0 == xmlStrcasecmp (name, _xml("gnm:MaxCol")) &&
346 XML_READER_TYPE_ELEMENT == sd->node_type)
348 sd->state = STATE_MAXCOL;
350 else if (0 == xmlStrcasecmp (name, _xml("gnm:Sheet")) &&
351 XML_READER_TYPE_END_ELEMENT == sd->node_type)
353 sd->state = STATE_INIT;
357 if (0 == xmlStrcasecmp (name, _xml("gnm:MaxRow")) &&
358 XML_READER_TYPE_END_ELEMENT == sd->node_type)
360 sd->state = STATE_SHEET_FOUND;
362 else if (sd->node_type == XML_READER_TYPE_TEXT)
364 xmlChar *value = xmlTextReaderValue (sd->xtr);
365 r->sheets[sd->current_sheet].maxrow = _xmlchar_to_int (value);
370 if (0 == xmlStrcasecmp (name, _xml("gnm:MaxCol")) &&
371 XML_READER_TYPE_END_ELEMENT == sd->node_type)
373 sd->state = STATE_SHEET_FOUND;
375 else if (sd->node_type == XML_READER_TYPE_TEXT)
377 xmlChar *value = xmlTextReaderValue (sd->xtr);
378 r->sheets[sd->current_sheet].maxcol = _xmlchar_to_int (value);
382 case STATE_CELLS_START:
383 if (0 == xmlStrcasecmp (name, _xml ("gnm:Cell")) &&
384 XML_READER_TYPE_ELEMENT == sd->node_type)
386 xmlChar *attr = NULL;
388 attr = xmlTextReaderGetAttribute (sd->xtr, _xml ("Col"));
389 sd->col = _xmlchar_to_int (attr);
392 if (sd->col < sd->min_col)
393 sd->min_col = sd->col;
395 attr = xmlTextReaderGetAttribute (sd->xtr, _xml ("Row"));
396 sd->row = _xmlchar_to_int (attr);
399 if (r->sheets[sd->current_sheet].start_row == -1)
401 r->sheets[sd->current_sheet].start_row = sd->row;
404 if (r->sheets[sd->current_sheet].start_col == -1)
406 r->sheets[sd->current_sheet].start_col = sd->col;
408 if (! xmlTextReaderIsEmptyElement (sd->xtr))
409 sd->state = STATE_CELL;
411 else if ( (0 == xmlStrcasecmp (name, _xml("gnm:Cells"))) && (XML_READER_TYPE_END_ELEMENT == sd->node_type) )
413 r->sheets[sd->current_sheet].stop_col = sd->col;
414 r->sheets[sd->current_sheet].stop_row = sd->row;
415 sd->state = STATE_SHEET_NAME;
419 if (0 == xmlStrcasecmp (name, _xml("gnm:Cell")) && XML_READER_TYPE_END_ELEMENT == sd->node_type)
421 sd->state = STATE_CELLS_START;
433 Sets the VAR of case C, to the value corresponding to the xml string XV
436 convert_xml_string_to_value (struct ccase *c, const struct variable *var,
439 union value *v = case_data_rw (c, var);
442 value_set_missing (v, var_get_width (var));
443 else if ( var_is_alpha (var))
444 value_copy_str_rpad (v, var_get_width (var), xv, ' ');
447 const char *text = CHAR_CAST (const char *, xv);
451 v->f = c_strtod (text, &endptr);
452 if ( errno != 0 || endptr == text)
461 xmlChar *first_value;
466 gnumeric_error_handler (void *ctx, const char *mesg,
467 UNUSED xmlParserSeverities sev, xmlTextReaderLocatorPtr loc)
469 struct gnumeric_reader *r = ctx;
471 msg (MW, _("There was a problem whilst reading the %s file `%s' (near line %d): `%s'"),
473 r->spreadsheet.file_name,
474 xmlTextReaderLocatorLineNumber (loc),
478 static struct gnumeric_reader *
479 gnumeric_reopen (struct gnumeric_reader *r, const char *filename, bool show_errors)
482 struct state_data *sd;
484 xmlTextReaderPtr xtr;
487 assert (r == NULL || filename == NULL);
491 gz = gzopen (filename, "r");
495 gz = gzopen (r->spreadsheet.file_name, "r");
502 xtr = xmlReaderForIO ((xmlInputReadCallback) gzread,
503 (xmlInputCloseCallback) gzclose, gz,
505 show_errors ? 0 : (XML_PARSE_NOERROR | XML_PARSE_NOWARNING) );
515 r = xzalloc (sizeof *r);
516 r->spreadsheet.n_sheets = -1;
517 r->spreadsheet.file_name = filename;
526 xmlTextReaderSetErrorHandler (xtr, gnumeric_error_handler, r);
528 r->target_sheet = NULL;
529 r->target_sheet_index = -1;
531 sd->row = sd->col = -1;
532 sd->state = STATE_PRE_INIT;
536 /* Advance to the start of the workbook.
537 This gives us some confidence that we are actually dealing with a gnumeric
540 while ( (sd->state != STATE_INIT )
541 && 1 == (ret = xmlTextReaderRead (sd->xtr)))
543 process_node (r, sd);
549 /* Does not seem to be a gnumeric file */
550 xmlFreeTextReader (sd->xtr);
555 r->spreadsheet.type = SPREADSHEET_GNUMERIC;
559 const xmlChar *enc = xmlTextReaderConstEncoding (sd->xtr);
560 xmlCharEncoding xce = xmlParseCharEncoding (CHAR_CAST (const char *, enc));
562 if ( XML_CHAR_ENCODING_UTF8 != xce)
564 /* I have been told that ALL gnumeric files are UTF8 encoded. If that is correct, this
566 msg (MW, _("The gnumeric file `%s' is encoded as %s instead of the usual UTF-8 encoding. "
567 "Any non-ascii characters will be incorrectly imported."),
568 r->spreadsheet.file_name,
578 gnumeric_probe (const char *filename, bool report_errors)
580 struct gnumeric_reader *r = gnumeric_reopen (NULL, filename, report_errors);
582 return &r->spreadsheet;
587 gnumeric_make_reader (struct spreadsheet *spreadsheet,
588 const struct spreadsheet_read_options *opts)
591 struct gnumeric_reader *r = NULL;
592 unsigned long int vstart = 0;
594 casenumber n_cases = CASENUMBER_MAX;
596 struct var_spec *var_spec = NULL;
599 r = (struct gnumeric_reader *) (spreadsheet);
601 if (r->rsd.row != -1)
602 r = gnumeric_reopen (r, NULL, true);
604 if ( opts->cell_range )
606 if ( ! convert_cell_ref (opts->cell_range,
607 &r->start_col, &r->start_row,
608 &r->stop_col, &r->stop_row))
610 msg (SE, _("Invalid cell range `%s'"),
623 r->target_sheet = BAD_CAST opts->sheet_name;
624 r->target_sheet_index = opts->sheet_index;
625 r->rsd.row = r->rsd.col = -1;
626 r->rsd.current_sheet = -1;
628 /* Advance to the start of the cells for the target sheet */
629 while ( (r->rsd.state != STATE_CELL || r->rsd.row < r->start_row )
630 && 1 == (ret = xmlTextReaderRead (r->rsd.xtr)))
633 process_node (r, &r->rsd);
634 value = xmlTextReaderValue (r->rsd.xtr);
636 if ( r->rsd.state == STATE_MAXROW && r->rsd.node_type == XML_READER_TYPE_TEXT)
638 n_cases = 1 + _xmlchar_to_int (value) ;
643 /* If a range has been given, then use that to calculate the number
645 if ( opts->cell_range)
647 n_cases = MIN (n_cases, r->stop_row - r->start_row + 1);
650 if ( opts->read_names )
656 /* Read in the first row of cells,
657 including the headers if read_names was set */
659 (( r->rsd.state == STATE_CELLS_START && r->rsd.row <= r->start_row) || r->rsd.state == STATE_CELL )
660 && (ret = xmlTextReaderRead (r->rsd.xtr))
664 process_node (r, &r->rsd);
666 if ( r->rsd.row > r->start_row ) break;
668 if ( r->rsd.col < r->start_col ||
669 (r->stop_col != -1 && r->rsd.col > r->stop_col))
672 idx = r->rsd.col - r->start_col;
674 if ( idx >= n_var_specs )
677 var_spec = xrealloc (var_spec, sizeof (*var_spec) * (idx + 1));
678 for (i = n_var_specs; i <= idx; ++i)
680 var_spec [i].name = NULL;
681 var_spec [i].width = -1;
682 var_spec [i].first_value = NULL;
684 n_var_specs = idx + 1 ;
687 if ( r->rsd.node_type == XML_READER_TYPE_TEXT )
689 xmlChar *value = xmlTextReaderValue (r->rsd.xtr);
690 const char *text = CHAR_CAST (const char *, value);
692 if ( r->rsd.row < r->start_row)
694 if ( opts->read_names )
696 var_spec [idx].name = xstrdup (text);
701 var_spec [idx].first_value = xmlStrdup (value);
703 if (-1 == var_spec [idx].width )
704 var_spec [idx].width = (opts->asw == -1) ?
705 ROUND_UP (strlen(text), SPREADSHEET_DEFAULT_WIDTH) : opts->asw;
710 else if ( r->rsd.node_type == XML_READER_TYPE_ELEMENT
711 && r->rsd.state == STATE_CELL)
713 if ( r->rsd.row == r->start_row )
716 xmlTextReaderGetAttribute (r->rsd.xtr, _xml ("ValueType"));
718 if ( NULL == attr || 60 != _xmlchar_to_int (attr))
719 var_spec [idx].width = 0;
727 const xmlChar *enc = xmlTextReaderConstEncoding (r->rsd.xtr);
730 /* Create the dictionary and populate it */
731 spreadsheet->dict = r->dict = dict_create (CHAR_CAST (const char *, enc));
734 for (i = 0 ; i < n_var_specs ; ++i )
738 if ( (var_spec[i].name == NULL) && (var_spec[i].first_value == NULL))
741 /* Probably no data exists for this variable, so allocate a
743 if ( var_spec[i].width == -1 )
744 var_spec[i].width = SPREADSHEET_DEFAULT_WIDTH;
746 name = dict_make_unique_var_name (r->dict, var_spec[i].name, &vstart);
747 dict_create_var (r->dict, name, var_spec[i].width);
751 /* Create the first case, and cache it */
752 r->used_first_case = false;
754 if ( n_var_specs == 0 )
756 msg (MW, _("Selected sheet or range of spreadsheet `%s' is empty."),
757 spreadsheet->file_name);
761 r->proto = caseproto_ref (dict_get_proto (r->dict));
762 r->first_case = case_create (r->proto);
763 case_set_missing (r->first_case);
766 for ( i = 0 ; i < n_var_specs ; ++i )
768 const struct variable *var;
770 if ( (var_spec[i].name == NULL) && (var_spec[i].first_value == NULL))
773 var = dict_get_var (r->dict, x++);
775 convert_xml_string_to_value (r->first_case, var,
776 var_spec[i].first_value);
779 for ( i = 0 ; i < n_var_specs ; ++i )
781 free (var_spec[i].first_value);
782 free (var_spec[i].name);
788 return casereader_create_sequential
792 &gnm_file_casereader_class, r);
796 for ( i = 0 ; i < n_var_specs ; ++i )
798 free (var_spec[i].first_value);
799 free (var_spec[i].name);
803 dict_destroy (spreadsheet->dict);
804 spreadsheet->dict = NULL;
806 gnm_file_casereader_destroy (NULL, r);
812 /* Reads and returns one case from READER's file. Returns a null
813 pointer on failure. */
814 static struct ccase *
815 gnm_file_casereader_read (struct casereader *reader UNUSED, void *r_)
820 struct gnumeric_reader *r = r_;
821 int current_row = r->rsd.row;
823 if ( !r->used_first_case )
825 r->used_first_case = true;
826 return r->first_case;
829 c = case_create (r->proto);
830 case_set_missing (c);
832 if (r->start_col == -1)
833 r->start_col = r->rsd.min_col;
835 while ((r->rsd.state == STATE_CELL || r->rsd.state == STATE_CELLS_START )
836 && r->rsd.row == current_row && (ret = xmlTextReaderRead (r->rsd.xtr)))
838 process_node (r, &r->rsd);
840 if ( r->rsd.col < r->start_col || (r->stop_col != -1 &&
841 r->rsd.col > r->stop_col))
844 if ( r->rsd.col - r->start_col >= caseproto_get_n_widths (r->proto))
847 if ( r->stop_row != -1 && r->rsd.row > r->stop_row)
850 if ( r->rsd.node_type == XML_READER_TYPE_TEXT )
852 xmlChar *value = xmlTextReaderValue (r->rsd.xtr);
854 const int idx = r->rsd.col - r->start_col;
856 const struct variable *var = dict_get_var (r->dict, idx);
858 convert_xml_string_to_value (c, var, value);
874 #endif /* GNM_SUPPORT */