1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
26 #include "dictionary.h"
28 #include "file-handle.h"
33 #include "pfm-write.h"
36 #include "sfm-write.h"
38 #include "value-labels.h"
43 #include "debug-print.h"
45 /* Rearranging and reducing a dictionary. */
46 static void start_case_map (struct dictionary *);
47 static struct case_map *finish_case_map (struct dictionary *);
48 static void map_case (const struct case_map *,
49 const struct ccase *, struct ccase *);
50 static void destroy_case_map (struct case_map *);
55 OP_READ, /* GET or IMPORT. */
56 OP_SAVE, /* SAVE or XSAVE. */
57 OP_EXPORT /* EXPORT. */
60 static bool trim_dictionary (struct dictionary *,
61 enum operation, int *compress);
63 /* GET input program. */
66 struct sfm_reader *reader; /* System file reader. */
67 struct case_map *map; /* Map from system file to active file dict. */
68 struct ccase bounce; /* Bounce buffer. */
71 static void get_pgm_free (struct get_pgm *);
73 /* Parses the GET command. */
77 struct get_pgm *pgm = NULL;
78 struct file_handle *fh;
79 struct dictionary *dict = NULL;
81 pgm = xmalloc (sizeof *pgm);
84 case_nullify (&pgm->bounce);
89 if (lex_match_id ("FILE"))
95 pgm->reader = sfm_open_reader (fh, &dict, NULL);
96 if (pgm->reader == NULL)
98 case_create (&pgm->bounce, dict_get_next_value_idx (dict));
100 start_case_map (dict);
101 if (!trim_dictionary (dict, OP_READ, NULL))
103 pgm->map = finish_case_map (dict);
105 dict_destroy (default_dict);
108 vfm_source = create_case_source (&get_source_class, pgm);
119 /* Frees a struct get_pgm. */
121 get_pgm_free (struct get_pgm *pgm)
125 sfm_close_reader (pgm->reader);
126 destroy_case_map (pgm->map);
127 case_destroy (&pgm->bounce);
132 /* Clears internal state related to GET input procedure. */
134 get_source_destroy (struct case_source *source)
136 struct get_pgm *pgm = source->aux;
140 /* Reads all the cases from the data file into C and passes them
141 to WRITE_CASE one by one, passing WC_DATA. */
143 get_source_read (struct case_source *source,
145 write_case_func *write_case, write_case_data wc_data)
147 struct get_pgm *pgm = source->aux;
152 if (pgm->map == NULL)
153 ok = sfm_read_case (pgm->reader, c);
156 ok = sfm_read_case (pgm->reader, &pgm->bounce);
158 map_case (pgm->map, &pgm->bounce, c);
162 ok = write_case (wc_data);
167 const struct case_source_class get_source_class =
175 /* XSAVE transformation and SAVE procedure. */
178 struct trns_header h;
179 struct sfm_writer *writer; /* System file writer. */
180 struct case_map *map; /* Map from active file to system file dict. */
181 struct ccase bounce; /* Bounce buffer. */
184 static int save_write_case_func (struct ccase *, void *);
185 static trns_proc_func save_trns_proc;
186 static trns_free_func save_trns_free;
188 /* Parses the SAVE or XSAVE command
189 and returns the parsed transformation. */
190 static struct save_trns *
191 cmd_save_internal (void)
193 struct file_handle *fh = NULL;
194 struct dictionary *dict = NULL;
195 struct save_trns *t = NULL;
196 int compress = get_scompression ();
197 const int default_version = 3;
198 int version = default_version;
199 short no_name_table = 0;
201 t = xmalloc (sizeof *t);
202 t->h.proc = save_trns_proc;
203 t->h.free = save_trns_free;
206 case_nullify (&t->bounce);
209 /* Read most of the subcommands. */
212 if (lex_match_id ("VERSION"))
215 if (lex_force_int ())
217 version = lex_integer ();
220 if (lex_match_id ("X"))
224 else if (lex_match_id ("OUTFILE"))
233 if ( ! lex_match('/') )
240 lex_error (_("expecting end of command"));
246 msg ( ME, _("The required %s subcommand was not present"), "OUTFILE");
250 if ( version != default_version )
252 msg (MW, _("Unsupported sysfile version: %d. Using version %d instead."),
253 version, default_version);
255 version = default_version;
258 dict = dict_clone (default_dict);
259 start_case_map (dict);
260 if (!trim_dictionary (dict, OP_SAVE, &compress))
262 t->map = finish_case_map (dict);
264 case_create (&t->bounce, dict_get_next_value_idx (dict));
266 t->writer = sfm_open_writer (fh, dict, compress, no_name_table);
267 if (t->writer == NULL)
277 save_trns_free (&t->h);
281 /* Parses and performs the SAVE procedure. */
285 struct save_trns *t = cmd_save_internal ();
288 procedure (save_write_case_func, t);
289 save_trns_free (&t->h);
297 /* Parses the XSAVE transformation command. */
301 struct save_trns *t = cmd_save_internal ();
304 add_transformation (&t->h);
311 /* Writes the given C to the file specified by T. */
313 do_write_case (struct save_trns *t, struct ccase *c)
316 sfm_write_case (t->writer, c);
319 map_case (t->map, c, &t->bounce);
320 sfm_write_case (t->writer, &t->bounce);
324 /* Writes case C to the system file specified on SAVE. */
326 save_write_case_func (struct ccase *c, void *aux UNUSED)
328 do_write_case (aux, c);
332 /* Writes case C to the system file specified on XSAVE. */
334 save_trns_proc (struct trns_header *h, struct ccase *c, int case_num UNUSED)
336 struct save_trns *t = (struct save_trns *) h;
337 do_write_case (t, c);
341 /* Frees a SAVE transformation. */
343 save_trns_free (struct trns_header *t_)
345 struct save_trns *t = (struct save_trns *) t_;
349 sfm_close_writer (t->writer);
350 destroy_case_map (t->map);
351 case_destroy (&t->bounce);
355 static bool rename_variables (struct dictionary *dict);
356 static bool drop_variables (struct dictionary *dict);
357 static bool keep_variables (struct dictionary *dict);
359 /* Commands that read and write system files share a great deal
360 of common syntactic structure for rearranging and dropping
361 variables. This function parses this syntax and modifies DICT
364 OP is the operation being performed. For operations that
365 write a system file, *COMPRESS is set to 1 if the system file
366 should be compressed, 0 otherwise.
368 Returns true on success, false on failure. */
370 trim_dictionary (struct dictionary *dict, enum operation op, int *compress)
372 assert ((compress != NULL) == (op == OP_SAVE));
373 if (get_scompression())
376 if (op == OP_SAVE || op == OP_EXPORT)
378 /* Delete all the scratch variables. */
383 v = xmalloc (sizeof *v * dict_get_var_cnt (dict));
385 for (i = 0; i < dict_get_var_cnt (dict); i++)
386 if (dict_class_from_id (dict_get_var (dict, i)->name) == DC_SCRATCH)
387 v[nv++] = dict_get_var (dict, i);
388 dict_delete_vars (dict, v, nv);
392 while (lex_match ('/'))
396 if (op == OP_SAVE && lex_match_id ("COMPRESSED"))
398 else if (op == OP_SAVE && lex_match_id ("UNCOMPRESSED"))
400 else if (lex_match_id ("DROP"))
401 ok = drop_variables (dict);
402 else if (lex_match_id ("KEEP"))
403 ok = keep_variables (dict);
404 else if (lex_match_id ("RENAME"))
405 ok = rename_variables (dict);
408 lex_error (_("expecting a valid subcommand"));
416 if (!lex_end_of_command ())
419 dict_compact_values (dict);
423 /* Parses and performs the RENAME subcommand of GET and SAVE. */
425 rename_variables (struct dictionary *dict)
443 v = parse_dict_variable (dict);
446 if (!lex_force_match ('=')
449 if (dict_lookup_var (dict, tokid) != NULL)
451 msg (SE, _("Cannot rename %s as %s because there already exists "
452 "a variable named %s. To rename variables with "
453 "overlapping names, use a single RENAME subcommand "
454 "such as \"/RENAME (A=B)(B=C)(C=A)\", or equivalently, "
455 "\"/RENAME (A B C=B C A)\"."), v->name, tokid, tokid);
459 dict_rename_var (dict, v, tokid);
468 while (lex_match ('('))
472 if (!parse_variables (dict, &v, &nv, PV_NO_DUPLICATE | PV_APPEND))
474 if (!lex_match ('='))
476 msg (SE, _("`=' expected after variable list."));
479 if (!parse_DATA_LIST_vars (&new_names, &nn, PV_APPEND | PV_NO_SCRATCH))
483 msg (SE, _("Number of variables on left side of `=' (%d) does not "
484 "match number of variables on right side (%d), in "
485 "parenthesized group %d of RENAME subcommand."),
486 nv - old_nv, nn - old_nv, group);
489 if (!lex_force_match (')'))
494 if (!dict_rename_vars (dict, v, new_names, nv, &err_name))
496 msg (SE, _("Requested renaming duplicates variable name %s."), err_name);
502 for (i = 0; i < nn; i++)
510 /* Parses and performs the DROP subcommand of GET and SAVE.
511 Returns true if successful, false on failure.*/
513 drop_variables (struct dictionary *dict)
519 if (!parse_variables (dict, &v, &nv, PV_NONE))
521 dict_delete_vars (dict, v, nv);
524 if (dict_get_var_cnt (dict) == 0)
526 msg (SE, _("Cannot DROP all variables from dictionary."));
532 /* Parses and performs the KEEP subcommand of GET and SAVE.
533 Returns true if successful, false on failure.*/
535 keep_variables (struct dictionary *dict)
542 if (!parse_variables (dict, &v, &nv, PV_NONE))
545 /* Move the specified variables to the beginning. */
546 dict_reorder_vars (dict, v, nv);
548 /* Delete the remaining variables. */
549 v = xrealloc (v, (dict_get_var_cnt (dict) - nv) * sizeof *v);
550 for (i = nv; i < dict_get_var_cnt (dict); i++)
551 v[i - nv] = dict_get_var (dict, i);
552 dict_delete_vars (dict, v, dict_get_var_cnt (dict) - nv);
558 /* EXPORT procedure. */
561 struct pfm_writer *writer; /* System file writer. */
562 struct case_map *map; /* Map from active file to system file dict. */
563 struct ccase bounce; /* Bounce buffer. */
566 static int export_write_case_func (struct ccase *, void *);
567 static void export_proc_free (struct export_proc *);
569 /* Parses the EXPORT command. */
570 /* FIXME: same as cmd_save_internal(). */
574 struct file_handle *fh;
575 struct dictionary *dict;
576 struct export_proc *proc;
578 proc = xmalloc (sizeof *proc);
581 case_nullify (&proc->bounce);
584 if (lex_match_id ("OUTFILE"))
590 dict = dict_clone (default_dict);
591 start_case_map (dict);
592 if (!trim_dictionary (dict, OP_EXPORT, NULL))
594 proc->map = finish_case_map (dict);
595 if (proc->map != NULL)
596 case_create (&proc->bounce, dict_get_next_value_idx (dict));
598 proc->writer = pfm_open_writer (fh, dict);
599 if (proc->writer == NULL)
604 procedure (export_write_case_func, proc);
605 export_proc_free (proc);
612 export_proc_free (proc);
617 /* Writes case C to the EXPORT file. */
619 export_write_case_func (struct ccase *c, void *aux)
621 struct export_proc *proc = aux;
622 if (proc->map == NULL)
623 pfm_write_case (proc->writer, c);
626 map_case (proc->map, c, &proc->bounce);
627 pfm_write_case (proc->writer, &proc->bounce);
633 export_proc_free (struct export_proc *proc)
637 pfm_close_writer (proc->writer);
638 destroy_case_map (proc->map);
639 case_destroy (&proc->bounce);
645 #include "debug-print.h"
650 MTF_FILE, /* Specified on FILE= subcommand. */
651 MTF_TABLE /* Specified on TABLE= subcommand. */
654 /* One of the files on MATCH FILES. */
657 struct mtf_file *next, *prev;
658 /* Next, previous in the list of files. */
659 struct mtf_file *next_min; /* Next in the chain of minimums. */
661 int type; /* One of MTF_*. */
662 struct variable **by; /* List of BY variables for this file. */
663 struct file_handle *handle; /* File handle. */
664 struct sfm_reader *reader; /* System file reader. */
665 struct dictionary *dict; /* Dictionary from system file. */
668 char *in_name; /* Variable name. */
669 struct variable *in_var; /* Variable (in master dictionary). */
671 struct ccase input; /* Input record. */
674 /* MATCH FILES procedure. */
677 struct mtf_file *head; /* First file mentioned on FILE or TABLE. */
678 struct mtf_file *tail; /* Last file mentioned on FILE or TABLE. */
680 size_t by_cnt; /* Number of variables on BY subcommand. */
682 /* Names of FIRST, LAST variables. */
683 char first[LONG_NAME_LEN + 1], last[LONG_NAME_LEN + 1];
685 struct dictionary *dict; /* Dictionary of output file. */
686 struct case_sink *sink; /* Sink to receive output. */
687 struct ccase mtf_case; /* Case used for output. */
689 unsigned seq_num; /* Have we initialized this variable? */
690 unsigned *seq_nums; /* Sequence numbers for each var in dict. */
693 static void mtf_free (struct mtf_proc *);
694 static void mtf_free_file (struct mtf_file *);
695 static int mtf_merge_dictionary (struct dictionary *const, struct mtf_file *);
696 static void mtf_delete_file_in_place (struct mtf_proc *, struct mtf_file **);
698 static void mtf_read_nonactive_records (void *);
699 static void mtf_processing_finish (void *);
700 static int mtf_processing (struct ccase *, void *);
702 static char *var_type_description (struct variable *);
704 static void set_master (struct variable *, struct variable *master);
705 static struct variable *get_master (struct variable *);
707 /* Parse and execute the MATCH FILES command. */
709 cmd_match_files (void)
712 struct mtf_file *first_table = NULL;
713 struct mtf_file *iter;
715 bool used_active_file = false;
716 bool saw_table = false;
719 mtf.head = mtf.tail = NULL;
723 mtf.dict = dict_create ();
725 case_nullify (&mtf.mtf_case);
728 dict_set_case_limit (mtf.dict, dict_get_case_limit (default_dict));
732 && (lex_id_match ("FILE", tokid) || lex_id_match ("TABLE", tokid)))
734 struct mtf_file *file = xmalloc (sizeof *file);
736 if (lex_match_id ("FILE"))
737 file->type = MTF_FILE;
738 else if (lex_match_id ("TABLE"))
740 file->type = MTF_TABLE;
751 file->in_name = NULL;
753 case_nullify (&file->input);
755 /* FILEs go first, then TABLEs. */
756 if (file->type == MTF_TABLE || first_table == NULL)
759 file->prev = mtf.tail;
761 mtf.tail->next = file;
763 if (mtf.head == NULL)
765 if (file->type == MTF_TABLE && first_table == NULL)
770 assert (file->type == MTF_FILE);
771 file->next = first_table;
772 file->prev = first_table->prev;
773 if (first_table->prev)
774 first_table->prev->next = file;
777 first_table->prev = file;
785 if (used_active_file)
787 msg (SE, _("The active file may not be specified more "
791 used_active_file = true;
793 assert (pgm_state != STATE_INPUT);
794 if (pgm_state == STATE_INIT)
796 msg (SE, _("Cannot specify the active file since no active "
797 "file has been defined."));
804 _("MATCH FILES may not be used after TEMPORARY when "
805 "the active file is an input source. "
806 "Temporary transformations will be made permanent."));
810 file->dict = default_dict;
814 file->handle = fh_parse ();
815 if (file->handle == NULL)
818 file->reader = sfm_open_reader (file->handle, &file->dict, NULL);
819 if (file->reader == NULL)
822 case_create (&file->input, dict_get_next_value_idx (file->dict));
825 while (lex_match ('/'))
826 if (lex_match_id ("RENAME"))
828 if (!rename_variables (file->dict))
831 else if (lex_match_id ("IN"))
840 if (file->in_name != NULL)
842 msg (SE, _("Multiple IN subcommands for a single FILE or "
846 file->in_name = xstrdup (tokid);
851 mtf_merge_dictionary (mtf.dict, file);
856 if (lex_match (T_BY))
858 struct variable **by;
862 msg (SE, _("BY may appear at most once."));
867 if (!parse_variables (mtf.dict, &by, &mtf.by_cnt,
868 PV_NO_DUPLICATE | PV_NO_SCRATCH))
871 for (iter = mtf.head; iter != NULL; iter = iter->next)
875 iter->by = xmalloc (sizeof *iter->by * mtf.by_cnt);
877 for (i = 0; i < mtf.by_cnt; i++)
879 iter->by[i] = dict_lookup_var (iter->dict, by[i]->name);
880 if (iter->by[i] == NULL)
882 msg (SE, _("File %s lacks BY variable %s."),
883 iter->handle ? handle_get_name (iter->handle) : "*",
891 else if (lex_match_id ("FIRST"))
893 if (mtf.first[0] != '\0')
895 msg (SE, _("FIRST may appear at most once."));
900 if (!lex_force_id ())
902 strcpy (mtf.first, tokid);
905 else if (lex_match_id ("LAST"))
907 if (mtf.last[0] != '\0')
909 msg (SE, _("LAST may appear at most once."));
914 if (!lex_force_id ())
916 strcpy (mtf.last, tokid);
919 else if (lex_match_id ("MAP"))
923 else if (lex_match_id ("DROP"))
925 if (!drop_variables (mtf.dict))
928 else if (lex_match_id ("KEEP"))
930 if (!keep_variables (mtf.dict))
939 if (!lex_match ('/') && token != '.')
941 lex_end_of_command ();
950 msg (SE, _("BY is required when TABLE is specified."));
955 msg (SE, _("BY is required when IN is specified."));
960 /* Set up mapping from each file's variables to master
962 for (iter = mtf.head; iter != NULL; iter = iter->next)
964 struct dictionary *d = iter->dict;
967 for (i = 0; i < dict_get_var_cnt (d); i++)
969 struct variable *v = dict_get_var (d, i);
970 struct variable *mv = dict_lookup_var (mtf.dict, v->name);
976 /* Add IN variables to master dictionary. */
977 for (iter = mtf.head; iter != NULL; iter = iter->next)
978 if (iter->in_name != NULL)
980 static const struct fmt_spec f1_0 = {FMT_F, 1, 0};
982 iter->in_var = dict_create_var (mtf.dict, iter->in_name, 0);
983 if (iter->in_var == NULL)
985 msg (SE, _("IN variable name %s duplicates an "
986 "existing variable name."),
990 iter->in_var->print = iter->in_var->write = f1_0;
993 /* MATCH FILES performs an n-way merge on all its input files.
996 1. Read one input record from every input FILE.
998 2. If no FILEs are left, stop. Otherwise, proceed to step 3.
1000 3. Find the FILE input record(s) that have minimum BY
1001 values. Store all the values from these input records into
1004 4. For every TABLE, read another record as long as the BY values
1005 on the TABLE's input record are less than the FILEs' BY values.
1006 If an exact match is found, store all the values from the TABLE
1007 input record into the output record.
1009 5. Write the output record.
1011 6. Read another record from each input file FILE and TABLE that
1012 we stored values from above. If we come to the end of one of the
1013 input files, remove it from the list of input files.
1015 7. Repeat from step 2.
1017 Unfortunately, this algorithm can't be implemented in a
1018 straightforward way because there's no function to read a
1019 record from the active file. Instead, it has to be written
1022 FIXME: For merging large numbers of files (more than 10?) a
1023 better algorithm would use a heap for finding minimum
1026 if (!used_active_file)
1027 discard_variables ();
1029 dict_compact_values (mtf.dict);
1030 mtf.sink = create_case_sink (&storage_sink_class, mtf.dict, NULL);
1031 if (mtf.sink->class->open != NULL)
1032 mtf.sink->class->open (mtf.sink);
1034 mtf.seq_nums = xcalloc (dict_get_var_cnt (mtf.dict) * sizeof *mtf.seq_nums);
1035 case_create (&mtf.mtf_case, dict_get_next_value_idx (mtf.dict));
1037 mtf_read_nonactive_records (&mtf);
1038 if (used_active_file)
1039 procedure (mtf_processing, &mtf);
1040 mtf_processing_finish (&mtf);
1042 dict_destroy (default_dict);
1043 default_dict = mtf.dict;
1045 vfm_source = mtf.sink->class->make_source (mtf.sink);
1046 free_case_sink (mtf.sink);
1056 /* Repeats 2...7 an arbitrary number of times. */
1058 mtf_processing_finish (void *mtf_)
1060 struct mtf_proc *mtf = mtf_;
1061 struct mtf_file *iter;
1063 /* Find the active file and delete it. */
1064 for (iter = mtf->head; iter; iter = iter->next)
1065 if (iter->handle == NULL)
1067 mtf_delete_file_in_place (mtf, &iter);
1071 while (mtf->head && mtf->head->type == MTF_FILE)
1072 if (!mtf_processing (NULL, mtf))
1076 /* Return a string in a static buffer describing V's variable type and
1079 var_type_description (struct variable *v)
1081 static char buf[2][32];
1088 if (v->type == NUMERIC)
1089 strcpy (s, "numeric");
1092 assert (v->type == ALPHA);
1093 sprintf (s, "string with width %d", v->width);
1098 /* Free FILE and associated data. */
1100 mtf_free_file (struct mtf_file *file)
1103 sfm_close_reader (file->reader);
1104 if (file->dict != default_dict)
1105 dict_destroy (file->dict);
1106 case_destroy (&file->input);
1107 free (file->in_name);
1111 /* Free all the data for the MATCH FILES procedure. */
1113 mtf_free (struct mtf_proc *mtf)
1115 struct mtf_file *iter, *next;
1117 for (iter = mtf->head; iter; iter = next)
1120 mtf_free_file (iter);
1124 dict_destroy (mtf->dict);
1125 case_destroy (&mtf->mtf_case);
1126 free (mtf->seq_nums);
1129 /* Remove *FILE from the mtf_file chain. Make *FILE point to the next
1130 file in the chain, or to NULL if was the last in the chain. */
1132 mtf_delete_file_in_place (struct mtf_proc *mtf, struct mtf_file **file)
1134 struct mtf_file *f = *file;
1138 f->prev->next = f->next;
1140 f->next->prev = f->prev;
1142 mtf->head = f->next;
1144 mtf->tail = f->prev;
1147 if (f->in_var != NULL)
1148 case_data_rw (&mtf->mtf_case, f->in_var->fv)->f = 0.;
1149 for (i = 0; i < dict_get_var_cnt (f->dict); i++)
1151 struct variable *v = dict_get_var (f->dict, i);
1152 struct variable *mv = get_master (v);
1155 union value *out = case_data_rw (&mtf->mtf_case, mv->fv);
1157 if (v->type == NUMERIC)
1160 memset (out->s, ' ', v->width);
1167 /* Read a record from every input file except the active file. */
1169 mtf_read_nonactive_records (void *mtf_)
1171 struct mtf_proc *mtf = mtf_;
1172 struct mtf_file *iter, *next;
1174 for (iter = mtf->head; iter != NULL; iter = next)
1177 if (iter->handle && !sfm_read_case (iter->reader, &iter->input))
1178 mtf_delete_file_in_place (mtf, &iter);
1182 /* Compare the BY variables for files A and B; return -1 if A < B, 0
1183 if A == B, 1 if A > B. */
1185 mtf_compare_BY_values (struct mtf_proc *mtf,
1186 struct mtf_file *a, struct mtf_file *b,
1189 struct ccase *ca = case_is_null (&a->input) ? c : &a->input;
1190 struct ccase *cb = case_is_null (&b->input) ? c : &b->input;
1191 assert ((a == NULL) + (b == NULL) + (c == NULL) <= 1);
1192 return case_compare_2dict (ca, cb, a->by, b->by, mtf->by_cnt);
1195 /* Perform one iteration of steps 3...7 above. */
1197 mtf_processing (struct ccase *c, void *mtf_)
1199 struct mtf_proc *mtf = mtf_;
1201 /* Do we need another record from the active file? */
1202 bool read_active_file;
1204 assert (mtf->head != NULL);
1205 assert (mtf->head->type == MTF_FILE);
1208 struct mtf_file *min_head, *min_tail; /* Files with minimum BY values. */
1209 struct mtf_file *max_head, *max_tail; /* Files with non-minimum BYs. */
1210 struct mtf_file *iter, *next;
1212 read_active_file = false;
1214 /* 3. Find the FILE input record(s) that have minimum BY
1215 values. Store all the values from these input records into
1216 the output record. */
1217 min_head = min_tail = mtf->head;
1218 max_head = max_tail = NULL;
1219 for (iter = mtf->head->next; iter && iter->type == MTF_FILE;
1221 switch (mtf_compare_BY_values (mtf, min_head, iter, c))
1225 max_tail = max_tail->next_min = iter;
1227 max_head = max_tail = iter;
1231 min_tail = min_tail->next_min = iter;
1237 max_tail->next_min = min_head;
1238 max_tail = min_tail;
1242 max_head = min_head;
1243 max_tail = min_tail;
1245 min_head = min_tail = iter;
1252 /* 4. For every TABLE, read another record as long as the BY
1253 values on the TABLE's input record are less than the FILEs'
1254 BY values. If an exact match is found, store all the values
1255 from the TABLE input record into the output record. */
1256 for (; iter != NULL; iter = next)
1258 assert (iter->type == MTF_TABLE);
1263 switch (mtf_compare_BY_values (mtf, min_head, iter, c))
1267 max_tail = max_tail->next_min = iter;
1269 max_head = max_tail = iter;
1273 min_tail = min_tail->next_min = iter;
1277 if (iter->handle == NULL)
1279 if (sfm_read_case (iter->reader, &iter->input))
1281 mtf_delete_file_in_place (mtf, &iter);
1289 /* Next sequence number. */
1292 /* Store data to all the records we are using. */
1294 min_tail->next_min = NULL;
1295 for (iter = min_head; iter; iter = iter->next_min)
1299 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1301 struct variable *v = dict_get_var (iter->dict, i);
1302 struct variable *mv = get_master (v);
1304 if (mv != NULL && mtf->seq_nums[mv->index] != mtf->seq_num)
1306 struct ccase *record
1307 = case_is_null (&iter->input) ? c : &iter->input;
1308 union value *out = case_data_rw (&mtf->mtf_case, mv->fv);
1310 mtf->seq_nums[mv->index] = mtf->seq_num;
1311 if (v->type == NUMERIC)
1312 out->f = case_num (record, v->fv);
1314 memcpy (out->s, case_str (record, v->fv), v->width);
1317 if (iter->in_var != NULL)
1318 case_data_rw (&mtf->mtf_case, iter->in_var->fv)->f = 1.;
1320 if (iter->type == MTF_FILE && iter->handle == NULL)
1321 read_active_file = true;
1324 /* Store missing values to all the records we're not
1327 max_tail->next_min = NULL;
1328 for (iter = max_head; iter; iter = iter->next_min)
1332 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1334 struct variable *v = dict_get_var (iter->dict, i);
1335 struct variable *mv = get_master (v);
1337 if (mv != NULL && mtf->seq_nums[mv->index] != mtf->seq_num)
1339 union value *out = case_data_rw (&mtf->mtf_case, mv->fv);
1340 mtf->seq_nums[mv->index] = mtf->seq_num;
1342 if (v->type == NUMERIC)
1345 memset (out->s, ' ', v->width);
1348 if (iter->in_var != NULL)
1349 case_data_rw (&mtf->mtf_case, iter->in_var->fv)->f = 0.;
1352 /* 5. Write the output record. */
1353 mtf->sink->class->write (mtf->sink, &mtf->mtf_case);
1355 /* 6. Read another record from each input file FILE and TABLE
1356 that we stored values from above. If we come to the end of
1357 one of the input files, remove it from the list of input
1359 for (iter = min_head; iter && iter->type == MTF_FILE; iter = next)
1361 next = iter->next_min;
1362 if (iter->reader != NULL
1363 && !sfm_read_case (iter->reader, &iter->input))
1364 mtf_delete_file_in_place (mtf, &iter);
1367 while (!read_active_file
1368 && mtf->head != NULL && mtf->head->type == MTF_FILE);
1370 return mtf->head != NULL && mtf->head->type == MTF_FILE;
1373 /* Merge the dictionary for file F into master dictionary M. */
1375 mtf_merge_dictionary (struct dictionary *const m, struct mtf_file *f)
1377 struct dictionary *d = f->dict;
1378 const char *d_docs, *m_docs;
1381 if (dict_get_label (m) == NULL)
1382 dict_set_label (m, dict_get_label (d));
1384 d_docs = dict_get_documents (d);
1385 m_docs = dict_get_documents (m);
1389 dict_set_documents (m, d_docs);
1395 new_len = strlen (m_docs) + strlen (d_docs);
1396 new_docs = xmalloc (new_len + 1);
1397 strcpy (new_docs, m_docs);
1398 strcat (new_docs, d_docs);
1399 dict_set_documents (m, new_docs);
1404 for (i = 0; i < dict_get_var_cnt (d); i++)
1406 struct variable *dv = dict_get_var (d, i);
1407 struct variable *mv = dict_lookup_var (m, dv->name);
1409 if (dict_class_from_id (dv->name) == DC_SCRATCH)
1414 if (mv->width != dv->width)
1416 msg (SE, _("Variable %s in file %s (%s) has different "
1417 "type or width from the same variable in "
1418 "earlier file (%s)."),
1419 dv->name, handle_get_name (f->handle),
1420 var_type_description (dv), var_type_description (mv));
1424 if (dv->width == mv->width)
1426 if (val_labs_count (dv->val_labs)
1427 && !val_labs_count (mv->val_labs))
1428 mv->val_labs = val_labs_copy (dv->val_labs);
1429 if (dv->miss_type != MISSING_NONE
1430 && mv->miss_type == MISSING_NONE)
1431 copy_missing_values (mv, dv);
1434 if (dv->label && !mv->label)
1435 mv->label = xstrdup (dv->label);
1438 mv = dict_clone_var_assert (m, dv, dv->name);
1444 /* Marks V's master variable as MASTER. */
1446 set_master (struct variable *v, struct variable *master)
1448 var_attach_aux (v, master, NULL);
1451 /* Returns the master variable corresponding to V,
1452 as set with set_master(). */
1453 static struct variable *
1454 get_master (struct variable *v)
1459 /* IMPORT command. */
1461 /* IMPORT input program. */
1464 struct pfm_reader *reader; /* Portable file reader. */
1465 struct case_map *map; /* Map from system file to active file dict. */
1466 struct ccase bounce; /* Bounce buffer. */
1469 static void import_pgm_free (struct import_pgm *);
1471 /* Parses the IMPORT command. */
1475 struct import_pgm *pgm = NULL;
1476 struct file_handle *fh = NULL;
1477 struct dictionary *dict = NULL;
1480 pgm = xmalloc (sizeof *pgm);
1483 case_nullify (&pgm->bounce);
1489 if (lex_match_id ("FILE") || token == T_STRING)
1497 else if (lex_match_id ("TYPE"))
1501 if (lex_match_id ("COMM"))
1503 else if (lex_match_id ("TAPE"))
1507 lex_error (_("expecting COMM or TAPE"));
1513 if (!lex_match ('/') && token != '.')
1519 discard_variables ();
1521 pgm->reader = pfm_open_reader (fh, &dict, NULL);
1522 if (pgm->reader == NULL)
1524 case_create (&pgm->bounce, dict_get_next_value_idx (dict));
1526 start_case_map (dict);
1527 if (!trim_dictionary (dict, OP_READ, NULL))
1529 pgm->map = finish_case_map (dict);
1531 dict_destroy (default_dict);
1532 default_dict = dict;
1534 vfm_source = create_case_source (&import_source_class, pgm);
1539 import_pgm_free (pgm);
1541 dict_destroy (dict);
1545 /* Frees a struct import_pgm. */
1547 import_pgm_free (struct import_pgm *pgm)
1551 pfm_close_reader (pgm->reader);
1552 destroy_case_map (pgm->map);
1553 case_destroy (&pgm->bounce);
1558 /* Clears internal state related to IMPORT input procedure. */
1560 import_source_destroy (struct case_source *source)
1562 struct import_pgm *pgm = source->aux;
1563 import_pgm_free (pgm);
1566 /* Reads all the cases from the data file into C and passes them
1567 to WRITE_CASE one by one, passing WC_DATA. */
1569 import_source_read (struct case_source *source,
1571 write_case_func *write_case, write_case_data wc_data)
1573 struct import_pgm *pgm = source->aux;
1578 if (pgm->map == NULL)
1579 ok = pfm_read_case (pgm->reader, c);
1582 ok = pfm_read_case (pgm->reader, &pgm->bounce);
1584 map_case (pgm->map, &pgm->bounce, c);
1588 ok = write_case (wc_data);
1593 const struct case_source_class import_source_class =
1598 import_source_destroy,
1604 A case map copies data from a case that corresponds for one
1605 dictionary to a case that corresponds to a second dictionary
1606 derived from the first by, optionally, deleting, reordering,
1607 or renaming variables. (No new variables may be created.)
1613 size_t value_cnt; /* Number of values in map. */
1614 int *map; /* For each destination index, the
1615 corresponding source index. */
1618 /* Prepares dictionary D for producing a case map. Afterward,
1619 the caller may delete, reorder, or rename variables within D
1620 at will before using finish_case_map() to produce the case
1623 Uses D's aux members, which may not otherwise be in use. */
1625 start_case_map (struct dictionary *d)
1627 size_t var_cnt = dict_get_var_cnt (d);
1630 for (i = 0; i < var_cnt; i++)
1632 struct variable *v = dict_get_var (d, i);
1633 int *src_fv = xmalloc (sizeof *src_fv);
1635 var_attach_aux (v, src_fv, var_dtor_free);
1639 /* Produces a case map from dictionary D, which must have been
1640 previously prepared with start_case_map().
1642 Does not retain any reference to D, and clears the aux members
1643 set up by start_case_map().
1645 Returns the new case map, or a null pointer if no mapping is
1646 required (that is, no data has changed position). */
1647 static struct case_map *
1648 finish_case_map (struct dictionary *d)
1650 struct case_map *map;
1651 size_t var_cnt = dict_get_var_cnt (d);
1655 map = xmalloc (sizeof *map);
1656 map->value_cnt = dict_get_next_value_idx (d);
1657 map->map = xmalloc (sizeof *map->map * map->value_cnt);
1658 for (i = 0; i < map->value_cnt; i++)
1662 for (i = 0; i < var_cnt; i++)
1664 struct variable *v = dict_get_var (d, i);
1665 int *src_fv = (int *) var_detach_aux (v);
1668 if (v->fv != *src_fv)
1671 for (idx = 0; idx < v->nv; idx++)
1673 int src_idx = *src_fv + idx;
1674 int dst_idx = v->fv + idx;
1676 assert (map->map[dst_idx] == -1);
1677 map->map[dst_idx] = src_idx;
1684 destroy_case_map (map);
1688 while (map->value_cnt > 0 && map->map[map->value_cnt - 1] == -1)
1694 /* Maps from SRC to DST, applying case map MAP. */
1696 map_case (const struct case_map *map,
1697 const struct ccase *src, struct ccase *dst)
1701 assert (map != NULL);
1702 assert (src != NULL);
1703 assert (dst != NULL);
1704 assert (src != dst);
1706 for (dst_idx = 0; dst_idx < map->value_cnt; dst_idx++)
1708 int src_idx = map->map[dst_idx];
1710 *case_data_rw (dst, dst_idx) = *case_data (src, src_idx);
1714 /* Destroys case map MAP. */
1716 destroy_case_map (struct case_map *map)