1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
26 #include "dictionary.h"
28 #include "file-handle.h"
33 #include "pfm-write.h"
36 #include "sfm-write.h"
38 #include "value-labels.h"
43 #include "debug-print.h"
45 /* Rearranging and reducing a dictionary. */
46 static void start_case_map (struct dictionary *);
47 static struct case_map *finish_case_map (struct dictionary *);
48 static void map_case (const struct case_map *,
49 const struct ccase *, struct ccase *);
50 static void destroy_case_map (struct case_map *);
55 OP_READ, /* GET or IMPORT. */
56 OP_SAVE, /* SAVE or XSAVE. */
57 OP_EXPORT /* EXPORT. */
60 static bool trim_dictionary (struct dictionary *,
61 enum operation, int *compress);
63 /* GET input program. */
66 struct sfm_reader *reader; /* System file reader. */
67 struct case_map *map; /* Map from system file to active file dict. */
68 struct ccase bounce; /* Bounce buffer. */
71 static void get_pgm_free (struct get_pgm *);
73 /* Parses the GET command. */
77 struct get_pgm *pgm = NULL;
78 struct file_handle *fh;
79 struct dictionary *dict = NULL;
81 pgm = xmalloc (sizeof *pgm);
84 case_nullify (&pgm->bounce);
89 if (lex_match_id ("FILE"))
95 pgm->reader = sfm_open_reader (fh, &dict, NULL);
96 if (pgm->reader == NULL)
98 case_create (&pgm->bounce, dict_get_next_value_idx (dict));
100 start_case_map (dict);
101 if (!trim_dictionary (dict, OP_READ, NULL))
103 pgm->map = finish_case_map (dict);
105 dict_destroy (default_dict);
108 vfm_source = create_case_source (&get_source_class, pgm);
119 /* Frees a struct get_pgm. */
121 get_pgm_free (struct get_pgm *pgm)
125 sfm_close_reader (pgm->reader);
126 destroy_case_map (pgm->map);
127 case_destroy (&pgm->bounce);
132 /* Clears internal state related to GET input procedure. */
134 get_source_destroy (struct case_source *source)
136 struct get_pgm *pgm = source->aux;
140 /* Reads all the cases from the data file into C and passes them
141 to WRITE_CASE one by one, passing WC_DATA. */
143 get_source_read (struct case_source *source,
145 write_case_func *write_case, write_case_data wc_data)
147 struct get_pgm *pgm = source->aux;
152 if (pgm->map == NULL)
153 ok = sfm_read_case (pgm->reader, c);
156 ok = sfm_read_case (pgm->reader, &pgm->bounce);
158 map_case (pgm->map, &pgm->bounce, c);
162 ok = write_case (wc_data);
167 const struct case_source_class get_source_class =
175 /* XSAVE transformation and SAVE procedure. */
178 struct trns_header h;
179 struct sfm_writer *writer; /* System file writer. */
180 struct case_map *map; /* Map from active file to system file dict. */
181 struct ccase bounce; /* Bounce buffer. */
184 static int save_write_case_func (struct ccase *, void *);
185 static trns_proc_func save_trns_proc;
186 static trns_free_func save_trns_free;
188 /* Parses the SAVE or XSAVE command
189 and returns the parsed transformation. */
190 static struct save_trns *
191 cmd_save_internal (void)
193 struct file_handle *fh = NULL;
194 struct dictionary *dict = NULL;
195 struct save_trns *t = NULL;
196 int compress = get_scompression ();
197 const int default_version = 3;
198 int version = default_version;
199 short no_name_table = 0;
201 t = xmalloc (sizeof *t);
202 t->h.proc = save_trns_proc;
203 t->h.free = save_trns_free;
206 case_nullify (&t->bounce);
209 /* Read most of the subcommands. */
212 if (lex_match_id ("VERSION"))
215 if (lex_force_int ())
217 version = lex_integer ();
220 if (lex_match_id ("X"))
224 else if (lex_match_id ("OUTFILE"))
233 if ( ! lex_match('/') )
240 lex_error (_("expecting end of command"));
246 msg ( ME, _("The required %s subcommand was not present"), "OUTFILE");
250 if ( version != default_version )
252 msg (MW, _("Unsupported sysfile version: %d. Using version %d instead."),
253 version, default_version);
255 version = default_version;
258 dict = dict_clone (default_dict);
259 start_case_map (dict);
260 if (!trim_dictionary (dict, OP_SAVE, &compress))
262 t->map = finish_case_map (dict);
264 case_create (&t->bounce, dict_get_next_value_idx (dict));
266 t->writer = sfm_open_writer (fh, dict, compress, no_name_table);
267 if (t->writer == NULL)
277 save_trns_free (&t->h);
281 /* Parses and performs the SAVE procedure. */
285 struct save_trns *t = cmd_save_internal ();
288 procedure (save_write_case_func, t);
289 save_trns_free (&t->h);
297 /* Parses the XSAVE transformation command. */
301 struct save_trns *t = cmd_save_internal ();
304 add_transformation (&t->h);
311 /* Writes the given C to the file specified by T. */
313 do_write_case (struct save_trns *t, struct ccase *c)
316 sfm_write_case (t->writer, c);
319 map_case (t->map, c, &t->bounce);
320 sfm_write_case (t->writer, &t->bounce);
324 /* Writes case C to the system file specified on SAVE. */
326 save_write_case_func (struct ccase *c, void *aux UNUSED)
328 do_write_case (aux, c);
332 /* Writes case C to the system file specified on XSAVE. */
334 save_trns_proc (struct trns_header *h, struct ccase *c, int case_num UNUSED)
336 struct save_trns *t = (struct save_trns *) h;
337 do_write_case (t, c);
341 /* Frees a SAVE transformation. */
343 save_trns_free (struct trns_header *t_)
345 struct save_trns *t = (struct save_trns *) t_;
349 sfm_close_writer (t->writer);
350 destroy_case_map (t->map);
351 case_destroy (&t->bounce);
355 static bool rename_variables (struct dictionary *dict);
356 static bool drop_variables (struct dictionary *dict);
357 static bool keep_variables (struct dictionary *dict);
359 /* Commands that read and write system files share a great deal
360 of common syntactic structure for rearranging and dropping
361 variables. This function parses this syntax and modifies DICT
364 OP is the operation being performed. For operations that
365 write a system file, *COMPRESS is set to 1 if the system file
366 should be compressed, 0 otherwise.
368 Returns true on success, false on failure. */
370 trim_dictionary (struct dictionary *dict, enum operation op, int *compress)
372 assert ((compress != NULL) == (op == OP_SAVE));
373 if (get_scompression())
376 if (op == OP_SAVE || op == OP_EXPORT)
378 /* Delete all the scratch variables. */
383 v = xmalloc (sizeof *v * dict_get_var_cnt (dict));
385 for (i = 0; i < dict_get_var_cnt (dict); i++)
386 if (dict_class_from_id (dict_get_var (dict, i)->name) == DC_SCRATCH)
387 v[nv++] = dict_get_var (dict, i);
388 dict_delete_vars (dict, v, nv);
392 while (lex_match ('/'))
396 if (op == OP_SAVE && lex_match_id ("COMPRESSED"))
398 else if (op == OP_SAVE && lex_match_id ("UNCOMPRESSED"))
400 else if (lex_match_id ("DROP"))
401 ok = drop_variables (dict);
402 else if (lex_match_id ("KEEP"))
403 ok = keep_variables (dict);
404 else if (lex_match_id ("RENAME"))
405 ok = rename_variables (dict);
408 lex_error (_("expecting a valid subcommand"));
416 if (!lex_end_of_command ())
419 dict_compact_values (dict);
423 /* Parses and performs the RENAME subcommand of GET and SAVE. */
425 rename_variables (struct dictionary *dict)
443 v = parse_dict_variable (dict);
446 if (!lex_force_match ('=')
449 if (dict_lookup_var (dict, tokid) != NULL)
451 msg (SE, _("Cannot rename %s as %s because there already exists "
452 "a variable named %s. To rename variables with "
453 "overlapping names, use a single RENAME subcommand "
454 "such as \"/RENAME (A=B)(B=C)(C=A)\", or equivalently, "
455 "\"/RENAME (A B C=B C A)\"."), v->name, tokid, tokid);
459 dict_rename_var (dict, v, tokid);
468 while (lex_match ('('))
472 if (!parse_variables (dict, &v, &nv, PV_NO_DUPLICATE | PV_APPEND))
474 if (!lex_match ('='))
476 msg (SE, _("`=' expected after variable list."));
479 if (!parse_DATA_LIST_vars (&new_names, &nn, PV_APPEND | PV_NO_SCRATCH))
483 msg (SE, _("Number of variables on left side of `=' (%d) does not "
484 "match number of variables on right side (%d), in "
485 "parenthesized group %d of RENAME subcommand."),
486 nv - old_nv, nn - old_nv, group);
489 if (!lex_force_match (')'))
494 if (!dict_rename_vars (dict, v, new_names, nv, &err_name))
496 msg (SE, _("Requested renaming duplicates variable name %s."), err_name);
502 for (i = 0; i < nn; i++)
510 /* Parses and performs the DROP subcommand of GET and SAVE.
511 Returns true if successful, false on failure.*/
513 drop_variables (struct dictionary *dict)
519 if (!parse_variables (dict, &v, &nv, PV_NONE))
521 dict_delete_vars (dict, v, nv);
524 if (dict_get_var_cnt (dict) == 0)
526 msg (SE, _("Cannot DROP all variables from dictionary."));
532 /* Parses and performs the KEEP subcommand of GET and SAVE.
533 Returns true if successful, false on failure.*/
535 keep_variables (struct dictionary *dict)
542 if (!parse_variables (dict, &v, &nv, PV_NONE))
545 /* Move the specified variables to the beginning. */
546 dict_reorder_vars (dict, v, nv);
548 /* Delete the remaining variables. */
549 v = xrealloc (v, (dict_get_var_cnt (dict) - nv) * sizeof *v);
550 for (i = nv; i < dict_get_var_cnt (dict); i++)
551 v[i - nv] = dict_get_var (dict, i);
552 dict_delete_vars (dict, v, dict_get_var_cnt (dict) - nv);
558 /* EXPORT procedure. */
561 struct pfm_writer *writer; /* System file writer. */
562 struct case_map *map; /* Map from active file to system file dict. */
563 struct ccase bounce; /* Bounce buffer. */
566 static int export_write_case_func (struct ccase *, void *);
567 static void export_proc_free (struct export_proc *);
569 /* Parses the EXPORT command. */
570 /* FIXME: same as cmd_save_internal(). */
574 struct file_handle *fh;
575 struct dictionary *dict;
576 struct export_proc *proc;
578 proc = xmalloc (sizeof *proc);
581 case_nullify (&proc->bounce);
584 if (lex_match_id ("OUTFILE"))
590 dict = dict_clone (default_dict);
591 start_case_map (dict);
592 if (!trim_dictionary (dict, OP_EXPORT, NULL))
594 proc->map = finish_case_map (dict);
595 if (proc->map != NULL)
596 case_create (&proc->bounce, dict_get_next_value_idx (dict));
598 proc->writer = pfm_open_writer (fh, dict);
599 if (proc->writer == NULL)
604 procedure (export_write_case_func, proc);
605 export_proc_free (proc);
612 export_proc_free (proc);
617 /* Writes case C to the EXPORT file. */
619 export_write_case_func (struct ccase *c, void *aux)
621 struct export_proc *proc = aux;
622 if (proc->map == NULL)
623 pfm_write_case (proc->writer, c);
626 map_case (proc->map, c, &proc->bounce);
627 pfm_write_case (proc->writer, &proc->bounce);
633 export_proc_free (struct export_proc *proc)
637 pfm_close_writer (proc->writer);
638 destroy_case_map (proc->map);
639 case_destroy (&proc->bounce);
645 #include "debug-print.h"
650 MTF_FILE, /* Specified on FILE= subcommand. */
651 MTF_TABLE /* Specified on TABLE= subcommand. */
654 /* One of the files on MATCH FILES. */
657 struct mtf_file *next, *prev;
658 /* Next, previous in the list of files. */
659 struct mtf_file *next_min; /* Next in the chain of minimums. */
661 int type; /* One of MTF_*. */
662 struct variable **by; /* List of BY variables for this file. */
663 struct file_handle *handle; /* File handle. */
664 struct sfm_reader *reader; /* System file reader. */
665 struct dictionary *dict; /* Dictionary from system file. */
668 char *in_name; /* Variable name. */
669 struct variable *in_var; /* Variable (in master dictionary). */
671 struct ccase input; /* Input record. */
674 /* MATCH FILES procedure. */
677 struct mtf_file *head; /* First file mentioned on FILE or TABLE. */
678 struct mtf_file *tail; /* Last file mentioned on FILE or TABLE. */
680 size_t by_cnt; /* Number of variables on BY subcommand. */
682 /* Names of FIRST, LAST variables. */
683 char first[LONG_NAME_LEN + 1], last[LONG_NAME_LEN + 1];
685 struct dictionary *dict; /* Dictionary of output file. */
686 struct case_sink *sink; /* Sink to receive output. */
687 struct ccase mtf_case; /* Case used for output. */
689 unsigned seq_num; /* Have we initialized this variable? */
690 unsigned *seq_nums; /* Sequence numbers for each var in dict. */
693 static void mtf_free (struct mtf_proc *);
694 static void mtf_free_file (struct mtf_file *);
695 static int mtf_merge_dictionary (struct dictionary *const, struct mtf_file *);
696 static void mtf_delete_file_in_place (struct mtf_proc *, struct mtf_file **);
698 static void mtf_read_nonactive_records (void *);
699 static void mtf_processing_finish (void *);
700 static int mtf_processing (struct ccase *, void *);
702 static char *var_type_description (struct variable *);
704 static void set_master (struct variable *, struct variable *master);
705 static struct variable *get_master (struct variable *);
707 /* Parse and execute the MATCH FILES command. */
709 cmd_match_files (void)
712 struct mtf_file *first_table = NULL;
713 struct mtf_file *iter;
715 bool used_active_file = false;
716 bool saw_table = false;
719 mtf.head = mtf.tail = NULL;
723 mtf.dict = dict_create ();
725 case_nullify (&mtf.mtf_case);
728 dict_set_case_limit (mtf.dict, dict_get_case_limit (default_dict));
732 && (lex_id_match ("FILE", tokid) || lex_id_match ("TABLE", tokid)))
734 struct mtf_file *file = xmalloc (sizeof *file);
736 if (lex_match_id ("FILE"))
737 file->type = MTF_FILE;
738 else if (lex_match_id ("TABLE"))
740 file->type = MTF_TABLE;
751 file->in_name = NULL;
753 case_nullify (&file->input);
755 /* FILEs go first, then TABLEs. */
756 if (file->type == MTF_TABLE || first_table == NULL)
759 file->prev = mtf.tail;
761 mtf.tail->next = file;
763 if (mtf.head == NULL)
765 if (file->type == MTF_TABLE && first_table == NULL)
770 assert (file->type == MTF_FILE);
771 file->next = first_table;
772 file->prev = first_table->prev;
773 if (first_table->prev)
774 first_table->prev->next = file;
777 first_table->prev = file;
785 if (used_active_file)
787 msg (SE, _("The active file may not be specified more "
791 used_active_file = true;
793 assert (pgm_state != STATE_INPUT);
794 if (pgm_state == STATE_INIT)
796 msg (SE, _("Cannot specify the active file since no active "
797 "file has been defined."));
804 _("MATCH FILES may not be used after TEMPORARY when "
805 "the active file is an input source. "
806 "Temporary transformations will be made permanent."));
810 file->dict = default_dict;
814 file->handle = fh_parse ();
815 if (file->handle == NULL)
818 file->reader = sfm_open_reader (file->handle, &file->dict, NULL);
819 if (file->reader == NULL)
822 case_create (&file->input, dict_get_next_value_idx (file->dict));
825 while (lex_match ('/'))
826 if (lex_match_id ("RENAME"))
828 if (!rename_variables (file->dict))
831 else if (lex_match_id ("IN"))
840 if (file->in_name != NULL)
842 msg (SE, _("Multiple IN subcommands for a single FILE or "
846 file->in_name = xstrdup (tokid);
851 mtf_merge_dictionary (mtf.dict, file);
856 if (lex_match (T_BY))
858 struct variable **by;
862 msg (SE, _("BY may appear at most once."));
867 if (!parse_variables (mtf.dict, &by, &mtf.by_cnt,
868 PV_NO_DUPLICATE | PV_NO_SCRATCH))
871 for (iter = mtf.head; iter != NULL; iter = iter->next)
875 iter->by = xmalloc (sizeof *iter->by * mtf.by_cnt);
877 for (i = 0; i < mtf.by_cnt; i++)
879 iter->by[i] = dict_lookup_var (iter->dict, by[i]->name);
880 if (iter->by[i] == NULL)
882 msg (SE, _("File %s lacks BY variable %s."),
883 iter->handle ? handle_get_name (iter->handle) : "*",
891 else if (lex_match_id ("FIRST"))
893 if (mtf.first[0] != '\0')
895 msg (SE, _("FIRST may appear at most once."));
900 if (!lex_force_id ())
902 strcpy (mtf.first, tokid);
905 else if (lex_match_id ("LAST"))
907 if (mtf.last[0] != '\0')
909 msg (SE, _("LAST may appear at most once."));
914 if (!lex_force_id ())
916 strcpy (mtf.last, tokid);
919 else if (lex_match_id ("MAP"))
923 else if (lex_match_id ("DROP"))
925 if (!drop_variables (mtf.dict))
928 else if (lex_match_id ("KEEP"))
930 if (!keep_variables (mtf.dict))
939 if (!lex_match ('/') && token != '.')
941 lex_end_of_command ();
950 msg (SE, _("BY is required when TABLE is specified."));
955 msg (SE, _("BY is required when IN is specified."));
960 for (iter = mtf.head; iter != NULL; iter = iter->next)
962 struct dictionary *d = iter->dict;
965 for (i = 0; i < dict_get_var_cnt (d); i++)
967 struct variable *v = dict_get_var (d, i);
968 struct variable *mv = dict_lookup_var (mtf.dict, v->name);
974 for (iter = mtf.head; iter != NULL; iter = iter->next)
975 if (iter->in_name != NULL)
977 static const struct fmt_spec f1_0 = {FMT_F, 1, 0};
979 iter->in_var = dict_create_var (mtf.dict, iter->in_name, 0);
980 if (iter->in_var == NULL)
982 msg (SE, _("IN variable name %s duplicates an "
983 "existing variable name."),
987 iter->in_var->print = iter->in_var->write = f1_0;
990 /* MATCH FILES performs an n-way merge on all its input files.
993 1. Read one input record from every input FILE.
995 2. If no FILEs are left, stop. Otherwise, proceed to step 3.
997 3. Find the FILE input record(s) that have minimum BY
998 values. Store all the values from these input records into
1001 4. For every TABLE, read another record as long as the BY values
1002 on the TABLE's input record are less than the FILEs' BY values.
1003 If an exact match is found, store all the values from the TABLE
1004 input record into the output record.
1006 5. Write the output record.
1008 6. Read another record from each input file FILE and TABLE that
1009 we stored values from above. If we come to the end of one of the
1010 input files, remove it from the list of input files.
1012 7. Repeat from step 2.
1014 Unfortunately, this algorithm can't be implemented in a
1015 straightforward way because there's no function to read a
1016 record from the active file. Instead, it has to be written
1019 FIXME: For merging large numbers of files (more than 10?) a
1020 better algorithm would use a heap for finding minimum
1023 if (!used_active_file)
1024 discard_variables ();
1026 dict_compact_values (mtf.dict);
1027 mtf.sink = create_case_sink (&storage_sink_class, mtf.dict, NULL);
1028 if (mtf.sink->class->open != NULL)
1029 mtf.sink->class->open (mtf.sink);
1031 mtf.seq_nums = xcalloc (dict_get_var_cnt (mtf.dict) * sizeof *mtf.seq_nums);
1032 case_create (&mtf.mtf_case, dict_get_next_value_idx (mtf.dict));
1034 mtf_read_nonactive_records (&mtf);
1035 if (used_active_file)
1036 procedure (mtf_processing, &mtf);
1037 mtf_processing_finish (&mtf);
1039 dict_destroy (default_dict);
1040 default_dict = mtf.dict;
1042 vfm_source = mtf.sink->class->make_source (mtf.sink);
1043 free_case_sink (mtf.sink);
1053 /* Repeats 2...7 an arbitrary number of times. */
1055 mtf_processing_finish (void *mtf_)
1057 struct mtf_proc *mtf = mtf_;
1058 struct mtf_file *iter;
1060 /* Find the active file and delete it. */
1061 for (iter = mtf->head; iter; iter = iter->next)
1062 if (iter->handle == NULL)
1064 mtf_delete_file_in_place (mtf, &iter);
1068 while (mtf->head && mtf->head->type == MTF_FILE)
1069 if (!mtf_processing (NULL, mtf))
1073 /* Return a string in a static buffer describing V's variable type and
1076 var_type_description (struct variable *v)
1078 static char buf[2][32];
1085 if (v->type == NUMERIC)
1086 strcpy (s, "numeric");
1089 assert (v->type == ALPHA);
1090 sprintf (s, "string with width %d", v->width);
1095 /* Free FILE and associated data. */
1097 mtf_free_file (struct mtf_file *file)
1100 sfm_close_reader (file->reader);
1101 if (file->dict != default_dict)
1102 dict_destroy (file->dict);
1103 case_destroy (&file->input);
1104 free (file->in_name);
1108 /* Free all the data for the MATCH FILES procedure. */
1110 mtf_free (struct mtf_proc *mtf)
1112 struct mtf_file *iter, *next;
1114 for (iter = mtf->head; iter; iter = next)
1117 mtf_free_file (iter);
1121 dict_destroy (mtf->dict);
1122 case_destroy (&mtf->mtf_case);
1123 free (mtf->seq_nums);
1126 /* Remove *FILE from the mtf_file chain. Make *FILE point to the next
1127 file in the chain, or to NULL if was the last in the chain. */
1129 mtf_delete_file_in_place (struct mtf_proc *mtf, struct mtf_file **file)
1131 struct mtf_file *f = *file;
1135 f->prev->next = f->next;
1137 f->next->prev = f->prev;
1139 mtf->head = f->next;
1141 mtf->tail = f->prev;
1144 if (f->in_var != NULL)
1145 case_data_rw (&mtf->mtf_case, f->in_var->fv)->f = 0.;
1146 for (i = 0; i < dict_get_var_cnt (f->dict); i++)
1148 struct variable *v = dict_get_var (f->dict, i);
1149 struct variable *mv = get_master (v);
1152 union value *out = case_data_rw (&mtf->mtf_case, mv->fv);
1154 if (v->type == NUMERIC)
1157 memset (out->s, ' ', v->width);
1164 /* Read a record from every input file except the active file. */
1166 mtf_read_nonactive_records (void *mtf_)
1168 struct mtf_proc *mtf = mtf_;
1169 struct mtf_file *iter, *next;
1171 for (iter = mtf->head; iter != NULL; iter = next)
1174 if (iter->handle && !sfm_read_case (iter->reader, &iter->input))
1175 mtf_delete_file_in_place (mtf, &iter);
1179 /* Compare the BY variables for files A and B; return -1 if A < B, 0
1180 if A == B, 1 if A > B. */
1182 mtf_compare_BY_values (struct mtf_proc *mtf,
1183 struct mtf_file *a, struct mtf_file *b,
1186 struct ccase *ca = case_is_null (&a->input) ? c : &a->input;
1187 struct ccase *cb = case_is_null (&b->input) ? c : &b->input;
1188 assert ((a == NULL) + (b == NULL) + (c == NULL) <= 1);
1189 return case_compare_2dict (ca, cb, a->by, b->by, mtf->by_cnt);
1192 /* Perform one iteration of steps 3...7 above. */
1194 mtf_processing (struct ccase *c, void *mtf_)
1196 struct mtf_proc *mtf = mtf_;
1198 /* Do we need another record from the active file? */
1199 bool read_active_file;
1201 assert (mtf->head != NULL);
1202 assert (mtf->head->type == MTF_FILE);
1205 struct mtf_file *min_head, *min_tail; /* Files with minimum BY values. */
1206 struct mtf_file *max_head, *max_tail; /* Files with non-minimum BYs. */
1207 struct mtf_file *iter, *next;
1209 read_active_file = false;
1211 /* 3. Find the FILE input record(s) that have minimum BY
1212 values. Store all the values from these input records into
1213 the output record. */
1214 min_head = min_tail = mtf->head;
1215 max_head = max_tail = NULL;
1216 for (iter = mtf->head->next; iter && iter->type == MTF_FILE;
1218 switch (mtf_compare_BY_values (mtf, min_head, iter, c))
1222 max_tail = max_tail->next_min = iter;
1224 max_head = max_tail = iter;
1228 min_tail = min_tail->next_min = iter;
1234 max_tail->next_min = min_head;
1235 max_tail = min_tail;
1239 max_head = min_head;
1240 max_tail = min_tail;
1242 min_head = min_tail = iter;
1249 /* 4. For every TABLE, read another record as long as the BY
1250 values on the TABLE's input record are less than the FILEs'
1251 BY values. If an exact match is found, store all the values
1252 from the TABLE input record into the output record. */
1253 for (; iter != NULL; iter = next)
1255 assert (iter->type == MTF_TABLE);
1260 switch (mtf_compare_BY_values (mtf, min_head, iter, c))
1264 max_tail = max_tail->next_min = iter;
1266 max_head = max_tail = iter;
1270 min_tail = min_tail->next_min = iter;
1274 if (iter->handle == NULL)
1276 if (sfm_read_case (iter->reader, &iter->input))
1278 mtf_delete_file_in_place (mtf, &iter);
1286 /* Next sequence number. */
1289 /* Store data to all the records we are using. */
1291 min_tail->next_min = NULL;
1292 for (iter = min_head; iter; iter = iter->next_min)
1296 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1298 struct variable *v = dict_get_var (iter->dict, i);
1299 struct variable *mv = get_master (v);
1301 if (mv != NULL && mtf->seq_nums[mv->index] != mtf->seq_num)
1303 struct ccase *record
1304 = case_is_null (&iter->input) ? c : &iter->input;
1305 union value *out = case_data_rw (&mtf->mtf_case, mv->fv);
1307 mtf->seq_nums[mv->index] = mtf->seq_num;
1308 if (v->type == NUMERIC)
1309 out->f = case_num (record, v->fv);
1311 memcpy (out->s, case_str (record, v->fv), v->width);
1314 if (iter->in_var != NULL)
1315 case_data_rw (&mtf->mtf_case, iter->in_var->fv)->f = 1.;
1317 if (iter->type == MTF_FILE && iter->handle == NULL)
1318 read_active_file = true;
1321 /* Store missing values to all the records we're not
1324 max_tail->next_min = NULL;
1325 for (iter = max_head; iter; iter = iter->next_min)
1329 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1331 struct variable *v = dict_get_var (iter->dict, i);
1332 struct variable *mv = get_master (v);
1334 if (mv != NULL && mtf->seq_nums[mv->index] != mtf->seq_num)
1336 union value *out = case_data_rw (&mtf->mtf_case, mv->fv);
1337 mtf->seq_nums[mv->index] = mtf->seq_num;
1339 if (v->type == NUMERIC)
1342 memset (out->s, ' ', v->width);
1345 if (iter->in_var != NULL)
1346 case_data_rw (&mtf->mtf_case, iter->in_var->fv)->f = 0.;
1349 /* 5. Write the output record. */
1350 mtf->sink->class->write (mtf->sink, &mtf->mtf_case);
1352 /* 6. Read another record from each input file FILE and TABLE
1353 that we stored values from above. If we come to the end of
1354 one of the input files, remove it from the list of input
1356 for (iter = min_head; iter && iter->type == MTF_FILE; iter = next)
1358 next = iter->next_min;
1359 if (iter->reader != NULL
1360 && !sfm_read_case (iter->reader, &iter->input))
1361 mtf_delete_file_in_place (mtf, &iter);
1364 while (!read_active_file
1365 && mtf->head != NULL && mtf->head->type == MTF_FILE);
1367 return mtf->head != NULL && mtf->head->type == MTF_FILE;
1370 /* Merge the dictionary for file F into master dictionary M. */
1372 mtf_merge_dictionary (struct dictionary *const m, struct mtf_file *f)
1374 struct dictionary *d = f->dict;
1375 const char *d_docs, *m_docs;
1378 if (dict_get_label (m) == NULL)
1379 dict_set_label (m, dict_get_label (d));
1381 d_docs = dict_get_documents (d);
1382 m_docs = dict_get_documents (m);
1386 dict_set_documents (m, d_docs);
1392 new_len = strlen (m_docs) + strlen (d_docs);
1393 new_docs = xmalloc (new_len + 1);
1394 strcpy (new_docs, m_docs);
1395 strcat (new_docs, d_docs);
1396 dict_set_documents (m, new_docs);
1401 dict_compact_values (d);
1403 for (i = 0; i < dict_get_var_cnt (d); i++)
1405 struct variable *dv = dict_get_var (d, i);
1406 struct variable *mv = dict_lookup_var (m, dv->name);
1410 if (mv->width != dv->width)
1412 msg (SE, _("Variable %s in file %s (%s) has different "
1413 "type or width from the same variable in "
1414 "earlier file (%s)."),
1415 dv->name, handle_get_name (f->handle),
1416 var_type_description (dv), var_type_description (mv));
1420 if (dv->width == mv->width)
1422 if (val_labs_count (dv->val_labs)
1423 && !val_labs_count (mv->val_labs))
1424 mv->val_labs = val_labs_copy (dv->val_labs);
1425 if (dv->miss_type != MISSING_NONE
1426 && mv->miss_type == MISSING_NONE)
1427 copy_missing_values (mv, dv);
1430 if (dv->label && !mv->label)
1431 mv->label = xstrdup (dv->label);
1434 mv = dict_clone_var_assert (m, dv, dv->name);
1440 /* Marks V's master variable as MASTER. */
1442 set_master (struct variable *v, struct variable *master)
1444 var_attach_aux (v, master, NULL);
1447 /* Returns the master variable corresponding to V,
1448 as set with set_master(). */
1449 static struct variable *
1450 get_master (struct variable *v)
1455 /* IMPORT command. */
1457 /* IMPORT input program. */
1460 struct pfm_reader *reader; /* Portable file reader. */
1461 struct case_map *map; /* Map from system file to active file dict. */
1462 struct ccase bounce; /* Bounce buffer. */
1465 static void import_pgm_free (struct import_pgm *);
1467 /* Parses the IMPORT command. */
1471 struct import_pgm *pgm = NULL;
1472 struct file_handle *fh = NULL;
1473 struct dictionary *dict = NULL;
1476 pgm = xmalloc (sizeof *pgm);
1479 case_nullify (&pgm->bounce);
1485 if (lex_match_id ("FILE") || token == T_STRING)
1493 else if (lex_match_id ("TYPE"))
1497 if (lex_match_id ("COMM"))
1499 else if (lex_match_id ("TAPE"))
1503 lex_error (_("expecting COMM or TAPE"));
1509 if (!lex_match ('/') && token != '.')
1515 discard_variables ();
1517 pgm->reader = pfm_open_reader (fh, &dict, NULL);
1518 if (pgm->reader == NULL)
1520 case_create (&pgm->bounce, dict_get_next_value_idx (dict));
1522 start_case_map (dict);
1523 if (!trim_dictionary (dict, OP_READ, NULL))
1525 pgm->map = finish_case_map (dict);
1527 dict_destroy (default_dict);
1528 default_dict = dict;
1530 vfm_source = create_case_source (&import_source_class, pgm);
1535 import_pgm_free (pgm);
1537 dict_destroy (dict);
1541 /* Frees a struct import_pgm. */
1543 import_pgm_free (struct import_pgm *pgm)
1547 pfm_close_reader (pgm->reader);
1548 destroy_case_map (pgm->map);
1549 case_destroy (&pgm->bounce);
1554 /* Clears internal state related to IMPORT input procedure. */
1556 import_source_destroy (struct case_source *source)
1558 struct import_pgm *pgm = source->aux;
1559 import_pgm_free (pgm);
1562 /* Reads all the cases from the data file into C and passes them
1563 to WRITE_CASE one by one, passing WC_DATA. */
1565 import_source_read (struct case_source *source,
1567 write_case_func *write_case, write_case_data wc_data)
1569 struct import_pgm *pgm = source->aux;
1574 if (pgm->map == NULL)
1575 ok = pfm_read_case (pgm->reader, c);
1578 ok = pfm_read_case (pgm->reader, &pgm->bounce);
1580 map_case (pgm->map, &pgm->bounce, c);
1584 ok = write_case (wc_data);
1589 const struct case_source_class import_source_class =
1594 import_source_destroy,
1600 A case map copies data from a case that corresponds for one
1601 dictionary to a case that corresponds to a second dictionary
1602 derived from the first by, optionally, deleting, reordering,
1603 or renaming variables. (No new variables may be created.)
1609 size_t value_cnt; /* Number of values in map. */
1610 int *map; /* For each destination index, the
1611 corresponding source index. */
1614 /* Prepares dictionary D for producing a case map. Afterward,
1615 the caller may delete, reorder, or rename variables within D
1616 at will before using finish_case_map() to produce the case
1619 Uses D's aux members, which may not otherwise be in use. */
1621 start_case_map (struct dictionary *d)
1623 size_t var_cnt = dict_get_var_cnt (d);
1626 for (i = 0; i < var_cnt; i++)
1628 struct variable *v = dict_get_var (d, i);
1629 int *src_fv = xmalloc (sizeof *src_fv);
1631 var_attach_aux (v, src_fv, var_dtor_free);
1635 /* Produces a case map from dictionary D, which must have been
1636 previously prepared with start_case_map().
1638 Does not retain any reference to D, and clears the aux members
1639 set up by start_case_map().
1641 Returns the new case map, or a null pointer if no mapping is
1642 required (that is, no data has changed position). */
1643 static struct case_map *
1644 finish_case_map (struct dictionary *d)
1646 struct case_map *map;
1647 size_t var_cnt = dict_get_var_cnt (d);
1651 map = xmalloc (sizeof *map);
1652 map->value_cnt = dict_get_next_value_idx (d);
1653 map->map = xmalloc (sizeof *map->map * map->value_cnt);
1654 for (i = 0; i < map->value_cnt; i++)
1658 for (i = 0; i < var_cnt; i++)
1660 struct variable *v = dict_get_var (d, i);
1661 int *src_fv = (int *) var_detach_aux (v);
1664 if (v->fv != *src_fv)
1667 for (idx = 0; idx < v->nv; idx++)
1669 int src_idx = *src_fv + idx;
1670 int dst_idx = v->fv + idx;
1672 assert (map->map[dst_idx] == -1);
1673 map->map[dst_idx] = src_idx;
1680 destroy_case_map (map);
1684 while (map->value_cnt > 0 && map->map[map->value_cnt - 1] == -1)
1690 /* Maps from SRC to DST, applying case map MAP. */
1692 map_case (const struct case_map *map,
1693 const struct ccase *src, struct ccase *dst)
1697 assert (map != NULL);
1698 assert (src != NULL);
1699 assert (dst != NULL);
1700 assert (src != dst);
1702 for (dst_idx = 0; dst_idx < map->value_cnt; dst_idx++)
1704 int src_idx = map->map[dst_idx];
1706 *case_data_rw (dst, dst_idx) = *case_data (src, src_idx);
1710 /* Destroys case map MAP. */
1712 destroy_case_map (struct case_map *map)