1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
26 #include "dictionary.h"
28 #include "file-handle.h"
33 #include "pfm-write.h"
36 #include "sfm-write.h"
38 #include "value-labels.h"
43 #include "debug-print.h"
45 /* Rearranging and reducing a dictionary. */
46 static void start_case_map (struct dictionary *);
47 static struct case_map *finish_case_map (struct dictionary *);
48 static void map_case (const struct case_map *,
49 const struct ccase *, struct ccase *);
50 static void destroy_case_map (struct case_map *);
55 OP_READ, /* GET or IMPORT. */
56 OP_SAVE, /* SAVE or XSAVE. */
57 OP_EXPORT /* EXPORT. */
60 static bool trim_dictionary (struct dictionary *,
61 enum operation, int *compress);
63 /* GET input program. */
66 struct sfm_reader *reader; /* System file reader. */
67 struct case_map *map; /* Map from system file to active file dict. */
68 struct ccase bounce; /* Bounce buffer. */
71 static void get_pgm_free (struct get_pgm *);
73 /* Parses the GET command. */
77 struct get_pgm *pgm = NULL;
78 struct file_handle *fh;
79 struct dictionary *dict = NULL;
81 pgm = xmalloc (sizeof *pgm);
84 case_nullify (&pgm->bounce);
89 if (lex_match_id ("FILE"))
95 pgm->reader = sfm_open_reader (fh, &dict, NULL);
96 if (pgm->reader == NULL)
98 case_create (&pgm->bounce, dict_get_next_value_idx (dict));
100 start_case_map (dict);
101 if (!trim_dictionary (dict, OP_READ, NULL))
103 pgm->map = finish_case_map (dict);
105 dict_destroy (default_dict);
108 vfm_source = create_case_source (&get_source_class, pgm);
119 /* Frees a struct get_pgm. */
121 get_pgm_free (struct get_pgm *pgm)
125 sfm_close_reader (pgm->reader);
126 destroy_case_map (pgm->map);
127 case_destroy (&pgm->bounce);
132 /* Clears internal state related to GET input procedure. */
134 get_source_destroy (struct case_source *source)
136 struct get_pgm *pgm = source->aux;
140 /* Reads all the cases from the data file into C and passes them
141 to WRITE_CASE one by one, passing WC_DATA. */
143 get_source_read (struct case_source *source,
145 write_case_func *write_case, write_case_data wc_data)
147 struct get_pgm *pgm = source->aux;
152 if (pgm->map == NULL)
153 ok = sfm_read_case (pgm->reader, c);
156 ok = sfm_read_case (pgm->reader, &pgm->bounce);
158 map_case (pgm->map, &pgm->bounce, c);
162 ok = write_case (wc_data);
167 const struct case_source_class get_source_class =
175 /* XSAVE transformation and SAVE procedure. */
178 struct trns_header h;
179 struct sfm_writer *writer; /* System file writer. */
180 struct case_map *map; /* Map from active file to system file dict. */
181 struct ccase bounce; /* Bounce buffer. */
184 static int save_write_case_func (struct ccase *, void *);
185 static trns_proc_func save_trns_proc;
186 static trns_free_func save_trns_free;
188 /* Parses the SAVE or XSAVE command
189 and returns the parsed transformation. */
190 static struct save_trns *
191 cmd_save_internal (void)
193 struct file_handle *fh = NULL;
194 struct dictionary *dict = NULL;
195 struct save_trns *t = NULL;
196 int compress = get_scompression ();
197 const int default_version = 3;
198 int version = default_version;
199 short no_name_table = 0;
201 t = xmalloc (sizeof *t);
202 t->h.proc = save_trns_proc;
203 t->h.free = save_trns_free;
206 case_nullify (&t->bounce);
209 /* Read most of the subcommands. */
212 if (lex_match_id ("VERSION"))
215 if (lex_force_int ())
217 version = lex_integer ();
220 if (lex_match_id ("X"))
224 else if (lex_match_id ("OUTFILE"))
233 if ( ! lex_match('/') )
240 lex_error (_("expecting end of command"));
246 msg ( ME, _("The required %s subcommand was not present"), "OUTFILE");
250 if ( version != default_version )
252 msg (MW, _("Unsupported sysfile version: %d. Using version %d instead."),
253 version, default_version);
255 version = default_version;
258 dict = dict_clone (default_dict);
259 start_case_map (dict);
260 if (!trim_dictionary (dict, OP_SAVE, &compress))
262 t->map = finish_case_map (dict);
264 case_create (&t->bounce, dict_get_next_value_idx (dict));
266 t->writer = sfm_open_writer (fh, dict, compress, no_name_table);
267 if (t->writer == NULL)
277 save_trns_free (&t->h);
281 /* Parses and performs the SAVE procedure. */
285 struct save_trns *t = cmd_save_internal ();
288 procedure (save_write_case_func, t);
289 save_trns_free (&t->h);
297 /* Parses the XSAVE transformation command. */
301 struct save_trns *t = cmd_save_internal ();
304 add_transformation (&t->h);
311 /* Writes the given C to the file specified by T. */
313 do_write_case (struct save_trns *t, struct ccase *c)
316 sfm_write_case (t->writer, c);
319 map_case (t->map, c, &t->bounce);
320 sfm_write_case (t->writer, &t->bounce);
324 /* Writes case C to the system file specified on SAVE. */
326 save_write_case_func (struct ccase *c, void *aux UNUSED)
328 do_write_case (aux, c);
332 /* Writes case C to the system file specified on XSAVE. */
334 save_trns_proc (struct trns_header *h, struct ccase *c, int case_num UNUSED)
336 struct save_trns *t = (struct save_trns *) h;
337 do_write_case (t, c);
341 /* Frees a SAVE transformation. */
343 save_trns_free (struct trns_header *t_)
345 struct save_trns *t = (struct save_trns *) t_;
349 sfm_close_writer (t->writer);
350 destroy_case_map (t->map);
351 case_destroy (&t->bounce);
355 static bool rename_variables (struct dictionary *dict);
356 static bool drop_variables (struct dictionary *dict);
357 static bool keep_variables (struct dictionary *dict);
359 /* Commands that read and write system files share a great deal
360 of common syntactic structure for rearranging and dropping
361 variables. This function parses this syntax and modifies DICT
364 OP is the operation being performed. For operations that
365 write a system file, *COMPRESS is set to 1 if the system file
366 should be compressed, 0 otherwise.
368 Returns true on success, false on failure. */
370 trim_dictionary (struct dictionary *dict, enum operation op, int *compress)
372 assert ((compress != NULL) == (op == OP_SAVE));
373 if (get_scompression())
376 if (op == OP_SAVE || op == OP_EXPORT)
378 /* Delete all the scratch variables. */
383 v = xmalloc (sizeof *v * dict_get_var_cnt (dict));
385 for (i = 0; i < dict_get_var_cnt (dict); i++)
386 if (dict_class_from_id (dict_get_var (dict, i)->name) == DC_SCRATCH)
387 v[nv++] = dict_get_var (dict, i);
388 dict_delete_vars (dict, v, nv);
392 while (lex_match ('/'))
396 if (op == OP_SAVE && lex_match_id ("COMPRESSED"))
398 else if (op == OP_SAVE && lex_match_id ("UNCOMPRESSED"))
400 else if (lex_match_id ("DROP"))
401 ok = drop_variables (dict);
402 else if (lex_match_id ("KEEP"))
403 ok = keep_variables (dict);
404 else if (lex_match_id ("RENAME"))
405 ok = rename_variables (dict);
408 lex_error (_("expecting a valid subcommand"));
416 if (!lex_end_of_command ())
419 dict_compact_values (dict);
423 /* Parses and performs the RENAME subcommand of GET and SAVE. */
425 rename_variables (struct dictionary *dict)
443 v = parse_dict_variable (dict);
446 if (!lex_force_match ('=')
449 if (dict_lookup_var (dict, tokid) != NULL)
451 msg (SE, _("Cannot rename %s as %s because there already exists "
452 "a variable named %s. To rename variables with "
453 "overlapping names, use a single RENAME subcommand "
454 "such as \"/RENAME (A=B)(B=C)(C=A)\", or equivalently, "
455 "\"/RENAME (A B C=B C A)\"."), v->name, tokid, tokid);
459 dict_rename_var (dict, v, tokid);
468 while (lex_match ('('))
472 if (!parse_variables (dict, &v, &nv, PV_NO_DUPLICATE | PV_APPEND))
474 if (!lex_match ('='))
476 msg (SE, _("`=' expected after variable list."));
479 if (!parse_DATA_LIST_vars (&new_names, &nn, PV_APPEND | PV_NO_SCRATCH))
483 msg (SE, _("Number of variables on left side of `=' (%d) does not "
484 "match number of variables on right side (%d), in "
485 "parenthesized group %d of RENAME subcommand."),
486 nv - old_nv, nn - old_nv, group);
489 if (!lex_force_match (')'))
494 if (!dict_rename_vars (dict, v, new_names, nv, &err_name))
496 msg (SE, _("Requested renaming duplicates variable name %s."), err_name);
502 for (i = 0; i < nn; i++)
510 /* Parses and performs the DROP subcommand of GET and SAVE.
511 Returns true if successful, false on failure.*/
513 drop_variables (struct dictionary *dict)
519 if (!parse_variables (dict, &v, &nv, PV_NONE))
521 dict_delete_vars (dict, v, nv);
524 if (dict_get_var_cnt (dict) == 0)
526 msg (SE, _("Cannot DROP all variables from dictionary."));
532 /* Parses and performs the KEEP subcommand of GET and SAVE.
533 Returns true if successful, false on failure.*/
535 keep_variables (struct dictionary *dict)
542 if (!parse_variables (dict, &v, &nv, PV_NONE))
545 /* Move the specified variables to the beginning. */
546 dict_reorder_vars (dict, v, nv);
548 /* Delete the remaining variables. */
549 v = xrealloc (v, (dict_get_var_cnt (dict) - nv) * sizeof *v);
550 for (i = nv; i < dict_get_var_cnt (dict); i++)
551 v[i - nv] = dict_get_var (dict, i);
552 dict_delete_vars (dict, v, dict_get_var_cnt (dict) - nv);
558 /* EXPORT procedure. */
561 struct pfm_writer *writer; /* System file writer. */
562 struct case_map *map; /* Map from active file to system file dict. */
563 struct ccase bounce; /* Bounce buffer. */
566 static int export_write_case_func (struct ccase *, void *);
567 static void export_proc_free (struct export_proc *);
569 /* Parses the EXPORT command. */
570 /* FIXME: same as cmd_save_internal(). */
574 struct file_handle *fh;
575 struct dictionary *dict;
576 struct export_proc *proc;
578 proc = xmalloc (sizeof *proc);
581 case_nullify (&proc->bounce);
584 if (lex_match_id ("OUTFILE"))
590 dict = dict_clone (default_dict);
591 start_case_map (dict);
592 if (!trim_dictionary (dict, OP_EXPORT, NULL))
594 proc->map = finish_case_map (dict);
595 if (proc->map != NULL)
596 case_create (&proc->bounce, dict_get_next_value_idx (dict));
598 proc->writer = pfm_open_writer (fh, dict);
599 if (proc->writer == NULL)
604 procedure (export_write_case_func, proc);
605 export_proc_free (proc);
612 export_proc_free (proc);
617 /* Writes case C to the EXPORT file. */
619 export_write_case_func (struct ccase *c, void *aux)
621 struct export_proc *proc = aux;
622 if (proc->map == NULL)
623 pfm_write_case (proc->writer, c);
626 map_case (proc->map, c, &proc->bounce);
627 pfm_write_case (proc->writer, &proc->bounce);
633 export_proc_free (struct export_proc *proc)
637 pfm_close_writer (proc->writer);
638 destroy_case_map (proc->map);
639 case_destroy (&proc->bounce);
645 #include "debug-print.h"
650 MTF_FILE, /* Specified on FILE= subcommand. */
651 MTF_TABLE /* Specified on TABLE= subcommand. */
654 /* One of the files on MATCH FILES. */
657 struct mtf_file *next, *prev;
658 /* Next, previous in the list of files. */
659 struct mtf_file *next_min; /* Next in the chain of minimums. */
661 int type; /* One of MTF_*. */
662 struct variable **by; /* List of BY variables for this file. */
663 struct file_handle *handle; /* File handle. */
664 struct sfm_reader *reader; /* System file reader. */
665 struct dictionary *dict; /* Dictionary from system file. */
668 char *in_name; /* Variable name. */
669 struct variable *in_var; /* Variable (in master dictionary). */
671 struct ccase input; /* Input record. */
674 /* MATCH FILES procedure. */
677 struct mtf_file *head; /* First file mentioned on FILE or TABLE. */
678 struct mtf_file *tail; /* Last file mentioned on FILE or TABLE. */
680 size_t by_cnt; /* Number of variables on BY subcommand. */
682 /* Names of FIRST, LAST variables. */
683 char first[LONG_NAME_LEN + 1], last[LONG_NAME_LEN + 1];
685 struct dictionary *dict; /* Dictionary of output file. */
686 struct case_sink *sink; /* Sink to receive output. */
687 struct ccase mtf_case; /* Case used for output. */
689 unsigned seq_num; /* Have we initialized this variable? */
690 unsigned *seq_nums; /* Sequence numbers for each var in dict. */
693 static void mtf_free (struct mtf_proc *);
694 static void mtf_free_file (struct mtf_file *);
695 static int mtf_merge_dictionary (struct dictionary *const, struct mtf_file *);
696 static void mtf_delete_file_in_place (struct mtf_proc *, struct mtf_file **);
698 static void mtf_read_nonactive_records (void *);
699 static void mtf_processing_finish (void *);
700 static int mtf_processing (struct ccase *, void *);
702 static char *var_type_description (struct variable *);
704 static void set_master (struct variable *, struct variable *master);
705 static struct variable *get_master (struct variable *);
707 /* Parse and execute the MATCH FILES command. */
709 cmd_match_files (void)
712 struct mtf_file *first_table = NULL;
713 struct mtf_file *iter;
715 bool used_active_file = false;
716 bool saw_table = false;
719 mtf.head = mtf.tail = NULL;
723 mtf.dict = dict_create ();
725 case_nullify (&mtf.mtf_case);
728 dict_set_case_limit (mtf.dict, dict_get_case_limit (default_dict));
732 && (lex_id_match ("FILE", tokid) || lex_id_match ("TABLE", tokid)))
734 struct mtf_file *file = xmalloc (sizeof *file);
736 if (lex_match_id ("FILE"))
737 file->type = MTF_FILE;
738 else if (lex_match_id ("TABLE"))
740 file->type = MTF_TABLE;
751 file->in_name = NULL;
753 case_nullify (&file->input);
755 /* FILEs go first, then TABLEs. */
756 if (file->type == MTF_TABLE || first_table == NULL)
759 file->prev = mtf.tail;
761 mtf.tail->next = file;
763 if (mtf.head == NULL)
765 if (file->type == MTF_TABLE && first_table == NULL)
770 assert (file->type == MTF_FILE);
771 file->next = first_table;
772 file->prev = first_table->prev;
773 if (first_table->prev)
774 first_table->prev->next = file;
777 first_table->prev = file;
785 if (used_active_file)
787 msg (SE, _("The active file may not be specified more "
791 used_active_file = true;
793 assert (pgm_state != STATE_INPUT);
794 if (pgm_state == STATE_INIT)
796 msg (SE, _("Cannot specify the active file since no active "
797 "file has been defined."));
804 _("MATCH FILES may not be used after TEMPORARY when "
805 "the active file is an input source. "
806 "Temporary transformations will be made permanent."));
810 file->dict = default_dict;
814 file->handle = fh_parse ();
815 if (file->handle == NULL)
818 file->reader = sfm_open_reader (file->handle, &file->dict, NULL);
819 if (file->reader == NULL)
822 case_create (&file->input, dict_get_next_value_idx (file->dict));
825 while (lex_match ('/'))
826 if (lex_match_id ("RENAME"))
828 if (!rename_variables (file->dict))
831 else if (lex_match_id ("IN"))
840 if (file->in_name != NULL)
842 msg (SE, _("Multiple IN subcommands for a single FILE or "
846 file->in_name = xstrdup (tokid);
851 mtf_merge_dictionary (mtf.dict, file);
856 if (lex_match (T_BY))
858 struct variable **by;
862 msg (SE, _("BY may appear at most once."));
867 if (!parse_variables (mtf.dict, &by, &mtf.by_cnt,
868 PV_NO_DUPLICATE | PV_NO_SCRATCH))
871 for (iter = mtf.head; iter != NULL; iter = iter->next)
875 iter->by = xmalloc (sizeof *iter->by * mtf.by_cnt);
877 for (i = 0; i < mtf.by_cnt; i++)
879 iter->by[i] = dict_lookup_var (iter->dict, by[i]->name);
880 if (iter->by[i] == NULL)
882 msg (SE, _("File %s lacks BY variable %s."),
883 iter->handle ? handle_get_name (iter->handle) : "*",
891 else if (lex_match_id ("FIRST"))
893 if (mtf.first[0] != '\0')
895 msg (SE, _("FIRST may appear at most once."));
900 if (!lex_force_id ())
902 strcpy (mtf.first, tokid);
905 else if (lex_match_id ("LAST"))
907 if (mtf.last[0] != '\0')
909 msg (SE, _("LAST may appear at most once."));
914 if (!lex_force_id ())
916 strcpy (mtf.last, tokid);
919 else if (lex_match_id ("MAP"))
923 else if (lex_match_id ("DROP"))
925 if (!drop_variables (mtf.dict))
928 else if (lex_match_id ("KEEP"))
930 if (!keep_variables (mtf.dict))
939 if (!lex_match ('/') && token != '.')
941 lex_end_of_command ();
950 msg (SE, _("BY is required when TABLE is specified."));
955 msg (SE, _("BY is required when IN is specified."));
960 /* Set up mapping from each file's variables to master
962 for (iter = mtf.head; iter != NULL; iter = iter->next)
964 struct dictionary *d = iter->dict;
967 for (i = 0; i < dict_get_var_cnt (d); i++)
969 struct variable *v = dict_get_var (d, i);
970 struct variable *mv = dict_lookup_var (mtf.dict, v->name);
976 /* Add IN variables to master dictionary. */
977 for (iter = mtf.head; iter != NULL; iter = iter->next)
978 if (iter->in_name != NULL)
980 iter->in_var = dict_create_var (mtf.dict, iter->in_name, 0);
981 if (iter->in_var == NULL)
983 msg (SE, _("IN variable name %s duplicates an "
984 "existing variable name."),
988 iter->in_var->print = iter->in_var->write
989 = make_output_format (FMT_F, 1, 0);
992 /* MATCH FILES performs an n-way merge on all its input files.
995 1. Read one input record from every input FILE.
997 2. If no FILEs are left, stop. Otherwise, proceed to step 3.
999 3. Find the FILE input record(s) that have minimum BY
1000 values. Store all the values from these input records into
1003 4. For every TABLE, read another record as long as the BY values
1004 on the TABLE's input record are less than the FILEs' BY values.
1005 If an exact match is found, store all the values from the TABLE
1006 input record into the output record.
1008 5. Write the output record.
1010 6. Read another record from each input file FILE and TABLE that
1011 we stored values from above. If we come to the end of one of the
1012 input files, remove it from the list of input files.
1014 7. Repeat from step 2.
1016 Unfortunately, this algorithm can't be implemented in a
1017 straightforward way because there's no function to read a
1018 record from the active file. Instead, it has to be written
1021 FIXME: For merging large numbers of files (more than 10?) a
1022 better algorithm would use a heap for finding minimum
1025 if (!used_active_file)
1026 discard_variables ();
1028 dict_compact_values (mtf.dict);
1029 mtf.sink = create_case_sink (&storage_sink_class, mtf.dict, NULL);
1030 if (mtf.sink->class->open != NULL)
1031 mtf.sink->class->open (mtf.sink);
1033 mtf.seq_nums = xcalloc (dict_get_var_cnt (mtf.dict), sizeof *mtf.seq_nums);
1034 case_create (&mtf.mtf_case, dict_get_next_value_idx (mtf.dict));
1036 mtf_read_nonactive_records (&mtf);
1037 if (used_active_file)
1038 procedure (mtf_processing, &mtf);
1039 mtf_processing_finish (&mtf);
1041 dict_destroy (default_dict);
1042 default_dict = mtf.dict;
1044 vfm_source = mtf.sink->class->make_source (mtf.sink);
1045 free_case_sink (mtf.sink);
1055 /* Repeats 2...7 an arbitrary number of times. */
1057 mtf_processing_finish (void *mtf_)
1059 struct mtf_proc *mtf = mtf_;
1060 struct mtf_file *iter;
1062 /* Find the active file and delete it. */
1063 for (iter = mtf->head; iter; iter = iter->next)
1064 if (iter->handle == NULL)
1066 mtf_delete_file_in_place (mtf, &iter);
1070 while (mtf->head && mtf->head->type == MTF_FILE)
1071 if (!mtf_processing (NULL, mtf))
1075 /* Return a string in a static buffer describing V's variable type and
1078 var_type_description (struct variable *v)
1080 static char buf[2][32];
1087 if (v->type == NUMERIC)
1088 strcpy (s, "numeric");
1091 assert (v->type == ALPHA);
1092 sprintf (s, "string with width %d", v->width);
1097 /* Free FILE and associated data. */
1099 mtf_free_file (struct mtf_file *file)
1102 sfm_close_reader (file->reader);
1103 if (file->dict != default_dict)
1104 dict_destroy (file->dict);
1105 case_destroy (&file->input);
1106 free (file->in_name);
1110 /* Free all the data for the MATCH FILES procedure. */
1112 mtf_free (struct mtf_proc *mtf)
1114 struct mtf_file *iter, *next;
1116 for (iter = mtf->head; iter; iter = next)
1119 mtf_free_file (iter);
1123 dict_destroy (mtf->dict);
1124 case_destroy (&mtf->mtf_case);
1125 free (mtf->seq_nums);
1128 /* Remove *FILE from the mtf_file chain. Make *FILE point to the next
1129 file in the chain, or to NULL if was the last in the chain. */
1131 mtf_delete_file_in_place (struct mtf_proc *mtf, struct mtf_file **file)
1133 struct mtf_file *f = *file;
1137 f->prev->next = f->next;
1139 f->next->prev = f->prev;
1141 mtf->head = f->next;
1143 mtf->tail = f->prev;
1146 if (f->in_var != NULL)
1147 case_data_rw (&mtf->mtf_case, f->in_var->fv)->f = 0.;
1148 for (i = 0; i < dict_get_var_cnt (f->dict); i++)
1150 struct variable *v = dict_get_var (f->dict, i);
1151 struct variable *mv = get_master (v);
1154 union value *out = case_data_rw (&mtf->mtf_case, mv->fv);
1156 if (v->type == NUMERIC)
1159 memset (out->s, ' ', v->width);
1166 /* Read a record from every input file except the active file. */
1168 mtf_read_nonactive_records (void *mtf_)
1170 struct mtf_proc *mtf = mtf_;
1171 struct mtf_file *iter, *next;
1173 for (iter = mtf->head; iter != NULL; iter = next)
1176 if (iter->handle && !sfm_read_case (iter->reader, &iter->input))
1177 mtf_delete_file_in_place (mtf, &iter);
1181 /* Compare the BY variables for files A and B; return -1 if A < B, 0
1182 if A == B, 1 if A > B. */
1184 mtf_compare_BY_values (struct mtf_proc *mtf,
1185 struct mtf_file *a, struct mtf_file *b,
1188 struct ccase *ca = case_is_null (&a->input) ? c : &a->input;
1189 struct ccase *cb = case_is_null (&b->input) ? c : &b->input;
1190 assert ((a == NULL) + (b == NULL) + (c == NULL) <= 1);
1191 return case_compare_2dict (ca, cb, a->by, b->by, mtf->by_cnt);
1194 /* Perform one iteration of steps 3...7 above. */
1196 mtf_processing (struct ccase *c, void *mtf_)
1198 struct mtf_proc *mtf = mtf_;
1200 /* Do we need another record from the active file? */
1201 bool read_active_file;
1203 assert (mtf->head != NULL);
1204 if (mtf->head->type == MTF_TABLE)
1209 struct mtf_file *min_head, *min_tail; /* Files with minimum BY values. */
1210 struct mtf_file *max_head, *max_tail; /* Files with non-minimum BYs. */
1211 struct mtf_file *iter, *next;
1213 read_active_file = false;
1215 /* 3. Find the FILE input record(s) that have minimum BY
1216 values. Store all the values from these input records into
1217 the output record. */
1218 min_head = min_tail = mtf->head;
1219 max_head = max_tail = NULL;
1220 for (iter = mtf->head->next; iter && iter->type == MTF_FILE;
1222 switch (mtf_compare_BY_values (mtf, min_head, iter, c))
1226 max_tail = max_tail->next_min = iter;
1228 max_head = max_tail = iter;
1232 min_tail = min_tail->next_min = iter;
1238 max_tail->next_min = min_head;
1239 max_tail = min_tail;
1243 max_head = min_head;
1244 max_tail = min_tail;
1246 min_head = min_tail = iter;
1253 /* 4. For every TABLE, read another record as long as the BY
1254 values on the TABLE's input record are less than the FILEs'
1255 BY values. If an exact match is found, store all the values
1256 from the TABLE input record into the output record. */
1257 for (; iter != NULL; iter = next)
1259 assert (iter->type == MTF_TABLE);
1264 switch (mtf_compare_BY_values (mtf, min_head, iter, c))
1268 max_tail = max_tail->next_min = iter;
1270 max_head = max_tail = iter;
1274 min_tail = min_tail->next_min = iter;
1278 if (iter->handle == NULL)
1280 if (sfm_read_case (iter->reader, &iter->input))
1282 mtf_delete_file_in_place (mtf, &iter);
1290 /* Next sequence number. */
1293 /* Store data to all the records we are using. */
1295 min_tail->next_min = NULL;
1296 for (iter = min_head; iter; iter = iter->next_min)
1300 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1302 struct variable *v = dict_get_var (iter->dict, i);
1303 struct variable *mv = get_master (v);
1305 if (mv != NULL && mtf->seq_nums[mv->index] != mtf->seq_num)
1307 struct ccase *record
1308 = case_is_null (&iter->input) ? c : &iter->input;
1309 union value *out = case_data_rw (&mtf->mtf_case, mv->fv);
1311 mtf->seq_nums[mv->index] = mtf->seq_num;
1312 if (v->type == NUMERIC)
1313 out->f = case_num (record, v->fv);
1315 memcpy (out->s, case_str (record, v->fv), v->width);
1318 if (iter->in_var != NULL)
1319 case_data_rw (&mtf->mtf_case, iter->in_var->fv)->f = 1.;
1321 if (iter->type == MTF_FILE && iter->handle == NULL)
1322 read_active_file = true;
1325 /* Store missing values to all the records we're not
1328 max_tail->next_min = NULL;
1329 for (iter = max_head; iter; iter = iter->next_min)
1333 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1335 struct variable *v = dict_get_var (iter->dict, i);
1336 struct variable *mv = get_master (v);
1338 if (mv != NULL && mtf->seq_nums[mv->index] != mtf->seq_num)
1340 union value *out = case_data_rw (&mtf->mtf_case, mv->fv);
1341 mtf->seq_nums[mv->index] = mtf->seq_num;
1343 if (v->type == NUMERIC)
1346 memset (out->s, ' ', v->width);
1349 if (iter->in_var != NULL)
1350 case_data_rw (&mtf->mtf_case, iter->in_var->fv)->f = 0.;
1353 /* 5. Write the output record. */
1354 mtf->sink->class->write (mtf->sink, &mtf->mtf_case);
1356 /* 6. Read another record from each input file FILE and TABLE
1357 that we stored values from above. If we come to the end of
1358 one of the input files, remove it from the list of input
1360 for (iter = min_head; iter && iter->type == MTF_FILE; iter = next)
1362 next = iter->next_min;
1363 if (iter->reader != NULL
1364 && !sfm_read_case (iter->reader, &iter->input))
1365 mtf_delete_file_in_place (mtf, &iter);
1368 while (!read_active_file
1369 && mtf->head != NULL && mtf->head->type == MTF_FILE);
1371 return mtf->head != NULL && mtf->head->type == MTF_FILE;
1374 /* Merge the dictionary for file F into master dictionary M. */
1376 mtf_merge_dictionary (struct dictionary *const m, struct mtf_file *f)
1378 struct dictionary *d = f->dict;
1379 const char *d_docs, *m_docs;
1382 if (dict_get_label (m) == NULL)
1383 dict_set_label (m, dict_get_label (d));
1385 d_docs = dict_get_documents (d);
1386 m_docs = dict_get_documents (m);
1390 dict_set_documents (m, d_docs);
1396 new_len = strlen (m_docs) + strlen (d_docs);
1397 new_docs = xmalloc (new_len + 1);
1398 strcpy (new_docs, m_docs);
1399 strcat (new_docs, d_docs);
1400 dict_set_documents (m, new_docs);
1405 for (i = 0; i < dict_get_var_cnt (d); i++)
1407 struct variable *dv = dict_get_var (d, i);
1408 struct variable *mv = dict_lookup_var (m, dv->name);
1410 if (dict_class_from_id (dv->name) == DC_SCRATCH)
1415 if (mv->width != dv->width)
1417 msg (SE, _("Variable %s in file %s (%s) has different "
1418 "type or width from the same variable in "
1419 "earlier file (%s)."),
1420 dv->name, handle_get_name (f->handle),
1421 var_type_description (dv), var_type_description (mv));
1425 if (dv->width == mv->width)
1427 if (val_labs_count (dv->val_labs)
1428 && !val_labs_count (mv->val_labs))
1429 mv->val_labs = val_labs_copy (dv->val_labs);
1430 if (dv->miss_type != MISSING_NONE
1431 && mv->miss_type == MISSING_NONE)
1432 copy_missing_values (mv, dv);
1435 if (dv->label && !mv->label)
1436 mv->label = xstrdup (dv->label);
1439 mv = dict_clone_var_assert (m, dv, dv->name);
1445 /* Marks V's master variable as MASTER. */
1447 set_master (struct variable *v, struct variable *master)
1449 var_attach_aux (v, master, NULL);
1452 /* Returns the master variable corresponding to V,
1453 as set with set_master(). */
1454 static struct variable *
1455 get_master (struct variable *v)
1460 /* IMPORT command. */
1462 /* IMPORT input program. */
1465 struct pfm_reader *reader; /* Portable file reader. */
1466 struct case_map *map; /* Map from system file to active file dict. */
1467 struct ccase bounce; /* Bounce buffer. */
1470 static void import_pgm_free (struct import_pgm *);
1472 /* Parses the IMPORT command. */
1476 struct import_pgm *pgm = NULL;
1477 struct file_handle *fh = NULL;
1478 struct dictionary *dict = NULL;
1481 pgm = xmalloc (sizeof *pgm);
1484 case_nullify (&pgm->bounce);
1490 if (lex_match_id ("FILE") || token == T_STRING)
1498 else if (lex_match_id ("TYPE"))
1502 if (lex_match_id ("COMM"))
1504 else if (lex_match_id ("TAPE"))
1508 lex_error (_("expecting COMM or TAPE"));
1514 if (!lex_match ('/') && token != '.')
1520 discard_variables ();
1522 pgm->reader = pfm_open_reader (fh, &dict, NULL);
1523 if (pgm->reader == NULL)
1525 case_create (&pgm->bounce, dict_get_next_value_idx (dict));
1527 start_case_map (dict);
1528 if (!trim_dictionary (dict, OP_READ, NULL))
1530 pgm->map = finish_case_map (dict);
1532 dict_destroy (default_dict);
1533 default_dict = dict;
1535 vfm_source = create_case_source (&import_source_class, pgm);
1540 import_pgm_free (pgm);
1542 dict_destroy (dict);
1546 /* Frees a struct import_pgm. */
1548 import_pgm_free (struct import_pgm *pgm)
1552 pfm_close_reader (pgm->reader);
1553 destroy_case_map (pgm->map);
1554 case_destroy (&pgm->bounce);
1559 /* Clears internal state related to IMPORT input procedure. */
1561 import_source_destroy (struct case_source *source)
1563 struct import_pgm *pgm = source->aux;
1564 import_pgm_free (pgm);
1567 /* Reads all the cases from the data file into C and passes them
1568 to WRITE_CASE one by one, passing WC_DATA. */
1570 import_source_read (struct case_source *source,
1572 write_case_func *write_case, write_case_data wc_data)
1574 struct import_pgm *pgm = source->aux;
1579 if (pgm->map == NULL)
1580 ok = pfm_read_case (pgm->reader, c);
1583 ok = pfm_read_case (pgm->reader, &pgm->bounce);
1585 map_case (pgm->map, &pgm->bounce, c);
1589 ok = write_case (wc_data);
1594 const struct case_source_class import_source_class =
1599 import_source_destroy,
1605 A case map copies data from a case that corresponds for one
1606 dictionary to a case that corresponds to a second dictionary
1607 derived from the first by, optionally, deleting, reordering,
1608 or renaming variables. (No new variables may be created.)
1614 size_t value_cnt; /* Number of values in map. */
1615 int *map; /* For each destination index, the
1616 corresponding source index. */
1619 /* Prepares dictionary D for producing a case map. Afterward,
1620 the caller may delete, reorder, or rename variables within D
1621 at will before using finish_case_map() to produce the case
1624 Uses D's aux members, which may not otherwise be in use. */
1626 start_case_map (struct dictionary *d)
1628 size_t var_cnt = dict_get_var_cnt (d);
1631 for (i = 0; i < var_cnt; i++)
1633 struct variable *v = dict_get_var (d, i);
1634 int *src_fv = xmalloc (sizeof *src_fv);
1636 var_attach_aux (v, src_fv, var_dtor_free);
1640 /* Produces a case map from dictionary D, which must have been
1641 previously prepared with start_case_map().
1643 Does not retain any reference to D, and clears the aux members
1644 set up by start_case_map().
1646 Returns the new case map, or a null pointer if no mapping is
1647 required (that is, no data has changed position). */
1648 static struct case_map *
1649 finish_case_map (struct dictionary *d)
1651 struct case_map *map;
1652 size_t var_cnt = dict_get_var_cnt (d);
1656 map = xmalloc (sizeof *map);
1657 map->value_cnt = dict_get_next_value_idx (d);
1658 map->map = xmalloc (sizeof *map->map * map->value_cnt);
1659 for (i = 0; i < map->value_cnt; i++)
1663 for (i = 0; i < var_cnt; i++)
1665 struct variable *v = dict_get_var (d, i);
1666 int *src_fv = (int *) var_detach_aux (v);
1669 if (v->fv != *src_fv)
1672 for (idx = 0; idx < v->nv; idx++)
1674 int src_idx = *src_fv + idx;
1675 int dst_idx = v->fv + idx;
1677 assert (map->map[dst_idx] == -1);
1678 map->map[dst_idx] = src_idx;
1685 destroy_case_map (map);
1689 while (map->value_cnt > 0 && map->map[map->value_cnt - 1] == -1)
1695 /* Maps from SRC to DST, applying case map MAP. */
1697 map_case (const struct case_map *map,
1698 const struct ccase *src, struct ccase *dst)
1702 assert (map != NULL);
1703 assert (src != NULL);
1704 assert (dst != NULL);
1705 assert (src != dst);
1707 for (dst_idx = 0; dst_idx < map->value_cnt; dst_idx++)
1709 int src_idx = map->map[dst_idx];
1711 *case_data_rw (dst, dst_idx) = *case_data (src, src_idx);
1715 /* Destroys case map MAP. */
1717 destroy_case_map (struct case_map *map)