1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
26 #include "file-handle.h"
34 #include "value-labels.h"
39 #include "debug-print.h"
41 /* GET or IMPORT input program. */
44 struct file_handle *handle; /* File to GET or IMPORT from. */
45 size_t case_size; /* Case size in bytes. */
48 /* XSAVE transformation (and related SAVE, EXPORT procedures). */
52 struct file_handle *f; /* Associated system file. */
53 int nvar; /* Number of variables. */
54 struct variable **var; /* Variables. */
55 flt64 *case_buf; /* Case transfer buffer. */
58 /* Options bits set by trim_dictionary(). */
59 #define GTSV_OPT_COMPRESSED 001 /* Compression; (X)SAVE only. */
60 #define GTSV_OPT_SAVE 002 /* The SAVE/XSAVE/EXPORT procedures. */
61 #define GTSV_OPT_MATCH_FILES 004 /* The MATCH FILES procedure. */
62 #define GTSV_OPT_NONE 0
64 static int trim_dictionary (struct dictionary * dict, int *options);
65 static int save_write_case_func (struct ccase *, void *);
66 static trns_proc_func save_trns_proc;
67 static trns_free_func save_trns_free;
70 void dump_dict_variables (struct dictionary *);
73 /* Parses the GET command. */
77 struct file_handle *handle;
78 struct dictionary *dict;
80 int options = GTSV_OPT_NONE;
86 if (lex_match_id ("FILE"))
89 handle = fh_parse_file_handle ();
93 dict = sfm_read_dictionary (handle, NULL);
98 dump_dict_variables (dict);
100 if (0 == trim_dictionary (dict, &options))
102 fh_close_handle (handle);
106 dump_dict_variables (dict);
109 dict_compact_values (dict);
112 printf (_("GET translation table from file to memory:\n"));
113 for (i = 0; i < dict->nvar; i++)
115 struct variable *v = dict->var[i];
117 printf (_(" %8s from %3d,%3d to %3d,%3d\n"), v->name,
118 v->get.fv, v->get.nv, v->fv, v->nv);
122 dict_destroy (default_dict);
125 pgm = xmalloc (sizeof *pgm);
126 pgm->handle = handle;
127 pgm->case_size = dict_get_case_size (default_dict);
128 vfm_source = create_case_source (&get_source_class, default_dict, pgm);
133 /* SAVE or XSAVE command? */
140 /* Parses the SAVE and XSAVE commands. */
142 cmd_save_internal (enum save_cmd save_cmd)
144 struct file_handle *handle;
145 struct dictionary *dict;
146 int options = GTSV_OPT_SAVE;
149 struct sfm_write_info inf;
153 lex_match_id ("SAVE");
156 if (lex_match_id ("OUTFILE"))
159 handle = fh_parse_file_handle ();
163 dict = dict_clone (default_dict);
165 dump_dict_variables (dict);
167 for (i = 0; i < dict_get_var_cnt (dict); i++)
168 dict_get_var (dict, i)->aux = dict_get_var (default_dict, i);
169 if (0 == trim_dictionary (dict, &options))
171 fh_close_handle (handle);
176 dump_dict_variables (dict);
179 /* Write dictionary. */
182 inf.compress = !!(options & GTSV_OPT_COMPRESSED);
183 if (!sfm_write_dictionary (&inf))
186 fh_close_handle (handle);
190 /* Fill in transformation structure. */
191 t = xmalloc (sizeof *t);
192 t->h.proc = save_trns_proc;
193 t->h.free = save_trns_free;
195 t->nvar = dict_get_var_cnt (dict);
196 t->var = xmalloc (sizeof *t->var * t->nvar);
197 for (i = 0; i < t->nvar; i++)
198 t->var[i] = dict_get_var (dict, i)->aux;
199 t->case_buf = xmalloc (sizeof *t->case_buf * inf.case_size);
202 if (save_cmd == CMD_SAVE)
204 procedure (NULL, save_write_case_func, NULL, t);
205 save_trns_free (&t->h);
209 assert (save_cmd == CMD_XSAVE);
210 add_transformation (&t->h);
216 /* Parses and performs the SAVE procedure. */
220 return cmd_save_internal (CMD_SAVE);
223 /* Parses the XSAVE transformation command. */
227 return cmd_save_internal (CMD_XSAVE);
230 /* Writes the given C to the file specified by T. */
232 do_write_case (struct save_trns *t, struct ccase *c)
234 flt64 *p = t->case_buf;
237 for (i = 0; i < t->nvar; i++)
239 struct variable *v = t->var[i];
240 if (v->type == NUMERIC)
242 double src = c->data[v->fv].f;
250 memcpy (p, c->data[v->fv].s, v->width);
251 memset (&((char *) p)[v->width], ' ',
252 REM_RND_UP (v->width, sizeof *p));
253 p += DIV_RND_UP (v->width, sizeof *p);
257 sfm_write_case (t->f, t->case_buf, p - t->case_buf);
260 /* Writes case C to the system file specified on SAVE. */
262 save_write_case_func (struct ccase *c, void *aux UNUSED)
264 do_write_case (aux, c);
268 /* Writes case C to the system file specified on XSAVE. */
270 save_trns_proc (struct trns_header *h, struct ccase *c, int case_num UNUSED)
272 struct save_trns *t = (struct save_trns *) h;
273 do_write_case (t, c);
277 /* Frees a SAVE transformation. */
279 save_trns_free (struct trns_header *pt)
281 struct save_trns *t = (struct save_trns *) pt;
283 fh_close_handle (t->f);
289 static int rename_variables (struct dictionary * dict);
291 /* The GET and SAVE commands have a common structure after the
292 FILE/OUTFILE subcommand. This function parses this structure and
293 returns nonzero on success, zero on failure. It both reads
294 *OPTIONS, for the GTSV_OPT_SAVE bit, and writes it, for the
295 GTSV_OPT_COMPRESSED bit. */
296 /* FIXME: IN, FIRST, LAST, MAP. */
297 /* FIXME? Should we call dict_compact_values() on dict as a
300 trim_dictionary (struct dictionary *dict, int *options)
302 if (set_scompression)
303 *options |= GTSV_OPT_COMPRESSED;
305 if (*options & GTSV_OPT_SAVE)
307 /* Delete all the scratch variables. */
312 v = xmalloc (sizeof *v * dict_get_var_cnt (dict));
314 for (i = 0; i < dict_get_var_cnt (dict); i++)
315 if (dict_class_from_id (dict_get_var (dict, i)->name) == DC_SCRATCH)
316 v[nv++] = dict_get_var (dict, i);
317 dict_delete_vars (dict, v, nv);
321 while ((*options & GTSV_OPT_MATCH_FILES) || lex_match ('/'))
323 if (!(*options & GTSV_OPT_MATCH_FILES) && lex_match_id ("COMPRESSED"))
324 *options |= GTSV_OPT_COMPRESSED;
325 else if (!(*options & GTSV_OPT_MATCH_FILES) && lex_match_id ("UNCOMPRESSED"))
326 *options &= ~GTSV_OPT_COMPRESSED;
327 else if (lex_match_id ("DROP"))
333 if (!parse_variables (dict, &v, &nv, PV_NONE))
335 dict_delete_vars (dict, v, nv);
338 else if (lex_match_id ("KEEP"))
345 if (!parse_variables (dict, &v, &nv, PV_NONE))
348 /* Move the specified variables to the beginning. */
349 dict_reorder_vars (dict, v, nv);
351 /* Delete the remaining variables. */
352 v = xrealloc (v, (dict_get_var_cnt (dict) - nv) * sizeof *v);
353 for (i = nv; i < dict_get_var_cnt (dict); i++)
354 v[i - nv] = dict_get_var (dict, i);
355 dict_delete_vars (dict, v, dict_get_var_cnt (dict) - nv);
358 else if (lex_match_id ("RENAME"))
360 if (!rename_variables (dict))
365 lex_error (_("while expecting a valid subcommand"));
369 if (dict_get_var_cnt (dict) == 0)
371 msg (SE, _("All variables deleted from system file dictionary."));
375 if (*options & GTSV_OPT_MATCH_FILES)
381 lex_error (_("expecting end of command"));
388 /* Parses and performs the RENAME subcommand of GET and SAVE. */
390 rename_variables (struct dictionary * dict)
408 v = parse_dict_variable (dict);
411 if (!lex_force_match ('=')
414 if (!strncmp (tokid, v->name, 8))
416 if (dict_lookup_var (dict, tokid) != NULL)
418 msg (SE, _("Cannot rename %s as %s because there already exists "
419 "a variable named %s. To rename variables with "
420 "overlapping names, use a single RENAME subcommand "
421 "such as \"/RENAME (A=B)(B=C)(C=A)\", or equivalently, "
422 "\"/RENAME (A B C=B C A)\"."), v->name, tokid, tokid);
426 dict_rename_var (dict, v, tokid);
435 while (lex_match ('('))
439 if (!parse_variables (dict, &v, &nv, PV_NO_DUPLICATE | PV_APPEND))
441 if (!lex_match ('='))
443 msg (SE, _("`=' expected after variable list."));
446 if (!parse_DATA_LIST_vars (&new_names, &nn, PV_APPEND | PV_NO_SCRATCH))
450 msg (SE, _("Number of variables on left side of `=' (%d) does not "
451 "match number of variables on right side (%d), in "
452 "parenthesized group %d of RENAME subcommand."),
453 nv - old_nv, nn - old_nv, group);
456 if (!lex_force_match (')'))
461 if (!dict_rename_vars (dict, v, new_names, nv, &err_name))
463 msg (SE, _("Requested renaming duplicates variable name %s."), err_name);
469 for (i = 0; i < nn; i++)
479 dump_dict_variables (struct dictionary * dict)
483 printf (_("\nVariables in dictionary:\n"));
484 for (i = 0; i < dict->nvar; i++)
485 printf ("%s, ", dict->var[i]->name);
490 /* Clears internal state related to GET input procedure. */
492 get_source_destroy (struct case_source *source)
494 struct get_pgm *pgm = source->aux;
496 /* It is not necessary to destroy the dictionary because if we get
497 to this point then the dictionary is default_dict. */
498 fh_close_handle (pgm->handle);
502 /* Reads all the cases from the data file into C and passes them
503 to WRITE_CASE one by one, passing WC_DATA. */
505 get_source_read (struct case_source *source,
507 write_case_func *write_case, write_case_data wc_data)
509 struct get_pgm *pgm = source->aux;
511 while (sfm_read_case (pgm->handle, c->data, default_dict)
512 && write_case (wc_data))
516 const struct case_source_class get_source_class =
527 #include "debug-print.h"
532 MTF_FILE, /* Specified on FILE= subcommand. */
533 MTF_TABLE /* Specified on TABLE= subcommand. */
536 /* One of the files on MATCH FILES. */
539 struct mtf_file *next, *prev;
540 /* Next, previous in the list of files. */
541 struct mtf_file *next_min; /* Next in the chain of minimums. */
543 int type; /* One of MTF_*. */
544 struct variable **by; /* List of BY variables for this file. */
545 struct file_handle *handle; /* File handle for the file. */
546 struct dictionary *dict; /* Dictionary from system file. */
547 char in[9]; /* Name of the variable from IN=. */
548 char first[9], last[9]; /* Name of the variables from FIRST=, LAST=. */
549 union value *input; /* Input record. */
552 /* All the files mentioned on FILE or TABLE. */
553 static struct mtf_file *mtf_head, *mtf_tail;
555 /* Variables on the BY subcommand. */
556 static struct variable **mtf_by;
559 /* Master dictionary. */
560 static struct dictionary *mtf_master;
562 /* Used to determine whether we've already initialized this
564 static unsigned mtf_seq_num;
566 /* Sequence numbers for each variable in mtf_master. */
567 static unsigned *mtf_seq_nums;
569 static void mtf_free (void);
570 static void mtf_free_file (struct mtf_file *file);
571 static int mtf_merge_dictionary (struct mtf_file *f);
572 static void mtf_delete_file_in_place (struct mtf_file **file);
574 static void mtf_read_nonactive_records (void *);
575 static void mtf_processing_finish (void *);
576 static int mtf_processing (struct ccase *, void *);
578 static char *var_type_description (struct variable *);
580 /* Parse and execute the MATCH FILES command. */
582 cmd_match_files (void)
584 struct mtf_file *first_table = NULL;
588 lex_match_id ("MATCH");
589 lex_match_id ("FILES");
591 mtf_head = mtf_tail = NULL;
594 mtf_master = dict_create ();
597 dict_set_case_limit (mtf_master, dict_get_case_limit (default_dict));
603 if (lex_match (T_BY))
607 msg (SE, _("The BY subcommand may be given once at most."));
613 if (!parse_variables (mtf_master, &mtf_by, &mtf_n_by,
614 PV_NO_DUPLICATE | PV_NO_SCRATCH))
617 else if (token != T_ID)
622 else if (lex_id_match ("FILE", tokid) || lex_id_match ("TABLE", tokid))
624 struct mtf_file *file = xmalloc (sizeof *file);
626 file->in[0] = file->first[0] = file->last[0] = '\0';
631 if (lex_match_id ("FILE"))
632 file->type = MTF_FILE;
633 else if (lex_match_id ("TABLE"))
635 file->type = MTF_TABLE;
641 /* FILEs go first, then TABLEs. */
642 if (file->type == MTF_TABLE || first_table == NULL)
645 file->prev = mtf_tail;
647 mtf_tail->next = file;
649 if (mtf_head == NULL)
651 if (file->type == MTF_TABLE && first_table == NULL)
656 assert (file->type == MTF_FILE);
657 file->next = first_table;
658 file->prev = first_table->prev;
659 if (first_table->prev)
660 first_table->prev->next = file;
663 first_table->prev = file;
674 msg (SE, _("The active file may not be specified more "
680 assert (pgm_state != STATE_INPUT);
681 if (pgm_state == STATE_INIT)
683 msg (SE, _("Cannot specify the active file since no active "
684 "file has been defined."));
690 file->handle = fh_parse_file_handle ();
697 file->dict = sfm_read_dictionary (file->handle, NULL);
702 file->dict = default_dict;
703 if (!mtf_merge_dictionary (file))
706 else if (lex_id_match ("IN", tokid)
707 || lex_id_match ("FIRST", tokid)
708 || lex_id_match ("LAST", tokid))
713 if (mtf_tail == NULL)
715 msg (SE, _("IN, FIRST, and LAST subcommands may not occur "
716 "before the first FILE or TABLE."));
720 if (lex_match_id ("IN"))
725 else if (lex_match_id ("FIRST"))
727 name = mtf_tail->first;
730 else if (lex_match_id ("LAST"))
732 name = mtf_tail->last;
747 msg (SE, _("Multiple %s subcommands for a single FILE or "
752 strcpy (name, tokid);
755 if (!dict_create_var (mtf_master, name, 0))
757 msg (SE, _("Duplicate variable name %s while creating %s "
763 else if (lex_id_match ("RENAME", tokid)
764 || lex_id_match ("KEEP", tokid)
765 || lex_id_match ("DROP", tokid))
767 int options = GTSV_OPT_MATCH_FILES;
769 if (mtf_tail == NULL)
771 msg (SE, _("RENAME, KEEP, and DROP subcommands may not occur "
772 "before the first FILE or TABLE."));
776 if (!trim_dictionary (mtf_tail->dict, &options))
779 else if (lex_match_id ("MAP"))
789 while (token != '.');
795 msg (SE, _("The BY subcommand is required when a TABLE subcommand "
803 struct mtf_file *iter;
805 for (iter = mtf_head; iter; iter = iter->next)
809 iter->by = xmalloc (sizeof *iter->by * mtf_n_by);
811 for (i = 0; i < mtf_n_by; i++)
813 iter->by[i] = dict_lookup_var (iter->dict, mtf_by[i]->name);
814 if (iter->by[i] == NULL)
816 msg (SE, _("File %s lacks BY variable %s."),
817 iter->handle ? fh_handle_name (iter->handle) : "*",
827 /* From sfm-read.c. */
828 extern void dump_dictionary (struct dictionary *);
830 dump_dictionary (mtf_master);
834 /* MATCH FILES performs an n-way merge on all its input files.
837 1. Read one input record from every input FILE.
839 2. If no FILEs are left, stop. Otherwise, proceed to step 3.
841 3. Find the FILE input record with minimum BY values. Store all
842 the values from this input record into the output record.
844 4. Find all the FILE input records with BY values identical to
845 the minimums. Store all the values from these input records into
848 5. For every TABLE, read another record as long as the BY values
849 on the TABLE's input record are less than the FILEs' BY values.
850 If an exact match is found, store all the values from the TABLE
851 input record into the output record.
853 6. Write the output record.
855 7. Read another record from each input file FILE and TABLE that
856 we stored values from above. If we come to the end of one of the
857 input files, remove it from the list of input files.
859 8. Repeat from step 2.
861 Unfortunately, this algorithm can't be directly implemented
862 because there's no function to read a record from the active
863 file; instead, it has to be done using callbacks.
865 FIXME: For merging large numbers of files (more than 10?) a
866 better algorithm would use a heap for finding minimum
870 discard_variables ();
873 temp_dict = mtf_master;
876 mtf_seq_nums = xmalloc (dict_get_var_cnt (mtf_master)
877 * sizeof *mtf_seq_nums);
878 memset (mtf_seq_nums, 0,
879 dict_get_var_cnt (mtf_master) * sizeof *mtf_seq_nums);
881 process_active_file (mtf_read_nonactive_records, mtf_processing,
882 mtf_processing_finish, NULL);
893 /* Repeats 2...8 an arbitrary number of times. */
895 mtf_processing_finish (void *aux UNUSED)
897 /* Find the active file and delete it. */
899 struct mtf_file *iter;
901 for (iter = mtf_head; iter; iter = iter->next)
902 if (iter->handle == NULL)
904 mtf_delete_file_in_place (&iter);
909 while (mtf_head && mtf_head->type == MTF_FILE)
910 if (!mtf_processing (NULL, NULL))
914 /* Return a string in a static buffer describing V's variable type and
917 var_type_description (struct variable *v)
919 static char buf[2][32];
926 if (v->type == NUMERIC)
927 strcpy (s, "numeric");
930 assert (v->type == ALPHA);
931 sprintf (s, "string with width %d", v->width);
936 /* Free FILE and associated data. */
938 mtf_free_file (struct mtf_file *file)
940 fh_close_handle (file->handle);
941 if (file->dict != NULL && file->dict != default_dict)
942 dict_destroy (file->dict);
949 /* Free all the data for the MATCH FILES procedure. */
953 struct mtf_file *iter, *next;
955 for (iter = mtf_head; iter; iter = next)
959 mtf_free_file (iter);
964 dict_destroy (mtf_master);
968 /* Remove *FILE from the mtf_file chain. Make *FILE point to the next
969 file in the chain, or to NULL if was the last in the chain. */
971 mtf_delete_file_in_place (struct mtf_file **file)
973 struct mtf_file *f = *file;
976 f->prev->next = f->next;
978 f->next->prev = f->prev;
988 for (i = 0; i < dict_get_var_cnt (f->dict); i++)
990 struct variable *v = dict_get_var (f->dict, i);
992 if (v->type == NUMERIC)
993 compaction_case->data[v->p.mtf.master->fv].f = SYSMIS;
995 memset (compaction_case->data[v->p.mtf.master->fv].s, ' ',
1003 /* Read a record from every input file except the active file. */
1005 mtf_read_nonactive_records (void *aux UNUSED)
1007 struct mtf_file *iter;
1009 for (iter = mtf_head; iter; )
1013 assert (iter->input == NULL);
1014 iter->input = xmalloc (dict_get_case_size (iter->dict));
1016 if (!sfm_read_case (iter->handle, iter->input, iter->dict))
1017 mtf_delete_file_in_place (&iter);
1026 /* Compare the BY variables for files A and B; return -1 if A < B, 0
1027 if A == B, 1 if A > B. */
1029 mtf_compare_BY_values (struct mtf_file *a, struct mtf_file *b,
1032 union value *a_input, *b_input;
1035 assert ((a == NULL) + (b == NULL) + (c == NULL) <= 1);
1036 a_input = a->input != NULL ? a->input : c->data;
1037 b_input = b->input != NULL ? b->input : c->data;
1038 for (i = 0; i < mtf_n_by; i++)
1040 assert (a->by[i]->type == b->by[i]->type);
1041 assert (a->by[i]->width == b->by[i]->width);
1043 if (a->by[i]->type == NUMERIC)
1045 double af = a_input[a->by[i]->fv].f;
1046 double bf = b_input[b->by[i]->fv].f;
1057 assert (a->by[i]->type == ALPHA);
1058 result = memcmp (a_input[a->by[i]->fv].s,
1059 b_input[b->by[i]->fv].s,
1063 else if (result > 0)
1070 /* Perform one iteration of steps 3...7 above. */
1072 mtf_processing (struct ccase *c, void *aux UNUSED)
1074 /* List of files with minimum BY values. */
1075 struct mtf_file *min_head, *min_tail;
1077 /* List of files with non-minimum BY values. */
1078 struct mtf_file *max_head, *max_tail;
1081 struct mtf_file *iter;
1085 /* If the active file doesn't have the minimum BY values, don't
1086 return because that would cause a record to be skipped. */
1089 if (mtf_head->type == MTF_TABLE)
1092 /* 3. Find the FILE input record with minimum BY values. Store
1093 all the values from this input record into the output record.
1095 4. Find all the FILE input records with BY values identical
1096 to the minimums. Store all the values from these input
1097 records into the output record. */
1098 min_head = min_tail = mtf_head;
1099 max_head = max_tail = NULL;
1100 for (iter = mtf_head->next; iter && iter->type == MTF_FILE;
1102 switch (mtf_compare_BY_values (min_head, iter, c))
1106 max_tail = max_tail->next_min = iter;
1108 max_head = max_tail = iter;
1112 min_tail = min_tail->next_min = iter;
1118 max_tail->next_min = min_head;
1119 max_tail = min_tail;
1123 max_head = min_head;
1124 max_tail = min_tail;
1126 min_head = min_tail = iter;
1133 /* 5. For every TABLE, read another record as long as the BY
1134 values on the TABLE's input record are less than the FILEs'
1135 BY values. If an exact match is found, store all the values
1136 from the TABLE input record into the output record. */
1139 struct mtf_file *next = iter->next;
1141 assert (iter->type == MTF_TABLE);
1143 if (iter->handle == NULL)
1147 switch (mtf_compare_BY_values (min_head, iter, c))
1151 max_tail = max_tail->next_min = iter;
1153 max_head = max_tail = iter;
1157 min_tail = min_tail->next_min = iter;
1161 if (iter->handle == NULL)
1163 if (sfm_read_case (iter->handle, iter->input, iter->dict))
1165 mtf_delete_file_in_place (&iter);
1175 /* Next sequence number. */
1178 /* Store data to all the records we are using. */
1180 min_tail->next_min = NULL;
1181 for (iter = min_head; iter; iter = iter->next_min)
1185 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1187 struct variable *v = dict_get_var (iter->dict, i);
1188 union value *record;
1190 if (mtf_seq_nums[v->p.mtf.master->index] == mtf_seq_num)
1192 mtf_seq_nums[v->p.mtf.master->index] = mtf_seq_num;
1194 record = iter->input != NULL ? iter->input : c->data;
1196 assert (v->type == NUMERIC || v->type == ALPHA);
1197 if (v->type == NUMERIC)
1198 compaction_case->data[v->p.mtf.master->fv].f = record[v->fv].f;
1200 memcpy (compaction_case->data[v->p.mtf.master->fv].s,
1201 record[v->fv].s, v->width);
1205 /* Store missing values to all the records we're not using. */
1207 max_tail->next_min = NULL;
1208 for (iter = max_head; iter; iter = iter->next_min)
1212 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1214 struct variable *v = dict_get_var (iter->dict, i);
1216 if (mtf_seq_nums[v->p.mtf.master->index] == mtf_seq_num)
1218 mtf_seq_nums[v->p.mtf.master->index] = mtf_seq_num;
1221 printf ("%s/%s: dest-fv=%d\n",
1222 fh_handle_name (iter->handle),
1224 v->p.mtf.master->fv);
1226 if (v->type == NUMERIC)
1227 compaction_case->data[v->p.mtf.master->fv].f = SYSMIS;
1229 memset (compaction_case->data[v->p.mtf.master->fv].s, ' ',
1233 if (iter->handle == NULL)
1237 /* 6. Write the output record. */
1238 process_active_file_output_case (compaction_case);
1240 /* 7. Read another record from each input file FILE and TABLE
1241 that we stored values from above. If we come to the end of
1242 one of the input files, remove it from the list of input
1244 for (iter = min_head; iter && iter->type == MTF_FILE; )
1246 struct mtf_file *next = iter->next_min;
1250 assert (iter->input != NULL);
1252 if (!sfm_read_case (iter->handle, iter->input, iter->dict))
1253 mtf_delete_file_in_place (&iter);
1263 return (mtf_head && mtf_head->type != MTF_TABLE);
1266 /* Merge the dictionary for file F into the master dictionary
1269 mtf_merge_dictionary (struct mtf_file *f)
1271 struct dictionary *const m = mtf_master;
1272 struct dictionary *d = f->dict;
1273 const char *d_docs, *m_docs;
1275 if (dict_get_label (m) == NULL)
1276 dict_set_label (m, dict_get_label (d));
1278 d_docs = dict_get_documents (d);
1279 m_docs = dict_get_documents (m);
1283 dict_set_documents (m, d_docs);
1289 new_len = strlen (m_docs) + strlen (d_docs);
1290 new_docs = xmalloc (new_len + 1);
1291 strcpy (new_docs, m_docs);
1292 strcat (new_docs, d_docs);
1293 dict_set_documents (m, new_docs);
1298 dict_compact_values (d);
1303 for (i = 0; i < dict_get_var_cnt (d); i++)
1305 struct variable *dv = dict_get_var (d, i);
1306 struct variable *mv = dict_lookup_var (m, dv->name);
1308 assert (dv->type == ALPHA || dv->width == 0);
1309 assert (!mv || mv->type == ALPHA || mv->width == 0);
1310 if (mv && dv->width == mv->width)
1312 if (val_labs_count (dv->val_labs)
1313 && !val_labs_count (mv->val_labs))
1314 mv->val_labs = val_labs_copy (dv->val_labs);
1315 if (dv->miss_type != MISSING_NONE
1316 && mv->miss_type == MISSING_NONE)
1317 copy_missing_values (mv, dv);
1319 if (mv && dv->label && !mv->label)
1320 mv->label = xstrdup (dv->label);
1323 mv = dict_clone_var (m, dv, dv->name);
1324 assert (mv != NULL);
1326 else if (mv->width != dv->width)
1328 msg (SE, _("Variable %s in file %s (%s) has different "
1329 "type or width from the same variable in "
1330 "earlier file (%s)."),
1331 dv->name, fh_handle_name (f->handle),
1332 var_type_description (dv), var_type_description (mv));
1335 dv->p.mtf.master = mv;
1342 /* IMPORT command. */
1344 /* Parses the IMPORT command. */
1348 struct file_handle *handle = NULL;
1349 struct dictionary *dict;
1350 struct get_pgm *pgm;
1351 int options = GTSV_OPT_NONE;
1354 lex_match_id ("IMPORT");
1360 if (lex_match_id ("FILE") || token == T_STRING)
1364 handle = fh_parse_file_handle ();
1368 else if (lex_match_id ("TYPE"))
1372 if (lex_match_id ("COMM"))
1374 else if (lex_match_id ("TAPE"))
1378 lex_error (_("expecting COMM or TAPE"));
1384 if (!lex_match ('/') && token != '.')
1390 discard_variables ();
1392 dict = pfm_read_dictionary (handle, NULL);
1397 dump_dict_variables (dict);
1399 if (0 == trim_dictionary (dict, &options))
1401 fh_close_handle (handle);
1405 dump_dict_variables (dict);
1408 dict_compact_values (dict);
1411 printf (_("IMPORT translation table from file to memory:\n"));
1412 for (i = 0; i < dict->nvar; i++)
1414 struct variable *v = dict->var[i];
1416 printf (_(" %8s from %3d,%3d to %3d,%3d\n"), v->name,
1417 v->get.fv, v->get.nv, v->fv, v->nv);
1421 dict_destroy (default_dict);
1422 default_dict = dict;
1424 pgm = xmalloc (sizeof *pgm);
1425 pgm->handle = handle;
1426 pgm->case_size = dict_get_case_size (default_dict);
1427 vfm_source = create_case_source (&import_source_class, default_dict, pgm);
1432 /* Reads all the cases from the data file and passes them to
1435 import_source_read (struct case_source *source,
1437 write_case_func *write_case, write_case_data wc_data)
1439 struct get_pgm *pgm = source->aux;
1441 while (pfm_read_case (pgm->handle, c->data, default_dict))
1442 if (!write_case (wc_data))
1446 const struct case_source_class import_source_class =
1454 static int export_write_case_func (struct ccase *c, void *);
1456 /* Parses the EXPORT command. */
1457 /* FIXME: same as cmd_save_internal(). */
1461 struct file_handle *handle;
1462 struct dictionary *dict;
1463 int options = GTSV_OPT_SAVE;
1465 struct save_trns *t;
1469 lex_match_id ("EXPORT");
1472 if (lex_match_id ("OUTFILE"))
1475 handle = fh_parse_file_handle ();
1479 dict = dict_clone (default_dict);
1481 dump_dict_variables (dict);
1483 for (i = 0; i < dict_get_var_cnt (dict); i++)
1484 dict_get_var (dict, i)->aux = dict_get_var (default_dict, i);
1485 if (0 == trim_dictionary (dict, &options))
1487 fh_close_handle (handle);
1492 dump_dict_variables (dict);
1495 /* Write dictionary. */
1496 if (!pfm_write_dictionary (handle, dict))
1498 dict_destroy (dict);
1499 fh_close_handle (handle);
1503 /* Fill in transformation structure. */
1504 t = xmalloc (sizeof *t);
1505 t->h.proc = save_trns_proc;
1506 t->h.free = save_trns_free;
1508 t->nvar = dict_get_var_cnt (dict);
1509 t->var = xmalloc (sizeof *t->var * t->nvar);
1510 for (i = 0; i < t->nvar; i++)
1511 t->var[i] = dict_get_var (dict, i)->aux;
1512 t->case_buf = xmalloc (sizeof *t->case_buf * t->nvar);
1513 dict_destroy (dict);
1515 procedure (NULL, export_write_case_func, NULL, t);
1516 save_trns_free (&t->h);
1521 /* Writes case C to the EXPORT file. */
1523 export_write_case_func (struct ccase *c, void *aux)
1525 struct save_trns *t = aux;
1526 union value *p = (union value *) t->case_buf;
1529 for (i = 0; i < t->nvar; i++)
1531 struct variable *v = t->var[i];
1533 if (v->type == NUMERIC)
1534 *p++ = c->data[v->fv];
1536 (*p++).c = c->data[v->fv].s;
1539 pfm_write_case (t->f, (union value *) t->case_buf);