1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
26 #include "file-handle.h"
34 #include "value-labels.h"
39 #include "debug-print.h"
41 /* XSAVE transformation (and related SAVE, EXPORT procedures). */
45 struct file_handle *f; /* Associated system file. */
46 int nvar; /* Number of variables. */
47 struct variable **var; /* Variables. */
48 flt64 *case_buf; /* Case transfer buffer. */
51 /* Options bits set by trim_dictionary(). */
52 #define GTSV_OPT_COMPRESSED 001 /* Compression; (X)SAVE only. */
53 #define GTSV_OPT_SAVE 002 /* The SAVE/XSAVE/EXPORT procedures. */
54 #define GTSV_OPT_MATCH_FILES 004 /* The MATCH FILES procedure. */
55 #define GTSV_OPT_NONE 0
57 /* The file being read by the input program. */
58 static struct file_handle *get_file;
60 /* The transformation being used by the SAVE procedure. */
61 static struct save_trns *trns;
63 static int trim_dictionary (struct dictionary * dict, int *options);
64 static int save_write_case_func (struct ccase *, void *);
65 static int save_trns_proc (struct trns_header *, struct ccase *);
66 static void save_trns_free (struct trns_header *);
69 void dump_dict_variables (struct dictionary *);
72 /* Parses the GET command. */
76 struct file_handle *handle;
77 struct dictionary *dict;
78 int options = GTSV_OPT_NONE;
84 if (lex_match_id ("FILE"))
87 handle = fh_parse_file_handle ();
91 dict = sfm_read_dictionary (handle, NULL);
96 dump_dict_variables (dict);
98 if (0 == trim_dictionary (dict, &options))
100 fh_close_handle (handle);
104 dump_dict_variables (dict);
107 dict_compact_values (dict);
110 printf (_("GET translation table from file to memory:\n"));
111 for (i = 0; i < dict->nvar; i++)
113 struct variable *v = dict->var[i];
115 printf (_(" %8s from %3d,%3d to %3d,%3d\n"), v->name,
116 v->get.fv, v->get.nv, v->fv, v->nv);
120 dict_destroy (default_dict);
123 vfm_source = &get_source;
129 /* Parses the SAVE (for XSAVE==0) and XSAVE (for XSAVE==1)
132 cmd_save_internal (int xsave)
134 struct file_handle *handle;
135 struct dictionary *dict;
136 int options = GTSV_OPT_SAVE;
139 struct sfm_write_info inf;
143 lex_match_id ("SAVE");
146 if (lex_match_id ("OUTFILE"))
149 handle = fh_parse_file_handle ();
153 dict = dict_clone (default_dict);
155 dump_dict_variables (dict);
157 for (i = 0; i < dict_get_var_cnt (dict); i++)
158 dict_get_var (dict, i)->aux = dict_get_var (default_dict, i);
159 if (0 == trim_dictionary (dict, &options))
161 fh_close_handle (handle);
166 dump_dict_variables (dict);
169 /* Write dictionary. */
172 inf.compress = !!(options & GTSV_OPT_COMPRESSED);
173 if (!sfm_write_dictionary (&inf))
176 fh_close_handle (handle);
180 /* Fill in transformation structure. */
181 t = trns = xmalloc (sizeof *t);
182 t->h.proc = save_trns_proc;
183 t->h.free = save_trns_free;
185 t->nvar = dict_get_var_cnt (dict);
186 t->var = xmalloc (sizeof *t->var * t->nvar);
187 for (i = 0; i < t->nvar; i++)
188 t->var[i] = dict_get_var (dict, i)->aux;
189 t->case_buf = xmalloc (sizeof *t->case_buf * inf.case_size);
195 procedure (NULL, save_write_case_func, NULL, NULL);
196 save_trns_free (&t->h);
200 add_transformation (&t->h);
205 /* Parses and performs the SAVE procedure. */
209 return cmd_save_internal (0);
212 /* Parses the XSAVE transformation command. */
216 return cmd_save_internal (1);
220 save_write_case_func (struct ccase * c, void *aux UNUSED)
222 save_trns_proc (&trns->h, c);
227 save_trns_proc (struct trns_header * t UNUSED, struct ccase * c)
229 flt64 *p = trns->case_buf;
232 for (i = 0; i < trns->nvar; i++)
234 struct variable *v = trns->var[i];
235 if (v->type == NUMERIC)
237 double src = c->data[v->fv].f;
245 memcpy (p, c->data[v->fv].s, v->width);
246 memset (&((char *) p)[v->width], ' ',
247 REM_RND_UP (v->width, sizeof *p));
248 p += DIV_RND_UP (v->width, sizeof *p);
252 sfm_write_case (trns->f, trns->case_buf, p - trns->case_buf);
257 save_trns_free (struct trns_header *pt)
259 struct save_trns *t = (struct save_trns *) pt;
261 fh_close_handle (t->f);
267 static int rename_variables (struct dictionary * dict);
269 /* The GET and SAVE commands have a common structure after the
270 FILE/OUTFILE subcommand. This function parses this structure and
271 returns nonzero on success, zero on failure. It both reads
272 *OPTIONS, for the GTSV_OPT_SAVE bit, and writes it, for the
273 GTSV_OPT_COMPRESSED bit. */
274 /* FIXME: IN, FIRST, LAST, MAP. */
275 /* FIXME? Should we call dict_compact_values() on dict as a
278 trim_dictionary (struct dictionary *dict, int *options)
280 if (set_scompression)
281 *options |= GTSV_OPT_COMPRESSED;
283 if (*options & GTSV_OPT_SAVE)
285 /* Delete all the scratch variables. */
290 v = xmalloc (sizeof *v * dict_get_var_cnt (dict));
292 for (i = 0; i < dict_get_var_cnt (dict); i++)
293 if (dict_get_var (dict, i)->name[0] == '#')
294 v[nv++] = dict_get_var (dict, i);
295 dict_delete_vars (dict, v, nv);
299 while ((*options & GTSV_OPT_MATCH_FILES) || lex_match ('/'))
301 if (!(*options & GTSV_OPT_MATCH_FILES) && lex_match_id ("COMPRESSED"))
302 *options |= GTSV_OPT_COMPRESSED;
303 else if (!(*options & GTSV_OPT_MATCH_FILES) && lex_match_id ("UNCOMPRESSED"))
304 *options &= ~GTSV_OPT_COMPRESSED;
305 else if (lex_match_id ("DROP"))
311 if (!parse_variables (dict, &v, &nv, PV_NONE))
313 dict_delete_vars (dict, v, nv);
316 else if (lex_match_id ("KEEP"))
323 if (!parse_variables (dict, &v, &nv, PV_NONE))
326 /* Move the specified variables to the beginning. */
327 dict_reorder_vars (dict, v, nv);
329 /* Delete the remaining variables. */
330 v = xrealloc (v, (dict_get_var_cnt (dict) - nv) * sizeof *v);
331 for (i = nv; i < dict_get_var_cnt (dict); i++)
332 v[i - nv] = dict_get_var (dict, i);
333 dict_delete_vars (dict, v, dict_get_var_cnt (dict) - nv);
336 else if (lex_match_id ("RENAME"))
338 if (!rename_variables (dict))
343 lex_error (_("while expecting a valid subcommand"));
347 if (dict_get_var_cnt (dict) == 0)
349 msg (SE, _("All variables deleted from system file dictionary."));
353 if (*options & GTSV_OPT_MATCH_FILES)
359 lex_error (_("expecting end of command"));
366 /* Parses and performs the RENAME subcommand of GET and SAVE. */
368 rename_variables (struct dictionary * dict)
386 v = parse_dict_variable (dict);
389 if (!lex_force_match ('=')
392 if (!strncmp (tokid, v->name, 8))
394 if (dict_lookup_var (dict, tokid) != NULL)
396 msg (SE, _("Cannot rename %s as %s because there already exists "
397 "a variable named %s. To rename variables with "
398 "overlapping names, use a single RENAME subcommand "
399 "such as \"/RENAME (A=B)(B=C)(C=A)\", or equivalently, "
400 "\"/RENAME (A B C=B C A)\"."), v->name, tokid, tokid);
404 dict_rename_var (dict, v, tokid);
413 while (lex_match ('('))
417 if (!parse_variables (dict, &v, &nv, PV_NO_DUPLICATE | PV_APPEND))
419 if (!lex_match ('='))
421 msg (SE, _("`=' expected after variable list."));
424 if (!parse_DATA_LIST_vars (&new_names, &nn, PV_APPEND | PV_NO_SCRATCH))
428 msg (SE, _("Number of variables on left side of `=' (%d) does not "
429 "match number of variables on right side (%d), in "
430 "parenthesized group %d of RENAME subcommand."),
431 nv - old_nv, nn - old_nv, group);
434 if (!lex_force_match (')'))
439 if (!dict_rename_vars (dict, v, new_names, nv, &err_name))
441 msg (SE, _("Requested renaming duplicates variable name %s."), err_name);
447 for (i = 0; i < nn; i++)
457 dump_dict_variables (struct dictionary * dict)
461 printf (_("\nVariables in dictionary:\n"));
462 for (i = 0; i < dict->nvar; i++)
463 printf ("%s, ", dict->var[i]->name);
468 /* Clears internal state related to GET input procedure. */
470 get_source_destroy_source (void)
472 /* It is not necessary to destroy the dictionary because if we get
473 to this point then the dictionary is default_dict. */
474 fh_close_handle (get_file);
477 /* Reads all the cases from the data file and passes them to
480 get_source_read (write_case_func *write_case, write_case_data wc_data)
482 while (sfm_read_case (get_file, temp_case->data, default_dict)
483 && write_case (wc_data))
485 get_source_destroy_source ();
488 struct case_stream get_source =
494 get_source_destroy_source,
502 #include "debug-print.h"
507 MTF_FILE, /* Specified on FILE= subcommand. */
508 MTF_TABLE /* Specified on TABLE= subcommand. */
511 /* One of the files on MATCH FILES. */
514 struct mtf_file *next, *prev;
515 /* Next, previous in the list of files. */
516 struct mtf_file *next_min; /* Next in the chain of minimums. */
518 int type; /* One of MTF_*. */
519 struct variable **by; /* List of BY variables for this file. */
520 struct file_handle *handle; /* File handle for the file. */
521 struct dictionary *dict; /* Dictionary from system file. */
522 char in[9]; /* Name of the variable from IN=. */
523 char first[9], last[9]; /* Name of the variables from FIRST=, LAST=. */
524 union value *input; /* Input record. */
527 /* All the files mentioned on FILE or TABLE. */
528 static struct mtf_file *mtf_head, *mtf_tail;
530 /* Variables on the BY subcommand. */
531 static struct variable **mtf_by;
534 /* Master dictionary. */
535 static struct dictionary *mtf_master;
537 /* Used to determine whether we've already initialized this
539 static unsigned mtf_seq_num;
541 /* Sequence numbers for each variable in mtf_master. */
542 static unsigned *mtf_seq_nums;
544 static void mtf_free (void);
545 static void mtf_free_file (struct mtf_file *file);
546 static int mtf_merge_dictionary (struct mtf_file *f);
547 static void mtf_delete_file_in_place (struct mtf_file **file);
549 static void mtf_read_nonactive_records (void *);
550 static void mtf_processing_finish (void *);
551 static int mtf_processing (struct ccase *, void *);
553 static char *var_type_description (struct variable *);
555 /* Parse and execute the MATCH FILES command. */
557 cmd_match_files (void)
559 struct mtf_file *first_table = NULL;
563 lex_match_id ("MATCH");
564 lex_match_id ("FILES");
566 mtf_head = mtf_tail = NULL;
569 mtf_master = dict_create ();
572 dict_set_case_limit (mtf_master, dict_get_case_limit (default_dict));
578 if (lex_match (T_BY))
582 msg (SE, _("The BY subcommand may be given once at most."));
588 if (!parse_variables (mtf_master, &mtf_by, &mtf_n_by,
589 PV_NO_DUPLICATE | PV_NO_SCRATCH))
592 else if (token != T_ID)
597 else if (lex_id_match ("FILE", tokid) || lex_id_match ("TABLE", tokid))
599 struct mtf_file *file = xmalloc (sizeof *file);
601 file->in[0] = file->first[0] = file->last[0] = '\0';
606 if (lex_match_id ("FILE"))
607 file->type = MTF_FILE;
608 else if (lex_match_id ("TABLE"))
610 file->type = MTF_TABLE;
616 /* FILEs go first, then TABLEs. */
617 if (file->type == MTF_TABLE || first_table == NULL)
620 file->prev = mtf_tail;
622 mtf_tail->next = file;
624 if (mtf_head == NULL)
626 if (file->type == MTF_TABLE && first_table == NULL)
631 assert (file->type == MTF_FILE);
632 file->next = first_table;
633 file->prev = first_table->prev;
634 if (first_table->prev)
635 first_table->prev->next = file;
638 first_table->prev = file;
649 msg (SE, _("The active file may not be specified more "
655 assert (pgm_state != STATE_INPUT);
656 if (pgm_state == STATE_INIT)
658 msg (SE, _("Cannot specify the active file since no active "
659 "file has been defined."));
665 file->handle = fh_parse_file_handle ();
672 file->dict = sfm_read_dictionary (file->handle, NULL);
677 file->dict = default_dict;
678 if (!mtf_merge_dictionary (file))
681 else if (lex_id_match ("IN", tokid)
682 || lex_id_match ("FIRST", tokid)
683 || lex_id_match ("LAST", tokid))
688 if (mtf_tail == NULL)
690 msg (SE, _("IN, FIRST, and LAST subcommands may not occur "
691 "before the first FILE or TABLE."));
695 if (lex_match_id ("IN"))
700 else if (lex_match_id ("FIRST"))
702 name = mtf_tail->first;
705 else if (lex_match_id ("LAST"))
707 name = mtf_tail->last;
722 msg (SE, _("Multiple %s subcommands for a single FILE or "
727 strcpy (name, tokid);
730 if (!dict_create_var (mtf_master, name, 0))
732 msg (SE, _("Duplicate variable name %s while creating %s "
738 else if (lex_id_match ("RENAME", tokid)
739 || lex_id_match ("KEEP", tokid)
740 || lex_id_match ("DROP", tokid))
742 int options = GTSV_OPT_MATCH_FILES;
744 if (mtf_tail == NULL)
746 msg (SE, _("RENAME, KEEP, and DROP subcommands may not occur "
747 "before the first FILE or TABLE."));
751 if (!trim_dictionary (mtf_tail->dict, &options))
754 else if (lex_match_id ("MAP"))
764 while (token != '.');
770 msg (SE, _("The BY subcommand is required when a TABLE subcommand "
778 struct mtf_file *iter;
780 for (iter = mtf_head; iter; iter = iter->next)
784 iter->by = xmalloc (sizeof *iter->by * mtf_n_by);
786 for (i = 0; i < mtf_n_by; i++)
788 iter->by[i] = dict_lookup_var (iter->dict, mtf_by[i]->name);
789 if (iter->by[i] == NULL)
791 msg (SE, _("File %s lacks BY variable %s."),
792 iter->handle ? fh_handle_name (iter->handle) : "*",
802 /* From sfm-read.c. */
803 extern void dump_dictionary (struct dictionary *);
805 dump_dictionary (mtf_master);
809 /* MATCH FILES performs an n-way merge on all its input files.
812 1. Read one input record from every input FILE.
814 2. If no FILEs are left, stop. Otherwise, proceed to step 3.
816 3. Find the FILE input record with minimum BY values. Store all
817 the values from this input record into the output record.
819 4. Find all the FILE input records with BY values identical to
820 the minimums. Store all the values from these input records into
823 5. For every TABLE, read another record as long as the BY values
824 on the TABLE's input record are less than the FILEs' BY values.
825 If an exact match is found, store all the values from the TABLE
826 input record into the output record.
828 6. Write the output record.
830 7. Read another record from each input file FILE and TABLE that
831 we stored values from above. If we come to the end of one of the
832 input files, remove it from the list of input files.
834 8. Repeat from step 2.
836 Unfortunately, this algorithm can't be directly implemented
837 because there's no function to read a record from the active
838 file; instead, it has to be done using callbacks.
840 FIXME: For merging large numbers of files (more than 10?) a
841 better algorithm would use a heap for finding minimum
842 values, or replacement selection, as described by Knuth in
843 _Art of Computer Programming, Vol. 3_. The SORT CASES
844 procedure does this, and perhaps some of its code could be
848 discard_variables ();
851 temp_dict = mtf_master;
854 mtf_seq_nums = xmalloc (dict_get_var_cnt (mtf_master)
855 * sizeof *mtf_seq_nums);
856 memset (mtf_seq_nums, 0,
857 dict_get_var_cnt (mtf_master) * sizeof *mtf_seq_nums);
859 process_active_file (mtf_read_nonactive_records, mtf_processing,
860 mtf_processing_finish, NULL);
871 /* Repeats 2...8 an arbitrary number of times. */
873 mtf_processing_finish (void *aux UNUSED)
875 /* Find the active file and delete it. */
877 struct mtf_file *iter;
879 for (iter = mtf_head; iter; iter = iter->next)
880 if (iter->handle == NULL)
882 mtf_delete_file_in_place (&iter);
887 while (mtf_head && mtf_head->type == MTF_FILE)
888 if (!mtf_processing (temp_case, NULL))
892 /* Return a string in a static buffer describing V's variable type and
895 var_type_description (struct variable *v)
897 static char buf[2][32];
904 if (v->type == NUMERIC)
905 strcpy (s, "numeric");
908 assert (v->type == ALPHA);
909 sprintf (s, "string with width %d", v->width);
914 /* Free FILE and associated data. */
916 mtf_free_file (struct mtf_file *file)
918 fh_close_handle (file->handle);
919 if (file->dict != NULL && file->dict != default_dict)
920 dict_destroy (file->dict);
927 /* Free all the data for the MATCH FILES procedure. */
931 struct mtf_file *iter, *next;
933 for (iter = mtf_head; iter; iter = next)
937 mtf_free_file (iter);
942 dict_destroy (mtf_master);
946 /* Remove *FILE from the mtf_file chain. Make *FILE point to the next
947 file in the chain, or to NULL if was the last in the chain. */
949 mtf_delete_file_in_place (struct mtf_file **file)
951 struct mtf_file *f = *file;
954 f->prev->next = f->next;
956 f->next->prev = f->prev;
966 for (i = 0; i < dict_get_var_cnt (f->dict); i++)
968 struct variable *v = dict_get_var (f->dict, i);
970 if (v->type == NUMERIC)
971 compaction_case->data[v->p.mtf.master->fv].f = SYSMIS;
973 memset (compaction_case->data[v->p.mtf.master->fv].s, ' ',
981 /* Read a record from every input file except the active file. */
983 mtf_read_nonactive_records (void *aux UNUSED)
985 struct mtf_file *iter;
987 for (iter = mtf_head; iter; )
991 assert (iter->input == NULL);
992 iter->input = xmalloc (dict_get_case_size (iter->dict));
994 if (!sfm_read_case (iter->handle, iter->input, iter->dict))
995 mtf_delete_file_in_place (&iter);
1001 iter->input = temp_case->data;
1007 /* Compare the BY variables for files A and B; return -1 if A < B, 0
1008 if A == B, 1 if A > B. */
1010 mtf_compare_BY_values (struct mtf_file *a, struct mtf_file *b)
1014 for (i = 0; i < mtf_n_by; i++)
1016 assert (a->by[i]->type == b->by[i]->type);
1017 assert (a->by[i]->width == b->by[i]->width);
1019 if (a->by[i]->type == NUMERIC)
1021 double af = a->input[a->by[i]->fv].f;
1022 double bf = b->input[b->by[i]->fv].f;
1033 assert (a->by[i]->type == ALPHA);
1034 result = memcmp (a->input[a->by[i]->fv].s,
1035 b->input[b->by[i]->fv].s,
1039 else if (result > 0)
1046 /* Perform one iteration of steps 3...7 above. */
1048 mtf_processing (struct ccase *c UNUSED, void *aux UNUSED)
1050 /* List of files with minimum BY values. */
1051 struct mtf_file *min_head, *min_tail;
1053 /* List of files with non-minimum BY values. */
1054 struct mtf_file *max_head, *max_tail;
1057 struct mtf_file *iter;
1061 /* If the active file doesn't have the minimum BY values, don't
1062 return because that would cause a record to be skipped. */
1065 if (mtf_head->type == MTF_TABLE)
1068 /* 3. Find the FILE input record with minimum BY values. Store
1069 all the values from this input record into the output record.
1071 4. Find all the FILE input records with BY values identical
1072 to the minimums. Store all the values from these input
1073 records into the output record. */
1074 min_head = min_tail = mtf_head;
1075 max_head = max_tail = NULL;
1076 for (iter = mtf_head->next; iter && iter->type == MTF_FILE;
1078 switch (mtf_compare_BY_values (min_head, iter))
1082 max_tail = max_tail->next_min = iter;
1084 max_head = max_tail = iter;
1088 min_tail = min_tail->next_min = iter;
1094 max_tail->next_min = min_head;
1095 max_tail = min_tail;
1099 max_head = min_head;
1100 max_tail = min_tail;
1102 min_head = min_tail = iter;
1109 /* 5. For every TABLE, read another record as long as the BY
1110 values on the TABLE's input record are less than the FILEs'
1111 BY values. If an exact match is found, store all the values
1112 from the TABLE input record into the output record. */
1115 struct mtf_file *next = iter->next;
1117 assert (iter->type == MTF_TABLE);
1119 if (iter->handle == NULL)
1123 switch (mtf_compare_BY_values (min_head, iter))
1127 max_tail = max_tail->next_min = iter;
1129 max_head = max_tail = iter;
1133 min_tail = min_tail->next_min = iter;
1137 if (iter->handle == NULL)
1139 if (sfm_read_case (iter->handle, iter->input, iter->dict))
1141 mtf_delete_file_in_place (&iter);
1151 /* Next sequence number. */
1154 /* Store data to all the records we are using. */
1156 min_tail->next_min = NULL;
1157 for (iter = min_head; iter; iter = iter->next_min)
1161 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1163 struct variable *v = dict_get_var (iter->dict, i);
1165 if (mtf_seq_nums[v->p.mtf.master->index] == mtf_seq_num)
1167 mtf_seq_nums[v->p.mtf.master->index] = mtf_seq_num;
1170 printf ("%s/%s: dest-fv=%d, src-fv=%d\n",
1171 fh_handle_name (iter->handle),
1173 v->p.mtf.master->fv, v->fv);
1175 if (v->type == NUMERIC)
1176 compaction_case->data[v->p.mtf.master->fv].f
1177 = iter->input[v->fv].f;
1180 assert (v->type == ALPHA);
1181 memcpy (compaction_case->data[v->p.mtf.master->fv].s,
1182 iter->input[v->fv].s, v->width);
1187 /* Store missing values to all the records we're not using. */
1189 max_tail->next_min = NULL;
1190 for (iter = max_head; iter; iter = iter->next_min)
1194 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1196 struct variable *v = dict_get_var (iter->dict, i);
1198 if (mtf_seq_nums[v->p.mtf.master->index] == mtf_seq_num)
1200 mtf_seq_nums[v->p.mtf.master->index] = mtf_seq_num;
1203 printf ("%s/%s: dest-fv=%d\n",
1204 fh_handle_name (iter->handle),
1206 v->p.mtf.master->fv);
1208 if (v->type == NUMERIC)
1209 compaction_case->data[v->p.mtf.master->fv].f = SYSMIS;
1211 memset (compaction_case->data[v->p.mtf.master->fv].s, ' ',
1215 if (iter->handle == NULL)
1219 /* 6. Write the output record. */
1220 process_active_file_output_case ();
1222 /* 7. Read another record from each input file FILE and TABLE
1223 that we stored values from above. If we come to the end of
1224 one of the input files, remove it from the list of input
1226 for (iter = min_head; iter && iter->type == MTF_FILE; )
1228 struct mtf_file *next = iter->next_min;
1232 assert (iter->input != NULL);
1234 if (!sfm_read_case (iter->handle, iter->input, iter->dict))
1235 mtf_delete_file_in_place (&iter);
1245 return (mtf_head && mtf_head->type != MTF_TABLE);
1248 /* Merge the dictionary for file F into the master dictionary
1251 mtf_merge_dictionary (struct mtf_file *f)
1253 struct dictionary *const m = mtf_master;
1254 struct dictionary *d = f->dict;
1255 const char *d_docs, *m_docs;
1257 if (dict_get_label (m) == NULL)
1258 dict_set_label (m, dict_get_label (d));
1260 d_docs = dict_get_documents (d);
1261 m_docs = dict_get_documents (m);
1265 dict_set_documents (m, d_docs);
1271 new_len = strlen (m_docs) + strlen (d_docs);
1272 new_docs = xmalloc (new_len + 1);
1273 strcpy (new_docs, m_docs);
1274 strcat (new_docs, d_docs);
1275 dict_set_documents (m, new_docs);
1280 dict_compact_values (d);
1285 for (i = 0; i < dict_get_var_cnt (d); i++)
1287 struct variable *dv = dict_get_var (d, i);
1288 struct variable *mv = dict_lookup_var (m, dv->name);
1290 assert (dv->type == ALPHA || dv->width == 0);
1291 assert (!mv || mv->type == ALPHA || mv->width == 0);
1292 if (mv && dv->width == mv->width)
1294 if (val_labs_count (dv->val_labs)
1295 && !val_labs_count (mv->val_labs))
1296 mv->val_labs = val_labs_copy (dv->val_labs);
1297 if (dv->miss_type != MISSING_NONE
1298 && mv->miss_type == MISSING_NONE)
1299 copy_missing_values (mv, dv);
1301 if (mv && dv->label && !mv->label)
1302 mv->label = xstrdup (dv->label);
1305 mv = dict_clone_var (m, dv, dv->name);
1306 assert (mv != NULL);
1308 else if (mv->width != dv->width)
1310 msg (SE, _("Variable %s in file %s (%s) has different "
1311 "type or width from the same variable in "
1312 "earlier file (%s)."),
1313 dv->name, fh_handle_name (f->handle),
1314 var_type_description (dv), var_type_description (mv));
1317 dv->p.mtf.master = mv;
1324 /* IMPORT command. */
1326 /* Parses the IMPORT command. */
1330 struct file_handle *handle = NULL;
1331 struct dictionary *dict;
1332 int options = GTSV_OPT_NONE;
1335 lex_match_id ("IMPORT");
1341 if (lex_match_id ("FILE") || token == T_STRING)
1345 handle = fh_parse_file_handle ();
1349 else if (lex_match_id ("TYPE"))
1353 if (lex_match_id ("COMM"))
1355 else if (lex_match_id ("TAPE"))
1359 lex_error (_("expecting COMM or TAPE"));
1365 if (!lex_match ('/') && token != '.')
1371 discard_variables ();
1373 dict = pfm_read_dictionary (handle, NULL);
1378 dump_dict_variables (dict);
1380 if (0 == trim_dictionary (dict, &options))
1382 fh_close_handle (handle);
1386 dump_dict_variables (dict);
1389 dict_compact_values (dict);
1392 printf (_("IMPORT translation table from file to memory:\n"));
1393 for (i = 0; i < dict->nvar; i++)
1395 struct variable *v = dict->var[i];
1397 printf (_(" %8s from %3d,%3d to %3d,%3d\n"), v->name,
1398 v->get.fv, v->get.nv, v->fv, v->nv);
1402 dict_destroy (default_dict);
1403 default_dict = dict;
1405 vfm_source = &import_source;
1411 /* Reads all the cases from the data file and passes them to
1414 import_source_read (write_case_func *write_case, write_case_data wc_data)
1416 while (pfm_read_case (get_file, temp_case->data, default_dict)
1417 && write_case (wc_data))
1419 get_source_destroy_source ();
1422 struct case_stream import_source =
1428 get_source_destroy_source,
1433 static int export_write_case_func (struct ccase *c, void *);
1435 /* Parses the EXPORT command. */
1436 /* FIXME: same as cmd_save_internal(). */
1440 struct file_handle *handle;
1441 struct dictionary *dict;
1442 int options = GTSV_OPT_SAVE;
1444 struct save_trns *t;
1448 lex_match_id ("EXPORT");
1451 if (lex_match_id ("OUTFILE"))
1454 handle = fh_parse_file_handle ();
1458 dict = dict_clone (default_dict);
1460 dump_dict_variables (dict);
1462 for (i = 0; i < dict_get_var_cnt (dict); i++)
1463 dict_get_var (dict, i)->aux = dict_get_var (default_dict, i);
1464 if (0 == trim_dictionary (dict, &options))
1466 fh_close_handle (handle);
1471 dump_dict_variables (dict);
1474 /* Write dictionary. */
1475 if (!pfm_write_dictionary (handle, dict))
1477 dict_destroy (dict);
1478 fh_close_handle (handle);
1482 /* Fill in transformation structure. */
1483 t = trns = xmalloc (sizeof *t);
1484 t->h.proc = save_trns_proc;
1485 t->h.free = save_trns_free;
1487 t->nvar = dict_get_var_cnt (dict);
1488 t->var = xmalloc (sizeof *t->var * t->nvar);
1489 for (i = 0; i < t->nvar; i++)
1490 t->var[i] = dict_get_var (dict, i)->aux;
1491 t->case_buf = xmalloc (sizeof *t->case_buf * t->nvar);
1492 dict_destroy (dict);
1494 procedure (NULL, export_write_case_func, NULL, NULL);
1495 save_trns_free (&t->h);
1501 export_write_case_func (struct ccase *c, void *aux UNUSED)
1503 union value *p = (union value *) trns->case_buf;
1506 for (i = 0; i < trns->nvar; i++)
1508 struct variable *v = trns->var[i];
1510 if (v->type == NUMERIC)
1511 *p++ = c->data[v->fv];
1513 (*p++).c = c->data[v->fv].s;
1516 pfm_write_case (trns->f, (union value *) trns->case_buf);