1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
26 #include "file-handle.h"
34 #include "value-labels.h"
39 #include "debug-print.h"
41 /* XSAVE transformation (and related SAVE, EXPORT procedures). */
45 struct file_handle *f; /* Associated system file. */
46 int nvar; /* Number of variables. */
47 struct variable **var; /* Variables. */
48 flt64 *case_buf; /* Case transfer buffer. */
51 /* Options bits set by trim_dictionary(). */
52 #define GTSV_OPT_COMPRESSED 001 /* Compression; (X)SAVE only. */
53 #define GTSV_OPT_SAVE 002 /* The SAVE/XSAVE/EXPORT procedures. */
54 #define GTSV_OPT_MATCH_FILES 004 /* The MATCH FILES procedure. */
55 #define GTSV_OPT_NONE 0
57 /* The file being read by the input program. */
58 static struct file_handle *get_file;
60 /* The transformation being used by the SAVE procedure. */
61 static struct save_trns *trns;
63 static int trim_dictionary (struct dictionary * dict, int *options);
64 static int save_write_case_func (struct ccase *, void *);
65 static int save_trns_proc (struct trns_header *, struct ccase *);
66 static void save_trns_free (struct trns_header *);
69 void dump_dict_variables (struct dictionary *);
72 /* Parses the GET command. */
76 struct file_handle *handle;
77 struct dictionary *dict;
78 int options = GTSV_OPT_NONE;
84 if (lex_match_id ("FILE"))
87 handle = fh_parse_file_handle ();
91 dict = sfm_read_dictionary (handle, NULL);
96 dump_dict_variables (dict);
98 if (0 == trim_dictionary (dict, &options))
100 fh_close_handle (handle);
104 dump_dict_variables (dict);
107 dict_compact_values (dict);
110 printf (_("GET translation table from file to memory:\n"));
111 for (i = 0; i < dict->nvar; i++)
113 struct variable *v = dict->var[i];
115 printf (_(" %8s from %3d,%3d to %3d,%3d\n"), v->name,
116 v->get.fv, v->get.nv, v->fv, v->nv);
120 dict_destroy (default_dict);
123 vfm_source = &get_source;
129 /* Parses the SAVE (for XSAVE==0) and XSAVE (for XSAVE==1)
132 cmd_save_internal (int xsave)
134 struct file_handle *handle;
135 struct dictionary *dict;
136 int options = GTSV_OPT_SAVE;
139 struct sfm_write_info inf;
143 lex_match_id ("SAVE");
146 if (lex_match_id ("OUTFILE"))
149 handle = fh_parse_file_handle ();
153 dict = dict_clone (default_dict);
155 dump_dict_variables (dict);
157 for (i = 0; i < dict_get_var_cnt (dict); i++)
158 dict_get_var (dict, i)->aux = dict_get_var (default_dict, i);
159 if (0 == trim_dictionary (dict, &options))
161 fh_close_handle (handle);
166 dump_dict_variables (dict);
169 /* Write dictionary. */
172 inf.compress = !!(options & GTSV_OPT_COMPRESSED);
173 if (!sfm_write_dictionary (&inf))
176 fh_close_handle (handle);
180 /* Fill in transformation structure. */
181 t = trns = xmalloc (sizeof *t);
182 t->h.proc = save_trns_proc;
183 t->h.free = save_trns_free;
185 t->nvar = dict_get_var_cnt (dict);
186 t->var = xmalloc (sizeof *t->var * t->nvar);
187 for (i = 0; i < t->nvar; i++)
188 t->var[i] = dict_get_var (dict, i)->aux;
189 t->case_buf = xmalloc (sizeof *t->case_buf * inf.case_size);
195 procedure (NULL, save_write_case_func, NULL, NULL);
196 save_trns_free (&t->h);
200 add_transformation (&t->h);
205 /* Parses and performs the SAVE procedure. */
209 return cmd_save_internal (0);
212 /* Parses the XSAVE transformation command. */
216 return cmd_save_internal (1);
220 save_write_case_func (struct ccase * c, void *aux UNUSED)
222 save_trns_proc (&trns->h, c);
227 save_trns_proc (struct trns_header * t UNUSED, struct ccase * c)
229 flt64 *p = trns->case_buf;
232 for (i = 0; i < trns->nvar; i++)
234 struct variable *v = trns->var[i];
235 if (v->type == NUMERIC)
237 double src = c->data[v->fv].f;
245 memcpy (p, c->data[v->fv].s, v->width);
246 memset (&((char *) p)[v->width], ' ',
247 REM_RND_UP (v->width, sizeof *p));
248 p += DIV_RND_UP (v->width, sizeof *p);
252 sfm_write_case (trns->f, trns->case_buf, p - trns->case_buf);
257 save_trns_free (struct trns_header *pt)
259 struct save_trns *t = (struct save_trns *) pt;
261 fh_close_handle (t->f);
267 static int rename_variables (struct dictionary * dict);
269 /* The GET and SAVE commands have a common structure after the
270 FILE/OUTFILE subcommand. This function parses this structure and
271 returns nonzero on success, zero on failure. It both reads
272 *OPTIONS, for the GTSV_OPT_SAVE bit, and writes it, for the
273 GTSV_OPT_COMPRESSED bit. */
274 /* FIXME: IN, FIRST, LAST, MAP. */
275 /* FIXME? Should we call dict_compact_values() on dict as a
278 trim_dictionary (struct dictionary *dict, int *options)
280 if (set_scompression)
281 *options |= GTSV_OPT_COMPRESSED;
283 if (*options & GTSV_OPT_SAVE)
285 /* Delete all the scratch variables. */
290 v = xmalloc (sizeof *v * dict_get_var_cnt (dict));
292 for (i = 0; i < dict_get_var_cnt (dict); i++)
293 if (dict_get_var (dict, i)->name[0] == '#')
294 v[nv++] = dict_get_var (dict, i);
295 dict_delete_vars (dict, v, nv);
299 while ((*options & GTSV_OPT_MATCH_FILES) || lex_match ('/'))
301 if (!(*options & GTSV_OPT_MATCH_FILES) && lex_match_id ("COMPRESSED"))
302 *options |= GTSV_OPT_COMPRESSED;
303 else if (!(*options & GTSV_OPT_MATCH_FILES) && lex_match_id ("UNCOMPRESSED"))
304 *options &= ~GTSV_OPT_COMPRESSED;
305 else if (lex_match_id ("DROP"))
311 if (!parse_variables (dict, &v, &nv, PV_NONE))
313 dict_delete_vars (dict, v, nv);
316 else if (lex_match_id ("KEEP"))
323 if (!parse_variables (dict, &v, &nv, PV_NONE))
326 /* Move the specified variables to the beginning. */
327 dict_reorder_vars (dict, v, nv);
329 /* Delete the remaining variables. */
330 v = xrealloc (v, (dict_get_var_cnt (dict) - nv) * sizeof *v);
331 for (i = nv; i < dict_get_var_cnt (dict); i++)
332 v[i - nv] = dict_get_var (dict, i);
333 dict_delete_vars (dict, v, dict_get_var_cnt (dict) - nv);
336 else if (lex_match_id ("RENAME"))
338 if (!rename_variables (dict))
343 lex_error (_("while expecting a valid subcommand"));
347 if (dict_get_var_cnt (dict) == 0)
349 msg (SE, _("All variables deleted from system file dictionary."));
353 if (*options & GTSV_OPT_MATCH_FILES)
359 lex_error (_("expecting end of command"));
366 /* Parses and performs the RENAME subcommand of GET and SAVE. */
368 rename_variables (struct dictionary * dict)
386 v = parse_dict_variable (dict);
389 if (!lex_force_match ('=')
392 if (!strncmp (tokid, v->name, 8))
394 if (dict_lookup_var (dict, tokid) != NULL)
396 msg (SE, _("Cannot rename %s as %s because there already exists "
397 "a variable named %s. To rename variables with "
398 "overlapping names, use a single RENAME subcommand "
399 "such as \"/RENAME (A=B)(B=C)(C=A)\", or equivalently, "
400 "\"/RENAME (A B C=B C A)\"."), v->name, tokid, tokid);
404 dict_rename_var (dict, v, tokid);
413 while (lex_match ('('))
417 if (!parse_variables (dict, &v, &nv, PV_NO_DUPLICATE | PV_APPEND))
419 if (!lex_match ('='))
421 msg (SE, _("`=' expected after variable list."));
424 if (!parse_DATA_LIST_vars (&new_names, &nn, PV_APPEND | PV_NO_SCRATCH))
428 msg (SE, _("Number of variables on left side of `=' (%d) does not "
429 "match number of variables on right side (%d), in "
430 "parenthesized group %d of RENAME subcommand."),
431 nv - old_nv, nn - old_nv, group);
434 if (!lex_force_match (')'))
439 if (!dict_rename_vars (dict, v, new_names, nv, &err_name))
441 msg (SE, _("Requested renaming duplicates variable name %s."), err_name);
447 for (i = 0; i < nn; i++)
457 dump_dict_variables (struct dictionary * dict)
461 printf (_("\nVariables in dictionary:\n"));
462 for (i = 0; i < dict->nvar; i++)
463 printf ("%s, ", dict->var[i]->name);
468 /* Clears internal state related to GET input procedure. */
470 get_source_destroy_source (void)
472 /* It is not necessary to destroy the dictionary because if we get
473 to this point then the dictionary is default_dict. */
474 fh_close_handle (get_file);
477 /* Reads all the cases from the data file and passes them to
480 get_source_read (write_case_func *write_case, write_case_data wc_data)
482 while (sfm_read_case (get_file, temp_case->data, default_dict)
483 && write_case (wc_data))
485 get_source_destroy_source ();
488 struct case_stream get_source =
494 get_source_destroy_source,
502 #include "debug-print.h"
507 MTF_FILE, /* Specified on FILE= subcommand. */
508 MTF_TABLE /* Specified on TABLE= subcommand. */
511 /* One of the files on MATCH FILES. */
514 struct mtf_file *next, *prev;
515 /* Next, previous in the list of files. */
516 struct mtf_file *next_min; /* Next in the chain of minimums. */
518 int type; /* One of MTF_*. */
519 struct variable **by; /* List of BY variables for this file. */
520 struct file_handle *handle; /* File handle for the file. */
521 struct dictionary *dict; /* Dictionary from system file. */
522 char in[9]; /* Name of the variable from IN=. */
523 char first[9], last[9]; /* Name of the variables from FIRST=, LAST=. */
524 union value *input; /* Input record. */
527 /* All the files mentioned on FILE or TABLE. */
528 static struct mtf_file *mtf_head, *mtf_tail;
530 /* Variables on the BY subcommand. */
531 static struct variable **mtf_by;
534 /* Master dictionary. */
535 static struct dictionary *mtf_master;
537 /* Used to determine whether we've already initialized this
539 static unsigned mtf_seq_num;
541 /* Sequence numbers for each variable in mtf_master. */
542 static unsigned *mtf_seq_nums;
544 static void mtf_free (void);
545 static void mtf_free_file (struct mtf_file *file);
546 static int mtf_merge_dictionary (struct mtf_file *f);
547 static void mtf_delete_file_in_place (struct mtf_file **file);
549 static void mtf_read_nonactive_records (void *);
550 static void mtf_processing_finish (void *);
551 static int mtf_processing (struct ccase *, void *);
553 static char *var_type_description (struct variable *);
555 /* Parse and execute the MATCH FILES command. */
557 cmd_match_files (void)
559 struct mtf_file *first_table = NULL;
563 lex_match_id ("MATCH");
564 lex_match_id ("FILES");
566 mtf_head = mtf_tail = NULL;
569 mtf_master = dict_create ();
572 dict_set_case_limit (mtf_master, dict_get_case_limit (default_dict));
578 if (lex_match (T_BY))
582 msg (SE, _("The BY subcommand may be given once at most."));
588 if (!parse_variables (mtf_master, &mtf_by, &mtf_n_by,
589 PV_NO_DUPLICATE | PV_NO_SCRATCH))
592 else if (token != T_ID)
597 else if (lex_id_match ("FILE", tokid) || lex_id_match ("TABLE", tokid))
599 struct mtf_file *file = xmalloc (sizeof *file);
601 file->in[0] = file->first[0] = file->last[0] = '\0';
606 if (lex_match_id ("FILE"))
607 file->type = MTF_FILE;
608 else if (lex_match_id ("TABLE"))
610 file->type = MTF_TABLE;
616 /* FILEs go first, then TABLEs. */
617 if (file->type == MTF_TABLE || first_table == NULL)
620 file->prev = mtf_tail;
622 mtf_tail->next = file;
624 if (mtf_head == NULL)
626 if (file->type == MTF_TABLE && first_table == NULL)
631 assert (file->type == MTF_FILE);
632 file->next = first_table;
633 file->prev = first_table->prev;
634 if (first_table->prev)
635 first_table->prev->next = file;
638 first_table->prev = file;
649 msg (SE, _("The active file may not be specified more "
655 assert (pgm_state != STATE_INPUT);
656 if (pgm_state == STATE_INIT)
658 msg (SE, _("Cannot specify the active file since no active "
659 "file has been defined."));
665 file->handle = fh_parse_file_handle ();
672 file->dict = sfm_read_dictionary (file->handle, NULL);
677 file->dict = default_dict;
678 if (!mtf_merge_dictionary (file))
681 else if (lex_id_match ("IN", tokid)
682 || lex_id_match ("FIRST", tokid)
683 || lex_id_match ("LAST", tokid))
688 if (mtf_tail == NULL)
690 msg (SE, _("IN, FIRST, and LAST subcommands may not occur "
691 "before the first FILE or TABLE."));
695 if (lex_match_id ("IN"))
700 else if (lex_match_id ("FIRST"))
702 name = mtf_tail->first;
705 else if (lex_match_id ("LAST"))
707 name = mtf_tail->last;
722 msg (SE, _("Multiple %s subcommands for a single FILE or "
727 strcpy (name, tokid);
730 if (!dict_create_var (mtf_master, name, 0))
732 msg (SE, _("Duplicate variable name %s while creating %s "
738 else if (lex_id_match ("RENAME", tokid)
739 || lex_id_match ("KEEP", tokid)
740 || lex_id_match ("DROP", tokid))
742 int options = GTSV_OPT_MATCH_FILES;
744 if (mtf_tail == NULL)
746 msg (SE, _("RENAME, KEEP, and DROP subcommands may not occur "
747 "before the first FILE or TABLE."));
751 if (!trim_dictionary (mtf_tail->dict, &options))
754 else if (lex_match_id ("MAP"))
764 while (token != '.');
770 msg (SE, _("The BY subcommand is required when a TABLE subcommand "
778 struct mtf_file *iter;
780 for (iter = mtf_head; iter; iter = iter->next)
784 iter->by = xmalloc (sizeof *iter->by * mtf_n_by);
786 for (i = 0; i < mtf_n_by; i++)
788 iter->by[i] = dict_lookup_var (iter->dict, mtf_by[i]->name);
789 if (iter->by[i] == NULL)
791 msg (SE, _("File %s lacks BY variable %s."),
792 iter->handle ? fh_handle_name (iter->handle) : "*",
802 /* From sfm-read.c. */
803 extern void dump_dictionary (struct dictionary *);
805 dump_dictionary (mtf_master);
809 /* MATCH FILES performs an n-way merge on all its input files.
812 1. Read one input record from every input FILE.
814 2. If no FILEs are left, stop. Otherwise, proceed to step 3.
816 3. Find the FILE input record with minimum BY values. Store all
817 the values from this input record into the output record.
819 4. Find all the FILE input records with BY values identical to
820 the minimums. Store all the values from these input records into
823 5. For every TABLE, read another record as long as the BY values
824 on the TABLE's input record are less than the FILEs' BY values.
825 If an exact match is found, store all the values from the TABLE
826 input record into the output record.
828 6. Write the output record.
830 7. Read another record from each input file FILE and TABLE that
831 we stored values from above. If we come to the end of one of the
832 input files, remove it from the list of input files.
834 8. Repeat from step 2.
836 Unfortunately, this algorithm can't be directly implemented
837 because there's no function to read a record from the active
838 file; instead, it has to be done using callbacks.
840 FIXME: For merging large numbers of files (more than 10?) a
841 better algorithm would use a heap for finding minimum
842 values, or replacement selection, as described by Knuth in
843 _Art of Computer Programming, Vol. 3_. The SORT CASES
844 procedure does this, and perhaps some of its code could be
848 discard_variables ();
851 temp_dict = mtf_master;
854 mtf_seq_nums = xmalloc (dict_get_var_cnt (mtf_master)
855 * sizeof *mtf_seq_nums);
856 memset (mtf_seq_nums, 0,
857 dict_get_var_cnt (mtf_master) * sizeof *mtf_seq_nums);
859 process_active_file (mtf_read_nonactive_records, mtf_processing,
860 mtf_processing_finish, NULL);
871 /* Repeats 2...8 an arbitrary number of times. */
873 mtf_processing_finish (void *aux UNUSED)
875 /* Find the active file and delete it. */
877 struct mtf_file *iter;
879 for (iter = mtf_head; iter; iter = iter->next)
880 if (iter->handle == NULL)
882 mtf_delete_file_in_place (&iter);
887 while (mtf_head && mtf_head->type == MTF_FILE)
888 if (!mtf_processing (temp_case, NULL))
892 /* Return a string in a static buffer describing V's variable type and
895 var_type_description (struct variable *v)
897 static char buf[2][32];
904 if (v->type == NUMERIC)
905 strcpy (s, "numeric");
908 assert (v->type == ALPHA);
909 sprintf (s, "string with width %d", v->width);
914 /* Free FILE and associated data. */
916 mtf_free_file (struct mtf_file *file)
918 fh_close_handle (file->handle);
919 if (file->dict != NULL && file->dict != default_dict)
920 dict_destroy (file->dict);
927 /* Free all the data for the MATCH FILES procedure. */
931 struct mtf_file *iter, *next;
933 for (iter = mtf_head; iter; iter = next)
937 mtf_free_file (iter);
942 dict_destroy (mtf_master);
946 /* Remove *FILE from the mtf_file chain. Make *FILE point to the next
947 file in the chain, or to NULL if was the last in the chain. */
949 mtf_delete_file_in_place (struct mtf_file **file)
951 struct mtf_file *f = *file;
954 f->prev->next = f->next;
956 f->next->prev = f->prev;
966 for (i = 0; i < dict_get_var_cnt (f->dict); i++)
968 struct variable *v = dict_get_var (f->dict, i);
970 if (v->type == NUMERIC)
971 compaction_case->data[v->p.mtf.master->fv].f = SYSMIS;
973 memset (compaction_case->data[v->p.mtf.master->fv].s, ' ',
981 /* Read a record from every input file except the active file. */
983 mtf_read_nonactive_records (void *aux UNUSED)
985 struct mtf_file *iter;
987 for (iter = mtf_head; iter; )
991 assert (iter->input == NULL);
992 iter->input = xmalloc (sizeof *iter->input
993 * dict_get_value_cnt (iter->dict));
995 if (!sfm_read_case (iter->handle, iter->input, iter->dict))
996 mtf_delete_file_in_place (&iter);
1002 iter->input = temp_case->data;
1008 /* Compare the BY variables for files A and B; return -1 if A < B, 0
1009 if A == B, 1 if A > B. */
1011 mtf_compare_BY_values (struct mtf_file *a, struct mtf_file *b)
1015 for (i = 0; i < mtf_n_by; i++)
1017 assert (a->by[i]->type == b->by[i]->type);
1018 assert (a->by[i]->width == b->by[i]->width);
1020 if (a->by[i]->type == NUMERIC)
1022 double af = a->input[a->by[i]->fv].f;
1023 double bf = b->input[b->by[i]->fv].f;
1034 assert (a->by[i]->type == ALPHA);
1035 result = memcmp (a->input[a->by[i]->fv].s,
1036 b->input[b->by[i]->fv].s,
1040 else if (result > 0)
1047 /* Perform one iteration of steps 3...7 above. */
1049 mtf_processing (struct ccase *c UNUSED, void *aux UNUSED)
1051 /* List of files with minimum BY values. */
1052 struct mtf_file *min_head, *min_tail;
1054 /* List of files with non-minimum BY values. */
1055 struct mtf_file *max_head, *max_tail;
1058 struct mtf_file *iter;
1062 /* If the active file doesn't have the minimum BY values, don't
1063 return because that would cause a record to be skipped. */
1066 if (mtf_head->type == MTF_TABLE)
1069 /* 3. Find the FILE input record with minimum BY values. Store
1070 all the values from this input record into the output record.
1072 4. Find all the FILE input records with BY values identical
1073 to the minimums. Store all the values from these input
1074 records into the output record. */
1075 min_head = min_tail = mtf_head;
1076 max_head = max_tail = NULL;
1077 for (iter = mtf_head->next; iter && iter->type == MTF_FILE;
1079 switch (mtf_compare_BY_values (min_head, iter))
1083 max_tail = max_tail->next_min = iter;
1085 max_head = max_tail = iter;
1089 min_tail = min_tail->next_min = iter;
1095 max_tail->next_min = min_head;
1096 max_tail = min_tail;
1100 max_head = min_head;
1101 max_tail = min_tail;
1103 min_head = min_tail = iter;
1110 /* 5. For every TABLE, read another record as long as the BY
1111 values on the TABLE's input record are less than the FILEs'
1112 BY values. If an exact match is found, store all the values
1113 from the TABLE input record into the output record. */
1116 struct mtf_file *next = iter->next;
1118 assert (iter->type == MTF_TABLE);
1120 if (iter->handle == NULL)
1124 switch (mtf_compare_BY_values (min_head, iter))
1128 max_tail = max_tail->next_min = iter;
1130 max_head = max_tail = iter;
1134 min_tail = min_tail->next_min = iter;
1138 if (iter->handle == NULL)
1140 if (sfm_read_case (iter->handle, iter->input, iter->dict))
1142 mtf_delete_file_in_place (&iter);
1152 /* Next sequence number. */
1155 /* Store data to all the records we are using. */
1157 min_tail->next_min = NULL;
1158 for (iter = min_head; iter; iter = iter->next_min)
1162 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1164 struct variable *v = dict_get_var (iter->dict, i);
1166 if (mtf_seq_nums[v->p.mtf.master->index] == mtf_seq_num)
1168 mtf_seq_nums[v->p.mtf.master->index] = mtf_seq_num;
1171 printf ("%s/%s: dest-fv=%d, src-fv=%d\n",
1172 fh_handle_name (iter->handle),
1174 v->p.mtf.master->fv, v->fv);
1176 if (v->type == NUMERIC)
1177 compaction_case->data[v->p.mtf.master->fv].f
1178 = iter->input[v->fv].f;
1181 assert (v->type == ALPHA);
1182 memcpy (compaction_case->data[v->p.mtf.master->fv].s,
1183 iter->input[v->fv].s, v->width);
1188 /* Store missing values to all the records we're not using. */
1190 max_tail->next_min = NULL;
1191 for (iter = max_head; iter; iter = iter->next_min)
1195 for (i = 0; i < dict_get_var_cnt (iter->dict); i++)
1197 struct variable *v = dict_get_var (iter->dict, i);
1199 if (mtf_seq_nums[v->p.mtf.master->index] == mtf_seq_num)
1201 mtf_seq_nums[v->p.mtf.master->index] = mtf_seq_num;
1204 printf ("%s/%s: dest-fv=%d\n",
1205 fh_handle_name (iter->handle),
1207 v->p.mtf.master->fv);
1209 if (v->type == NUMERIC)
1210 compaction_case->data[v->p.mtf.master->fv].f = SYSMIS;
1212 memset (compaction_case->data[v->p.mtf.master->fv].s, ' ',
1216 if (iter->handle == NULL)
1220 /* 6. Write the output record. */
1221 process_active_file_output_case ();
1223 /* 7. Read another record from each input file FILE and TABLE
1224 that we stored values from above. If we come to the end of
1225 one of the input files, remove it from the list of input
1227 for (iter = min_head; iter && iter->type == MTF_FILE; )
1229 struct mtf_file *next = iter->next_min;
1233 assert (iter->input != NULL);
1235 if (!sfm_read_case (iter->handle, iter->input, iter->dict))
1236 mtf_delete_file_in_place (&iter);
1246 return (mtf_head && mtf_head->type != MTF_TABLE);
1249 /* Merge the dictionary for file F into the master dictionary
1252 mtf_merge_dictionary (struct mtf_file *f)
1254 struct dictionary *const m = mtf_master;
1255 struct dictionary *d = f->dict;
1256 const char *d_docs, *m_docs;
1258 if (dict_get_label (m) == NULL)
1259 dict_set_label (m, dict_get_label (d));
1261 d_docs = dict_get_documents (d);
1262 m_docs = dict_get_documents (m);
1266 dict_set_documents (m, d_docs);
1272 new_len = strlen (m_docs) + strlen (d_docs);
1273 new_docs = xmalloc (new_len + 1);
1274 strcpy (new_docs, m_docs);
1275 strcat (new_docs, d_docs);
1276 dict_set_documents (m, new_docs);
1281 dict_compact_values (d);
1286 for (i = 0; i < dict_get_var_cnt (d); i++)
1288 struct variable *dv = dict_get_var (d, i);
1289 struct variable *mv = dict_lookup_var (m, dv->name);
1291 assert (dv->type == ALPHA || dv->width == 0);
1292 assert (!mv || mv->type == ALPHA || mv->width == 0);
1293 if (mv && dv->width == mv->width)
1295 if (val_labs_count (dv->val_labs)
1296 && !val_labs_count (mv->val_labs))
1297 mv->val_labs = val_labs_copy (dv->val_labs);
1298 if (dv->miss_type != MISSING_NONE
1299 && mv->miss_type == MISSING_NONE)
1300 copy_missing_values (mv, dv);
1302 if (mv && dv->label && !mv->label)
1303 mv->label = xstrdup (dv->label);
1306 mv = dict_clone_var (m, dv, dv->name);
1307 assert (mv != NULL);
1309 else if (mv->width != dv->width)
1311 msg (SE, _("Variable %s in file %s (%s) has different "
1312 "type or width from the same variable in "
1313 "earlier file (%s)."),
1314 dv->name, fh_handle_name (f->handle),
1315 var_type_description (dv), var_type_description (mv));
1318 dv->p.mtf.master = mv;
1325 /* IMPORT command. */
1327 /* Parses the IMPORT command. */
1331 struct file_handle *handle = NULL;
1332 struct dictionary *dict;
1333 int options = GTSV_OPT_NONE;
1336 lex_match_id ("IMPORT");
1342 if (lex_match_id ("FILE") || token == T_STRING)
1346 handle = fh_parse_file_handle ();
1350 else if (lex_match_id ("TYPE"))
1354 if (lex_match_id ("COMM"))
1356 else if (lex_match_id ("TAPE"))
1360 lex_error (_("expecting COMM or TAPE"));
1366 if (!lex_match ('/') && token != '.')
1372 discard_variables ();
1374 dict = pfm_read_dictionary (handle, NULL);
1379 dump_dict_variables (dict);
1381 if (0 == trim_dictionary (dict, &options))
1383 fh_close_handle (handle);
1387 dump_dict_variables (dict);
1390 dict_compact_values (dict);
1393 printf (_("IMPORT translation table from file to memory:\n"));
1394 for (i = 0; i < dict->nvar; i++)
1396 struct variable *v = dict->var[i];
1398 printf (_(" %8s from %3d,%3d to %3d,%3d\n"), v->name,
1399 v->get.fv, v->get.nv, v->fv, v->nv);
1403 dict_destroy (default_dict);
1404 default_dict = dict;
1406 vfm_source = &import_source;
1412 /* Reads all the cases from the data file and passes them to
1415 import_source_read (write_case_func *write_case, write_case_data wc_data)
1417 while (pfm_read_case (get_file, temp_case->data, default_dict)
1418 && write_case (wc_data))
1420 get_source_destroy_source ();
1423 struct case_stream import_source =
1429 get_source_destroy_source,
1434 static int export_write_case_func (struct ccase *c, void *);
1436 /* Parses the EXPORT command. */
1437 /* FIXME: same as cmd_save_internal(). */
1441 struct file_handle *handle;
1442 struct dictionary *dict;
1443 int options = GTSV_OPT_SAVE;
1445 struct save_trns *t;
1449 lex_match_id ("EXPORT");
1452 if (lex_match_id ("OUTFILE"))
1455 handle = fh_parse_file_handle ();
1459 dict = dict_clone (default_dict);
1461 dump_dict_variables (dict);
1463 for (i = 0; i < dict_get_var_cnt (dict); i++)
1464 dict_get_var (dict, i)->aux = dict_get_var (default_dict, i);
1465 if (0 == trim_dictionary (dict, &options))
1467 fh_close_handle (handle);
1472 dump_dict_variables (dict);
1475 /* Write dictionary. */
1476 if (!pfm_write_dictionary (handle, dict))
1478 dict_destroy (dict);
1479 fh_close_handle (handle);
1483 /* Fill in transformation structure. */
1484 t = trns = xmalloc (sizeof *t);
1485 t->h.proc = save_trns_proc;
1486 t->h.free = save_trns_free;
1488 t->nvar = dict_get_var_cnt (dict);
1489 t->var = xmalloc (sizeof *t->var * t->nvar);
1490 for (i = 0; i < t->nvar; i++)
1491 t->var[i] = dict_get_var (dict, i)->aux;
1492 t->case_buf = xmalloc (sizeof *t->case_buf * t->nvar);
1493 dict_destroy (dict);
1495 procedure (NULL, export_write_case_func, NULL, NULL);
1496 save_trns_free (&t->h);
1502 export_write_case_func (struct ccase *c, void *aux UNUSED)
1504 union value *p = (union value *) trns->case_buf;
1507 for (i = 0; i < trns->nvar; i++)
1509 struct variable *v = trns->var[i];
1511 if (v->type == NUMERIC)
1512 *p++ = c->data[v->fv];
1514 (*p++).c = c->data[v->fv].s;
1517 pfm_write_case (trns->f, (union value *) trns->case_buf);