1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
26 #include "file-handle.h"
34 #include "value-labels.h"
39 #include "debug-print.h"
41 /* XSAVE transformation (and related SAVE, EXPORT procedures). */
45 struct file_handle *f; /* Associated system file. */
46 int nvar; /* Number of variables. */
47 int *var; /* Indices of variables. */
48 flt64 *case_buf; /* Case transfer buffer. */
51 /* Options bits set by trim_dictionary(). */
52 #define GTSV_OPT_COMPRESSED 001 /* Compression; (X)SAVE only. */
53 #define GTSV_OPT_SAVE 002 /* The SAVE/XSAVE/EXPORT procedures. */
54 #define GTSV_OPT_MATCH_FILES 004 /* The MATCH FILES procedure. */
55 #define GTSV_OPT_NONE 0
57 /* The file being read by the input program. */
58 static struct file_handle *get_file;
60 /* The transformation being used by the SAVE procedure. */
61 static struct save_trns *trns;
63 static int trim_dictionary (struct dictionary * dict, int *options);
64 static int save_write_case_func (struct ccase *);
65 static int save_trns_proc (struct trns_header *, struct ccase *);
66 static void save_trns_free (struct trns_header *);
69 void dump_dict_variables (struct dictionary *);
72 /* Parses the GET command. */
76 struct file_handle *handle;
77 struct dictionary *dict;
78 int options = GTSV_OPT_NONE;
87 if (lex_match_id ("FILE"))
90 handle = fh_parse_file_handle ();
94 dict = sfm_read_dictionary (handle, NULL);
99 dump_dict_variables (dict);
101 if (0 == trim_dictionary (dict, &options))
103 fh_close_handle (handle);
107 dump_dict_variables (dict);
110 /* Set the fv and lv elements of all variables remaining in the
113 for (i = 0; i < dict->nvar; i++)
115 struct variable *v = dict->var[i];
124 printf (_("GET translation table from file to memory:\n"));
125 for (i = 0; i < dict->nvar; i++)
127 struct variable *v = dict->var[i];
129 printf (_(" %8s from %3d,%3d to %3d,%3d\n"), v->name,
130 v->get.fv, v->get.nv, v->fv, v->nv);
134 restore_dictionary (dict);
136 vfm_source = &get_source;
142 /* Parses the SAVE (for X==0) and XSAVE (for X==1) commands. */
143 /* FIXME: save_dictionary() is too expensive. It would make more
144 sense to copy just the first few fields of each variables (up to
145 `foo'): that's a SMOP. */
147 cmd_save_internal (int x)
149 struct file_handle *handle;
150 struct dictionary *dict;
151 int options = GTSV_OPT_SAVE;
154 struct sfm_write_info inf;
158 lex_match_id ("SAVE");
161 if (lex_match_id ("OUTFILE"))
164 handle = fh_parse_file_handle ();
168 dict = save_dictionary ();
170 dump_dict_variables (dict);
172 for (i = 0; i < dict->nvar; i++)
173 dict->var[i]->foo = i;
174 if (0 == trim_dictionary (dict, &options))
176 fh_close_handle (handle);
181 dump_dict_variables (dict);
184 /* Write dictionary. */
187 inf.compress = !!(options & GTSV_OPT_COMPRESSED);
188 if (!sfm_write_dictionary (&inf))
190 free_dictionary (dict);
191 fh_close_handle (handle);
195 /* Fill in transformation structure. */
196 t = trns = xmalloc (sizeof *t);
197 t->h.proc = save_trns_proc;
198 t->h.free = save_trns_free;
200 t->nvar = dict->nvar;
201 t->var = xmalloc (sizeof *t->var * dict->nvar);
202 for (i = 0; i < dict->nvar; i++)
203 t->var[i] = dict->var[i]->foo;
204 t->case_buf = xmalloc (sizeof *t->case_buf * inf.case_size);
205 free_dictionary (dict);
210 procedure (NULL, save_write_case_func, NULL);
211 save_trns_free ((struct trns_header *) t);
215 add_transformation ((struct trns_header *) t);
220 /* Parses and performs the SAVE procedure. */
224 return cmd_save_internal (0);
227 /* Parses the XSAVE transformation command. */
231 return cmd_save_internal (1);
235 save_write_case_func (struct ccase * c)
237 save_trns_proc ((struct trns_header *) trns, c);
242 save_trns_proc (struct trns_header * t unused, struct ccase * c)
244 flt64 *p = trns->case_buf;
247 for (i = 0; i < trns->nvar; i++)
249 struct variable *v = default_dict.var[trns->var[i]];
250 if (v->type == NUMERIC)
252 double src = c->data[v->fv].f;
260 memcpy (p, c->data[v->fv].s, v->width);
261 memset (&((char *) p)[v->width], ' ',
262 REM_RND_UP (v->width, sizeof *p));
263 p += DIV_RND_UP (v->width, sizeof *p);
267 sfm_write_case (trns->f, trns->case_buf, p - trns->case_buf);
272 save_trns_free (struct trns_header *pt)
274 struct save_trns *t = (struct save_trns *) pt;
276 fh_close_handle (t->f);
282 /* Deletes NV variables from DICT, starting at index FIRST. The
283 variables must have consecutive indices. The variables are cleared
286 dict_delete_run (struct dictionary *dict, int first, int nv)
290 for (i = first; i < first + nv; i++)
292 clear_variable (dict, dict->var[i]);
295 for (i = first; i < dict->nvar - nv; i++)
297 dict->var[i] = dict->var[i + nv];
298 dict->var[i]->index -= nv;
303 static int rename_variables (struct dictionary * dict);
305 /* The GET and SAVE commands have a common structure after the
306 FILE/OUTFILE subcommand. This function parses this structure and
307 returns nonzero on success, zero on failure. It both reads
308 *OPTIONS, for the GTSV_OPT_SAVE bit, and writes it, for the
309 GTSV_OPT_COMPRESSED bit. */
310 /* FIXME: IN, FIRST, LAST, MAP. */
312 trim_dictionary (struct dictionary *dict, int *options)
314 if (set_scompression)
315 *options |= GTSV_OPT_COMPRESSED;
317 if (*options & GTSV_OPT_SAVE)
321 /* Delete all the scratch variables. */
322 for (i = 0; i < dict->nvar; i++)
326 if (dict->var[i]->name[0] != '#')
329 /* Find a run of variables to be deleted. */
330 for (j = i + 1; j < dict->nvar; j++)
331 if (dict->var[j]->name[0] != '#')
334 /* Actually delete 'em. */
335 dict_delete_run (dict, i, j - i);
339 while ((*options & GTSV_OPT_MATCH_FILES) || lex_match ('/'))
341 if (!(*options & GTSV_OPT_MATCH_FILES) && lex_match_id ("COMPRESSED"))
342 *options |= GTSV_OPT_COMPRESSED;
343 else if (!(*options & GTSV_OPT_MATCH_FILES) && lex_match_id ("UNCOMPRESSED"))
344 *options &= ~GTSV_OPT_COMPRESSED;
345 else if (lex_match_id ("DROP"))
352 if (!parse_variables (dict, &v, &nv, PV_NONE))
355 /* Loop through the variables to delete. */
360 /* Find a run of variables to be deleted. */
361 for (j = i + 1; j < nv; j++)
362 if (v[j]->index != v[j - 1]->index + 1)
365 /* Actually delete 'em. */
366 dict_delete_run (dict, v[i]->index, j - i);
370 else if (lex_match_id ("KEEP"))
376 if (!parse_variables (dict, &v, &nv, PV_NONE))
379 /* Reorder the dictionary so that the kept variables are at
384 for (i1 = 0; i1 < nv; i1++)
386 int i2 = v[i1]->index;
388 /* Swap variables with indices i1 and i2. */
389 struct variable *t = dict->var[i1];
390 dict->var[i1] = dict->var[i2];
392 dict->var[i1]->index = i1;
393 dict->var[i2]->index = i2;
399 /* Delete all but the first NV variables from the
403 for (i = nv; i < dict->nvar; i++)
405 clear_variable (dict, dict->var[i]);
409 dict->var = xrealloc (dict->var, sizeof *dict->var * nv);
412 else if (lex_match_id ("RENAME"))
414 if (!rename_variables (dict))
419 lex_error (_("while expecting a valid subcommand"));
425 msg (SE, _("All variables deleted from system file dictionary."));
429 if (*options & GTSV_OPT_MATCH_FILES)
435 lex_error (_("expecting end of command"));
442 /* Parses and performs the RENAME subcommand of GET and SAVE. */
444 rename_variables (struct dictionary * dict)
461 v = parse_dict_variable (dict);
464 if (!lex_force_match ('=')
467 if (!strncmp (tokid, v->name, 8))
469 if (is_dict_varname (dict, tokid))
471 msg (SE, _("Cannot rename %s as %s because there already exists "
472 "a variable named %s. To rename variables with "
473 "overlapping names, use a single RENAME subcommand "
474 "such as \"/RENAME (A=B)(B=C)(C=A)\", or equivalently, "
475 "\"/RENAME (A B C=B C A)\"."), v->name, tokid, tokid);
479 rename_variable (dict, v, tokid);
488 while (lex_match ('('))
492 if (!parse_variables (dict, &v, &nv, PV_NO_DUPLICATE | PV_APPEND))
494 if (!lex_match ('='))
496 msg (SE, _("`=' expected after variable list."));
499 if (!parse_DATA_LIST_vars (&new_names, &nn, PV_APPEND | PV_NO_SCRATCH))
503 msg (SE, _("Number of variables on left side of `=' (%d) do not "
504 "match number of variables on right side (%d), in "
505 "parenthesized group %d of RENAME subcommand."),
506 nv - old_nv, nn - old_nv, group);
509 if (!lex_force_match (')'))
514 for (i = 0; i < nv; i++)
515 hsh_force_delete (dict->name_tab, v[i]);
516 for (i = 0; i < nv; i++)
518 strcpy (v[i]->name, new_names[i]);
519 if (NULL != hsh_insert (dict->name_tab, v[i]))
521 msg (SE, _("Duplicate variables name %s."), v[i]->name);
528 /* The label is a bit of a misnomer, we actually come here on any
530 for (i = 0; i < nn; i++)
540 dump_dict_variables (struct dictionary * dict)
544 printf (_("\nVariables in dictionary:\n"));
545 for (i = 0; i < dict->nvar; i++)
546 printf ("%s, ", dict->var[i]->name);
551 /* Clears internal state related to GET input procedure. */
553 get_source_destroy_source (void)
555 /* It is not necessary to destroy the dictionary because if we get
556 to this point then the dictionary is default_dict. */
557 fh_close_handle (get_file);
560 /* Reads all the cases from the data file and passes them to
563 get_source_read (void)
565 while (sfm_read_case (get_file, temp_case->data, &default_dict)
568 get_source_destroy_source ();
571 struct case_stream get_source =
577 get_source_destroy_source,
585 #include "debug-print.h"
590 MTF_FILE, /* Specified on FILE= subcommand. */
591 MTF_TABLE /* Specified on TABLE= subcommand. */
594 /* One of the files on MATCH FILES. */
597 struct mtf_file *next, *prev;
598 /* Next, previous in the list of files. */
599 struct mtf_file *next_min; /* Next in the chain of minimums. */
601 int type; /* One of MTF_*. */
602 struct variable **by; /* List of BY variables for this file. */
603 struct file_handle *handle; /* File handle for the file. */
604 struct dictionary *dict; /* Dictionary from system file. */
605 char in[9]; /* Name of the variable from IN=. */
606 char first[9], last[9]; /* Name of the variables from FIRST=, LAST=. */
607 union value *input; /* Input record. */
610 /* All the files mentioned on FILE= or TABLE=. */
611 static struct mtf_file *mtf_head, *mtf_tail;
613 /* Variables on the BY subcommand. */
614 static struct variable **mtf_by;
617 /* Master dictionary. */
618 static struct dictionary *mtf_master;
620 static void mtf_free (void);
621 static void mtf_free_file (struct mtf_file *file);
622 static int mtf_merge_dictionary (struct mtf_file *f);
623 static void mtf_delete_file_in_place (struct mtf_file **file);
625 static void mtf_read_nonactive_records (void);
626 static void mtf_processing_finish (void);
627 static int mtf_processing (struct ccase *);
629 static char *var_type_description (struct variable *);
631 /* Parse and execute the MATCH FILES command. */
633 cmd_match_files (void)
635 struct mtf_file *first_table = NULL;
639 lex_match_id ("MATCH");
640 lex_match_id ("FILES");
642 mtf_head = mtf_tail = NULL;
645 mtf_master = new_dictionary (0);
646 mtf_master->N = default_dict.N;
652 if (lex_match (T_BY))
656 msg (SE, _("The BY subcommand may be given once at most."));
662 if (!parse_variables (mtf_master, &mtf_by, &mtf_n_by,
663 PV_NO_DUPLICATE | PV_NO_SCRATCH))
666 else if (token != T_ID)
671 else if (lex_id_match ("FILE", tokid) || lex_id_match ("TABLE", tokid))
673 struct mtf_file *file = xmalloc (sizeof *file);
675 file->in[0] = file->first[0] = file->last[0] = '\0';
680 if (lex_match_id ("FILE"))
681 file->type = MTF_FILE;
682 else if (lex_match_id ("TABLE"))
684 file->type = MTF_TABLE;
690 /* FILEs go first, then TABLEs. */
691 if (file->type == MTF_TABLE || first_table == NULL)
694 file->prev = mtf_tail;
696 mtf_tail->next = file;
698 if (mtf_head == NULL)
700 if (file->type == MTF_TABLE && first_table == NULL)
705 assert (file->type == MTF_FILE);
706 file->next = first_table;
707 file->prev = first_table->prev;
708 if (first_table->prev)
709 first_table->prev->next = file;
712 first_table->prev = file;
723 msg (SE, _("The active file may not be specified more "
729 assert (pgm_state != STATE_INPUT);
730 if (pgm_state == STATE_INIT)
732 msg (SE, _("Cannot specify the active file since no active "
733 "file has been defined."));
739 file->handle = fh_parse_file_handle ();
746 file->dict = sfm_read_dictionary (file->handle, NULL);
751 file->dict = &default_dict;
752 if (!mtf_merge_dictionary (file))
755 else if (lex_id_match ("IN", tokid)
756 || lex_id_match ("FIRST", tokid)
757 || lex_id_match ("LAST", tokid))
762 if (mtf_tail == NULL)
764 msg (SE, _("IN, FIRST, and LAST subcommands may not occur "
765 "before the first FILE or TABLE."));
769 if (lex_match_id ("IN"))
774 else if (lex_match_id ("FIRST"))
776 name = mtf_tail->first;
779 else if (lex_match_id ("LAST"))
781 name = mtf_tail->last;
796 msg (SE, _("Multiple %s subcommands for a single FILE or "
801 strcpy (name, tokid);
804 if (!create_variable (mtf_master, name, NUMERIC, 0))
806 msg (SE, _("Duplicate variable name %s while creating %s "
812 else if (lex_id_match ("RENAME", tokid)
813 || lex_id_match ("KEEP", tokid)
814 || lex_id_match ("DROP", tokid))
816 int options = GTSV_OPT_MATCH_FILES;
818 if (mtf_tail == NULL)
820 msg (SE, _("RENAME, KEEP, and DROP subcommands may not occur "
821 "before the first FILE or TABLE."));
825 if (!trim_dictionary (mtf_tail->dict, &options))
828 else if (lex_match_id ("MAP"))
838 while (token != '.');
844 msg (SE, _("The BY subcommand is required when a TABLE subcommand "
852 struct mtf_file *iter;
854 for (iter = mtf_head; iter; iter = iter->next)
858 iter->by = xmalloc (sizeof *iter->by * mtf_n_by);
860 for (i = 0; i < mtf_n_by; i++)
862 iter->by[i] = find_dict_variable (iter->dict, mtf_by[i]->name);
863 if (iter->by[i] == NULL)
865 msg (SE, _("File %s lacks BY variable %s."),
866 iter->handle ? fh_handle_name (iter->handle) : "*",
876 /* From sfm-read.c. */
877 extern void dump_dictionary (struct dictionary *);
879 dump_dictionary (mtf_master);
883 /* MATCH FILES performs an n-way merge on all its input files.
886 1. Read one input record from every input FILE.
888 2. If no FILEs are left, stop. Otherwise, proceed to step 3.
890 3. Find the FILE input record with minimum BY values. Store all
891 the values from this input record into the output record.
893 4. Find all the FILE input records with BY values identical to
894 the minimums. Store all the values from these input records into
897 5. For every TABLE, read another record as long as the BY values
898 on the TABLE's input record are less than the FILEs' BY values.
899 If an exact match is found, store all the values from the TABLE
900 input record into the output record.
902 6. Write the output record.
904 7. Read another record from each input file FILE and TABLE that
905 we stored values from above. If we come to the end of one of the
906 input files, remove it from the list of input files.
908 8. Repeat from step 2.
910 Unfortunately, this algorithm can't be directly implemented
911 because there's no function to read a record from the active
912 file; instead, it has to be done using callbacks.
914 FIXME: A better algorithm would use a heap for finding minimum
915 values, or replacement selection, as described by Knuth in _Art
916 of Computer Programming, Vol. 3_. The SORT CASES procedure does
917 this, and perhaps some of its code could be adapted. */
920 discard_variables ();
923 temp_dict = mtf_master;
926 process_active_file (mtf_read_nonactive_records, mtf_processing,
927 mtf_processing_finish);
938 /* Repeats 2...8 an arbitrary number of times. */
940 mtf_processing_finish (void)
942 /* Find the active file and delete it. */
944 struct mtf_file *iter;
946 for (iter = mtf_head; iter; iter = iter->next)
947 if (iter->handle == NULL)
949 mtf_delete_file_in_place (&iter);
954 while (mtf_head && mtf_head->type == MTF_FILE)
955 if (!mtf_processing (temp_case))
959 /* Return a string in a static buffer describing V's variable type and
962 var_type_description (struct variable *v)
964 static char buf[2][32];
971 if (v->type == NUMERIC)
972 strcpy (s, "numeric");
975 assert (v->type == ALPHA);
976 sprintf (s, "string with width %d", v->width);
981 /* Free FILE and associated data. */
983 mtf_free_file (struct mtf_file *file)
985 fh_close_handle (file->handle);
986 if (file->dict && file->dict != &default_dict)
987 free_dictionary (file->dict);
994 /* Free all the data for the MATCH FILES procedure. */
998 struct mtf_file *iter, *next;
1000 for (iter = mtf_head; iter; iter = next)
1004 mtf_free_file (iter);
1009 free_dictionary (mtf_master);
1012 /* Remove *FILE from the mtf_file chain. Make *FILE point to the next
1013 file in the chain, or to NULL if was the last in the chain. */
1015 mtf_delete_file_in_place (struct mtf_file **file)
1017 struct mtf_file *f = *file;
1020 f->prev->next = f->next;
1022 f->next->prev = f->prev;
1032 for (i = 0; i < f->dict->nvar; i++)
1034 struct variable *v = f->dict->var[i];
1036 if (v->type == NUMERIC)
1037 compaction_case->data[v->p.mtf.master->fv].f = SYSMIS;
1039 memset (compaction_case->data[v->p.mtf.master->fv].s, ' ',
1047 /* Read a record from every input file except the active file. */
1049 mtf_read_nonactive_records (void)
1051 struct mtf_file *iter;
1053 for (iter = mtf_head; iter; )
1057 assert (iter->input == NULL);
1058 iter->input = xmalloc (sizeof *iter->input * iter->dict->nval);
1060 if (!sfm_read_case (iter->handle, iter->input, iter->dict))
1061 mtf_delete_file_in_place (&iter);
1067 iter->input = temp_case->data;
1073 /* Compare the BY variables for files A and B; return -1 if A < B, 0
1074 if A == B, 1 if A > B. */
1076 mtf_compare_BY_values (struct mtf_file *a, struct mtf_file *b)
1080 for (i = 0; i < mtf_n_by; i++)
1082 assert (a->by[i]->type == b->by[i]->type);
1083 assert (a->by[i]->width == b->by[i]->width);
1085 if (a->by[i]->type == NUMERIC)
1087 double af = a->input[a->by[i]->fv].f;
1088 double bf = b->input[b->by[i]->fv].f;
1099 assert (a->by[i]->type == ALPHA);
1100 result = memcmp (a->input[a->by[i]->fv].s,
1101 b->input[b->by[i]->fv].s,
1105 else if (result > 0)
1112 /* Used to determine whether we've already initialized this
1114 static int mtf_seq_no = 0;
1116 /* Perform one iteration of steps 3...7 above. */
1118 mtf_processing (struct ccase *c unused)
1120 /* List of files with minimum BY values. */
1121 struct mtf_file *min_head, *min_tail;
1123 /* List of files with non-minimum BY values. */
1124 struct mtf_file *max_head, *max_tail;
1127 struct mtf_file *iter;
1131 /* If the active file doesn't have the minimum BY values, don't
1132 return because that would cause a record to be skipped. */
1135 if (mtf_head->type == MTF_TABLE)
1138 /* 3. Find the FILE input record with minimum BY values. Store
1139 all the values from this input record into the output record.
1141 4. Find all the FILE input records with BY values identical
1142 to the minimums. Store all the values from these input
1143 records into the output record. */
1144 min_head = min_tail = mtf_head;
1145 max_head = max_tail = NULL;
1146 for (iter = mtf_head->next; iter && iter->type == MTF_FILE;
1148 switch (mtf_compare_BY_values (min_head, iter))
1152 max_tail = max_tail->next_min = iter;
1154 max_head = max_tail = iter;
1158 min_tail = min_tail->next_min = iter;
1164 max_tail->next_min = min_head;
1165 max_tail = min_tail;
1169 max_head = min_head;
1170 max_tail = min_tail;
1172 min_head = min_tail = iter;
1179 /* 5. For every TABLE, read another record as long as the BY
1180 values on the TABLE's input record are less than the FILEs'
1181 BY values. If an exact match is found, store all the values
1182 from the TABLE input record into the output record. */
1185 struct mtf_file *next = iter->next;
1187 assert (iter->type == MTF_TABLE);
1189 if (iter->handle == NULL)
1193 switch (mtf_compare_BY_values (min_head, iter))
1197 max_tail = max_tail->next_min = iter;
1199 max_head = max_tail = iter;
1203 min_tail = min_tail->next_min = iter;
1207 if (iter->handle == NULL)
1209 if (sfm_read_case (iter->handle, iter->input, iter->dict))
1211 mtf_delete_file_in_place (&iter);
1221 /* Next sequence number. */
1224 /* Store data to all the records we are using. */
1226 min_tail->next_min = NULL;
1227 for (iter = min_head; iter; iter = iter->next_min)
1231 for (i = 0; i < iter->dict->nvar; i++)
1233 struct variable *v = iter->dict->var[i];
1235 if (v->p.mtf.master->foo == mtf_seq_no)
1237 v->p.mtf.master->foo = mtf_seq_no;
1240 printf ("%s/%s: dest-fv=%d, src-fv=%d\n",
1241 fh_handle_name (iter->handle),
1243 v->p.mtf.master->fv, v->fv);
1245 if (v->type == NUMERIC)
1246 compaction_case->data[v->p.mtf.master->fv].f
1247 = iter->input[v->fv].f;
1250 assert (v->type == ALPHA);
1251 memcpy (compaction_case->data[v->p.mtf.master->fv].s,
1252 iter->input[v->fv].s, v->width);
1257 /* Store missing values to all the records we're not using. */
1259 max_tail->next_min = NULL;
1260 for (iter = max_head; iter; iter = iter->next_min)
1264 for (i = 0; i < iter->dict->nvar; i++)
1266 struct variable *v = iter->dict->var[i];
1268 if (v->p.mtf.master->foo == mtf_seq_no)
1270 v->p.mtf.master->foo = mtf_seq_no;
1273 printf ("%s/%s: dest-fv=%d\n",
1274 fh_handle_name (iter->handle),
1276 v->p.mtf.master->fv);
1278 if (v->type == NUMERIC)
1279 compaction_case->data[v->p.mtf.master->fv].f = SYSMIS;
1281 memset (compaction_case->data[v->p.mtf.master->fv].s, ' ',
1285 if (iter->handle == NULL)
1289 /* 6. Write the output record. */
1290 process_active_file_output_case ();
1292 /* 7. Read another record from each input file FILE and TABLE
1293 that we stored values from above. If we come to the end of
1294 one of the input files, remove it from the list of input
1296 for (iter = min_head; iter && iter->type == MTF_FILE; )
1298 struct mtf_file *next = iter->next_min;
1302 assert (iter->input != NULL);
1304 if (!sfm_read_case (iter->handle, iter->input, iter->dict))
1305 mtf_delete_file_in_place (&iter);
1315 return (mtf_head && mtf_head->type != MTF_TABLE);
1318 /* Merge the dictionary for file F into the master dictionary
1321 mtf_merge_dictionary (struct mtf_file *f)
1323 struct dictionary *const m = mtf_master;
1324 struct dictionary *d = f->dict;
1326 if (d->label && m->label == NULL)
1327 m->label = xstrdup (d->label);
1331 m->documents = xrealloc (m->documents,
1332 80 * (m->n_documents + d->n_documents));
1333 memcpy (&m->documents[80 * m->n_documents],
1334 d->documents, 80 * d->n_documents);
1335 m->n_documents += d->n_documents;
1342 for (i = 0; i < d->nvar; i++)
1344 struct variable *dv = d->var[i];
1345 struct variable *mv = find_dict_variable (m, dv->name);
1350 assert (dv->type == ALPHA || dv->width == 0);
1351 assert (!mv || mv->type == ALPHA || mv->width == 0);
1352 if (mv && dv->width == mv->width)
1354 if (val_labs_count (dv->val_labs)
1355 && !val_labs_count (mv->val_labs))
1356 mv->val_labs = val_labs_copy (dv->val_labs);
1357 if (dv->miss_type != MISSING_NONE && mv->miss_type == MISSING_NONE)
1358 copy_missing_values (mv, dv);
1360 if (mv && dv->label && !mv->label)
1361 mv->label = xstrdup (dv->label);
1364 mv = force_dup_variable (m, dv, dv->name);
1366 /* Used to make sure we initialize each variable in the
1367 master dictionary exactly once per case. */
1368 mv->foo = mtf_seq_no;
1370 else if (mv->width != dv->width)
1372 msg (SE, _("Variable %s in file %s (%s) has different "
1373 "type or width from the same variable in "
1374 "earlier file (%s)."),
1375 dv->name, fh_handle_name (f->handle),
1376 var_type_description (dv), var_type_description (mv));
1379 dv->p.mtf.master = mv;
1386 /* IMPORT command. */
1388 /* Parses the IMPORT command. */
1392 struct file_handle *handle = NULL;
1393 struct dictionary *dict;
1394 int options = GTSV_OPT_NONE;
1400 lex_match_id ("IMPORT");
1406 if (lex_match_id ("FILE") || token == T_STRING)
1410 handle = fh_parse_file_handle ();
1414 else if (lex_match_id ("TYPE"))
1418 if (lex_match_id ("COMM"))
1420 else if (lex_match_id ("TAPE"))
1424 lex_error (_("expecting COMM or TAPE"));
1430 if (!lex_match ('/') && token != '.')
1436 discard_variables ();
1438 dict = pfm_read_dictionary (handle, NULL);
1443 dump_dict_variables (dict);
1445 if (0 == trim_dictionary (dict, &options))
1447 fh_close_handle (handle);
1451 dump_dict_variables (dict);
1454 /* Set the fv and lv elements of all variables remaining in the
1457 for (i = 0; i < dict->nvar; i++)
1459 struct variable *v = dict->var[i];
1468 printf (_("IMPORT translation table from file to memory:\n"));
1469 for (i = 0; i < dict->nvar; i++)
1471 struct variable *v = dict->var[i];
1473 printf (_(" %8s from %3d,%3d to %3d,%3d\n"), v->name,
1474 v->get.fv, v->get.nv, v->fv, v->nv);
1478 restore_dictionary (dict);
1480 vfm_source = &import_source;
1486 /* Reads all the cases from the data file and passes them to
1489 import_source_read (void)
1491 while (pfm_read_case (get_file, temp_case->data, &default_dict)
1494 get_source_destroy_source ();
1497 struct case_stream import_source =
1503 get_source_destroy_source,
1508 static int export_write_case_func (struct ccase *c);
1510 /* Parses the EXPORT command. */
1511 /* FIXME: same as cmd_save_internal(). */
1515 struct file_handle *handle;
1516 struct dictionary *dict;
1517 int options = GTSV_OPT_SAVE;
1519 struct save_trns *t;
1523 lex_match_id ("EXPORT");
1526 if (lex_match_id ("OUTFILE"))
1529 handle = fh_parse_file_handle ();
1533 dict = save_dictionary ();
1535 dump_dict_variables (dict);
1537 for (i = 0; i < dict->nvar; i++)
1538 dict->var[i]->foo = i;
1539 if (0 == trim_dictionary (dict, &options))
1541 fh_close_handle (handle);
1546 dump_dict_variables (dict);
1549 /* Write dictionary. */
1550 if (!pfm_write_dictionary (handle, dict))
1552 free_dictionary (dict);
1553 fh_close_handle (handle);
1557 /* Fill in transformation structure. */
1558 t = trns = xmalloc (sizeof *t);
1559 t->h.proc = save_trns_proc;
1560 t->h.free = save_trns_free;
1562 t->nvar = dict->nvar;
1563 t->var = xmalloc (sizeof *t->var * dict->nvar);
1564 for (i = 0; i < dict->nvar; i++)
1565 t->var[i] = dict->var[i]->foo;
1566 t->case_buf = xmalloc (sizeof *t->case_buf * dict->nvar);
1567 free_dictionary (dict);
1569 procedure (NULL, export_write_case_func, NULL);
1570 save_trns_free ((struct trns_header *) t);
1576 export_write_case_func (struct ccase *c)
1578 union value *p = (union value *) trns->case_buf;
1581 for (i = 0; i < trns->nvar; i++)
1583 struct variable *v = default_dict.var[trns->var[i]];
1585 if (v->type == NUMERIC)
1586 *p++ = c->data[v->fv];
1588 (*p++).c = c->data[v->fv].s;
1594 pfm_write_case (trns->f, (union value *) trns->case_buf);