1 /* PSPP - computes sample statistics.
2 Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
3 Written by Ben Pfaff <blp@gnu.org>.
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License as
7 published by the Free Software Foundation; either version 2 of the
8 License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
27 #include "file-handle.h"
38 #include "debug-print.h"
40 /* XSAVE transformation (and related SAVE, EXPORT procedures). */
44 struct file_handle *f; /* Associated system file. */
45 int nvar; /* Number of variables. */
46 int *var; /* Indices of variables. */
47 flt64 *case_buf; /* Case transfer buffer. */
50 /* Options bits set by trim_dictionary(). */
51 #define GTSV_OPT_COMPRESSED 001 /* Compression; (X)SAVE only. */
52 #define GTSV_OPT_SAVE 002 /* The SAVE/XSAVE/EXPORT procedures. */
53 #define GTSV_OPT_MATCH_FILES 004 /* The MATCH FILES procedure. */
54 #define GTSV_OPT_NONE 0
56 /* The file being read by the input program. */
57 static struct file_handle *get_file;
59 /* The transformation being used by the SAVE procedure. */
60 static struct save_trns *trns;
62 static int trim_dictionary (struct dictionary * dict, int *options);
63 static int save_write_case_func (struct ccase *);
64 static int save_trns_proc (struct trns_header *, struct ccase *);
65 static void save_trns_free (struct trns_header *);
68 void dump_dict_variables (struct dictionary *);
71 /* Parses the GET command. */
75 struct file_handle *handle;
76 struct dictionary *dict;
77 int options = GTSV_OPT_NONE;
86 if (lex_match_id ("FILE"))
89 handle = fh_parse_file_handle ();
93 dict = sfm_read_dictionary (handle, NULL);
98 dump_dict_variables (dict);
100 if (0 == trim_dictionary (dict, &options))
102 fh_close_handle (handle);
106 dump_dict_variables (dict);
109 /* Set the fv and lv elements of all variables remaining in the
112 for (i = 0; i < dict->nvar; i++)
114 struct variable *v = dict->var[i];
123 printf (_("GET translation table from file to memory:\n"));
124 for (i = 0; i < dict->nvar; i++)
126 struct variable *v = dict->var[i];
128 printf (_(" %8s from %3d,%3d to %3d,%3d\n"), v->name,
129 v->get.fv, v->get.nv, v->fv, v->nv);
133 restore_dictionary (dict);
135 vfm_source = &get_source;
141 /* Parses the SAVE (for X==0) and XSAVE (for X==1) commands. */
142 /* FIXME: save_dictionary() is too expensive. It would make more
143 sense to copy just the first few fields of each variables (up to
144 `foo'): that's a SMOP. */
146 cmd_save_internal (int x)
148 struct file_handle *handle;
149 struct dictionary *dict;
150 int options = GTSV_OPT_SAVE;
153 struct sfm_write_info inf;
157 lex_match_id ("SAVE");
160 if (lex_match_id ("OUTFILE"))
163 handle = fh_parse_file_handle ();
167 dict = save_dictionary ();
169 dump_dict_variables (dict);
171 for (i = 0; i < dict->nvar; i++)
172 dict->var[i]->foo = i;
173 if (0 == trim_dictionary (dict, &options))
175 fh_close_handle (handle);
180 dump_dict_variables (dict);
183 /* Write dictionary. */
186 inf.compress = !!(options & GTSV_OPT_COMPRESSED);
187 if (!sfm_write_dictionary (&inf))
189 free_dictionary (dict);
190 fh_close_handle (handle);
194 /* Fill in transformation structure. */
195 t = trns = xmalloc (sizeof *t);
196 t->h.proc = save_trns_proc;
197 t->h.free = save_trns_free;
199 t->nvar = dict->nvar;
200 t->var = xmalloc (sizeof *t->var * dict->nvar);
201 for (i = 0; i < dict->nvar; i++)
202 t->var[i] = dict->var[i]->foo;
203 t->case_buf = xmalloc (sizeof *t->case_buf * inf.case_size);
204 free_dictionary (dict);
209 procedure (NULL, save_write_case_func, NULL);
210 save_trns_free ((struct trns_header *) t);
214 add_transformation ((struct trns_header *) t);
219 /* Parses and performs the SAVE procedure. */
223 return cmd_save_internal (0);
226 /* Parses the XSAVE transformation command. */
230 return cmd_save_internal (1);
234 save_write_case_func (struct ccase * c)
236 save_trns_proc ((struct trns_header *) trns, c);
241 save_trns_proc (struct trns_header * t unused, struct ccase * c)
243 flt64 *p = trns->case_buf;
246 for (i = 0; i < trns->nvar; i++)
248 struct variable *v = default_dict.var[trns->var[i]];
249 if (v->type == NUMERIC)
251 double src = c->data[v->fv].f;
259 memcpy (p, c->data[v->fv].s, v->width);
260 memset (&((char *) p)[v->width], ' ',
261 REM_RND_UP (v->width, sizeof *p));
262 p += DIV_RND_UP (v->width, sizeof *p);
266 sfm_write_case (trns->f, trns->case_buf, p - trns->case_buf);
271 save_trns_free (struct trns_header *pt)
273 struct save_trns *t = (struct save_trns *) pt;
275 fh_close_handle (t->f);
281 /* Deletes NV variables from DICT, starting at index FIRST. The
282 variables must have consecutive indices. The variables are cleared
285 dict_delete_run (struct dictionary *dict, int first, int nv)
289 for (i = first; i < first + nv; i++)
291 clear_variable (dict, dict->var[i]);
294 for (i = first; i < dict->nvar - nv; i++)
296 dict->var[i] = dict->var[i + nv];
297 dict->var[i]->index -= nv;
302 static int rename_variables (struct dictionary * dict);
304 /* The GET and SAVE commands have a common structure after the
305 FILE/OUTFILE subcommand. This function parses this structure and
306 returns nonzero on success, zero on failure. It both reads
307 *OPTIONS, for the GTSV_OPT_SAVE bit, and writes it, for the
308 GTSV_OPT_COMPRESSED bit. */
309 /* FIXME: IN, FIRST, LAST, MAP. */
311 trim_dictionary (struct dictionary *dict, int *options)
313 if (set_scompression)
314 *options |= GTSV_OPT_COMPRESSED;
316 if (*options & GTSV_OPT_SAVE)
320 /* Delete all the scratch variables. */
321 for (i = 0; i < dict->nvar; i++)
325 if (dict->var[i]->name[0] != '#')
328 /* Find a run of variables to be deleted. */
329 for (j = i + 1; j < dict->nvar; j++)
330 if (dict->var[j]->name[0] != '#')
333 /* Actually delete 'em. */
334 dict_delete_run (dict, i, j - i);
338 while ((*options & GTSV_OPT_MATCH_FILES) || lex_match ('/'))
340 if (!(*options & GTSV_OPT_MATCH_FILES) && lex_match_id ("COMPRESSED"))
341 *options |= GTSV_OPT_COMPRESSED;
342 else if (!(*options & GTSV_OPT_MATCH_FILES) && lex_match_id ("UNCOMPRESSED"))
343 *options &= ~GTSV_OPT_COMPRESSED;
344 else if (lex_match_id ("DROP"))
351 if (!parse_variables (dict, &v, &nv, PV_NONE))
354 /* Loop through the variables to delete. */
359 /* Find a run of variables to be deleted. */
360 for (j = i + 1; j < nv; j++)
361 if (v[j]->index != v[j - 1]->index + 1)
364 /* Actually delete 'em. */
365 dict_delete_run (dict, v[i]->index, j - i);
369 else if (lex_match_id ("KEEP"))
375 if (!parse_variables (dict, &v, &nv, PV_NONE))
378 /* Reorder the dictionary so that the kept variables are at
383 for (i1 = 0; i1 < nv; i1++)
385 int i2 = v[i1]->index;
387 /* Swap variables with indices i1 and i2. */
388 struct variable *t = dict->var[i1];
389 dict->var[i1] = dict->var[i2];
391 dict->var[i1]->index = i1;
392 dict->var[i2]->index = i2;
398 /* Delete all but the first NV variables from the
402 for (i = nv; i < dict->nvar; i++)
404 clear_variable (dict, dict->var[i]);
408 dict->var = xrealloc (dict->var, sizeof *dict->var * nv);
411 else if (lex_match_id ("RENAME"))
413 if (!rename_variables (dict))
418 lex_error (_("while expecting a valid subcommand"));
424 msg (SE, _("All variables deleted from system file dictionary."));
428 if (*options & GTSV_OPT_MATCH_FILES)
434 lex_error (_("expecting end of command"));
441 /* Parses and performs the RENAME subcommand of GET and SAVE. */
443 rename_variables (struct dictionary * dict)
460 v = parse_dict_variable (dict);
463 if (!lex_force_match ('=')
466 if (!strncmp (tokid, v->name, 8))
468 if (is_dict_varname (dict, tokid))
470 msg (SE, _("Cannot rename %s as %s because there already exists "
471 "a variable named %s. To rename variables with "
472 "overlapping names, use a single RENAME subcommand "
473 "such as \"/RENAME (A=B)(B=C)(C=A)\", or equivalently, "
474 "\"/RENAME (A B C=B C A)\"."), v->name, tokid, tokid);
478 rename_variable (dict, v, tokid);
487 while (lex_match ('('))
491 if (!parse_variables (dict, &v, &nv, PV_NO_DUPLICATE | PV_APPEND))
493 if (!lex_match ('='))
495 msg (SE, _("`=' expected after variable list."));
498 if (!parse_DATA_LIST_vars (&new_names, &nn, PV_APPEND | PV_NO_SCRATCH))
502 msg (SE, _("Number of variables on left side of `=' (%d) do not "
503 "match number of variables on right side (%d), in "
504 "parenthesized group %d of RENAME subcommand."),
505 nv - old_nv, nn - old_nv, group);
508 if (!lex_force_match (')'))
513 for (i = 0; i < nv; i++)
514 avl_force_delete (dict->var_by_name, v[i]);
515 for (i = 0; i < nv; i++)
517 strcpy (v[i]->name, new_names[i]);
518 if (NULL != avl_insert (dict->var_by_name, v[i]))
520 msg (SE, _("Duplicate variables name %s."), v[i]->name);
527 /* The label is a bit of a misnomer, we actually come here on any
529 for (i = 0; i < nn; i++)
539 dump_dict_variables (struct dictionary * dict)
543 printf (_("\nVariables in dictionary:\n"));
544 for (i = 0; i < dict->nvar; i++)
545 printf ("%s, ", dict->var[i]->name);
550 /* Clears internal state related to GET input procedure. */
552 get_source_destroy_source (void)
554 /* It is not necessary to destroy the dictionary because if we get
555 to this point then the dictionary is default_dict. */
556 fh_close_handle (get_file);
559 /* Reads all the cases from the data file and passes them to
562 get_source_read (void)
564 while (sfm_read_case (get_file, temp_case->data, &default_dict)
567 get_source_destroy_source ();
570 struct case_stream get_source =
576 get_source_destroy_source,
584 #include "debug-print.h"
589 MTF_FILE, /* Specified on FILE= subcommand. */
590 MTF_TABLE /* Specified on TABLE= subcommand. */
593 /* One of the files on MATCH FILES. */
596 struct mtf_file *next, *prev;
597 /* Next, previous in the list of files. */
598 struct mtf_file *next_min; /* Next in the chain of minimums. */
600 int type; /* One of MTF_*. */
601 struct variable **by; /* List of BY variables for this file. */
602 struct file_handle *handle; /* File handle for the file. */
603 struct dictionary *dict; /* Dictionary from system file. */
604 char in[9]; /* Name of the variable from IN=. */
605 char first[9], last[9]; /* Name of the variables from FIRST=, LAST=. */
606 union value *input; /* Input record. */
609 /* All the files mentioned on FILE= or TABLE=. */
610 static struct mtf_file *mtf_head, *mtf_tail;
612 /* Variables on the BY subcommand. */
613 static struct variable **mtf_by;
616 /* Master dictionary. */
617 static struct dictionary *mtf_master;
619 static void mtf_free (void);
620 static void mtf_free_file (struct mtf_file *file);
621 static int mtf_merge_dictionary (struct mtf_file *f);
622 static void mtf_delete_file_in_place (struct mtf_file **file);
624 static void mtf_read_nonactive_records (void);
625 static void mtf_processing_finish (void);
626 static int mtf_processing (struct ccase *);
628 static char *var_type_description (struct variable *);
630 /* Parse and execute the MATCH FILES command. */
632 cmd_match_files (void)
634 struct mtf_file *first_table = NULL;
638 lex_match_id ("MATCH");
639 lex_match_id ("FILES");
641 mtf_head = mtf_tail = NULL;
644 mtf_master = new_dictionary (0);
645 mtf_master->N = default_dict.N;
651 if (lex_match (T_BY))
655 msg (SE, _("The BY subcommand may be given once at most."));
661 if (!parse_variables (mtf_master, &mtf_by, &mtf_n_by,
662 PV_NO_DUPLICATE | PV_NO_SCRATCH))
665 else if (token != T_ID)
670 else if (lex_id_match ("FILE", tokid) || lex_id_match ("TABLE", tokid))
672 struct mtf_file *file = xmalloc (sizeof *file);
674 file->in[0] = file->first[0] = file->last[0] = '\0';
679 if (lex_match_id ("FILE"))
680 file->type = MTF_FILE;
681 else if (lex_match_id ("TABLE"))
683 file->type = MTF_TABLE;
689 /* FILEs go first, then TABLEs. */
690 if (file->type == MTF_TABLE || first_table == NULL)
693 file->prev = mtf_tail;
695 mtf_tail->next = file;
697 if (mtf_head == NULL)
699 if (file->type == MTF_TABLE && first_table == NULL)
704 assert (file->type == MTF_FILE);
705 file->next = first_table;
706 file->prev = first_table->prev;
707 if (first_table->prev)
708 first_table->prev->next = file;
711 first_table->prev = file;
722 msg (SE, _("The active file may not be specified more "
728 assert (pgm_state != STATE_INPUT);
729 if (pgm_state == STATE_INIT)
731 msg (SE, _("Cannot specify the active file since no active "
732 "file has been defined."));
738 file->handle = fh_parse_file_handle ();
745 file->dict = sfm_read_dictionary (file->handle, NULL);
750 file->dict = &default_dict;
751 if (!mtf_merge_dictionary (file))
754 else if (lex_id_match ("IN", tokid)
755 || lex_id_match ("FIRST", tokid)
756 || lex_id_match ("LAST", tokid))
761 if (mtf_tail == NULL)
763 msg (SE, _("IN, FIRST, and LAST subcommands may not occur "
764 "before the first FILE or TABLE."));
768 if (lex_match_id ("IN"))
773 else if (lex_match_id ("FIRST"))
775 name = mtf_tail->first;
778 else if (lex_match_id ("LAST"))
780 name = mtf_tail->last;
795 msg (SE, _("Multiple %s subcommands for a single FILE or "
800 strcpy (name, tokid);
803 if (!create_variable (mtf_master, name, NUMERIC, 0))
805 msg (SE, _("Duplicate variable name %s while creating %s "
811 else if (lex_id_match ("RENAME", tokid)
812 || lex_id_match ("KEEP", tokid)
813 || lex_id_match ("DROP", tokid))
815 int options = GTSV_OPT_MATCH_FILES;
817 if (mtf_tail == NULL)
819 msg (SE, _("RENAME, KEEP, and DROP subcommands may not occur "
820 "before the first FILE or TABLE."));
824 if (!trim_dictionary (mtf_tail->dict, &options))
827 else if (lex_match_id ("MAP"))
837 while (token != '.');
843 msg (SE, _("The BY subcommand is required when a TABLE subcommand "
851 struct mtf_file *iter;
853 for (iter = mtf_head; iter; iter = iter->next)
857 iter->by = xmalloc (sizeof *iter->by * mtf_n_by);
859 for (i = 0; i < mtf_n_by; i++)
861 iter->by[i] = find_dict_variable (iter->dict, mtf_by[i]->name);
862 if (iter->by[i] == NULL)
864 msg (SE, _("File %s lacks BY variable %s."),
865 iter->handle ? fh_handle_name (iter->handle) : "*",
875 /* From sfm-read.c. */
876 extern void dump_dictionary (struct dictionary *);
878 dump_dictionary (mtf_master);
882 /* MATCH FILES performs an n-way merge on all its input files.
885 1. Read one input record from every input FILE.
887 2. If no FILEs are left, stop. Otherwise, proceed to step 3.
889 3. Find the FILE input record with minimum BY values. Store all
890 the values from this input record into the output record.
892 4. Find all the FILE input records with BY values identical to
893 the minimums. Store all the values from these input records into
896 5. For every TABLE, read another record as long as the BY values
897 on the TABLE's input record are less than the FILEs' BY values.
898 If an exact match is found, store all the values from the TABLE
899 input record into the output record.
901 6. Write the output record.
903 7. Read another record from each input file FILE and TABLE that
904 we stored values from above. If we come to the end of one of the
905 input files, remove it from the list of input files.
907 8. Repeat from step 2.
909 Unfortunately, this algorithm can't be directly implemented
910 because there's no function to read a record from the active
911 file; instead, it has to be done using callbacks.
913 FIXME: A better algorithm would use a heap for finding minimum
914 values, or replacement selection, as described by Knuth in _Art
915 of Computer Programming, Vol. 3_. The SORT CASES procedure does
916 this, and perhaps some of its code could be adapted. */
919 discard_variables ();
922 temp_dict = mtf_master;
925 process_active_file (mtf_read_nonactive_records, mtf_processing,
926 mtf_processing_finish);
937 /* Repeats 2...8 an arbitrary number of times. */
939 mtf_processing_finish (void)
941 /* Find the active file and delete it. */
943 struct mtf_file *iter;
945 for (iter = mtf_head; iter; iter = iter->next)
946 if (iter->handle == NULL)
948 mtf_delete_file_in_place (&iter);
953 while (mtf_head && mtf_head->type == MTF_FILE)
954 if (!mtf_processing (temp_case))
958 /* Return a string in a static buffer describing V's variable type and
961 var_type_description (struct variable *v)
963 static char buf[2][32];
970 if (v->type == NUMERIC)
971 strcpy (s, "numeric");
974 assert (v->type == ALPHA);
975 sprintf (s, "string with width %d", v->width);
980 /* Free FILE and associated data. */
982 mtf_free_file (struct mtf_file *file)
984 fh_close_handle (file->handle);
985 if (file->dict && file->dict != &default_dict)
986 free_dictionary (file->dict);
993 /* Free all the data for the MATCH FILES procedure. */
997 struct mtf_file *iter, *next;
999 for (iter = mtf_head; iter; iter = next)
1003 mtf_free_file (iter);
1008 free_dictionary (mtf_master);
1011 /* Remove *FILE from the mtf_file chain. Make *FILE point to the next
1012 file in the chain, or to NULL if was the last in the chain. */
1014 mtf_delete_file_in_place (struct mtf_file **file)
1016 struct mtf_file *f = *file;
1019 f->prev->next = f->next;
1021 f->next->prev = f->prev;
1031 for (i = 0; i < f->dict->nvar; i++)
1033 struct variable *v = f->dict->var[i];
1035 if (v->type == NUMERIC)
1036 compaction_case->data[v->p.mtf.master->fv].f = SYSMIS;
1038 memset (compaction_case->data[v->p.mtf.master->fv].s, ' ',
1046 /* Read a record from every input file except the active file. */
1048 mtf_read_nonactive_records (void)
1050 struct mtf_file *iter;
1052 for (iter = mtf_head; iter; )
1056 assert (iter->input == NULL);
1057 iter->input = xmalloc (sizeof *iter->input * iter->dict->nval);
1059 if (!sfm_read_case (iter->handle, iter->input, iter->dict))
1060 mtf_delete_file_in_place (&iter);
1066 iter->input = temp_case->data;
1072 /* Compare the BY variables for files A and B; return -1 if A < B, 0
1073 if A == B, 1 if A > B. */
1075 mtf_compare_BY_values (struct mtf_file *a, struct mtf_file *b)
1079 for (i = 0; i < mtf_n_by; i++)
1081 assert (a->by[i]->type == b->by[i]->type);
1082 assert (a->by[i]->width == b->by[i]->width);
1084 if (a->by[i]->type == NUMERIC)
1086 double af = a->input[a->by[i]->fv].f;
1087 double bf = b->input[b->by[i]->fv].f;
1098 assert (a->by[i]->type == ALPHA);
1099 result = memcmp (a->input[a->by[i]->fv].s,
1100 b->input[b->by[i]->fv].s,
1104 else if (result > 0)
1111 /* Used to determine whether we've already initialized this
1113 static int mtf_seq_no = 0;
1115 /* Perform one iteration of steps 3...7 above. */
1117 mtf_processing (struct ccase *c unused)
1119 /* List of files with minimum BY values. */
1120 struct mtf_file *min_head, *min_tail;
1122 /* List of files with non-minimum BY values. */
1123 struct mtf_file *max_head, *max_tail;
1126 struct mtf_file *iter;
1130 /* If the active file doesn't have the minimum BY values, don't
1131 return because that would cause a record to be skipped. */
1134 if (mtf_head->type == MTF_TABLE)
1137 /* 3. Find the FILE input record with minimum BY values. Store
1138 all the values from this input record into the output record.
1140 4. Find all the FILE input records with BY values identical
1141 to the minimums. Store all the values from these input
1142 records into the output record. */
1143 min_head = min_tail = mtf_head;
1144 max_head = max_tail = NULL;
1145 for (iter = mtf_head->next; iter && iter->type == MTF_FILE;
1147 switch (mtf_compare_BY_values (min_head, iter))
1151 max_tail = max_tail->next_min = iter;
1153 max_head = max_tail = iter;
1157 min_tail = min_tail->next_min = iter;
1163 max_tail->next_min = min_head;
1164 max_tail = min_tail;
1168 max_head = min_head;
1169 max_tail = min_tail;
1171 min_head = min_tail = iter;
1178 /* 5. For every TABLE, read another record as long as the BY
1179 values on the TABLE's input record are less than the FILEs'
1180 BY values. If an exact match is found, store all the values
1181 from the TABLE input record into the output record. */
1184 struct mtf_file *next = iter->next;
1186 assert (iter->type == MTF_TABLE);
1188 if (iter->handle == NULL)
1192 switch (mtf_compare_BY_values (min_head, iter))
1196 max_tail = max_tail->next_min = iter;
1198 max_head = max_tail = iter;
1202 min_tail = min_tail->next_min = iter;
1206 if (iter->handle == NULL)
1208 if (sfm_read_case (iter->handle, iter->input, iter->dict))
1210 mtf_delete_file_in_place (&iter);
1220 /* Next sequence number. */
1223 /* Store data to all the records we are using. */
1225 min_tail->next_min = NULL;
1226 for (iter = min_head; iter; iter = iter->next_min)
1230 for (i = 0; i < iter->dict->nvar; i++)
1232 struct variable *v = iter->dict->var[i];
1234 if (v->p.mtf.master->foo == mtf_seq_no)
1236 v->p.mtf.master->foo = mtf_seq_no;
1239 printf ("%s/%s: dest-fv=%d, src-fv=%d\n",
1240 fh_handle_name (iter->handle),
1242 v->p.mtf.master->fv, v->fv);
1244 if (v->type == NUMERIC)
1245 compaction_case->data[v->p.mtf.master->fv].f
1246 = iter->input[v->fv].f;
1249 assert (v->type == ALPHA);
1250 memcpy (compaction_case->data[v->p.mtf.master->fv].s,
1251 iter->input[v->fv].s, v->width);
1256 /* Store missing values to all the records we're not using. */
1258 max_tail->next_min = NULL;
1259 for (iter = max_head; iter; iter = iter->next_min)
1263 for (i = 0; i < iter->dict->nvar; i++)
1265 struct variable *v = iter->dict->var[i];
1267 if (v->p.mtf.master->foo == mtf_seq_no)
1269 v->p.mtf.master->foo = mtf_seq_no;
1272 printf ("%s/%s: dest-fv=%d\n",
1273 fh_handle_name (iter->handle),
1275 v->p.mtf.master->fv);
1277 if (v->type == NUMERIC)
1278 compaction_case->data[v->p.mtf.master->fv].f = SYSMIS;
1280 memset (compaction_case->data[v->p.mtf.master->fv].s, ' ',
1284 if (iter->handle == NULL)
1288 /* 6. Write the output record. */
1289 process_active_file_output_case ();
1291 /* 7. Read another record from each input file FILE and TABLE
1292 that we stored values from above. If we come to the end of
1293 one of the input files, remove it from the list of input
1295 for (iter = min_head; iter && iter->type == MTF_FILE; )
1297 struct mtf_file *next = iter->next_min;
1301 assert (iter->input != NULL);
1303 if (!sfm_read_case (iter->handle, iter->input, iter->dict))
1304 mtf_delete_file_in_place (&iter);
1314 return (mtf_head && mtf_head->type != MTF_TABLE);
1317 /* Merge the dictionary for file F into the master dictionary
1320 mtf_merge_dictionary (struct mtf_file *f)
1322 struct dictionary *const m = mtf_master;
1323 struct dictionary *d = f->dict;
1325 if (d->label && m->label == NULL)
1326 m->label = xstrdup (d->label);
1330 m->documents = xrealloc (m->documents,
1331 80 * (m->n_documents + d->n_documents));
1332 memcpy (&m->documents[80 * m->n_documents],
1333 d->documents, 80 * d->n_documents);
1334 m->n_documents += d->n_documents;
1341 for (i = 0; i < d->nvar; i++)
1343 struct variable *dv = d->var[i];
1344 struct variable *mv = find_dict_variable (m, dv->name);
1349 assert (dv->type == ALPHA || dv->width == 0);
1350 assert (!mv || mv->type == ALPHA || mv->width == 0);
1351 if (mv && dv->width == mv->width)
1353 if (dv->val_lab && !mv->val_lab)
1354 mv->val_lab = copy_value_labels (dv->val_lab);
1355 if (dv->miss_type != MISSING_NONE && mv->miss_type == MISSING_NONE)
1356 copy_missing_values (mv, dv);
1358 if (mv && dv->label && !mv->label)
1359 mv->label = xstrdup (dv->label);
1362 mv = force_dup_variable (m, dv, dv->name);
1364 /* Used to make sure we initialize each variable in the
1365 master dictionary exactly once per case. */
1366 mv->foo = mtf_seq_no;
1368 else if (mv->width != dv->width)
1370 msg (SE, _("Variable %s in file %s (%s) has different "
1371 "type or width from the same variable in "
1372 "earlier file (%s)."),
1373 dv->name, fh_handle_name (f->handle),
1374 var_type_description (dv), var_type_description (mv));
1377 dv->p.mtf.master = mv;
1384 /* IMPORT command. */
1386 /* Parses the IMPORT command. */
1390 struct file_handle *handle = NULL;
1391 struct dictionary *dict;
1392 int options = GTSV_OPT_NONE;
1398 lex_match_id ("IMPORT");
1404 if (lex_match_id ("FILE") || token == T_STRING)
1408 handle = fh_parse_file_handle ();
1412 else if (lex_match_id ("TYPE"))
1416 if (lex_match_id ("COMM"))
1418 else if (lex_match_id ("TAPE"))
1422 lex_error (_("expecting COMM or TAPE"));
1428 if (!lex_match ('/') && token != '.')
1434 discard_variables ();
1436 dict = pfm_read_dictionary (handle, NULL);
1441 dump_dict_variables (dict);
1443 if (0 == trim_dictionary (dict, &options))
1445 fh_close_handle (handle);
1449 dump_dict_variables (dict);
1452 /* Set the fv and lv elements of all variables remaining in the
1455 for (i = 0; i < dict->nvar; i++)
1457 struct variable *v = dict->var[i];
1466 printf (_("IMPORT translation table from file to memory:\n"));
1467 for (i = 0; i < dict->nvar; i++)
1469 struct variable *v = dict->var[i];
1471 printf (_(" %8s from %3d,%3d to %3d,%3d\n"), v->name,
1472 v->get.fv, v->get.nv, v->fv, v->nv);
1476 restore_dictionary (dict);
1478 vfm_source = &import_source;
1484 /* Reads all the cases from the data file and passes them to
1487 import_source_read (void)
1489 while (pfm_read_case (get_file, temp_case->data, &default_dict)
1492 get_source_destroy_source ();
1495 struct case_stream import_source =
1501 get_source_destroy_source,
1506 static int export_write_case_func (struct ccase *c);
1508 /* Parses the EXPORT command. */
1509 /* FIXME: same as cmd_save_internal(). */
1513 struct file_handle *handle;
1514 struct dictionary *dict;
1515 int options = GTSV_OPT_SAVE;
1517 struct save_trns *t;
1521 lex_match_id ("EXPORT");
1524 if (lex_match_id ("OUTFILE"))
1527 handle = fh_parse_file_handle ();
1531 dict = save_dictionary ();
1533 dump_dict_variables (dict);
1535 for (i = 0; i < dict->nvar; i++)
1536 dict->var[i]->foo = i;
1537 if (0 == trim_dictionary (dict, &options))
1539 fh_close_handle (handle);
1544 dump_dict_variables (dict);
1547 /* Write dictionary. */
1548 if (!pfm_write_dictionary (handle, dict))
1550 free_dictionary (dict);
1551 fh_close_handle (handle);
1555 /* Fill in transformation structure. */
1556 t = trns = xmalloc (sizeof *t);
1557 t->h.proc = save_trns_proc;
1558 t->h.free = save_trns_free;
1560 t->nvar = dict->nvar;
1561 t->var = xmalloc (sizeof *t->var * dict->nvar);
1562 for (i = 0; i < dict->nvar; i++)
1563 t->var[i] = dict->var[i]->foo;
1564 t->case_buf = xmalloc (sizeof *t->case_buf * dict->nvar);
1565 free_dictionary (dict);
1567 procedure (NULL, export_write_case_func, NULL);
1568 save_trns_free ((struct trns_header *) t);
1574 export_write_case_func (struct ccase *c)
1576 union value *p = (union value *) trns->case_buf;
1579 for (i = 0; i < trns->nvar; i++)
1581 struct variable *v = default_dict.var[trns->var[i]];
1583 if (v->type == NUMERIC)
1584 *p++ = c->data[v->fv];
1586 (*p++).c = c->data[v->fv].s;
1592 pfm_write_case (trns->f, (union value *) trns->case_buf);