X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Fautorecode.c;h=499f149d22a14c4003f2d2ae1a9a42d368202798;hb=8a0397328b6230fd49724e1c6d91a5a545d2fb4b;hp=aabef142e10731a4b299b5917872485e661a7014;hpb=d55697401a3cd6ee1609c9f4f8252930c23bca1e;p=pspp diff --git a/src/language/stats/autorecode.c b/src/language/stats/autorecode.c index aabef142e1..499f149d22 100644 --- a/src/language/stats/autorecode.c +++ b/src/language/stats/autorecode.c @@ -1,5 +1,6 @@ /* PSPP - a program for statistical analysis. - Copyright (C) 1997-9, 2000, 2009, 2010, 2012, 2013, 2014 Free Software Foundation, Inc. + Copyright (C) 1997-9, 2000, 2009, 2010, 2012, 2013, 2014 + 2021, Free Software Foundation, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -36,16 +37,16 @@ #include "libpspp/message.h" #include "libpspp/pool.h" #include "libpspp/str.h" +#include "output/pivot-table.h" #include "gl/xalloc.h" #include "gl/c-xvasprintf.h" #include "gl/mbiter.h" - +#include "gl/size_max.h" #include "gettext.h" #define _(msgid) gettext (msgid) - -/* FIXME: Implement PRINT subcommand. */ +#define N_(msgid) (msgid) /* Explains how to recode one value. */ struct arc_item @@ -53,8 +54,10 @@ struct arc_item struct hmap_node hmap_node; /* Element in "struct arc_spec" hash table. */ union value from; /* Original value. */ int width; /* Width of the original value */ + bool missing; /* Is 'from' missing in its source varible? */ + char *value_label; /* Value label in source variable, if any. */ - double to; /* Recoded value. */ + double to; /* Recoded value. */ }; /* Explains how to recode an AUTORECODE variable. */ @@ -62,7 +65,11 @@ struct arc_spec { int width; /* Variable width. */ int src_idx; /* Case index of source variable. */ - struct variable *dst; /* Target variable. */ + char *src_name; /* Name of source variable. */ + struct fmt_spec format; /* Print format in source variable. */ + struct variable *dst; /* Target variable. */ + struct missing_values mv; /* Missing values of source variable. */ + char *label; /* Variable label of source variable. */ struct rec_items *items; }; @@ -76,7 +83,6 @@ enum arc_direction struct rec_items { struct hmap ht; /* Hash table of "struct arc_item"s. */ - int refcnt; }; @@ -87,63 +93,45 @@ struct autorecode_pgm struct arc_spec *specs; size_t n_specs; - /* Hash table of "struct arc_item"s. */ - struct rec_items *global_items; - bool blank_valid; }; -static trns_proc_func autorecode_trns_proc; -static trns_free_func autorecode_trns_free; +static const struct trns_class autorecode_trns_class; static int compare_arc_items (const void *, const void *, const void *aux); static void arc_free (struct autorecode_pgm *); -static struct arc_item *find_arc_item (const struct arc_spec *, const union value *, - size_t hash); +static struct arc_item *find_arc_item ( + const struct rec_items *, const union value *, int width, + size_t hash); -static bool -value_is_blank (const union value *val, int width, const struct dictionary *dict) +/* Returns WIDTH with any trailing spaces in VALUE trimmed off (except that a + minimum width of 1 is always returned because otherwise the width would + indicate a numeric type). */ +static int +value_trim_spaces (const union value *value, int width) { - mbi_iterator_t iter; - const char *str = CHAR_CAST_BUG (const char*, value_str (val, width)); - char *text = recode_string (UTF8, dict_get_encoding (dict), str, width); - - for (mbi_init (iter, text, width); mbi_avail (iter); mbi_advance (iter)) - { - mbchar_t c = mbi_cur (iter); - - if ( ! mb_isblank (c)) - { - free (text); - return false; - } - } - - free (text); - return true; + while (width > 1 && value->s[width - 1] == ' ') + width--; + return width; } /* Performs the AUTORECODE procedure. */ int cmd_autorecode (struct lexer *lexer, struct dataset *ds) { - struct autorecode_pgm *arc = NULL; - struct dictionary *dict = dataset_dict (ds); + const struct variable **src_vars = NULL; - char **dst_names = NULL; size_t n_srcs = 0; + + char **dst_names = NULL; size_t n_dsts = 0; enum arc_direction direction = ASCENDING; - - struct casereader *input; - struct ccase *c; - - bool ok; + bool print = false; /* Create procedure. */ - arc = xzalloc (sizeof *arc); + struct autorecode_pgm *arc = XZALLOC (struct autorecode_pgm); arc->blank_valid = true; /* Parse variable lists. */ @@ -180,37 +168,32 @@ cmd_autorecode (struct lexer *lexer, struct dataset *ds) } /* Parse options. */ + bool group = false; while (lex_match (lexer, T_SLASH)) { if (lex_match_id (lexer, "DESCENDING")) - direction = DESCENDING; + direction = DESCENDING; else if (lex_match_id (lexer, "PRINT")) - { - /* Not yet implemented. */ - } + print = true; else if (lex_match_id (lexer, "GROUP")) - { - arc->global_items = xmalloc (sizeof (*arc->global_items)); - arc->global_items->refcnt = 1; - hmap_init (&arc->global_items->ht); - } + group = true; else if (lex_match_id (lexer, "BLANK")) - { - lex_match (lexer, T_EQUALS); - if (lex_match_id (lexer, "VALID")) - { - arc->blank_valid = true; - } - else if (lex_match_id (lexer, "MISSING")) - { - arc->blank_valid = false; - } - else + { + lex_match (lexer, T_EQUALS); + if (lex_match_id (lexer, "VALID")) + { + arc->blank_valid = true; + } + else if (lex_match_id (lexer, "MISSING")) + { + arc->blank_valid = false; + } + else { lex_error_expecting (lexer, "VALID", "MISSING"); goto error; } - } + } else { lex_error_expecting (lexer, "DESCENDING", "PRINT", "GROUP", "BLANK"); @@ -224,57 +207,118 @@ cmd_autorecode (struct lexer *lexer, struct dataset *ds) goto error; } - arc->specs = xmalloc (n_dsts * sizeof *arc->specs); - arc->n_specs = n_dsts; + /* If GROUP is specified, verify that the variables are all string or all + numeric. */ + if (group) + { + enum val_type type = var_get_type (src_vars[0]); + for (size_t i = 1; i < n_dsts; i++) + { + if (var_get_type (src_vars[i]) != type) + { + size_t string_idx = type == VAL_STRING ? 0 : i; + size_t numeric_idx = type == VAL_STRING ? i : 0; + lex_error (lexer, _("With GROUP, variables may not mix string " + "variables (such as %s) and numeric " + "variables (such as %s)."), + var_get_name (src_vars[string_idx]), + var_get_name (src_vars[numeric_idx])); + goto error; + } + } + } + /* Allocate all the specs and the rec_items that they point to. + If GROUP is specified, there is only a single global rec_items, with the + maximum width 'width', and all of the specs point to it; otherwise each + spec has its own rec_items. */ + arc->specs = xmalloc (n_dsts * sizeof *arc->specs); + arc->n_specs = n_dsts; for (size_t i = 0; i < n_dsts; i++) { struct arc_spec *spec = &arc->specs[i]; spec->width = var_get_width (src_vars[i]); spec->src_idx = var_get_case_index (src_vars[i]); + spec->src_name = xstrdup (var_get_name (src_vars[i])); + spec->format = *var_get_print_format (src_vars[i]); + + const char *label = var_get_label (src_vars[i]); + spec->label = xstrdup_if_nonnull (label); - if (arc->global_items) - { - arc->global_items->refcnt++; - spec->items = arc->global_items; - } + if (group && i > 0) + spec->items = arc->specs[0].items; else - { - spec->items = xzalloc (sizeof (*spec->items)); - spec->items->refcnt = 1; - hmap_init (&spec->items->ht); - } + { + spec->items = xzalloc (sizeof (*spec->items)); + hmap_init (&spec->items->ht); + } } + /* Initialize specs[*]->mv to the user-missing values for each + source variable. */ + if (group) + { + /* Use the first source variable that has any user-missing values. */ + size_t mv_idx = 0; + for (size_t i = 0; i < n_dsts; i++) + if (var_has_missing_values (src_vars[i])) + { + mv_idx = i; + break; + } + + for (size_t i = 0; i < n_dsts; i++) + mv_copy (&arc->specs[i].mv, var_get_missing_values (src_vars[mv_idx])); + } + else + { + /* Each variable uses its own user-missing values. */ + for (size_t i = 0; i < n_dsts; i++) + mv_copy (&arc->specs[i].mv, var_get_missing_values (src_vars[i])); + } /* Execute procedure. */ - input = proc_open (ds); + struct casereader *input = proc_open (ds); + struct ccase *c; for (; (c = casereader_read (input)) != NULL; case_unref (c)) for (size_t i = 0; i < arc->n_specs; i++) { struct arc_spec *spec = &arc->specs[i]; - int width = spec->width; const union value *value = case_data_idx (c, spec->src_idx); - size_t hash = value_hash (value, width, 0); - struct arc_item *item; - - item = find_arc_item (spec, value, hash); - if ( (item == NULL) - && - ( arc->blank_valid - || val_type_from_width (spec->width) == VAL_NUMERIC - || ! value_is_blank (value, width, dict)) - ) + if (spec->width == 0 && value->f == SYSMIS) { - item = xmalloc (sizeof *item); - item->width = width; - value_clone (&item->from, value, width); - hmap_insert (&spec->items->ht, &item->hmap_node, hash); + /* AUTORECODE never changes the system-missing value. + (Leaving it out of the translation table has this + effect automatically because values not found in the + translation table get translated to system-missing.) */ + continue; } + + int width = value_trim_spaces (value, spec->width); + if (width == 1 && value->s[0] == ' ' && !arc->blank_valid) + continue; + + size_t hash = value_hash (value, width, 0); + if (find_arc_item (spec->items, value, width, hash)) + continue; + + struct string value_label = DS_EMPTY_INITIALIZER; + var_append_value_name__ (src_vars[i], value, + SETTINGS_VALUE_SHOW_LABEL, &value_label); + + struct arc_item *item = xmalloc (sizeof *item); + item->width = width; + value_clone (&item->from, value, width); + item->missing = mv_is_value_missing_varwidth (&spec->mv, value, + spec->width); + item->value_label = ds_steal_cstr (&value_label); + hmap_insert (&spec->items->ht, &item->hmap_node, hash); + + ds_destroy (&value_label); } - ok = casereader_destroy (input); + bool ok = casereader_destroy (input); ok = proc_commit (ds) && ok; /* Re-fetch dictionary because it might have changed (if TEMPORARY was in @@ -285,67 +329,127 @@ cmd_autorecode (struct lexer *lexer, struct dataset *ds) for (size_t i = 0; i < arc->n_specs; i++) { struct arc_spec *spec = &arc->specs[i]; - struct arc_item **items; - struct arc_item *item; - size_t n_items; - size_t j; /* Create destination variable. */ spec->dst = dict_create_var_assert (dict, dst_names[i], 0); + var_set_label (spec->dst, spec->label); + + /* Set print format. */ + size_t n_items = hmap_count (&spec->items->ht); + char *longest_value = xasprintf ("%zu", n_items); + struct fmt_spec format = { .type = FMT_F, .w = strlen (longest_value) }; + var_set_both_formats (spec->dst, &format); + free (longest_value); /* Create array of pointers to items. */ - n_items = hmap_count (&spec->items->ht); - items = xmalloc (n_items * sizeof *items); - j = 0; + struct arc_item **items = xmalloc (n_items * sizeof *items); + struct arc_item *item; + size_t j = 0; HMAP_FOR_EACH (item, struct arc_item, hmap_node, &spec->items->ht) items[j++] = item; - assert (j == n_items); /* Sort array by value. */ - sort (items, n_items, sizeof *items, compare_arc_items, NULL); + sort (items, n_items, sizeof *items, compare_arc_items, &direction); /* Assign recoded values in sorted order. */ for (j = 0; j < n_items; j++) - { - const union value *from = &items[j]->from; - char *recoded_value = NULL; - const int src_width = items[j]->width; - union value to_val; - size_t len; - - value_init (&to_val, 0); - - items[j]->to = direction == ASCENDING ? j + 1 : n_items - j; + items[j]->to = j + 1; - to_val.f = items[j]->to; - - /* Add value labels to the destination variable which indicate - the source value from whence the new value comes. */ - if (src_width > 0) - { - const char *str = CHAR_CAST_BUG (const char*, value_str (from, src_width)); + if (print && (!group || i == 0)) + { + struct pivot_value *title + = (group + ? pivot_value_new_text (N_("Recoding grouped variables.")) + : spec->label && spec->label[0] + ? pivot_value_new_text_format (N_("Recoding %s into %s (%s)."), + spec->src_name, + var_get_name (spec->dst), + spec->label) + : pivot_value_new_text_format (N_("Recoding %s into %s."), + spec->src_name, + var_get_name (spec->dst))); + struct pivot_table *table = pivot_table_create__ (title, "Recoding"); + + pivot_dimension_create ( + table, PIVOT_AXIS_COLUMN, N_("Attributes"), + N_("New Value"), N_("Value Label")); + + struct pivot_dimension *old_values = pivot_dimension_create ( + table, PIVOT_AXIS_ROW, N_("Old Value")); + old_values->root->show_label = true; + + for (size_t k = 0; k < n_items; k++) + { + const struct arc_item *item = items[k]; + int old_value_idx = pivot_category_create_leaf ( + old_values->root, pivot_value_new_value ( + &item->from, item->width, + (item->width + ? &(struct fmt_spec) { .type = FMT_F, .w = item->width } + : &spec->format), + dict_get_encoding (dict))); + pivot_table_put2 (table, 0, old_value_idx, + pivot_value_new_integer (item->to)); + + const char *value_label = item->value_label; + if (value_label && value_label[0]) + pivot_table_put2 (table, 1, old_value_idx, + pivot_value_new_user_text (value_label, -1)); + } - recoded_value = recode_string (UTF8, dict_get_encoding (dict), - str, src_width); - } - else - recoded_value = c_xasprintf ("%.*g", DBL_DIG + 1, from->f); + pivot_table_submit (table); + } - /* Remove trailing whitespace */ - len = strlen (recoded_value); - while (len > 0 && recoded_value[len - 1] == ' ') - recoded_value[--len] = '\0'; + /* Assign user-missing values. + + User-missing values in the source variable(s) must be marked + as user-missing values in the destination variable. There + might be an arbitrary number of missing values, since the + source variable might have a range. Our sort function always + puts missing values together at the top of the range, so that + means that we can use a missing value range to cover all of + the user-missing values in any case (but we avoid it unless + necessary because user-missing value ranges are an obscure + feature). */ + size_t n_missing = n_items; + for (size_t k = 0; k < n_items; k++) + if (!items[n_items - k - 1]->missing) + { + n_missing = k; + break; + } + if (n_missing > 0) + { + size_t lo = n_items - (n_missing - 1); + size_t hi = n_items; + + struct missing_values mv; + mv_init (&mv, 0); + if (n_missing > 3) + mv_add_range (&mv, lo, hi); + else + for (size_t k = 0; k < n_missing; k++) + mv_add_num (&mv, lo + k); + var_set_missing_values (spec->dst, &mv); + mv_destroy (&mv); + } - var_add_value_label (spec->dst, &to_val, recoded_value); - value_destroy (&to_val, 0); - free (recoded_value); - } + /* Add value labels to the destination variable. */ + for (j = 0; j < n_items; j++) + { + const char *value_label = items[j]->value_label; + if (value_label && value_label[0]) + { + union value to_val = { .f = items[j]->to }; + var_add_value_label (spec->dst, &to_val, value_label); + } + } /* Free array. */ free (items); } - add_transformation (ds, autorecode_trns_proc, autorecode_trns_free, arc); + add_transformation (ds, &autorecode_trns_class, arc); for (size_t i = 0; i < n_dsts; i++) free (dst_names[i]); @@ -373,31 +477,30 @@ arc_free (struct autorecode_pgm *arc) struct arc_spec *spec = &arc->specs[i]; struct arc_item *item, *next; - HMAP_FOR_EACH_SAFE (item, next, struct arc_item, hmap_node, - &spec->items->ht) - { - value_destroy (&item->from, item->width); - hmap_delete (&spec->items->ht, &item->hmap_node); - free (item); - } + HMAP_FOR_EACH_SAFE (item, next, struct arc_item, hmap_node, + &spec->items->ht) + { + value_destroy (&item->from, item->width); + free (item->value_label); + hmap_delete (&spec->items->ht, &item->hmap_node); + free (item); + } + free (spec->label); + free (spec->src_name); + mv_destroy (&spec->mv); } - for (size_t i = 0; i < arc->n_specs; i++) - { - struct arc_spec *spec = &arc->specs[i]; - - if (--spec->items->refcnt == 0) - { - hmap_destroy (&spec->items->ht); - free (spec->items); - } - } - - if (arc->global_items && --arc->global_items->refcnt == 0) - { - hmap_destroy (&arc->global_items->ht); - free (arc->global_items); - } + size_t n_rec_items = + (arc->n_specs >= 2 && arc->specs[0].items == arc->specs[1].items + ? 1 + : arc->n_specs); + + for (size_t i = 0; i < n_rec_items; i++) + { + struct arc_spec *spec = &arc->specs[i]; + hmap_destroy (&spec->items->ht); + free (spec->items); + } free (arc->specs); free (arc); @@ -405,39 +508,51 @@ arc_free (struct autorecode_pgm *arc) } static struct arc_item * -find_arc_item (const struct arc_spec *spec, const union value *value, +find_arc_item (const struct rec_items *items, + const union value *value, int width, size_t hash) { struct arc_item *item; - HMAP_FOR_EACH_WITH_HASH (item, struct arc_item, hmap_node, hash, &spec->items->ht) - if (value_equal (value, &item->from, spec->width)) + HMAP_FOR_EACH_WITH_HASH (item, struct arc_item, hmap_node, hash, &items->ht) + if (item->width == width && value_equal (value, &item->from, width)) return item; return NULL; } static int -compare_arc_items (const void *a_, const void *b_, const void *aux UNUSED) +compare_arc_items (const void *a_, const void *b_, const void *direction_) { - const struct arc_item *const *a = a_; - const struct arc_item *const *b = b_; - int width_a = (*a)->width; - int width_b = (*b)->width; - - if ( width_a == width_b) - return value_compare_3way (&(*a)->from, &(*b)->from, width_a); - - if ( width_a == 0 && width_b != 0) - return -1; - - if ( width_b == 0 && width_a != 0) - return +1; + const struct arc_item *const *ap = a_; + const struct arc_item *const *bp = b_; + const struct arc_item *a = *ap; + const struct arc_item *b = *bp; + + /* User-missing values always sort to the highest target values + (regardless of sort direction). */ + if (a->missing != b->missing) + return a->missing < b->missing ? -1 : 1; + + /* Otherwise, compare the data. */ + int aw = a->width; + int bw = b->width; + int cmp; + if (aw == bw) + cmp = value_compare_3way (&a->from, &b->from, aw); + else + { + assert (aw && bw); + cmp = buf_compare_rpad (CHAR_CAST_BUG (const char *, a->from.s), aw, + CHAR_CAST_BUG (const char *, b->from.s), bw); + } - return buf_compare_rpad (CHAR_CAST_BUG (const char *, value_str (&(*a)->from, width_a)), width_a, - CHAR_CAST_BUG (const char *, value_str (&(*b)->from, width_b)), width_b); + /* Then apply sort direction. */ + const enum arc_direction *directionp = direction_; + enum arc_direction direction = *directionp; + return direction == ASCENDING ? cmp : -cmp; } -static int +static enum trns_result autorecode_trns_proc (void *arc_, struct ccase **c, casenumber case_idx UNUSED) { @@ -448,10 +563,11 @@ autorecode_trns_proc (void *arc_, struct ccase **c, { const struct arc_spec *spec = &arc->specs[i]; const union value *value = case_data_idx (*c, spec->src_idx); - unsigned int hash = value_hash (value, spec->width, 0); - const struct arc_item *item = find_arc_item (spec, value, hash); - - case_data_rw (*c, spec->dst)->f = item ? item->to : SYSMIS; + int width = value_trim_spaces (value, spec->width); + size_t hash = value_hash (value, width, 0); + const struct arc_item *item = find_arc_item (spec->items, value, width, + hash); + *case_num_rw (*c, spec->dst) = item ? item->to : SYSMIS; } return TRNS_CONTINUE; @@ -465,3 +581,9 @@ autorecode_trns_free (void *arc_) arc_free (arc); return true; } + +static const struct trns_class autorecode_trns_class = { + .name = "AUTORECODE", + .execute = autorecode_trns_proc, + .destroy = autorecode_trns_free, +};