-/* PSPP - computes sample statistics.
- Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
- Written by Ben Pfaff <blp@gnu.org>.
+/* PSPP - a program for statistical analysis.
+ Copyright (C) 1997-2000, 2009-2014 Free Software Foundation, Inc.
- This program is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License as
- published by the Free Software Foundation; either version 2 of the
- License, or (at your option) any later version.
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
- This program is distributed in the hope that it will be useful, but
- WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- General Public License for more details.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- 02110-1301, USA. */
-
-/* FIXME: Many possible optimizations. */
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */
#include <config.h>
+#include <float.h>
#include <limits.h>
#include <math.h>
#include <stdlib.h>
-#include <data/case.h>
-#include <data/casefile.h>
-#include <data/dictionary.h>
-#include <data/procedure.h>
-#include <data/transformations.h>
-#include <data/variable.h>
-#include <language/command.h>
-#include <language/lexer/lexer.h>
-#include <libpspp/alloc.h>
-#include <libpspp/array.h>
-#include <libpspp/compiler.h>
-#include <libpspp/magic.h>
-#include <libpspp/message.h>
-#include <math/moments.h>
-#include <output/manager.h>
-#include <output/table.h>
+#include "data/casegrouper.h"
+#include "data/casereader.h"
+#include "data/casewriter.h"
+#include "data/dataset.h"
+#include "data/dictionary.h"
+#include "data/transformations.h"
+#include "data/variable.h"
+#include "language/command.h"
+#include "language/dictionary/split-file.h"
+#include "language/lexer/lexer.h"
+#include "language/lexer/variable-parser.h"
+#include "libpspp/array.h"
+#include "libpspp/assertion.h"
+#include "libpspp/compiler.h"
+#include "libpspp/i18n.h"
+#include "libpspp/message.h"
+#include "math/moments.h"
+#include "output/pivot-table.h"
+
+#include "gl/xalloc.h"
#include "gettext.h"
#define _(msgid) gettext (msgid)
calculating a Z-score. */
struct dsc_z_score
{
- int src_idx; /* Source index into case data. */
- int dst_idx; /* Destination index into case data. */
+ const struct variable *src_var; /* Variable on which z-score is based. */
+ struct variable *z_var; /* New z-score variable. */
double mean; /* Distribution mean. */
double std_dev; /* Distribution standard deviation. */
- struct variable *v; /* Variable on which z-score is based. */
};
/* DESCRIPTIVES transformation (for calculating Z-scores). */
struct dsc_trns
{
struct dsc_z_score *z_scores; /* Array of Z-scores. */
- int z_score_cnt; /* Number of Z-scores. */
- struct variable **vars; /* Variables for listwise missing checks. */
- size_t var_cnt; /* Number of variables. */
+ int n_z_scores; /* Number of Z-scores. */
+ const struct variable **vars; /* Variables for listwise missing checks. */
+ size_t n_vars; /* Number of variables. */
enum dsc_missing_type missing_type; /* Treatment of missing values. */
- int include_user_missing; /* Nonzero to include user-missing values. */
+ enum mv_class exclude; /* Classes of missing values to exclude. */
+ const struct variable *filter; /* Dictionary FILTER BY variable. */
+ struct casereader *z_reader; /* Reader for count, mean, stddev. */
+ casenumber count; /* Number left in this SPLIT FILE group.*/
+ bool ok;
};
/* Statistics. Used as bit indexes, so must be 32 or fewer. */
static const struct dsc_statistic_info dsc_info[DSC_N_STATS] =
{
{"MEAN", N_("Mean"), MOMENT_MEAN},
- {"SEMEAN", N_("S E Mean"), MOMENT_VARIANCE},
+ {"SEMEAN", N_("S.E. Mean"), MOMENT_VARIANCE},
{"STDDEV", N_("Std Dev"), MOMENT_VARIANCE},
{"VARIANCE", N_("Variance"), MOMENT_VARIANCE},
{"KURTOSIS", N_("Kurtosis"), MOMENT_KURTOSIS},
- {"SEKURTOSIS", N_("S E Kurt"), MOMENT_NONE},
+ {"SEKURTOSIS", N_("S.E. Kurt"), MOMENT_NONE},
{"SKEWNESS", N_("Skewness"), MOMENT_SKEWNESS},
- {"SESKEWNESS", N_("S E Skew"), MOMENT_NONE},
+ {"SESKEWNESS", N_("S.E. Skew"), MOMENT_NONE},
{"RANGE", N_("Range"), MOMENT_NONE},
{"MINIMUM", N_("Minimum"), MOMENT_NONE},
{"MAXIMUM", N_("Maximum"), MOMENT_NONE},
#define DEFAULT_STATS \
((1ul << DSC_MEAN) | (1ul << DSC_STDDEV) | (1ul << DSC_MIN) \
| (1ul << DSC_MAX))
-
+
/* A variable specified on DESCRIPTIVES. */
struct dsc_var
{
- struct variable *v; /* Variable to calculate on. */
- char z_name[LONG_NAME_LEN + 1]; /* Name for z-score variable. */
+ const struct variable *v; /* Variable to calculate on. */
+ char *z_name; /* Name for z-score variable. */
double valid, missing; /* Valid, missing counts. */
struct moments *moments; /* Moments. */
double min, max; /* Maximum and mimimum values. */
double stats[DSC_N_STATS]; /* All the stats' values. */
};
-/* Output format. */
-enum dsc_format
- {
- DSC_LINE, /* Abbreviated format. */
- DSC_SERIAL /* Long format. */
- };
-
/* A DESCRIPTIVES procedure. */
-struct dsc_proc
+struct dsc_proc
{
/* Per-variable info. */
+ struct dictionary *dict; /* Dictionary. */
struct dsc_var *vars; /* Variables. */
- size_t var_cnt; /* Number of variables. */
+ size_t n_vars; /* Number of variables. */
/* User options. */
enum dsc_missing_type missing_type; /* Treatment of missing values. */
- int include_user_missing; /* Nonzero to include user-missing values. */
- int show_var_labels; /* Nonzero to show variable labels. */
- int show_index; /* Nonzero to show variable index. */
- enum dsc_format format; /* Output format. */
+ enum mv_class exclude; /* Classes of missing values to exclude. */
/* Accumulated results. */
double missing_listwise; /* Sum of weights of cases missing listwise. */
double valid; /* Sum of weights of valid cases. */
- int bad_warn; /* Warn if bad weight found. */
+ bool bad_warn; /* Warn if bad weight found. */
enum dsc_statistic sort_by_stat; /* Statistic to sort by; -1: name. */
int sort_ascending; /* !0: ascending order; 0: descending. */
unsigned long show_stats; /* Statistics to display. */
unsigned long calc_stats; /* Statistics to calculate. */
enum moment max_moment; /* Highest moment needed for stats. */
+
+ /* Z scores. */
+ struct casewriter *z_writer; /* Mean and stddev per SPLIT FILE group. */
};
/* Parsing. */
-static enum dsc_statistic match_statistic (void);
+static enum dsc_statistic match_statistic (struct lexer *);
static void free_dsc_proc (struct dsc_proc *);
/* Z-score functions. */
-static int try_name (struct dsc_proc *dsc, char *name);
-static int generate_z_varname (struct dsc_proc *dsc, char *z_name,
- const char *name, size_t *z_cnt);
+static bool try_name (const struct dictionary *dict,
+ struct dsc_proc *dsc, const char *name);
+static char *generate_z_varname (const struct dictionary *dict,
+ struct dsc_proc *dsc,
+ const char *name, int *n_zs);
static void dump_z_table (struct dsc_proc *);
-static void setup_z_trns (struct dsc_proc *);
+static void setup_z_trns (struct dsc_proc *, struct dataset *);
/* Procedure execution functions. */
-static bool calc_descriptives (const struct casefile *, void *dsc_);
+static void calc_descriptives (struct dsc_proc *, struct casereader *,
+ struct dataset *);
static void display (struct dsc_proc *dsc);
\f
/* Parser and outline. */
/* Handles DESCRIPTIVES. */
int
-cmd_descriptives (void)
+cmd_descriptives (struct lexer *lexer, struct dataset *ds)
{
+ struct dictionary *dict = dataset_dict (ds);
struct dsc_proc *dsc;
- struct variable **vars = NULL;
- size_t var_cnt = 0;
+ const struct variable **vars = NULL;
+ size_t n_vars = 0;
int save_z_scores = 0;
- size_t z_cnt = 0;
+ int n_zs = 0;
size_t i;
bool ok;
+ struct casegrouper *grouper;
+ struct casereader *group;
+
/* Create and initialize dsc. */
dsc = xmalloc (sizeof *dsc);
+ dsc->dict = dict;
dsc->vars = NULL;
- dsc->var_cnt = 0;
+ dsc->n_vars = 0;
dsc->missing_type = DSC_VARIABLE;
- dsc->include_user_missing = 0;
- dsc->show_var_labels = 1;
- dsc->show_index = 0;
- dsc->format = DSC_LINE;
+ dsc->exclude = MV_ANY;
dsc->missing_listwise = 0.;
dsc->valid = 0.;
dsc->bad_warn = 1;
dsc->sort_by_stat = DSC_NONE;
dsc->sort_ascending = 1;
dsc->show_stats = dsc->calc_stats = DEFAULT_STATS;
+ dsc->z_writer = NULL;
/* Parse DESCRIPTIVES. */
- while (token != '.')
+ while (lex_token (lexer) != T_ENDCMD)
{
- if (lex_match_id ("MISSING"))
+ if (lex_match_id (lexer, "MISSING"))
{
- lex_match ('=');
- while (token != '.' && token != '/')
+ lex_match (lexer, T_EQUALS);
+ while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
{
- if (lex_match_id ("VARIABLE"))
+ if (lex_match_id (lexer, "VARIABLE"))
dsc->missing_type = DSC_VARIABLE;
- else if (lex_match_id ("LISTWISE"))
+ else if (lex_match_id (lexer, "LISTWISE"))
dsc->missing_type = DSC_LISTWISE;
- else if (lex_match_id ("INCLUDE"))
- dsc->include_user_missing = 1;
+ else if (lex_match_id (lexer, "INCLUDE"))
+ dsc->exclude = MV_SYSTEM;
else
{
- lex_error (NULL);
+ lex_error (lexer, NULL);
goto error;
}
- lex_match (',');
+ lex_match (lexer, T_COMMA);
}
}
- else if (lex_match_id ("SAVE"))
+ else if (lex_match_id (lexer, "SAVE"))
save_z_scores = 1;
- else if (lex_match_id ("FORMAT"))
+ else if (lex_match_id (lexer, "FORMAT"))
{
- lex_match ('=');
- while (token != '.' && token != '/')
+ lex_match (lexer, T_EQUALS);
+ while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
{
- if (lex_match_id ("LABELS"))
- dsc->show_var_labels = 1;
- else if (lex_match_id ("NOLABELS"))
- dsc->show_var_labels = 0;
- else if (lex_match_id ("INDEX"))
- dsc->show_index = 1;
- else if (lex_match_id ("NOINDEX"))
- dsc->show_index = 0;
- else if (lex_match_id ("LINE"))
- dsc->format = DSC_LINE;
- else if (lex_match_id ("SERIAL"))
- dsc->format = DSC_SERIAL;
+ if (lex_match_id (lexer, "LABELS")
+ || lex_match_id (lexer, "NOLABELS")
+ || lex_match_id (lexer, "INDEX")
+ || lex_match_id (lexer, "NOINDEX")
+ || lex_match_id (lexer, "LINE")
+ || lex_match_id (lexer, "SERIAL"))
+ {
+ /* Ignore. */
+ }
else
{
- lex_error (NULL);
+ lex_error (lexer, NULL);
goto error;
}
- lex_match (',');
+ lex_match (lexer, T_COMMA);
}
}
- else if (lex_match_id ("STATISTICS"))
+ else if (lex_match_id (lexer, "STATISTICS"))
{
- lex_match ('=');
+ lex_match (lexer, T_EQUALS);
dsc->show_stats = 0;
- while (token != '.' && token != '/')
+ while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
{
- if (lex_match (T_ALL))
+ if (lex_match (lexer, T_ALL))
dsc->show_stats |= (1ul << DSC_N_STATS) - 1;
- else if (lex_match_id ("DEFAULT"))
+ else if (lex_match_id (lexer, "DEFAULT"))
dsc->show_stats |= DEFAULT_STATS;
else
- dsc->show_stats |= 1ul << (match_statistic ());
- lex_match (',');
+ {
+ enum dsc_statistic s = match_statistic (lexer);
+ if (s == DSC_NONE)
+ {
+ lex_error (lexer, NULL);
+ goto error;
+ }
+ dsc->show_stats |= 1ul << s;
+ }
+ lex_match (lexer, T_COMMA);
}
if (dsc->show_stats == 0)
dsc->show_stats = DEFAULT_STATS;
}
- else if (lex_match_id ("SORT"))
+ else if (lex_match_id (lexer, "SORT"))
{
- lex_match ('=');
- if (lex_match_id ("NAME"))
+ lex_match (lexer, T_EQUALS);
+ if (lex_match_id (lexer, "NAME"))
dsc->sort_by_stat = DSC_NAME;
- else
+ else
{
- dsc->sort_by_stat = match_statistic ();
- if (dsc->sort_by_stat == DSC_NONE )
+ dsc->sort_by_stat = match_statistic (lexer);
+ if (dsc->sort_by_stat == DSC_NONE)
dsc->sort_by_stat = DSC_MEAN;
}
- if (lex_match ('('))
+ if (lex_match (lexer, T_LPAREN))
{
- if (lex_match_id ("A"))
+ if (lex_match_id (lexer, "A"))
dsc->sort_ascending = 1;
- else if (lex_match_id ("D"))
+ else if (lex_match_id (lexer, "D"))
dsc->sort_ascending = 0;
else
- lex_error (NULL);
- lex_force_match (')');
+ lex_error (lexer, NULL);
+ if (! lex_force_match (lexer, T_RPAREN))
+ goto error;
}
}
- else if (var_cnt == 0)
+ else if (n_vars == 0)
{
- if (lex_look_ahead () == '=')
+ if (lex_next_token (lexer, 1) == T_EQUALS)
{
- lex_match_id ("VARIABLES");
- lex_match ('=');
+ lex_match_id (lexer, "VARIABLES");
+ lex_match (lexer, T_EQUALS);
}
- while (token != '.' && token != '/')
+ while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
{
int i;
-
- if (!parse_variables (default_dict, &vars, &var_cnt,
+
+ if (!parse_variables_const (lexer, dict, &vars, &n_vars,
PV_APPEND | PV_NO_DUPLICATE | PV_NUMERIC))
goto error;
- dsc->vars = xnrealloc (dsc->vars, var_cnt, sizeof *dsc->vars);
- for (i = dsc->var_cnt; i < var_cnt; i++)
+ dsc->vars = xnrealloc ((void *)dsc->vars, n_vars, sizeof *dsc->vars);
+ for (i = dsc->n_vars; i < n_vars; i++)
{
struct dsc_var *dv = &dsc->vars[i];
dv->v = vars[i];
- dv->z_name[0] = '\0';
+ dv->z_name = NULL;
dv->moments = NULL;
}
- dsc->var_cnt = var_cnt;
+ dsc->n_vars = n_vars;
- if (lex_match ('('))
+ if (lex_match (lexer, T_LPAREN))
{
- if (token != T_ID)
+ if (lex_token (lexer) != T_ID)
{
- lex_error (NULL);
+ lex_error (lexer, NULL);
goto error;
}
- if (try_name (dsc, tokid))
+ if (try_name (dict, dsc, lex_tokcstr (lexer)))
{
- strcpy (dsc->vars[dsc->var_cnt - 1].z_name, tokid);
- z_cnt++;
+ struct dsc_var *dsc_var = &dsc->vars[dsc->n_vars - 1];
+ dsc_var->z_name = xstrdup (lex_tokcstr (lexer));
+ n_zs++;
}
else
msg (SE, _("Z-score variable name %s would be"
- " a duplicate variable name."), tokid);
- lex_get ();
- if (!lex_force_match (')'))
+ " a duplicate variable name."), lex_tokcstr (lexer));
+ lex_get (lexer);
+ if (!lex_force_match (lexer, T_RPAREN))
goto error;
}
}
}
- else
+ else
{
- lex_error (NULL);
- goto error;
+ lex_error (lexer, NULL);
+ goto error;
}
- lex_match ('/');
+ lex_match (lexer, T_SLASH);
}
- if (var_cnt == 0)
+ if (n_vars == 0)
{
msg (SE, _("No variables specified."));
goto error;
}
/* Construct z-score varnames, show translation table. */
- if (z_cnt || save_z_scores)
+ if (n_zs || save_z_scores)
{
- if (save_z_scores)
+ struct caseproto *proto;
+
+ if (save_z_scores)
{
- size_t gen_cnt = 0;
+ int n_gens = 0;
- for (i = 0; i < dsc->var_cnt; i++)
- if (dsc->vars[i].z_name[0] == 0)
- {
- if (!generate_z_varname (dsc, dsc->vars[i].z_name,
- dsc->vars[i].v->name, &gen_cnt))
- goto error;
- z_cnt++;
- }
+ for (i = 0; i < dsc->n_vars; i++)
+ {
+ struct dsc_var *dsc_var = &dsc->vars[i];
+ if (dsc_var->z_name == NULL)
+ {
+ const char *name = var_get_name (dsc_var->v);
+ dsc_var->z_name = generate_z_varname (dict, dsc, name,
+ &n_gens);
+ if (dsc_var->z_name == NULL)
+ goto error;
+
+ n_zs++;
+ }
+ }
}
+
+ /* It would be better to handle Z scores correctly (however we define
+ that) when TEMPORARY is in effect, but in the meantime this at least
+ prevents a use-after-free error. See bug #38786. */
+ if (proc_make_temporary_transformations_permanent (ds))
+ msg (SW, _("DESCRIPTIVES with Z scores ignores TEMPORARY. "
+ "Temporary transformations will be made permanent."));
+
+ proto = caseproto_create ();
+ for (i = 0; i < 1 + 2 * n_zs; i++)
+ proto = caseproto_add_width (proto, 0);
+ dsc->z_writer = autopaging_writer_create (proto);
+ caseproto_unref (proto);
+
dump_z_table (dsc);
}
/* Figure out which statistics to calculate. */
dsc->calc_stats = dsc->show_stats;
- if (z_cnt > 0)
+ if (n_zs > 0)
dsc->calc_stats |= (1ul << DSC_MEAN) | (1ul << DSC_STDDEV);
if (dsc->sort_by_stat >= 0)
dsc->calc_stats |= 1ul << dsc->sort_by_stat;
/* Figure out maximum moment needed and allocate moments for
the variables. */
dsc->max_moment = MOMENT_NONE;
- for (i = 0; i < DSC_N_STATS; i++)
+ for (i = 0; i < DSC_N_STATS; i++)
if (dsc->calc_stats & (1ul << i) && dsc_info[i].moment > dsc->max_moment)
dsc->max_moment = dsc_info[i].moment;
if (dsc->max_moment != MOMENT_NONE)
- for (i = 0; i < dsc->var_cnt; i++)
+ for (i = 0; i < dsc->n_vars; i++)
dsc->vars[i].moments = moments_create (dsc->max_moment);
/* Data pass. */
- ok = multipass_procedure_with_splits (calc_descriptives, dsc);
+ grouper = casegrouper_create_splits (proc_open_filtering (ds, false), dict);
+ while (casegrouper_get_next_group (grouper, &group))
+ calc_descriptives (dsc, group, ds);
+ ok = casegrouper_destroy (grouper);
+ ok = proc_commit (ds) && ok;
/* Z-scoring! */
- if (ok && z_cnt)
- setup_z_trns (dsc);
+ if (ok && n_zs)
+ setup_z_trns (dsc, ds);
/* Done. */
free (vars);
specifiers). Emits an error if the current token ID does not name a
statistic. */
static enum dsc_statistic
-match_statistic (void)
+match_statistic (struct lexer *lexer)
{
- if (token == T_ID)
+ if (lex_token (lexer) == T_ID)
{
enum dsc_statistic stat;
for (stat = 0; stat < DSC_N_STATS; stat++)
- if (lex_match_id (dsc_info[stat].identifier))
+ if (lex_match_id (lexer, dsc_info[stat].identifier))
return stat;
- lex_get();
- lex_error (_("expecting statistic name: reverting to default"));
+ lex_get (lexer);
+ lex_error (lexer, _("expecting statistic name: reverting to default"));
}
return DSC_NONE;
if (dsc == NULL)
return;
-
- for (i = 0; i < dsc->var_cnt; i++)
- moments_destroy (dsc->vars[i].moments);
+
+ for (i = 0; i < dsc->n_vars; i++)
+ {
+ struct dsc_var *dsc_var = &dsc->vars[i];
+ free (dsc_var->z_name);
+ moments_destroy (dsc_var->moments);
+ }
+ casewriter_destroy (dsc->z_writer);
free (dsc->vars);
free (dsc);
}
\f
/* Z scores. */
-/* Returns 0 if NAME is a duplicate of any existing variable name or
- of any previously-declared z-var name; otherwise returns 1. */
-static int
-try_name (struct dsc_proc *dsc, char *name)
+/* Returns false if NAME is a duplicate of any existing variable name or
+ of any previously-declared z-var name; otherwise returns true. */
+static bool
+try_name (const struct dictionary *dict, struct dsc_proc *dsc,
+ const char *name)
{
size_t i;
- if (dict_lookup_var (default_dict, name) != NULL)
- return 0;
- for (i = 0; i < dsc->var_cnt; i++)
- if (!strcasecmp (dsc->vars[i].z_name, name))
- return 0;
- return 1;
+ if (dict_lookup_var (dict, name) != NULL)
+ return false;
+ for (i = 0; i < dsc->n_vars; i++)
+ {
+ struct dsc_var *dsc_var = &dsc->vars[i];
+ if (dsc_var->z_name != NULL && !utf8_strcasecmp (dsc_var->z_name, name))
+ return false;
+ }
+ return true;
}
/* Generates a name for a Z-score variable based on a variable
named VAR_NAME, given that *Z_CNT generated variable names are
- known to already exist. If successful, returns nonzero and
- copies the new name into Z_NAME. On failure, returns zero. */
-static int
-generate_z_varname (struct dsc_proc *dsc, char *z_name,
- const char *var_name, size_t *z_cnt)
+ known to already exist. If successful, returns the new name
+ as a dynamically allocated string. On failure, returns NULL. */
+static char *
+generate_z_varname (const struct dictionary *dict, struct dsc_proc *dsc,
+ const char *var_name, int *n_zs)
{
- char name[LONG_NAME_LEN + 1];
+ char *z_name, *trunc_name;
/* Try a name based on the original variable name. */
- name[0] = 'Z';
- str_copy_trunc (name + 1, sizeof name - 1, var_name);
- if (try_name (dsc, name))
- {
- strcpy (z_name, name);
- return 1;
- }
+ z_name = xasprintf ("Z%s", var_name);
+ trunc_name = utf8_encoding_trunc (z_name, dict_get_encoding (dict),
+ ID_MAX_LEN);
+ free (z_name);
+ if (try_name (dict, dsc, trunc_name))
+ return trunc_name;
+ free (trunc_name);
/* Generate a synthetic name. */
for (;;)
{
- (*z_cnt)++;
-
- if (*z_cnt <= 99)
- sprintf (name, "ZSC%03d", *z_cnt);
- else if (*z_cnt <= 108)
- sprintf (name, "STDZ%02d", *z_cnt - 99);
- else if (*z_cnt <= 117)
- sprintf (name, "ZZZZ%02d", *z_cnt - 108);
- else if (*z_cnt <= 126)
- sprintf (name, "ZQZQ%02d", *z_cnt - 117);
+ char name[16];
+
+ (*n_zs)++;
+
+ if (*n_zs <= 99)
+ sprintf (name, "ZSC%03d", *n_zs);
+ else if (*n_zs <= 108)
+ sprintf (name, "STDZ%02d", *n_zs - 99);
+ else if (*n_zs <= 117)
+ sprintf (name, "ZZZZ%02d", *n_zs - 108);
+ else if (*n_zs <= 126)
+ sprintf (name, "ZQZQ%02d", *n_zs - 117);
else
{
msg (SE, _("Ran out of generic names for Z-score variables. "
"There are only 126 generic names: ZSC001-ZSC0999, "
"STDZ01-STDZ09, ZZZZ01-ZZZZ09, ZQZQ01-ZQZQ09."));
- return 0;
- }
-
- if (try_name (dsc, name))
- {
- strcpy (z_name, name);
- return 1;
+ return NULL;
}
+
+ if (try_name (dict, dsc, name))
+ return xstrdup (name);
}
+ NOT_REACHED();
}
/* Outputs a table describing the mapping between source
static void
dump_z_table (struct dsc_proc *dsc)
{
- size_t cnt = 0;
- struct tab_table *t;
-
- {
- size_t i;
-
- for (i = 0; i < dsc->var_cnt; i++)
- if (dsc->vars[i].z_name[0] != '\0')
- cnt++;
- }
-
- t = tab_create (2, cnt + 1, 0);
- tab_title (t, _("Mapping of variables to corresponding Z-scores."));
- tab_columns (t, SOM_COL_DOWN, 1);
- tab_headers (t, 0, 0, 1, 0);
- tab_box (t, TAL_1, TAL_1, TAL_0, TAL_1, 0, 0, 1, cnt);
- tab_hline (t, TAL_2, 0, 1, 1);
- tab_text (t, 0, 0, TAB_CENTER | TAT_TITLE, _("Source"));
- tab_text (t, 1, 0, TAB_CENTER | TAT_TITLE, _("Target"));
- tab_dim (t, tab_natural_dimensions);
+ struct pivot_table *table = pivot_table_create (
+ N_("Mapping of Variables to Z-scores"));
- {
- size_t i, y;
-
- for (i = 0, y = 1; i < dsc->var_cnt; i++)
- if (dsc->vars[i].z_name[0] != '\0')
- {
- tab_text (t, 0, y, TAB_LEFT, dsc->vars[i].v->name);
- tab_text (t, 1, y++, TAB_LEFT, dsc->vars[i].z_name);
- }
- }
-
- tab_submit (t);
+ pivot_dimension_create (table, PIVOT_AXIS_COLUMN, N_("Names"),
+ N_("Source"), N_("Target"));
+
+ struct pivot_dimension *names = pivot_dimension_create (
+ table, PIVOT_AXIS_ROW, N_("Variables"));
+ names->hide_all_labels = true;
+
+ for (size_t i = 0; i < dsc->n_vars; i++)
+ if (dsc->vars[i].z_name != NULL)
+ {
+ int row = pivot_category_create_leaf (names->root,
+ pivot_value_new_number (i));
+
+ pivot_table_put2 (table, 0, row,
+ pivot_value_new_variable (dsc->vars[i].v));
+ pivot_table_put2 (table, 1, row,
+ pivot_value_new_user_text (dsc->vars[i].z_name, -1));
+ }
+
+ pivot_table_submit (table);
+}
+
+static void
+descriptives_set_all_sysmis_zscores (const struct dsc_trns *t, struct ccase *c)
+{
+ const struct dsc_z_score *z;
+
+ for (z = t->z_scores; z < t->z_scores + t->n_z_scores; z++)
+ *case_num_rw (c, z->z_var) = SYSMIS;
}
/* Transformation function to calculate Z-scores. Will return SYSMIS if any of
analyis. 4) any of the variables in the original analysis were missing
(either system or user-missing values that weren't included).
*/
-static int
-descriptives_trns_proc (void *trns_, struct ccase * c,
- int case_idx UNUSED)
+static enum trns_result
+descriptives_trns_proc (void *trns_, struct ccase **c,
+ casenumber case_idx UNUSED)
{
struct dsc_trns *t = trns_;
struct dsc_z_score *z;
- struct variable **vars;
- int all_sysmis = 0;
+ const struct variable **vars;
+
+ *c = case_unshare (*c);
+
+ if (t->filter)
+ {
+ double f = case_num (*c, t->filter);
+ if (f == 0.0 || var_is_num_missing (t->filter, f))
+ {
+ descriptives_set_all_sysmis_zscores (t, *c);
+ return TRNS_CONTINUE;
+ }
+ }
+
+ if (t->count <= 0)
+ {
+ struct ccase *z_case;
+
+ z_case = casereader_read (t->z_reader);
+ if (z_case)
+ {
+ size_t z_idx = 0;
+
+ t->count = case_num_idx (z_case, z_idx++);
+ for (z = t->z_scores; z < t->z_scores + t->n_z_scores; z++)
+ {
+ z->mean = case_num_idx (z_case, z_idx++);
+ z->std_dev = case_num_idx (z_case, z_idx++);
+ }
+ case_unref (z_case);
+ }
+ else
+ {
+ if (t->ok)
+ {
+ msg (SE, _("Internal error processing Z scores. "
+ "Please report this to %s."),
+ PACKAGE_BUGREPORT);
+ t->ok = false;
+ }
+ descriptives_set_all_sysmis_zscores (t, *c);
+ return TRNS_CONTINUE;
+ }
+ }
+ t->count--;
if (t->missing_type == DSC_LISTWISE)
{
assert(t->vars);
- for (vars = t->vars; vars < t->vars + t->var_cnt; vars++)
+ for (vars = t->vars; vars < t->vars + t->n_vars; vars++)
{
- double score = case_num (c, (*vars)->fv);
- if ( score == SYSMIS
- || (!t->include_user_missing
- && mv_is_num_user_missing (&(*vars)->miss, score)))
+ double score = case_num (*c, *vars);
+ if (var_is_num_missing (*vars, score) & t->exclude)
{
- all_sysmis = 1;
- break;
+ descriptives_set_all_sysmis_zscores (t, *c);
+ return TRNS_CONTINUE;
}
}
}
-
- for (z = t->z_scores; z < t->z_scores + t->z_score_cnt; z++)
+
+ for (z = t->z_scores; z < t->z_scores + t->n_z_scores; z++)
{
- double input = case_num (c, z->src_idx);
- double *output = &case_data_rw (c, z->dst_idx)->f;
+ double input = case_num (*c, z->src_var);
+ double *output = case_num_rw (*c, z->z_var);
- if (z->mean == SYSMIS || z->std_dev == SYSMIS
- || all_sysmis || input == SYSMIS
- || (!t->include_user_missing
- && mv_is_num_user_missing (&z->v->miss, input)))
+ if (z->mean == SYSMIS || z->std_dev == SYSMIS
+ || var_is_num_missing (z->src_var, input) & t->exclude)
*output = SYSMIS;
else
*output = (input - z->mean) / z->std_dev;
descriptives_trns_free (void *trns_)
{
struct dsc_trns *t = trns_;
+ bool ok = t->ok && !casereader_error (t->z_reader);
free (t->z_scores);
+ casereader_destroy (t->z_reader);
assert((t->missing_type != DSC_LISTWISE) ^ (t->vars != NULL));
free (t->vars);
- return true;
+ free (t);
+
+ return ok;
}
+static const struct trns_class descriptives_trns_class = {
+ .name = "DESCRIPTIVES (Z scores)",
+ .execute = descriptives_trns_proc,
+ .destroy = descriptives_trns_free,
+};
+
/* Sets up a transformation to calculate Z scores. */
static void
-setup_z_trns (struct dsc_proc *dsc)
+setup_z_trns (struct dsc_proc *dsc, struct dataset *ds)
{
struct dsc_trns *t;
- size_t cnt, i;
+ size_t n, i;
- for (cnt = i = 0; i < dsc->var_cnt; i++)
- if (dsc->vars[i].z_name[0] != '\0')
- cnt++;
+ for (n = i = 0; i < dsc->n_vars; i++)
+ if (dsc->vars[i].z_name != NULL)
+ n++;
t = xmalloc (sizeof *t);
- t->z_scores = xnmalloc (cnt, sizeof *t->z_scores);
- t->z_score_cnt = cnt;
+ t->z_scores = xnmalloc (n, sizeof *t->z_scores);
+ t->n_z_scores = n;
t->missing_type = dsc->missing_type;
- t->include_user_missing = dsc->include_user_missing;
- if ( t->missing_type == DSC_LISTWISE )
+ t->exclude = dsc->exclude;
+ if (t->missing_type == DSC_LISTWISE)
{
- t->var_cnt = dsc->var_cnt;
- t->vars = xnmalloc (t->var_cnt, sizeof *t->vars);
- for (i = 0; i < t->var_cnt; i++)
+ t->n_vars = dsc->n_vars;
+ t->vars = xnmalloc (t->n_vars, sizeof *t->vars);
+ for (i = 0; i < t->n_vars; i++)
t->vars[i] = dsc->vars[i].v;
}
else
{
- t->var_cnt = 0;
+ t->n_vars = 0;
t->vars = NULL;
}
+ t->filter = dict_get_filter (dataset_dict (ds));
+ t->z_reader = casewriter_make_reader (dsc->z_writer);
+ t->count = 0;
+ t->ok = true;
+ dsc->z_writer = NULL;
- for (cnt = i = 0; i < dsc->var_cnt; i++)
+ for (n = i = 0; i < dsc->n_vars; i++)
{
struct dsc_var *dv = &dsc->vars[i];
- if (dv->z_name[0] != '\0')
+ if (dv->z_name != NULL)
{
struct dsc_z_score *z;
- char *cp;
struct variable *dst_var;
+ char *label;
- dst_var = dict_create_var_assert (default_dict, dv->z_name, 0);
- if (dv->v->label)
- {
- dst_var->label = xmalloc (strlen (dv->v->label) + 12);
- cp = stpcpy (dst_var->label, _("Z-score of "));
- strcpy (cp, dv->v->label);
- }
- else
- {
- dst_var->label = xmalloc (strlen (dv->v->name) + 12);
- cp = stpcpy (dst_var->label, _("Z-score of "));
- strcpy (cp, dv->v->name);
- }
+ dst_var = dict_create_var_assert (dataset_dict (ds), dv->z_name, 0);
+
+ label = xasprintf (_("Z-score of %s"),var_to_string (dv->v));
+ var_set_label (dst_var, label);
+ free (label);
- z = &t->z_scores[cnt++];
- z->src_idx = dv->v->fv;
- z->dst_idx = dst_var->fv;
- z->mean = dv->stats[DSC_MEAN];
- z->std_dev = dv->stats[DSC_STDDEV];
- z->v = dv->v;
+ z = &t->z_scores[n++];
+ z->src_var = dv->v;
+ z->z_var = dst_var;
}
}
- add_transformation (descriptives_trns_proc, descriptives_trns_free, t);
+ add_transformation (ds, &descriptives_trns_class, t);
}
\f
/* Statistical calculation. */
-static int listwise_missing (struct dsc_proc *dsc, const struct ccase *c);
+static bool listwise_missing (struct dsc_proc *dsc, const struct ccase *c);
/* Calculates and displays descriptive statistics for the cases
in CF. */
-static bool
-calc_descriptives (const struct casefile *cf, void *dsc_)
+static void
+calc_descriptives (struct dsc_proc *dsc, struct casereader *group,
+ struct dataset *ds)
{
- struct dsc_proc *dsc = dsc_;
- struct casereader *reader;
- struct ccase c;
+ const struct variable *filter = dict_get_filter (dataset_dict (ds));
+ struct casereader *pass1, *pass2;
+ casenumber count;
+ struct ccase *c;
+ size_t z_idx;
size_t i;
- for (i = 0; i < dsc->var_cnt; i++)
+ c = casereader_peek (group, 0);
+ if (c == NULL)
+ {
+ casereader_destroy (group);
+ return;
+ }
+ output_split_file_values (ds, c);
+ case_unref (c);
+
+ group = casereader_create_filter_weight (group, dataset_dict (ds),
+ NULL, NULL);
+
+ pass1 = group;
+ pass2 = dsc->max_moment <= MOMENT_MEAN ? NULL : casereader_clone (pass1);
+
+ for (i = 0; i < dsc->n_vars; i++)
{
struct dsc_var *dv = &dsc->vars[i];
-
+
dv->valid = dv->missing = 0.0;
if (dv->moments != NULL)
moments_clear (dv->moments);
dsc->valid = 0.;
/* First pass to handle most of the work. */
- for (reader = casefile_get_reader (cf);
- casereader_read (reader, &c);
- case_destroy (&c))
+ count = 0;
+ for (; (c = casereader_read (pass1)) != NULL; case_unref (c))
{
- double weight = dict_get_case_weight (default_dict, &c, &dsc->bad_warn);
- if (weight <= 0.0)
- continue;
-
+ double weight = dict_get_case_weight (dataset_dict (ds), c, NULL);
+
+ if (filter)
+ {
+ double f = case_num (c, filter);
+ if (f == 0.0 || var_is_num_missing (filter, f))
+ continue;
+ }
+
/* Check for missing values. */
- if (listwise_missing (dsc, &c))
+ if (listwise_missing (dsc, c))
{
dsc->missing_listwise += weight;
if (dsc->missing_type == DSC_LISTWISE)
- continue;
+ continue;
}
dsc->valid += weight;
- for (i = 0; i < dsc->var_cnt; i++)
+ for (i = 0; i < dsc->n_vars; i++)
{
struct dsc_var *dv = &dsc->vars[i];
- double x = case_num (&c, dv->v->fv);
-
- if (dsc->missing_type != DSC_LISTWISE
- && (x == SYSMIS
- || (!dsc->include_user_missing
- && mv_is_num_user_missing (&dv->v->miss, x))))
+ double x = case_num (c, dv->v);
+
+ if (var_is_num_missing (dv->v, x) & dsc->exclude)
{
dv->missing += weight;
continue;
}
- if (dv->moments != NULL)
+ if (dv->moments != NULL)
moments_pass_one (dv->moments, x, weight);
if (x < dv->min)
if (x > dv->max)
dv->max = x;
}
+
+ count++;
+ }
+ if (!casereader_destroy (pass1))
+ {
+ casereader_destroy (pass2);
+ return;
}
- casereader_destroy (reader);
/* Second pass for higher-order moments. */
- if (dsc->max_moment > MOMENT_MEAN)
+ if (dsc->max_moment > MOMENT_MEAN)
{
- for (reader = casefile_get_reader (cf);
- casereader_read (reader, &c);
- case_destroy (&c))
+ for (; (c = casereader_read (pass2)) != NULL; case_unref (c))
{
- double weight = dict_get_case_weight (default_dict, &c,
- &dsc->bad_warn);
- if (weight <= 0.0)
- continue;
-
+ double weight = dict_get_case_weight (dataset_dict (ds), c, NULL);
+
+ if (filter)
+ {
+ double f = case_num (c, filter);
+ if (f == 0.0 || var_is_num_missing (filter, f))
+ continue;
+ }
+
/* Check for missing values. */
- if (listwise_missing (dsc, &c)
- && dsc->missing_type == DSC_LISTWISE)
- continue;
+ if (dsc->missing_type == DSC_LISTWISE && listwise_missing (dsc, c))
+ continue;
- for (i = 0; i < dsc->var_cnt; i++)
+ for (i = 0; i < dsc->n_vars; i++)
{
struct dsc_var *dv = &dsc->vars[i];
- double x = case_num (&c, dv->v->fv);
-
- if (dsc->missing_type != DSC_LISTWISE
- && (x == SYSMIS
- || (!dsc->include_user_missing
- && mv_is_num_user_missing (&dv->v->miss, x))))
+ double x = case_num (c, dv->v);
+
+ if (var_is_num_missing (dv->v, x) & dsc->exclude)
continue;
if (dv->moments != NULL)
moments_pass_two (dv->moments, x, weight);
}
}
- casereader_destroy (reader);
+ if (!casereader_destroy (pass2))
+ return;
}
-
+
/* Calculate results. */
- for (i = 0; i < dsc->var_cnt; i++)
+ if (dsc->z_writer && count > 0)
+ {
+ c = case_create (casewriter_get_proto (dsc->z_writer));
+ z_idx = 0;
+ *case_num_rw_idx (c, z_idx++) = count;
+ }
+ else
+ c = NULL;
+
+ for (i = 0; i < dsc->n_vars; i++)
{
struct dsc_var *dv = &dsc->vars[i];
double W;
if (dsc->calc_stats & (1ul << DSC_STDDEV)
&& dv->stats[DSC_VARIANCE] != SYSMIS)
dv->stats[DSC_STDDEV] = sqrt (dv->stats[DSC_VARIANCE]);
- if (dsc->calc_stats & (1ul << DSC_SEKURT))
+ if (dsc->calc_stats & (1ul << DSC_SEKURT))
if (dv->stats[DSC_KURTOSIS] != SYSMIS)
dv->stats[DSC_SEKURT] = calc_sekurt (W);
if (dsc->calc_stats & (1ul << DSC_SESKEW)
dv->stats[DSC_MAX] = dv->max == -DBL_MAX ? SYSMIS : dv->max;
if (dsc->calc_stats & (1ul << DSC_SUM))
dv->stats[DSC_SUM] = W * dv->stats[DSC_MEAN];
+
+ if (dv->z_name && c != NULL)
+ {
+ *case_num_rw_idx (c, z_idx++) = dv->stats[DSC_MEAN];
+ *case_num_rw_idx (c, z_idx++) = dv->stats[DSC_STDDEV];
+ }
}
+ if (c != NULL)
+ casewriter_write (dsc->z_writer, c);
+
/* Output results. */
display (dsc);
-
- return true;
}
-/* Returns nonzero if any of the descriptives variables in DSC's
- variable list have missing values in case C, zero otherwise. */
-static int
-listwise_missing (struct dsc_proc *dsc, const struct ccase *c)
+/* Returns true if any of the descriptives variables in DSC's
+ variable list have missing values in case C, false otherwise. */
+static bool
+listwise_missing (struct dsc_proc *dsc, const struct ccase *c)
{
size_t i;
- for (i = 0; i < dsc->var_cnt; i++)
+ for (i = 0; i < dsc->n_vars; i++)
{
struct dsc_var *dv = &dsc->vars[i];
- double x = case_num (c, dv->v->fv);
+ double x = case_num (c, dv->v);
- if (x == SYSMIS
- || (!dsc->include_user_missing
- && mv_is_num_user_missing (&dv->v->miss, x)))
- return 1;
+ if (var_is_num_missing (dv->v, x) & dsc->exclude)
+ return true;
}
- return 0;
+ return false;
}
\f
/* Statistical display. */
static void
display (struct dsc_proc *dsc)
{
- size_t i;
- int nc;
- struct tab_table *t;
-
- nc = 1 + (dsc->format == DSC_SERIAL ? 2 : 1);
- for (i = 0; i < DSC_N_STATS; i++)
+ struct pivot_table *table = pivot_table_create (
+ N_("Descriptive Statistics"));
+ pivot_table_set_weight_var (table, dict_get_weight (dsc->dict));
+
+ struct pivot_dimension *statistics = pivot_dimension_create (
+ table, PIVOT_AXIS_COLUMN, N_("Statistics"));
+ pivot_category_create_leaf_rc (
+ statistics->root, pivot_value_new_text (N_("N")), PIVOT_RC_COUNT);
+ for (int i = 0; i < DSC_N_STATS; i++)
if (dsc->show_stats & (1ul << i))
- nc++;
+ pivot_category_create_leaf (statistics->root,
+ pivot_value_new_text (dsc_info[i].name));
if (dsc->sort_by_stat != DSC_NONE)
- sort (dsc->vars, dsc->var_cnt, sizeof *dsc->vars,
+ sort (dsc->vars, dsc->n_vars, sizeof *dsc->vars,
descriptives_compare_dsc_vars, dsc);
- t = tab_create (nc, dsc->var_cnt + 1, 0);
- tab_headers (t, 1, 0, 1, 0);
- tab_box (t, TAL_1, TAL_1, -1, -1, 0, 0, nc - 1, dsc->var_cnt);
- tab_box (t, -1, -1, -1, TAL_1, 1, 0, nc - 1, dsc->var_cnt);
- tab_hline (t, TAL_2, 0, nc - 1, 1);
- tab_vline (t, TAL_2, 1, 0, dsc->var_cnt);
- tab_dim (t, tab_natural_dimensions);
-
- nc = 0;
- tab_text (t, nc++, 0, TAB_LEFT | TAT_TITLE, _("Variable"));
- if (dsc->format == DSC_SERIAL)
+ struct pivot_dimension *variables = pivot_dimension_create (
+ table, PIVOT_AXIS_ROW, N_("Variable"));
+ for (size_t i = 0; i < dsc->n_vars; i++)
{
- tab_text (t, nc++, 0, TAB_CENTER | TAT_TITLE, _("Valid N"));
- tab_text (t, nc++, 0, TAB_CENTER | TAT_TITLE, _("Missing N"));
- }
- else
- tab_text (t, nc++, 0, TAB_CENTER | TAT_TITLE, "N");
+ const struct dsc_var *dv = &dsc->vars[i];
- for (i = 0; i < DSC_N_STATS; i++)
- if (dsc->show_stats & (1ul << i))
- {
- const char *title = gettext (dsc_info[i].name);
- tab_text (t, nc++, 0, TAB_CENTER | TAT_TITLE, title);
- }
+ int row = pivot_category_create_leaf (variables->root,
+ pivot_value_new_variable (dv->v));
- for (i = 0; i < dsc->var_cnt; i++)
- {
- struct dsc_var *dv = &dsc->vars[i];
- size_t j;
+ int column = 0;
+ pivot_table_put2 (table, column++, row,
+ pivot_value_new_number (dv->valid));
- nc = 0;
- tab_text (t, nc++, i + 1, TAB_LEFT, dv->v->name);
- tab_text (t, nc++, i + 1, TAT_PRINTF, "%g", dv->valid);
- if (dsc->format == DSC_SERIAL)
- tab_text (t, nc++, i + 1, TAT_PRINTF, "%g", dv->missing);
- for (j = 0; j < DSC_N_STATS; j++)
+ for (int j = 0; j < DSC_N_STATS; j++)
if (dsc->show_stats & (1ul << j))
- tab_float (t, nc++, i + 1, TAB_NONE, dv->stats[j], 10, 3);
+ {
+ union value v = { .f = dv->stats[j] };
+ struct pivot_value *pv = (j == DSC_MIN || j == DSC_MAX
+ ? pivot_value_new_var_value (dv->v, &v)
+ : pivot_value_new_number (dv->stats[j]));
+ pivot_table_put2 (table, column++, row, pv);
+ }
}
- tab_title (t, _("Valid cases = %g; cases with missing value(s) = %g."),
- dsc->valid, dsc->missing_listwise);
-
- tab_submit (t);
+ int row = pivot_category_create_leaves (
+ variables->root, N_("Valid N (listwise)"), N_("Missing N (listwise)"));
+ pivot_table_put2 (table, 0, row, pivot_value_new_number (dsc->valid));
+ pivot_table_put2 (table, 0, row + 1,
+ pivot_value_new_number (dsc->missing_listwise));
+ pivot_table_submit (table);
}
/* Compares `struct dsc_var's A and B according to the ordering
specified by CMD. */
static int
-descriptives_compare_dsc_vars (const void *a_, const void *b_, void *dsc_)
+descriptives_compare_dsc_vars (const void *a_, const void *b_, const void *dsc_)
{
const struct dsc_var *a = a_;
const struct dsc_var *b = b_;
- struct dsc_proc *dsc = dsc_;
+ const struct dsc_proc *dsc = dsc_;
int result;
if (dsc->sort_by_stat == DSC_NAME)
- result = strcasecmp (a->v->name, b->v->name);
- else
+ result = utf8_strcasecmp (var_get_name (a->v), var_get_name (b->v));
+ else
{
double as = a->stats[dsc->sort_by_stat];
double bs = b->stats[dsc->sort_by_stat];