/* PSPP - a program for statistical analysis.
- Copyright (C) 1997-9, 2000, 2009 Free Software Foundation, Inc.
+ Copyright (C) 1997-2000, 2009-2014 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
#include <config.h>
+#include <float.h>
#include <limits.h>
#include <math.h>
#include <stdlib.h>
-#include <data/casegrouper.h>
-#include <data/casereader.h>
-#include <data/dictionary.h>
-#include <data/procedure.h>
-#include <data/transformations.h>
-#include <data/variable.h>
-#include <language/command.h>
-#include <language/dictionary/split-file.h>
-#include <language/lexer/lexer.h>
-#include <language/lexer/variable-parser.h>
-#include <libpspp/array.h>
-#include <libpspp/compiler.h>
-#include <libpspp/message.h>
-#include <libpspp/assertion.h>
-#include <math/moments.h>
-#include <output/tab.h>
-
-#include "xalloc.h"
+#include "data/casegrouper.h"
+#include "data/casereader.h"
+#include "data/casewriter.h"
+#include "data/dataset.h"
+#include "data/dictionary.h"
+#include "data/transformations.h"
+#include "data/variable.h"
+#include "language/command.h"
+#include "language/dictionary/split-file.h"
+#include "language/lexer/lexer.h"
+#include "language/lexer/variable-parser.h"
+#include "libpspp/array.h"
+#include "libpspp/assertion.h"
+#include "libpspp/compiler.h"
+#include "libpspp/i18n.h"
+#include "libpspp/message.h"
+#include "math/moments.h"
+#include "output/tab.h"
+
+#include "gl/xalloc.h"
#include "gettext.h"
#define _(msgid) gettext (msgid)
size_t var_cnt; /* Number of variables. */
enum dsc_missing_type missing_type; /* Treatment of missing values. */
enum mv_class exclude; /* Classes of missing values to exclude. */
+ struct casereader *z_reader; /* Reader for count, mean, stddev. */
+ casenumber count; /* Number left in this SPLIT FILE group.*/
+ bool ok;
};
/* Statistics. Used as bit indexes, so must be 32 or fewer. */
struct dsc_var
{
const struct variable *v; /* Variable to calculate on. */
- char z_name[VAR_NAME_LEN + 1]; /* Name for z-score variable. */
+ char *z_name; /* Name for z-score variable. */
double valid, missing; /* Valid, missing counts. */
struct moments *moments; /* Moments. */
double min, max; /* Maximum and mimimum values. */
unsigned long show_stats; /* Statistics to display. */
unsigned long calc_stats; /* Statistics to calculate. */
enum moment max_moment; /* Highest moment needed for stats. */
+
+ /* Z scores. */
+ struct casewriter *z_writer; /* Mean and stddev per SPLIT FILE group. */
};
/* Parsing. */
/* Z-score functions. */
static bool try_name (const struct dictionary *dict,
struct dsc_proc *dsc, const char *name);
-static bool generate_z_varname (const struct dictionary *dict,
- struct dsc_proc *dsc, char *z_name,
- const char *name, int *z_cnt);
+static char *generate_z_varname (const struct dictionary *dict,
+ struct dsc_proc *dsc,
+ const char *name, int *z_cnt);
static void dump_z_table (struct dsc_proc *);
static void setup_z_trns (struct dsc_proc *, struct dataset *);
dsc->sort_by_stat = DSC_NONE;
dsc->sort_ascending = 1;
dsc->show_stats = dsc->calc_stats = DEFAULT_STATS;
+ dsc->z_writer = NULL;
/* Parse DESCRIPTIVES. */
- while (lex_token (lexer) != '.')
+ while (lex_token (lexer) != T_ENDCMD)
{
if (lex_match_id (lexer, "MISSING"))
{
- lex_match (lexer, '=');
- while (lex_token (lexer) != '.' && lex_token (lexer) != '/')
+ lex_match (lexer, T_EQUALS);
+ while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
{
if (lex_match_id (lexer, "VARIABLE"))
dsc->missing_type = DSC_VARIABLE;
lex_error (lexer, NULL);
goto error;
}
- lex_match (lexer, ',');
+ lex_match (lexer, T_COMMA);
}
}
else if (lex_match_id (lexer, "SAVE"))
save_z_scores = 1;
else if (lex_match_id (lexer, "FORMAT"))
{
- lex_match (lexer, '=');
- while (lex_token (lexer) != '.' && lex_token (lexer) != '/')
+ lex_match (lexer, T_EQUALS);
+ while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
{
if (lex_match_id (lexer, "LABELS"))
dsc->show_var_labels = 1;
lex_error (lexer, NULL);
goto error;
}
- lex_match (lexer, ',');
+ lex_match (lexer, T_COMMA);
}
}
else if (lex_match_id (lexer, "STATISTICS"))
{
- lex_match (lexer, '=');
+ lex_match (lexer, T_EQUALS);
dsc->show_stats = 0;
- while (lex_token (lexer) != '.' && lex_token (lexer) != '/')
+ while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
{
if (lex_match (lexer, T_ALL))
dsc->show_stats |= (1ul << DSC_N_STATS) - 1;
dsc->show_stats |= DEFAULT_STATS;
else
dsc->show_stats |= 1ul << (match_statistic (lexer));
- lex_match (lexer, ',');
+ lex_match (lexer, T_COMMA);
}
if (dsc->show_stats == 0)
dsc->show_stats = DEFAULT_STATS;
}
else if (lex_match_id (lexer, "SORT"))
{
- lex_match (lexer, '=');
+ lex_match (lexer, T_EQUALS);
if (lex_match_id (lexer, "NAME"))
dsc->sort_by_stat = DSC_NAME;
else
if (dsc->sort_by_stat == DSC_NONE )
dsc->sort_by_stat = DSC_MEAN;
}
- if (lex_match (lexer, '('))
+ if (lex_match (lexer, T_LPAREN))
{
if (lex_match_id (lexer, "A"))
dsc->sort_ascending = 1;
dsc->sort_ascending = 0;
else
lex_error (lexer, NULL);
- lex_force_match (lexer, ')');
+ lex_force_match (lexer, T_RPAREN);
}
}
else if (var_cnt == 0)
{
- if (lex_look_ahead (lexer) == '=')
+ if (lex_next_token (lexer, 1) == T_EQUALS)
{
lex_match_id (lexer, "VARIABLES");
- lex_match (lexer, '=');
+ lex_match (lexer, T_EQUALS);
}
- while (lex_token (lexer) != '.' && lex_token (lexer) != '/')
+ while (lex_token (lexer) != T_ENDCMD && lex_token (lexer) != T_SLASH)
{
int i;
{
struct dsc_var *dv = &dsc->vars[i];
dv->v = vars[i];
- dv->z_name[0] = '\0';
+ dv->z_name = NULL;
dv->moments = NULL;
}
dsc->var_cnt = var_cnt;
- if (lex_match (lexer, '('))
+ if (lex_match (lexer, T_LPAREN))
{
if (lex_token (lexer) != T_ID)
{
lex_error (lexer, NULL);
goto error;
}
- if (try_name (dict, dsc, lex_tokid (lexer)))
+ if (try_name (dict, dsc, lex_tokcstr (lexer)))
{
- strcpy (dsc->vars[dsc->var_cnt - 1].z_name, lex_tokid (lexer));
+ struct dsc_var *dsc_var = &dsc->vars[dsc->var_cnt - 1];
+ dsc_var->z_name = xstrdup (lex_tokcstr (lexer));
z_cnt++;
}
else
msg (SE, _("Z-score variable name %s would be"
- " a duplicate variable name."), lex_tokid (lexer));
+ " a duplicate variable name."), lex_tokcstr (lexer));
lex_get (lexer);
- if (!lex_force_match (lexer, ')'))
+ if (!lex_force_match (lexer, T_RPAREN))
goto error;
}
}
goto error;
}
- lex_match (lexer, '/');
+ lex_match (lexer, T_SLASH);
}
if (var_cnt == 0)
{
/* Construct z-score varnames, show translation table. */
if (z_cnt || save_z_scores)
{
+ struct caseproto *proto;
+
if (save_z_scores)
{
int gen_cnt = 0;
for (i = 0; i < dsc->var_cnt; i++)
- if (dsc->vars[i].z_name[0] == 0)
- {
- if (!generate_z_varname (dict, dsc, dsc->vars[i].z_name,
- var_get_name (dsc->vars[i].v),
- &gen_cnt))
- goto error;
- z_cnt++;
- }
+ {
+ struct dsc_var *dsc_var = &dsc->vars[i];
+ if (dsc_var->z_name == NULL)
+ {
+ const char *name = var_get_name (dsc_var->v);
+ dsc_var->z_name = generate_z_varname (dict, dsc, name,
+ &gen_cnt);
+ if (dsc_var->z_name == NULL)
+ goto error;
+
+ z_cnt++;
+ }
+ }
}
+
+ /* It would be better to handle Z scores correctly (however we define
+ that) when TEMPORARY is in effect, but in the meantime this at least
+ prevents a use-after-free error. See bug #38786. */
+ if (proc_make_temporary_transformations_permanent (ds))
+ msg (SW, _("DESCRIPTIVES with Z scores ignores TEMPORARY. "
+ "Temporary transformations will be made permanent."));
+
+ proto = caseproto_create ();
+ for (i = 0; i < 1 + 2 * z_cnt; i++)
+ proto = caseproto_add_width (proto, 0);
+ dsc->z_writer = autopaging_writer_create (proto);
+ caseproto_unref (proto);
+
dump_z_table (dsc);
}
dsc->vars[i].moments = moments_create (dsc->max_moment);
/* Data pass. */
- grouper = casegrouper_create_splits (proc_open (ds), dict);
+ grouper = casegrouper_create_splits (proc_open_filtering (ds, z_cnt == 0),
+ dict);
while (casegrouper_get_next_group (grouper, &group))
calc_descriptives (dsc, group, ds);
ok = casegrouper_destroy (grouper);
return;
for (i = 0; i < dsc->var_cnt; i++)
- moments_destroy (dsc->vars[i].moments);
+ {
+ struct dsc_var *dsc_var = &dsc->vars[i];
+ free (dsc_var->z_name);
+ moments_destroy (dsc_var->moments);
+ }
+ casewriter_destroy (dsc->z_writer);
free (dsc->vars);
free (dsc);
}
if (dict_lookup_var (dict, name) != NULL)
return false;
for (i = 0; i < dsc->var_cnt; i++)
- if (!strcasecmp (dsc->vars[i].z_name, name))
- return false;
+ {
+ struct dsc_var *dsc_var = &dsc->vars[i];
+ if (dsc_var->z_name != NULL && !utf8_strcasecmp (dsc_var->z_name, name))
+ return false;
+ }
return true;
}
/* Generates a name for a Z-score variable based on a variable
named VAR_NAME, given that *Z_CNT generated variable names are
- known to already exist. If successful, returns true and
- copies the new name into Z_NAME. On failure, returns false. */
-static bool
-generate_z_varname (const struct dictionary *dict, struct dsc_proc *dsc, char *z_name,
+ known to already exist. If successful, returns the new name
+ as a dynamically allocated string. On failure, returns NULL. */
+static char *
+generate_z_varname (const struct dictionary *dict, struct dsc_proc *dsc,
const char *var_name, int *z_cnt)
{
- char name[VAR_NAME_LEN + 1];
+ char *z_name, *trunc_name;
/* Try a name based on the original variable name. */
- name[0] = 'Z';
- str_copy_trunc (name + 1, sizeof name - 1, var_name);
- if (try_name (dict, dsc, name))
- {
- strcpy (z_name, name);
- return true;
- }
+ z_name = xasprintf ("Z%s", var_name);
+ trunc_name = utf8_encoding_trunc (z_name, dict_get_encoding (dict),
+ ID_MAX_LEN);
+ free (z_name);
+ if (try_name (dict, dsc, trunc_name))
+ return trunc_name;
+ free (trunc_name);
/* Generate a synthetic name. */
for (;;)
{
+ char name[8];
+
(*z_cnt)++;
if (*z_cnt <= 99)
msg (SE, _("Ran out of generic names for Z-score variables. "
"There are only 126 generic names: ZSC001-ZSC0999, "
"STDZ01-STDZ09, ZZZZ01-ZZZZ09, ZQZQ01-ZQZQ09."));
- return false;
+ return NULL;
}
if (try_name (dict, dsc, name))
- {
- strcpy (z_name, name);
- return true;
- }
+ return xstrdup (name);
}
NOT_REACHED();
}
size_t i;
for (i = 0; i < dsc->var_cnt; i++)
- if (dsc->vars[i].z_name[0] != '\0')
+ if (dsc->vars[i].z_name != NULL)
cnt++;
}
size_t i, y;
for (i = 0, y = 1; i < dsc->var_cnt; i++)
- if (dsc->vars[i].z_name[0] != '\0')
+ if (dsc->vars[i].z_name != NULL)
{
- tab_text (t, 0, y, TAB_LEFT, var_get_name (dsc->vars[i].v));
+ tab_text (t, 0, y, TAB_LEFT, var_to_string (dsc->vars[i].v));
tab_text (t, 1, y++, TAB_LEFT, dsc->vars[i].z_name);
}
}
const struct variable **vars;
int all_sysmis = 0;
+ if (t->count <= 0)
+ {
+ struct ccase *z_case;
+
+ z_case = casereader_read (t->z_reader);
+ if (z_case)
+ {
+ size_t z_idx = 0;
+
+ t->count = case_num_idx (z_case, z_idx++);
+ for (z = t->z_scores; z < t->z_scores + t->z_score_cnt; z++)
+ {
+ z->mean = case_num_idx (z_case, z_idx++);
+ z->std_dev = case_num_idx (z_case, z_idx++);
+ }
+ case_unref (z_case);
+ }
+ else
+ {
+ if (t->ok)
+ {
+ msg (SE, _("Internal error processing Z scores"));
+ t->ok = false;
+ }
+ for (z = t->z_scores; z < t->z_scores + t->z_score_cnt; z++)
+ z->mean = z->std_dev = SYSMIS;
+ }
+ }
+ t->count--;
+
if (t->missing_type == DSC_LISTWISE)
{
assert(t->vars);
descriptives_trns_free (void *trns_)
{
struct dsc_trns *t = trns_;
+ bool ok = t->ok && !casereader_error (t->z_reader);
free (t->z_scores);
+ casereader_destroy (t->z_reader);
assert((t->missing_type != DSC_LISTWISE) ^ (t->vars != NULL));
free (t->vars);
- return true;
+ free (t);
+
+ return ok;
}
/* Sets up a transformation to calculate Z scores. */
size_t cnt, i;
for (cnt = i = 0; i < dsc->var_cnt; i++)
- if (dsc->vars[i].z_name[0] != '\0')
+ if (dsc->vars[i].z_name != NULL)
cnt++;
t = xmalloc (sizeof *t);
t->var_cnt = 0;
t->vars = NULL;
}
+ t->z_reader = casewriter_make_reader (dsc->z_writer);
+ t->count = 0;
+ t->ok = true;
+ dsc->z_writer = NULL;
for (cnt = i = 0; i < dsc->var_cnt; i++)
{
struct dsc_var *dv = &dsc->vars[i];
- if (dv->z_name[0] != '\0')
+ if (dv->z_name != NULL)
{
struct dsc_z_score *z;
struct variable *dst_var;
+ char *label;
dst_var = dict_create_var_assert (dataset_dict (ds), dv->z_name, 0);
- var_set_label (dst_var, xasprintf (_("Z-score of %s"),
- var_to_string (dv->v)));
+
+ label = xasprintf (_("Z-score of %s"),var_to_string (dv->v));
+ var_set_label (dst_var, label);
+ free (label);
z = &t->z_scores[cnt++];
z->src_var = dv->v;
z->z_var = dst_var;
- z->mean = dv->stats[DSC_MEAN];
- z->std_dev = dv->stats[DSC_STDDEV];
}
}
struct dataset *ds)
{
struct casereader *pass1, *pass2;
+ casenumber count;
struct ccase *c;
+ size_t z_idx;
size_t i;
c = casereader_peek (group, 0);
dsc->valid = 0.;
/* First pass to handle most of the work. */
+ count = 0;
for (; (c = casereader_read (pass1)) != NULL; case_unref (c))
{
double weight = dict_get_case_weight (dataset_dict (ds), c, NULL);
if (x > dv->max)
dv->max = x;
}
+
+ count++;
}
if (!casereader_destroy (pass1))
{
}
/* Calculate results. */
+ if (dsc->z_writer)
+ {
+ c = case_create (casewriter_get_proto (dsc->z_writer));
+ z_idx = 0;
+ case_data_rw_idx (c, z_idx++)->f = count;
+ }
+ else
+ c = NULL;
+
for (i = 0; i < dsc->var_cnt; i++)
{
struct dsc_var *dv = &dsc->vars[i];
dv->stats[DSC_MAX] = dv->max == -DBL_MAX ? SYSMIS : dv->max;
if (dsc->calc_stats & (1ul << DSC_SUM))
dv->stats[DSC_SUM] = W * dv->stats[DSC_MEAN];
+
+ if (dv->z_name)
+ {
+ case_data_rw_idx (c, z_idx++)->f = dv->stats[DSC_MEAN];
+ case_data_rw_idx (c, z_idx++)->f = dv->stats[DSC_STDDEV];
+ }
}
+ if (c != NULL)
+ casewriter_write (dsc->z_writer, c);
+
/* Output results. */
display (dsc);
}
size_t j;
nc = 0;
- tab_text (t, nc++, i + 1, TAB_LEFT, var_get_name (dv->v));
- tab_text_format (t, nc++, i + 1, 0, "%g", dv->valid);
+ tab_text (t, nc++, i + 1, TAB_LEFT, var_to_string (dv->v));
+ tab_text_format (t, nc++, i + 1, 0, "%.*g", DBL_DIG + 1, dv->valid);
if (dsc->format == DSC_SERIAL)
- tab_text_format (t, nc++, i + 1, 0, "%g", dv->missing);
+ tab_text_format (t, nc++, i + 1, 0, "%.*g", DBL_DIG + 1, dv->missing);
for (j = 0; j < DSC_N_STATS; j++)
if (dsc->show_stats & (1ul << j))
tab_double (t, nc++, i + 1, TAB_NONE, dv->stats[j], NULL);
}
- tab_title (t, _("Valid cases = %g; cases with missing value(s) = %g."),
- dsc->valid, dsc->missing_listwise);
+ tab_title (t, _("Valid cases = %.*g; cases with missing value(s) = %.*g."),
+ DBL_DIG + 1, dsc->valid,
+ DBL_DIG + 1, dsc->missing_listwise);
tab_submit (t);
}
int result;
if (dsc->sort_by_stat == DSC_NAME)
- result = strcasecmp (var_get_name (a->v), var_get_name (b->v));
+ result = utf8_strcasecmp (var_get_name (a->v), var_get_name (b->v));
else
{
double as = a->stats[dsc->sort_by_stat];