X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Flanguage%2Fstats%2Ffrequencies.q;h=ef36f139d59b971403ad9d43557df81db145c5e0;hb=3c9825c825998664d992841d851e23a0adf8af6e;hp=5f0e96e9e1eeb3e7c6e24a6b8fba8bd211138576;hpb=92c09e564002d356d20fc1e2e131027ef89f6748;p=pspp

diff --git a/src/language/stats/frequencies.q b/src/language/stats/frequencies.q
index 5f0e96e9e1..ef36f139d5 100644
--- a/src/language/stats/frequencies.q
+++ b/src/language/stats/frequencies.q
@@ -1,26 +1,18 @@
-/* PSPP - computes sample statistics.
-   Copyright (C) 1997-9, 2000 Free Software Foundation, Inc.
+/* PSPP - a program for statistical analysis.
+   Copyright (C) 1997-9, 2000, 2007, 2009, 2010 Free Software Foundation, Inc.
 
-   This program is free software; you can redistribute it and/or
-   modify it under the terms of the GNU General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
 
-   This program is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
 
    You should have received a copy of the GNU General Public License
-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-   02110-1301, USA. */
-
-/*
-  TODO:
-
-  * Remember that histograms, bar charts need mean, stddev.
-*/
+   along with this program.  If not, see <http://www.gnu.org/licenses/>. */
 
 #include <config.h>
 
@@ -28,40 +20,36 @@
 #include <stdlib.h>
 #include <gsl/gsl_histogram.h>
 
-#include <data/case.h>
-#include <data/casegrouper.h>
-#include <data/casereader.h>
-#include <data/dictionary.h>
-#include <data/format.h>
-#include <data/procedure.h>
-#include <data/settings.h>
-#include <data/value-labels.h>
-#include <data/variable.h>
-#include <language/command.h>
-#include <language/dictionary/split-file.h>
-#include <language/lexer/lexer.h>
-#include <libpspp/alloc.h>
-#include <libpspp/array.h>
-#include <libpspp/bit-vector.h>
-#include <libpspp/compiler.h>
-#include <libpspp/hash.h>
-#include <libpspp/magic.h>
-#include <libpspp/message.h>
-#include <libpspp/misc.h>
-#include <libpspp/pool.h>
-#include <libpspp/str.h>
-#include <math/histogram.h>
-#include <math/moments.h>
-#include <output/chart.h>
-#include <output/charts/piechart.h>
-#include <output/charts/plot-hist.h>
-#include <output/manager.h>
-#include <output/output.h>
-#include <output/table.h>
-
-#include "freq.h"
-
-#include "minmax.h"
+#include "data/case.h"
+#include "data/casegrouper.h"
+#include "data/casereader.h"
+#include "data/dictionary.h"
+#include "data/format.h"
+#include "data/procedure.h"
+#include "data/settings.h"
+#include "data/value-labels.h"
+#include "data/variable.h"
+#include "language/command.h"
+#include "language/dictionary/split-file.h"
+#include "language/lexer/lexer.h"
+#include "language/stats/freq.h"
+#include "libpspp/array.h"
+#include "libpspp/bit-vector.h"
+#include "libpspp/compiler.h"
+#include "libpspp/hmap.h"
+#include "libpspp/message.h"
+#include "libpspp/misc.h"
+#include "libpspp/pool.h"
+#include "libpspp/str.h"
+#include "math/histogram.h"
+#include "math/moments.h"
+#include "output/chart-item.h"
+#include "output/charts/piechart.h"
+#include "output/charts/plot-hist.h"
+#include "output/tab.h"
+
+#include "gl/minmax.h"
+#include "gl/xalloc.h"
 
 #include "gettext.h"
 #define _(msgid) gettext (msgid)
@@ -72,35 +60,26 @@
 /* (specification)
    FREQUENCIES (frq_):
      *+variables=custom;
-     +format=cond:condense/onepage(*n:onepage_limit,"%s>=0")/!standard,
-	     table:limit(n:limit,"%s>0")/notable/!table, 
-	     labels:!labels/nolabels,
-	     sort:!avalue/dvalue/afreq/dfreq,
-	     spaces:!single/double,
-	     paging:newpage/!oldpage;
+     +format=table:limit(n:limit,"%s>0")/notable/!table,
+	     sort:!avalue/dvalue/afreq/dfreq;
      missing=miss:include/!exclude;
      barchart(ba_)=:minimum(d:min),
 	    :maximum(d:max),
 	    scale:freq(*n:freq,"%s>0")/percent(*n:pcnt,"%s>0");
      piechart(pie_)=:minimum(d:min),
 	    :maximum(d:max),
-	    missing:missing/!nomissing;
+	    missing:missing/!nomissing,
+	    scale:!freq/percent;
      histogram(hi_)=:minimum(d:min),
 	    :maximum(d:max),
 	    scale:freq(*n:freq,"%s>0")/percent(*n:pcnt,"%s>0"),
-	    norm:!nonormal/normal,
-	    incr:increment(d:inc,"%s>0");
-     hbar(hb_)=:minimum(d:min),
-	    :maximum(d:max),
-	    scale:freq(*n:freq,"%s>0")/percent(*n:pcnt,"%s>0"),
-	    norm:!nonormal/normal,
-	    incr:increment(d:inc,"%s>0");
+	    norm:!nonormal/normal;
      +grouped=custom;
      +ntiles=integer;
      +percentiles = double list;
-     +statistics[st_]=1|mean,2|semean,3|median,4|mode,5|stddev,6|variance,
- 	    7|kurtosis,8|skewness,9|range,10|minimum,11|maximum,12|sum,
- 	    13|default,14|seskewness,15|sekurtosis,all,none.
+     +statistics[st_]=mean,semean,median,mode,stddev,variance,
+ 	    kurtosis,skewness,range,minimum,maximum,sum,
+ 	    default,seskewness,sekurtosis,all,none.
 */
 /* (declarations) */
 /* (functions) */
@@ -148,17 +127,16 @@ struct percentile
   double value;    /* the %ile's value */
   double x1;       /* The datum value <= the percentile */
   double x2;       /* The datum value >= the percentile */
-  int flag;        
+  int flag;
   int flag2;       /* Set to 1 if this percentile value has been found */
+  bool show;       /* True to show this percentile in the statistics box. */
 };
 
 
-static void add_percentile (double x) ;
+static void add_percentile (double x, bool show);
 
 static struct percentile *percentiles;
-static int n_percentiles;
-
-static int implicit_50th ; 
+static int n_percentiles, n_show_percentiles;
 
 /* Groups of statistics. */
 #define BI          BIT_INDEX
@@ -175,63 +153,45 @@ static int implicit_50th ;
 static unsigned long stats;
 static int n_stats;
 
-/* Types of graphs. */
-enum
+struct frq_chart
   {
-    GFT_NONE,			/* Don't draw graphs. */
-    GFT_BAR,			/* Draw bar charts. */
-    GFT_HIST,			/* Draw histograms. */
-    GFT_PIE,                    /* Draw piechart */
-    GFT_HBAR			/* Draw bar charts or histograms at our discretion. */
+    double x_min;               /* X axis minimum value. */
+    double x_max;               /* X axis maximum value. */
+    int y_scale;                /* Y axis scale: FRQ_FREQ or FRQ_PERCENT. */
+
+    /* Histograms only. */
+    double y_max;               /* Y axis maximum value. */
+    bool draw_normal;           /* Whether to draw normal curve. */
+
+    /* Pie charts only. */
+    bool include_missing;       /* Whether to include missing values. */
   };
 
+/* Histogram and pie chart settings. */
+static struct frq_chart hist, pie;
+
 /* Parsed command. */
 static struct cmd_frequencies cmd;
 
-/* Summary of the barchart, histogram, and hbar subcommands. */
-/* FIXME: These should not be mututally exclusive */
-static int chart;		/* NONE/BAR/HIST/HBAR/PIE. */
-static double min, max;		/* Minimum, maximum on y axis. */
-static int format;		/* FREQ/PERCENT: Scaling of y axis. */
-static double scale, incr;	/* FIXME */
-static int normal;		/* FIXME */
-
 /* Variables for which to calculate statistics. */
 static size_t n_variables;
 static const struct variable **v_variables;
 
-/* Arenas used to store semi-permanent storage. */
-static struct pool *int_pool;	/* Integer mode. */
-static struct pool *gen_pool;	/* General mode. */
+/* Pools. */
+static struct pool *data_pool;  	/* For per-SPLIT FILE group data. */
+static struct pool *syntax_pool;        /* For syntax-related data. */
 
 /* Frequency tables. */
 
-/* Types of frequency tables. */
-enum
-  {
-    FRQM_GENERAL,
-    FRQM_INTEGER
-  };
-
 /* Entire frequency table. */
 struct freq_tab
   {
-    int mode;			/* FRQM_GENERAL or FRQM_INTEGER. */
-
-    /* General mode. */
-    struct hsh_table *data;	/* Undifferentiated data. */
-
-    /* Integer mode. */
-    double *vector;		/* Frequencies proper. */
-    int min, max;		/* The boundaries of the table. */
-    double out_of_range;	/* Sum of weights of out-of-range values. */
-    double sysmis;		/* Sum of weights of SYSMIS values. */
-
-    /* All modes. */
+    struct hmap data;           /* Hash table for accumulating counts. */
     struct freq *valid;         /* Valid freqs. */
     int n_valid;		/* Number of total freqs. */
+    const struct dictionary *dict; /* Source of entries in the table. */
 
-    struct freq *missing;	/* Missing freqs. */
+    struct freq *missing;       /* Missing freqs. */
     int n_missing;		/* Number of missing freqs. */
 
     /* Statistics. */
@@ -239,7 +199,6 @@ struct freq_tab
     double valid_cases;		/* Sum of weights of valid cases. */
   };
 
-
 /* Per-variable frequency data. */
 struct var_freqs
   {
@@ -253,11 +212,7 @@ struct var_freqs
     /* Statistics. */
     double stat[frq_n_stats];
 
-    /* Width and format for analysis and display.
-       This is normally the same as "width" and "print" in struct
-       variable, but in SPSS-compatible mode only the first
-       MAX_SHORT_STRING bytes of long string variables are
-       included. */
+    /* Variable attributes. */
     int width;
     struct fmt_spec print;
   };
@@ -274,24 +229,23 @@ static void calc_stats (const struct variable *v, double d[frq_n_stats]);
 
 static void precalc (struct casereader *, struct dataset *);
 static void calc (const struct ccase *, const struct dataset *);
-static void postcalc (void);
+static void postcalc (const struct dataset *);
 
 static void postprocess_freq_tab (const struct variable *);
-static void dump_full (const struct variable *);
-static void dump_condensed (const struct variable *);
-static void dump_statistics (const struct variable *, int show_varname);
+static void dump_freq_table (const struct variable *, const struct variable *);
+static void dump_statistics (const struct variable *, const struct variable *);
 static void cleanup_freq_tab (const struct variable *);
 
-static hsh_compare_func compare_value_numeric_a, compare_value_alpha_a;
-static hsh_compare_func compare_value_numeric_d, compare_value_alpha_d;
-static hsh_compare_func compare_freq_numeric_a, compare_freq_alpha_a;
-static hsh_compare_func compare_freq_numeric_d, compare_freq_alpha_d;
+static algo_compare_func compare_value_numeric_a, compare_value_alpha_a;
+static algo_compare_func compare_value_numeric_d, compare_value_alpha_d;
+static algo_compare_func compare_freq_numeric_a, compare_freq_alpha_a;
+static algo_compare_func compare_freq_numeric_d, compare_freq_alpha_d;
 
 
 static void do_piechart(const struct variable *var,
 			const struct freq_tab *frq_tab);
 
-gsl_histogram * 
+struct histogram *
 freq_tab_to_hist(const struct freq_tab *ft, const struct variable *var);
 
 
@@ -305,12 +259,12 @@ cmd_frequencies (struct lexer *lexer, struct dataset *ds)
 {
   int result;
 
-  int_pool = pool_create ();
+  syntax_pool = pool_create ();
   result = internal_cmd_frequencies (lexer, ds);
-  pool_destroy (int_pool);
-  int_pool=0;
-  pool_destroy (gen_pool);
-  gen_pool=0;
+  pool_destroy (syntax_pool);
+  syntax_pool=0;
+  pool_destroy (data_pool);
+  data_pool=0;
   free (v_variables);
   v_variables=0;
   return result;
@@ -325,6 +279,7 @@ internal_cmd_frequencies (struct lexer *lexer, struct dataset *ds)
   int i;
 
   n_percentiles = 0;
+  n_show_percentiles = 0;
   percentiles = NULL;
 
   n_variables = 0;
@@ -333,9 +288,6 @@ internal_cmd_frequencies (struct lexer *lexer, struct dataset *ds)
   if (!parse_frequencies (lexer, ds, &cmd, NULL))
     return CMD_FAILURE;
 
-  if (cmd.onepage_limit == NOT_LONG)
-    cmd.onepage_limit = 50;
-
   /* Figure out statistics to calculate. */
   stats = 0;
   if (cmd.a_statistics[FRQ_ST_DEFAULT] || !cmd.sbc_statistics)
@@ -343,14 +295,14 @@ internal_cmd_frequencies (struct lexer *lexer, struct dataset *ds)
   if (cmd.a_statistics[FRQ_ST_ALL])
     stats |= frq_all;
   if (cmd.sort != FRQ_AVALUE && cmd.sort != FRQ_DVALUE)
-    stats &= ~frq_median;
+    stats &= ~BIT_INDEX (frq_median);
   for (i = 0; i < frq_n_stats; i++)
     if (cmd.a_statistics[st_name[i].st_indx])
       stats |= BIT_INDEX (i);
   if (stats & frq_kurt)
-    stats |= frq_sekurt;
+    stats |= BIT_INDEX (frq_sekurt);
   if (stats & frq_skew)
-    stats |= frq_seskew;
+    stats |= BIT_INDEX (frq_seskew);
 
   /* Calculate n_stats. */
   n_stats = 0;
@@ -360,44 +312,56 @@ internal_cmd_frequencies (struct lexer *lexer, struct dataset *ds)
 
   /* Charting. */
   determine_charts ();
-  if (chart != GFT_NONE || cmd.sbc_ntiles)
+  if (cmd.sbc_histogram || cmd.sbc_piechart || cmd.sbc_ntiles)
     cmd.sort = FRQ_AVALUE;
 
   /* Work out what percentiles need to be calculated */
-  if ( cmd.sbc_percentiles ) 
+  if ( cmd.sbc_percentiles )
     {
-      for ( i = 0 ; i < MAXLISTS ; ++i ) 
+      for ( i = 0 ; i < MAXLISTS ; ++i )
 	{
 	  int pl;
 	  subc_list_double *ptl_list = &cmd.dl_percentiles[i];
 	  for ( pl = 0 ; pl < subc_list_double_count(ptl_list); ++pl)
-	      add_percentile (subc_list_double_at(ptl_list, pl) / 100.0 );
+            add_percentile (subc_list_double_at(ptl_list, pl) / 100.0, true);
 	}
     }
-  if ( cmd.sbc_ntiles ) 
+  if ( cmd.sbc_ntiles )
     {
-      for ( i = 0 ; i < cmd.sbc_ntiles ; ++i ) 
+      for ( i = 0 ; i < cmd.sbc_ntiles ; ++i )
 	{
 	  int j;
-	  for (j = 0; j <= cmd.n_ntiles[i]; ++j ) 
-	      add_percentile (j / (double) cmd.n_ntiles[i]);
+	  for (j = 0; j <= cmd.n_ntiles[i]; ++j )
+            add_percentile (j / (double) cmd.n_ntiles[i], true);
 	}
     }
-  
+  if (stats & BIT_INDEX (frq_median))
+    {
+      /* Treat the median as the 50% percentile.
+         We output it in the percentiles table as "50 (Median)." */
+      add_percentile (0.5, true);
+      stats &= ~BIT_INDEX (frq_median);
+      n_stats--;
+    }
+  if (cmd.sbc_histogram)
+    {
+      add_percentile (0.25, false);
+      add_percentile (0.75, false);
+    }
 
   /* Do it! */
   input = casereader_create_filter_weight (proc_open (ds), dataset_dict (ds),
                                            NULL, NULL);
   grouper = casegrouper_create_splits (input, dataset_dict (ds));
   for (; casegrouper_get_next_group (grouper, &group);
-       casereader_destroy (group)) 
+       casereader_destroy (group))
     {
-      struct ccase c;
-      
+      struct ccase *c;
+
       precalc (group, ds);
-      for (; casereader_read (group, &c); case_destroy (&c)) 
-        calc (&c, ds);
-      postcalc ();
+      for (; (c = casereader_read (group)) != NULL; case_unref (c))
+        calc (c, ds);
+      postcalc (ds);
     }
   ok = casegrouper_destroy (grouper);
   ok = proc_commit (ds) && ok;
@@ -411,105 +375,45 @@ internal_cmd_frequencies (struct lexer *lexer, struct dataset *ds)
 static void
 determine_charts (void)
 {
-  int count = (!!cmd.sbc_histogram) + (!!cmd.sbc_barchart) + 
-    (!!cmd.sbc_hbar) + (!!cmd.sbc_piechart);
-
-  if (!count)
-    {
-      chart = GFT_NONE;
-      return;
-    }
-  else if (count > 1)
-    {
-      chart = GFT_HBAR;
-      msg (SW, _("At most one of BARCHART, HISTOGRAM, or HBAR should be "
-	   "given.  HBAR will be assumed.  Argument values will be "
-	   "given precedence increasing along the order given."));
-    }
-  else if (cmd.sbc_histogram)
-    chart = GFT_HIST;
-  else if (cmd.sbc_barchart)
-    chart = GFT_BAR;
-  else if (cmd.sbc_piechart)
-    chart = GFT_PIE;
-  else
-    chart = GFT_HBAR;
-
-  min = max = SYSMIS;
-  format = FRQ_FREQ;
-  scale = SYSMIS;
-  incr = SYSMIS;
-  normal = 0;
-
   if (cmd.sbc_barchart)
-    {
-      if (cmd.ba_min != SYSMIS)
-	min = cmd.ba_min;
-      if (cmd.ba_max != SYSMIS)
-	max = cmd.ba_max;
-      if (cmd.ba_scale == FRQ_FREQ)
-	{
-	  format = FRQ_FREQ;
-	  scale = cmd.ba_freq;
-	}
-      else if (cmd.ba_scale == FRQ_PERCENT)
-	{
-	  format = FRQ_PERCENT;
-	  scale = cmd.ba_pcnt;
-	}
-    }
+    msg (SW, _("Bar charts are not implemented."));
 
   if (cmd.sbc_histogram)
     {
-      if (cmd.hi_min != SYSMIS)
-	min = cmd.hi_min;
-      if (cmd.hi_max != SYSMIS)
-	max = cmd.hi_max;
-      if (cmd.hi_scale == FRQ_FREQ)
-	{
-	  format = FRQ_FREQ;
-	  scale = cmd.hi_freq;
-	}
-      else if (cmd.hi_scale == FRQ_PERCENT)
-	{
-	  format = FRQ_PERCENT;
-	  scale = cmd.ba_pcnt;
-	}
-      if (cmd.hi_norm != FRQ_NONORMAL )
-	normal = 1;
-      if (cmd.hi_incr == FRQ_INCREMENT)
-	incr = cmd.hi_inc;
+      hist.x_min = cmd.hi_min;
+      hist.x_max = cmd.hi_max;
+      hist.y_scale = cmd.hi_scale;
+      hist.y_max = cmd.hi_scale == FRQ_FREQ ? cmd.hi_freq : cmd.hi_pcnt;
+      hist.draw_normal = cmd.hi_norm != FRQ_NONORMAL;
+      hist.include_missing = false;
+
+      if (hist.x_min != SYSMIS && hist.x_max != SYSMIS
+          && hist.x_min >= hist.x_max)
+        {
+          msg (SE, _("MAX for histogram must be greater than or equal to MIN, "
+                     "but MIN was specified as %.15g and MAX as %.15g.  "
+                     "MIN and MAX will be ignored."), hist.x_min, hist.x_max);
+          hist.x_min = hist.x_max = SYSMIS;
+        }
     }
 
-  if (cmd.sbc_hbar)
+  if (cmd.sbc_piechart)
     {
-      if (cmd.hb_min != SYSMIS)
-	min = cmd.hb_min;
-      if (cmd.hb_max != SYSMIS)
-	max = cmd.hb_max;
-      if (cmd.hb_scale == FRQ_FREQ)
-	{
-	  format = FRQ_FREQ;
-	  scale = cmd.hb_freq;
-	}
-      else if (cmd.hb_scale == FRQ_PERCENT)
-	{
-	  format = FRQ_PERCENT;
-	  scale = cmd.ba_pcnt;
-	}
-      if (cmd.hb_norm)
-	normal = 1;
-      if (cmd.hb_incr == FRQ_INCREMENT)
-	incr = cmd.hb_inc;
-    }
+      pie.x_min = cmd.pie_min;
+      pie.x_max = cmd.pie_max;
+      pie.y_scale = cmd.pie_scale;
+      pie.include_missing = cmd.pie_missing == FRQ_MISSING;
 
-  if (min != SYSMIS && max != SYSMIS && min >= max)
-    {
-      msg (SE, _("MAX must be greater than or equal to MIN, if both are "
-	   "specified.  However, MIN was specified as %g and MAX as %g.  "
-	   "MIN and MAX will be ignored."), min, max);
-      min = max = SYSMIS;
+      if (pie.x_min != SYSMIS && pie.x_max != SYSMIS
+          && pie.x_min >= pie.x_max)
+        {
+          msg (SE, _("MAX for pie chart must be greater than or equal to MIN, "
+                     "but MIN was specified as %.15g and MAX as %.15g.  "
+                     "MIN and MAX will be ignored."), pie.x_min, pie.x_max);
+          pie.x_min = pie.x_max = SYSMIS;
+        }
     }
+
 }
 
 /* Add data from case C to the frequency table. */
@@ -521,51 +425,20 @@ calc (const struct ccase *c, const struct dataset *ds)
 
   for (i = 0; i < n_variables; i++)
     {
-      const struct variable *v = v_variables[i];
-      const union value *val = case_data (c, v);
-      struct var_freqs *vf = get_var_freqs (v);
-      struct freq_tab *ft = &vf->tab;
+      const struct variable *var = v_variables[i];
+      int width = var_get_width (var);
 
-      switch (ft->mode)
-	{
-	  case FRQM_GENERAL:
-	    {
-	      /* General mode. */
-              struct freq target;
-	      struct freq **fpp;
-
-              target.value = (union value *) val;
-              fpp = (struct freq **) hsh_probe (ft->data, &target);
-
-	      if (*fpp != NULL)
-		(*fpp)->count += weight;
-	      else
-		{
-		  struct freq *fp = pool_alloc (gen_pool, sizeof *fp);
-                  fp->count = weight;
-                  fp->value = pool_clone (gen_pool,
-                                          val,
-                                          MAX (MAX_SHORT_STRING, vf->width));
-                  *fpp = fp;
-		}
-	    }
-	  break;
-	case FRQM_INTEGER:
-	  /* Integer mode. */
-	  if (val->f == SYSMIS)
-	    ft->sysmis += weight;
-	  else if (val->f > INT_MIN+1 && val->f < INT_MAX-1)
-	    {
-	      int i = val->f;
-	      if (i >= ft->min && i <= ft->max)
-		ft->vector[i - ft->min] += weight;
-	    }
-	  else
-	    ft->out_of_range += weight;
-	  break;
-	default:
-          NOT_REACHED ();
-	}
+      const union value *value = case_data (c, var);
+      size_t hash = value_hash (value, width, 0);
+
+      struct hmap *hmap = &get_var_freqs (var)->tab.data;
+      struct freq *f;
+
+      f = freq_hmap_search (hmap, value, width, hash);
+      if (f == NULL)
+        f = freq_hmap_insert (hmap, value, width, hash);
+
+      f->count += weight;
     }
 }
 
@@ -574,43 +447,35 @@ calc (const struct ccase *c, const struct dataset *ds)
 static void
 precalc (struct casereader *input, struct dataset *ds)
 {
-  struct ccase c;
+  struct ccase *c;
   size_t i;
 
-  if (!casereader_peek (input, 0, &c))
-    return;
-  output_split_file_values (ds, &c);
-  case_destroy (&c);
+  c = casereader_peek (input, 0);
+  if (c != NULL)
+    {
+      output_split_file_values (ds, c);
+      case_unref (c);
+    }
+
+  pool_destroy (data_pool);
+  data_pool = pool_create ();
 
-  pool_destroy (gen_pool);
-  gen_pool = pool_create ();
-  
   for (i = 0; i < n_variables; i++)
     {
       const struct variable *v = v_variables[i];
       struct freq_tab *ft = &get_var_freqs (v)->tab;
 
-      if (ft->mode == FRQM_GENERAL)
-	{
-	  ft->data = hsh_create (16, compare_freq, hash_freq, NULL, v);
-	}
-      else
-	{
-	  int j;
-
-	  for (j = (ft->max - ft->min); j >= 0; j--)
-	    ft->vector[j] = 0.0;
-	  ft->out_of_range = 0.0;
-	  ft->sysmis = 0.0;
-	}
+      hmap_init (&ft->data);
     }
 }
 
 /* Finishes up with the variables after frequencies have been
    calculated.  Displays statistics, percentiles, ... */
 static void
-postcalc (void)
+postcalc (const struct dataset *ds)
 {
+  const struct dictionary *dict = dataset_dict (ds);
+  const struct variable *wv = dict_get_weight (dict);
   size_t i;
 
   for (i = 0; i < n_variables; i++)
@@ -619,67 +484,40 @@ postcalc (void)
       struct var_freqs *vf = get_var_freqs (v);
       struct freq_tab *ft = &vf->tab;
       int n_categories;
-      int dumped_freq_tab = 1;
 
       postprocess_freq_tab (v);
 
       /* Frequencies tables. */
       n_categories = ft->n_valid + ft->n_missing;
-      if (cmd.table == FRQ_TABLE
-	  || (cmd.table == FRQ_LIMIT && n_categories <= cmd.limit))
-	switch (cmd.cond)
-	  {
-	  case FRQ_CONDENSE:
-	    dump_condensed (v);
-	    break;
-	  case FRQ_STANDARD:
-	    dump_full (v);
-	    break;
-	  case FRQ_ONEPAGE:
-	    if (n_categories > cmd.onepage_limit)
-	      dump_condensed (v);
-	    else
-	      dump_full (v);
-	    break;
-	  default:
-            NOT_REACHED ();
-	  }
-      else
-	dumped_freq_tab = 0;
+      if  (cmd.table == FRQ_TABLE
+           || (cmd.table == FRQ_LIMIT && n_categories <= cmd.limit))
+        dump_freq_table (v, wv);
 
       /* Statistics. */
       if (n_stats)
-	dump_statistics (v, !dumped_freq_tab);
-
+	dump_statistics (v, wv);
 
-
-      if ( chart == GFT_HIST) 
+      if (cmd.sbc_histogram && var_is_numeric (v) && ft->n_valid > 0)
 	{
 	  double d[frq_n_stats];
-	  struct normal_curve norm;
-	  gsl_histogram *hist ;
-
-
-	  norm.N = vf->tab.valid_cases;
+	  struct histogram *histogram;
 
 	  calc_stats (v, d);
-	  norm.mean = d[frq_mean];
-	  norm.stddev = d[frq_stddev];
 
-	  hist = freq_tab_to_hist(ft,v);
+	  histogram = freq_tab_to_hist (ft, v);
 
-	  histogram_plot(hist, var_to_string(v), &norm, normal);
+          chart_item_submit (histogram_chart_create (
+                               histogram->gsl_hist, var_to_string(v),
+                               vf->tab.valid_cases,
+                               d[frq_mean],
+                               d[frq_stddev],
+                               hist.draw_normal));
 
-	  gsl_histogram_free(hist);
+	  statistic_destroy (&histogram->parent);
 	}
 
-
-      if ( chart == GFT_PIE) 
-	{
-	  do_piechart(v_variables[i], ft);
-	}
-
-
+      if (cmd.sbc_piechart)
+        do_piechart(v_variables[i], ft);
 
       cleanup_freq_tab (v);
 
@@ -687,12 +525,12 @@ postcalc (void)
 }
 
 /* Returns the comparison function that should be used for
-   sorting a frequency table by FRQ_SORT using VAR_TYPE
-   variables. */
-static hsh_compare_func *
-get_freq_comparator (int frq_sort, enum var_type var_type) 
+   sorting a frequency table by FRQ_SORT using VAL_TYPE
+   values. */
+static algo_compare_func *
+get_freq_comparator (int frq_sort, enum val_type val_type)
 {
-  bool is_numeric = var_type == VAR_NUMERIC;
+  bool is_numeric = val_type == VAL_NUMERIC;
   switch (frq_sort)
     {
     case FRQ_AVALUE:
@@ -711,40 +549,30 @@ get_freq_comparator (int frq_sort, enum var_type var_type)
 /* Returns true iff the value in struct freq F is non-missing
    for variable V. */
 static bool
-not_missing (const void *f_, const void *v_) 
+not_missing (const void *f_, const void *v_)
 {
   const struct freq *f = f_;
   const struct variable *v = v_;
 
-  return !var_is_value_missing (v, f->value, MV_ANY);
+  return !var_is_value_missing (v, &f->value, MV_ANY);
 }
 
 /* Summarizes the frequency table data for variable V. */
 static void
 postprocess_freq_tab (const struct variable *v)
 {
-  hsh_compare_func *compare;
+  algo_compare_func *compare;
   struct freq_tab *ft;
   size_t count;
-  void *const *data;
   struct freq *freqs, *f;
   size_t i;
 
   ft = &get_var_freqs (v)->tab;
-  assert (ft->mode == FRQM_GENERAL);
   compare = get_freq_comparator (cmd.sort, var_get_type (v));
 
   /* Extract data from hash table. */
-  count = hsh_count (ft->data);
-  data = hsh_data (ft->data);
-
-  /* Copy dereferenced data into freqs. */
-  freqs = xnmalloc (count, sizeof *freqs);
-  for (i = 0; i < count; i++) 
-    {
-      struct freq *f = data[i];
-      freqs[i] = *f; 
-    }
+  count = hmap_count (&ft->data);
+  freqs = freq_hmap_extract (&ft->data);
 
   /* Put data into ft. */
   ft->valid = freqs;
@@ -758,15 +586,15 @@ postprocess_freq_tab (const struct variable *v)
 
   /* Summary statistics. */
   ft->valid_cases = 0.0;
-  for(i = 0 ;  i < ft->n_valid ; ++i ) 
+  for(i = 0 ;  i < ft->n_valid ; ++i )
     {
       f = &ft->valid[i];
       ft->valid_cases += f->count;
 
     }
 
-  ft->total_cases = ft->valid_cases ; 
-  for(i = 0 ;  i < ft->n_missing ; ++i ) 
+  ft->total_cases = ft->valid_cases ;
+  for(i = 0 ;  i < ft->n_missing ; ++i )
     {
       f = &ft->missing[i];
       ft->total_cases += f->count;
@@ -778,10 +606,9 @@ postprocess_freq_tab (const struct variable *v)
 static void
 cleanup_freq_tab (const struct variable *v)
 {
-  struct freq_tab *ft = &get_var_freqs (v)->tab;
-  assert (ft->mode == FRQM_GENERAL);
-  free (ft->valid);
-  hsh_destroy (ft->data);
+  struct var_freqs *vf = get_var_freqs (v);
+  free (vf->tab.valid);
+  freq_hmap_destroy (&vf->tab.data, vf->width);
 }
 
 /* Parses the VARIABLES subcommand, adding to
@@ -789,9 +616,6 @@ cleanup_freq_tab (const struct variable *v)
 static int
 frq_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_frequencies *cmd UNUSED, void *aux UNUSED)
 {
-  int mode;
-  int min = 0, max = 0;
-
   size_t old_n_variables = n_variables;
   size_t i;
 
@@ -804,31 +628,6 @@ frq_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_freque
 			PV_APPEND | PV_NO_SCRATCH))
     return 0;
 
-  if (!lex_match (lexer, '('))
-    mode = FRQM_GENERAL;
-  else
-    {
-      mode = FRQM_INTEGER;
-      if (!lex_force_int (lexer))
-	return 0;
-      min = lex_integer (lexer);
-      lex_get (lexer);
-      if (!lex_force_match (lexer, ','))
-	return 0;
-      if (!lex_force_int (lexer))
-	return 0;
-      max = lex_integer (lexer);
-      lex_get (lexer);
-      if (!lex_force_match (lexer, ')'))
-	return 0;
-      if (max < min)
-	{
-	  msg (SE, _("Upper limit of integer mode value range must be "
-		     "greater than lower limit."));
-	  return 0;
-	}
-    }
-
   for (i = old_n_variables; i < n_variables; i++)
     {
       const struct variable *v = v_variables[i];
@@ -840,35 +639,13 @@ frq_custom_variables (struct lexer *lexer, struct dataset *ds, struct cmd_freque
 		     "subcommand."), var_get_name (v));
 	  return 0;
 	}
-      if (mode == FRQM_INTEGER && !var_is_numeric (v))
-        {
-          msg (SE, _("Integer mode specified, but %s is not a numeric "
-                     "variable."), var_get_name (v));
-          return 0;
-        }
-
       vf = var_attach_aux (v, xmalloc (sizeof *vf), var_dtor_free);
-      vf->tab.mode = mode;
       vf->tab.valid = vf->tab.missing = NULL;
-      if (mode == FRQM_INTEGER)
-	{
-	  vf->tab.min = min;
-	  vf->tab.max = max;
-	  vf->tab.vector = pool_nalloc (int_pool,
-                                        max - min + 1, sizeof *vf->tab.vector);
-	}
-      else 
-        vf->tab.vector = NULL;
+      vf->tab.dict = dataset_dict (ds);
       vf->n_groups = 0;
       vf->groups = NULL;
       vf->width = var_get_width (v);
       vf->print = *var_get_print_format (v);
-      if (vf->width > MAX_SHORT_STRING && get_algorithm () == COMPATIBLE) 
-        {
-          enum fmt_type type = var_get_print_format (v)->type;
-          vf->width = MAX_SHORT_STRING;
-          vf->print.w = MAX_SHORT_STRING * (type == FMT_AHEX ? 2 : 1);
-        }
     }
   return 1;
 }
@@ -905,7 +682,7 @@ frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequenc
 		if (nl >= ml)
 		  {
 		    ml += 16;
-		    dl = pool_nrealloc (int_pool, dl, ml, sizeof *dl);
+		    dl = pool_nrealloc (syntax_pool, dl, ml, sizeof *dl);
 		  }
 		dl[nl++] = lex_tokval (lexer);
 		lex_get (lexer);
@@ -920,7 +697,7 @@ frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequenc
 		return 0;
 	      }
 	  }
-	else 
+	else
           {
             nl = 0;
             dl = NULL;
@@ -930,10 +707,10 @@ frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequenc
           if (var_get_aux (v[i]) == NULL)
             msg (SE, _("Variables %s specified on GROUPED but not on "
                        "VARIABLES."), var_get_name (v[i]));
-          else 
+          else
             {
               struct var_freqs *vf = get_var_freqs (v[i]);
-                
+
               if (vf->groups != NULL)
                 msg (SE, _("Variables %s specified multiple times on GROUPED "
                            "subcommand."), var_get_name (v[i]));
@@ -958,17 +735,25 @@ frq_custom_grouped (struct lexer *lexer, struct dataset *ds, struct cmd_frequenc
 }
 
 /* Adds X to the list of percentiles, keeping the list in proper
-   order. */
+   order.  If SHOW is true, the percentile will be shown in the statistics
+   box, otherwise it will be hidden. */
 static void
-add_percentile (double x)
+add_percentile (double x, bool show)
 {
   int i;
 
   for (i = 0; i < n_percentiles; i++)
     {
       /* Do nothing if it's already in the list */
-      if ( fabs(x - percentiles[i].p) < DBL_EPSILON ) 
-	return;
+      if ( fabs(x - percentiles[i].p) < DBL_EPSILON )
+        {
+          if (show && !percentiles[i].show)
+            {
+              n_show_percentiles++;
+              percentiles[i].show = true;
+            }
+          return;
+        }
 
       if (x < percentiles[i].p)
 	break;
@@ -976,15 +761,14 @@ add_percentile (double x)
 
   if (i >= n_percentiles || x != percentiles[i].p)
     {
-      percentiles = pool_nrealloc (int_pool, percentiles,
+      percentiles = pool_nrealloc (syntax_pool, percentiles,
                                    n_percentiles + 1, sizeof *percentiles);
-
-      if (i < n_percentiles)
-          memmove (&percentiles[i + 1], &percentiles[i],
-                   (n_percentiles - i) * sizeof (struct percentile) );
-
+      insert_element (percentiles, n_percentiles, sizeof *percentiles, i);
       percentiles[i].p = x;
+      percentiles[i].show = show;
       n_percentiles++;
+      if (show)
+        n_show_percentiles++;
     }
 }
 
@@ -997,9 +781,9 @@ compare_value_numeric_a (const void *a_, const void *b_, const void *aux UNUSED)
   const struct freq *a = a_;
   const struct freq *b = b_;
 
-  if (a->value[0].f > b->value[0].f)
+  if (a->value.f > b->value.f)
     return 1;
-  else if (a->value[0].f < b->value[0].f)
+  else if (a->value.f < b->value.f)
     return -1;
   else
     return 0;
@@ -1014,7 +798,7 @@ compare_value_alpha_a (const void *a_, const void *b_, const void *v_)
   const struct variable *v = v_;
   struct var_freqs *vf = get_var_freqs (v);
 
-  return memcmp (a->value[0].s, b->value[0].s, vf->width);
+  return value_compare_3way (&a->value, &b->value, vf->width);
 }
 
 /* Descending numeric compare of values. */
@@ -1044,9 +828,9 @@ compare_freq_numeric_a (const void *a_, const void *b_, const void *aux UNUSED)
   else if (a->count < b->count)
     return -1;
 
-  if (a->value[0].f > b->value[0].f)
+  if (a->value.f > b->value.f)
     return 1;
-  else if (a->value[0].f < b->value[0].f)
+  else if (a->value.f < b->value.f)
     return -1;
   else
     return 0;
@@ -1067,7 +851,7 @@ compare_freq_alpha_a (const void *a_, const void *b_, const void *v_)
   else if (a->count < b->count)
     return -1;
   else
-    return memcmp (a->value[0].s, b->value[0].s, vf->width);
+    return value_compare_3way (&a->value, &b->value, vf->width);
 }
 
 /* Descending numeric compare of frequency;
@@ -1083,9 +867,9 @@ compare_freq_numeric_d (const void *a_, const void *b_, const void *aux UNUSED)
   else if (a->count < b->count)
     return 1;
 
-  if (a->value[0].f > b->value[0].f)
+  if (a->value.f > b->value.f)
     return 1;
-  else if (a->value[0].f < b->value[0].f)
+  else if (a->value.f < b->value.f)
     return -1;
   else
     return 0;
@@ -1106,89 +890,47 @@ compare_freq_alpha_d (const void *a_, const void *b_, const void *v_)
   else if (a->count < b->count)
     return 1;
   else
-    return memcmp (a->value[0].s, b->value[0].s, vf->width);
+    return value_compare_3way (&a->value, &b->value, vf->width);
 }
 
 /* Frequency table display. */
 
-/* Sets the widths of all the columns and heights of all the rows in
-   table T for driver D. */
-static void
-full_dim (struct tab_table *t, struct outp_driver *d)
-{
-  int i = 0;
-  int columns = 5;
-
-  if (cmd.labels == FRQ_LABELS)
-    {
-    t->w[0] = MIN (tab_natural_width (t, d, 0), d->prop_em_width * 15);
-      i = 1;
-      columns ++;
-    }
-
-  for (;i < columns; i++)
-    t->w[i] = MAX (tab_natural_width (t, d, i), d->prop_em_width * 8);
-
-  for (i = 0; i < t->nr; i++)
-    t->h[i] = d->font_height;
-}
-
 /* Displays a full frequency table for variable V. */
 static void
-dump_full (const struct variable *v)
+dump_freq_table (const struct variable *v, const struct variable *wv)
 {
+  const struct fmt_spec *wfmt = wv ? var_get_print_format (wv) : &F_8_0;
   int n_categories;
   struct var_freqs *vf;
   struct freq_tab *ft;
   struct freq *f;
   struct tab_table *t;
-  int r;
+  int r, x;
   double cum_total = 0.0;
   double cum_freq = 0.0;
 
-  struct init
-    {
-      int c, r;
-      const char *s;
-    };
-
-  const struct init *p;
-
-  static const struct init vec[] =
-  {
-    {4, 0, N_("Valid")},
-    {5, 0, N_("Cum")},
-    {1, 1, N_("Value")},
-    {2, 1, N_("Frequency")},
-    {3, 1, N_("Percent")},
-    {4, 1, N_("Percent")},
-    {5, 1, N_("Percent")},
-    {0, 0, NULL},
-    {1, 0, NULL},
-    {2, 0, NULL},
-    {3, 0, NULL},
-    {-1, -1, NULL},
+  static const char *headings[] = {
+    N_("Value Label"),
+    N_("Value"),
+    N_("Frequency"),
+    N_("Percent"),
+    N_("Valid Percent"),
+    N_("Cum Percent")
   };
 
-  const bool lab = (cmd.labels == FRQ_LABELS);
-
   vf = get_var_freqs (v);
   ft = &vf->tab;
   n_categories = ft->n_valid + ft->n_missing;
-  t = tab_create (5 + lab, n_categories + 3, 0);
-  tab_headers (t, 0, 0, 2, 0);
-  tab_dim (t, full_dim);
-
-  if (lab)
-    tab_text (t, 0, 1, TAB_CENTER | TAT_TITLE, _("Value Label"));
+  t = tab_create (6, n_categories + 2);
+  tab_headers (t, 0, 0, 1, 0);
 
-  for (p = vec; p->s; p++)
-    tab_text (t, lab ? p->c : p->c - 1, p->r,
-		  TAB_CENTER | TAT_TITLE, gettext (p->s));
+  for (x = 0; x < 6; x++)
+    tab_text (t, x, 0, TAB_CENTER | TAT_TITLE, gettext (headings[x]));
 
-  r = 2;
+  r = 1;
   for (f = ft->valid; f < ft->missing; f++)
     {
+      const char *label;
       double percent, valid_percent;
 
       cum_freq += f->count;
@@ -1197,170 +939,71 @@ dump_full (const struct variable *v)
       valid_percent = f->count / ft->valid_cases * 100.0;
       cum_total += valid_percent;
 
-      if (lab)
-	{
-	  const char *label = var_lookup_value_label (v, &f->value[0]);
-	  if (label != NULL)
-	    tab_text (t, 0, r, TAB_LEFT, label);
-	}
+      label = var_lookup_value_label (v, &f->value);
+      if (label != NULL)
+        tab_text (t, 0, r, TAB_LEFT, label);
 
-      tab_value (t, 0 + lab, r, TAB_NONE, f->value, &vf->print);
-      tab_float (t, 1 + lab, r, TAB_NONE, f->count, 8, 0);
-      tab_float (t, 2 + lab, r, TAB_NONE, percent, 5, 1);
-      tab_float (t, 3 + lab, r, TAB_NONE, valid_percent, 5, 1);
-      tab_float (t, 4 + lab, r, TAB_NONE, cum_total, 5, 1);
+      tab_value (t, 1, r, TAB_NONE, &f->value, ft->dict, &vf->print);
+      tab_double (t, 2, r, TAB_NONE, f->count, wfmt);
+      tab_double (t, 3, r, TAB_NONE, percent, NULL);
+      tab_double (t, 4, r, TAB_NONE, valid_percent, NULL);
+      tab_double (t, 5, r, TAB_NONE, cum_total, NULL);
       r++;
     }
   for (; f < &ft->valid[n_categories]; f++)
     {
+      const char *label;
+
       cum_freq += f->count;
 
-      if (lab)
-	{
-	  const char *label = var_lookup_value_label (v, &f->value[0]);
-	  if (label != NULL)
-	    tab_text (t, 0, r, TAB_LEFT, label);
-	}
+      label = var_lookup_value_label (v, &f->value);
+      if (label != NULL)
+        tab_text (t, 0, r, TAB_LEFT, label);
 
-      tab_value (t, 0 + lab, r, TAB_NONE, f->value, &vf->print);
-      tab_float (t, 1 + lab, r, TAB_NONE, f->count, 8, 0);
-      tab_float (t, 2 + lab, r, TAB_NONE,
-		     f->count / ft->total_cases * 100.0, 5, 1);
-      tab_text (t, 3 + lab, r, TAB_NONE, _("Missing"));
+      tab_value (t, 1, r, TAB_NONE, &f->value, ft->dict, &vf->print);
+      tab_double (t, 2, r, TAB_NONE, f->count, wfmt);
+      tab_double (t, 3, r, TAB_NONE,
+		     f->count / ft->total_cases * 100.0, NULL);
+      tab_text (t, 4, r, TAB_NONE, _("Missing"));
       r++;
     }
 
-  tab_box (t, TAL_1, TAL_1,
-	   cmd.spaces == FRQ_SINGLE ? -1 : TAL_GAP, TAL_1,
-	   0, 0, 4 + lab, r);
-  tab_hline (t, TAL_2, 0, 4 + lab, 2);
-  tab_hline (t, TAL_2, 0, 4 + lab, r);
-  tab_joint_text (t, 0, r, 0 + lab, r, TAB_RIGHT | TAT_TITLE, _("Total"));
+  tab_box (t, TAL_1, TAL_1, -1, TAL_1, 0, 0, 5, r);
+  tab_hline (t, TAL_2, 0, 5, 1);
+  tab_hline (t, TAL_2, 0, 5, r);
+  tab_joint_text (t, 0, r, 1, r, TAB_RIGHT | TAT_TITLE, _("Total"));
   tab_vline (t, TAL_0, 1, r, r);
-  tab_float (t, 1 + lab, r, TAB_NONE, cum_freq, 8, 0);
-  tab_float (t, 2 + lab, r, TAB_NONE, 100.0, 5, 1);
-  tab_float (t, 3 + lab, r, TAB_NONE, 100.0, 5, 1);
+  tab_double (t, 2, r, TAB_NONE, cum_freq, wfmt);
+  tab_fixed (t, 3, r, TAB_NONE, 100.0, 5, 1);
+  tab_fixed (t, 4, r, TAB_NONE, 100.0, 5, 1);
 
   tab_title (t, "%s", var_to_string (v));
   tab_submit (t);
 }
-
-/* Sets the widths of all the columns and heights of all the rows in
-   table T for driver D. */
-static void
-condensed_dim (struct tab_table *t, struct outp_driver *d)
-{
-  int cum_w = MAX (outp_string_width (d, _("Cum"), OUTP_PROPORTIONAL),
-		   MAX (outp_string_width (d, _("Cum"), OUTP_PROPORTIONAL),
-			outp_string_width (d, "000", OUTP_PROPORTIONAL)));
-
-  int i;
-
-  for (i = 0; i < 2; i++)
-    t->w[i] = MAX (tab_natural_width (t, d, i), d->prop_em_width * 8);
-  for (i = 2; i < 4; i++)
-    t->w[i] = cum_w;
-  for (i = 0; i < t->nr; i++)
-    t->h[i] = d->font_height;
-}
-
-/* Display condensed frequency table for variable V. */
-static void
-dump_condensed (const struct variable *v)
-{
-  int n_categories;
-  struct var_freqs *vf;
-  struct freq_tab *ft;
-  struct freq *f;
-  struct tab_table *t;
-  int r;
-  double cum_total = 0.0;
-
-  vf = get_var_freqs (v);
-  ft = &vf->tab;
-  n_categories = ft->n_valid + ft->n_missing;
-  t = tab_create (4, n_categories + 2, 0);
-
-  tab_headers (t, 0, 0, 2, 0);
-  tab_text (t, 0, 1, TAB_CENTER | TAT_TITLE, _("Value"));
-  tab_text (t, 1, 1, TAB_CENTER | TAT_TITLE, _("Freq"));
-  tab_text (t, 2, 1, TAB_CENTER | TAT_TITLE, _("Pct"));
-  tab_text (t, 3, 0, TAB_CENTER | TAT_TITLE, _("Cum"));
-  tab_text (t, 3, 1, TAB_CENTER | TAT_TITLE, _("Pct"));
-  tab_dim (t, condensed_dim);
-
-  r = 2;
-  for (f = ft->valid; f < ft->missing; f++)
-    {
-      double percent;
-
-      percent = f->count / ft->total_cases * 100.0;
-      cum_total += f->count / ft->valid_cases * 100.0;
-
-      tab_value (t, 0, r, TAB_NONE, f->value, &vf->print);
-      tab_float (t, 1, r, TAB_NONE, f->count, 8, 0);
-      tab_float (t, 2, r, TAB_NONE, percent, 3, 0);
-      tab_float (t, 3, r, TAB_NONE, cum_total, 3, 0);
-      r++;
-    }
-  for (; f < &ft->valid[n_categories]; f++)
-    {
-      tab_value (t, 0, r, TAB_NONE, f->value, &vf->print);
-      tab_float (t, 1, r, TAB_NONE, f->count, 8, 0);
-      tab_float (t, 2, r, TAB_NONE,
-		 f->count / ft->total_cases * 100.0, 3, 0);
-      r++;
-    }
-
-  tab_box (t, TAL_1, TAL_1,
-	   cmd.spaces == FRQ_SINGLE ? -1 : TAL_GAP, TAL_1,
-	   0, 0, 3, r - 1);
-  tab_hline (t, TAL_2, 0, 3, 2);
-  tab_title (t, "%s", var_to_string (v));
-  tab_columns (t, SOM_COL_DOWN, 1);
-  tab_submit (t);
-}
 
 /* Statistical display. */
 
-/* Calculates all the pertinent statistics for variable V, putting
-   them in array D[].  FIXME: This could be made much more optimal. */
+/* Calculates all the pertinent statistics for variable V, putting them in
+   array D[]. */
 static void
 calc_stats (const struct variable *v, double d[frq_n_stats])
 {
   struct freq_tab *ft = &get_var_freqs (v)->tab;
   double W = ft->valid_cases;
   struct moments *m;
-  struct freq *f=0; 
+  struct freq *f=0;
   int most_often;
   double X_mode;
 
   double rank;
   int i = 0;
   int idx;
-  double *median_value;
 
   /* Calculate percentiles. */
 
-  /* If the 50th percentile was not explicitly requested then we must 
-     calculate it anyway --- it's the median */
-  median_value = 0 ;
-  for (i = 0; i < n_percentiles; i++) 
-    {
-      if (percentiles[i].p == 0.5)
-	{
-	  median_value = &percentiles[i].value;
-	  break;
-	}
-    }
-
-  if ( 0 == median_value )  
-    {
-      add_percentile (0.5);
-      implicit_50th = 1;
-    }
+  assert (ft->n_valid > 0);
 
-  for (i = 0; i < n_percentiles; i++) 
+  for (i = 0; i < n_percentiles; i++)
     {
       percentiles[i].flag = 0;
       percentiles[i].flag2 = 0;
@@ -1370,33 +1013,33 @@ calc_stats (const struct variable *v, double d[frq_n_stats])
   for (idx = 0; idx < ft->n_valid; ++idx)
     {
       static double prev_value = SYSMIS;
-      f = &ft->valid[idx]; 
+      f = &ft->valid[idx];
       rank += f->count ;
-      for (i = 0; i < n_percentiles; i++) 
+      for (i = 0; i < n_percentiles; i++)
         {
 	  double tp;
-	  if ( percentiles[i].flag2  ) continue ; 
+	  if ( percentiles[i].flag2  ) continue ;
 
-	  if ( get_algorithm() != COMPATIBLE ) 
-	    tp = 
+	  if ( settings_get_algorithm () != COMPATIBLE )
+	    tp =
 	      (ft->valid_cases - 1) *  percentiles[i].p;
 	  else
-	    tp = 
+	    tp =
 	      (ft->valid_cases + 1) *  percentiles[i].p - 1;
 
-	  if ( percentiles[i].flag ) 
+	  if ( percentiles[i].flag )
 	    {
-	      percentiles[i].x2 = f->value[0].f;
+	      percentiles[i].x2 = f->value.f;
 	      percentiles[i].x1 = prev_value;
 	      percentiles[i].flag2 = 1;
 	      continue;
 	    }
 
-          if (rank >  tp ) 
+          if (rank >  tp )
 	  {
-	    if ( f->count > 1 && rank - (f->count - 1) > tp ) 
+	    if ( f->count > 1 && rank - (f->count - 1) > tp )
 	      {
-		percentiles[i].x2 = percentiles[i].x1 = f->value[0].f;
+		percentiles[i].x2 = percentiles[i].x1 = f->value.f;
 		percentiles[i].flag2 = 1;
 	      }
 	    else
@@ -1407,14 +1050,14 @@ calc_stats (const struct variable *v, double d[frq_n_stats])
 	    continue;
 	  }
         }
-      prev_value = f->value[0].f;
+      prev_value = f->value.f;
     }
 
-  for (i = 0; i < n_percentiles; i++) 
+  for (i = 0; i < n_percentiles; i++)
     {
       /* Catches the case when p == 100% */
-      if ( ! percentiles[i].flag2 ) 
-	percentiles[i].x1 = percentiles[i].x2 = f->value[0].f;
+      if ( ! percentiles[i].flag2 )
+	percentiles[i].x1 = percentiles[i].x2 = f->value.f;
 
       /*
       printf("percentile %d (p==%.2f); X1 = %g; X2 = %g\n",
@@ -1422,13 +1065,13 @@ calc_stats (const struct variable *v, double d[frq_n_stats])
       */
     }
 
-  for (i = 0; i < n_percentiles; i++) 
+  for (i = 0; i < n_percentiles; i++)
     {
       struct freq_tab *ft = &get_var_freqs (v)->tab;
       double s;
 
       double dummy;
-      if ( get_algorithm() != COMPATIBLE ) 
+      if ( settings_get_algorithm () != COMPATIBLE )
 	{
 	  s = modf((ft->valid_cases - 1) * percentiles[i].p , &dummy);
 	}
@@ -1437,11 +1080,8 @@ calc_stats (const struct variable *v, double d[frq_n_stats])
 	  s = modf((ft->valid_cases + 1) * percentiles[i].p -1, &dummy);
 	}
 
-      percentiles[i].value = percentiles[i].x1 + 
-	( percentiles[i].x2 - percentiles[i].x1) * s ; 
-
-      if ( percentiles[i].p == 0.50) 
-	median_value = &percentiles[i].value; 
+      percentiles[i].value = percentiles[i].x1 +
+	( percentiles[i].x2 - percentiles[i].x1) * s ;
     }
 
 
@@ -1450,12 +1090,12 @@ calc_stats (const struct variable *v, double d[frq_n_stats])
   X_mode = SYSMIS;
   for (f = ft->valid; f < ft->missing; f++)
     {
-      if (most_often < f->count) 
+      if (most_often < f->count)
         {
           most_often = f->count;
-          X_mode = f->value[0].f;
+          X_mode = f->value.f;
         }
-      else if (most_often == f->count) 
+      else if (most_often == f->count)
         {
           /* A duplicate mode is undefined.
              FIXME: keep track of *all* the modes. */
@@ -1466,19 +1106,18 @@ calc_stats (const struct variable *v, double d[frq_n_stats])
   /* Calculate moments. */
   m = moments_create (MOMENT_KURTOSIS);
   for (f = ft->valid; f < ft->missing; f++)
-    moments_pass_one (m, f->value[0].f, f->count);
+    moments_pass_one (m, f->value.f, f->count);
   for (f = ft->valid; f < ft->missing; f++)
-    moments_pass_two (m, f->value[0].f, f->count);
+    moments_pass_two (m, f->value.f, f->count);
   moments_calculate (m, NULL, &d[frq_mean], &d[frq_variance],
                      &d[frq_skew], &d[frq_kurt]);
   moments_destroy (m);
-                     
+
   /* Formulas below are taken from _SPSS Statistical Algorithms_. */
-  d[frq_min] = ft->valid[0].value[0].f;
-  d[frq_max] = ft->valid[ft->n_valid - 1].value[0].f;
+  d[frq_min] = ft->valid[0].value.f;
+  d[frq_max] = ft->valid[ft->n_valid - 1].value.f;
   d[frq_mode] = X_mode;
   d[frq_range] = d[frq_max] - d[frq_min];
-  d[frq_median] = *median_value;
   d[frq_sum] = d[frq_mean] * W;
   d[frq_stddev] = sqrt (d[frq_variance]);
   d[frq_semean] = d[frq_stddev] / sqrt (W);
@@ -1488,18 +1127,14 @@ calc_stats (const struct variable *v, double d[frq_n_stats])
 
 /* Displays a table of all the statistics requested for variable V. */
 static void
-dump_statistics (const struct variable *v, int show_varname)
+dump_statistics (const struct variable *v, const struct variable *wv)
 {
+  const struct fmt_spec *wfmt = wv ? var_get_print_format (wv) : &F_8_0;
   struct freq_tab *ft;
   double stat_value[frq_n_stats];
   struct tab_table *t;
   int i, r;
 
-  int n_explicit_percentiles = n_percentiles;
-
-  if ( implicit_50th && n_percentiles > 0 ) 
-    --n_percentiles;
-
   if (var_is_alpha (v))
     return;
   ft = &get_var_freqs (v)->tab;
@@ -1511,15 +1146,14 @@ dump_statistics (const struct variable *v, int show_varname)
     }
   calc_stats (v, stat_value);
 
-  t = tab_create (3, n_stats + n_explicit_percentiles + 2, 0);
-  tab_dim (t, tab_natural_dimensions);
+  t = tab_create (3, n_stats + n_show_percentiles + 2);
 
   tab_box (t, TAL_1, TAL_1, -1, -1 , 0 , 0 , 2, tab_nr(t) - 1) ;
 
 
   tab_vline (t, TAL_1 , 2, 0, tab_nr(t) - 1);
   tab_vline (t, TAL_GAP , 1, 0, tab_nr(t) - 1 ) ;
-  
+
   r=2; /* N missing and N valid are always dumped */
 
   for (i = 0; i < frq_n_stats; i++)
@@ -1527,7 +1161,7 @@ dump_statistics (const struct variable *v, int show_varname)
       {
 	tab_text (t, 0, r, TAB_LEFT | TAT_TITLE,
 		      gettext (st_name[i].s10));
-	tab_float (t, 2, r, TAB_NONE, stat_value[i], 11, 3);
+	tab_double (t, 2, r, TAB_NONE, stat_value[i], NULL);
 	r++;
       }
 
@@ -1535,100 +1169,155 @@ dump_statistics (const struct variable *v, int show_varname)
   tab_text (t, 1, 0, TAB_LEFT | TAT_TITLE, _("Valid"));
   tab_text (t, 1, 1, TAB_LEFT | TAT_TITLE, _("Missing"));
 
-  tab_float(t, 2, 0, TAB_NONE, ft->valid_cases, 11, 0);
-  tab_float(t, 2, 1, TAB_NONE, ft->total_cases - ft->valid_cases, 11, 0);
+  tab_double (t, 2, 0, TAB_NONE, ft->valid_cases, wfmt);
+  tab_double (t, 2, 1, TAB_NONE, ft->total_cases - ft->valid_cases, wfmt);
 
-
-  for (i = 0; i < n_explicit_percentiles; i++, r++) 
+  for (i = 0; i < n_percentiles; i++, r++)
     {
-      if ( i == 0 ) 
-	{ 
+      if (!percentiles[i].show)
+        continue;
+
+      if ( i == 0 )
+	{
 	  tab_text (t, 0, r, TAB_LEFT | TAT_TITLE, _("Percentiles"));
 	}
 
-      tab_float (t, 1, r, TAB_LEFT, percentiles[i].p * 100, 3, 0 );
-      tab_float (t, 2, r, TAB_NONE, percentiles[i].value, 11, 3);
-
+      if (percentiles[i].p == 0.5)
+        tab_text (t, 1, r, TAB_LEFT, _("50 (Median)"));
+      else
+        tab_fixed (t, 1, r, TAB_LEFT, percentiles[i].p * 100, 3, 0);
+      tab_double (t, 2, r, TAB_NONE, percentiles[i].value,
+		  var_get_print_format (v));
     }
 
-  tab_columns (t, SOM_COL_DOWN, 1);
-  if (show_varname)
-    tab_title (t, "%s", var_to_string (v));
-  else
-    tab_flags (t, SOMF_NO_TITLE);
-
+  tab_title (t, "%s", var_to_string (v));
 
   tab_submit (t);
 }
 
+static double
+calculate_iqr (void)
+{
+  double q1 = SYSMIS;
+  double q3 = SYSMIS;
+  int i;
+
+  for (i = 0; i < n_percentiles; i++)
+    {
+      if (fabs (0.25 - percentiles[i].p) < DBL_EPSILON)
+        q1 = percentiles[i].value;
+      else if (fabs (0.75 - percentiles[i].p) < DBL_EPSILON)
+        q3 = percentiles[i].value;
+    }
+
+  return q1 == SYSMIS || q3 == SYSMIS ? SYSMIS : q3 - q1;
+}
+
+static bool
+chart_includes_value (const struct frq_chart *chart,
+                      const struct variable *var,
+                      const union value *value)
+{
+  if (!chart->include_missing && var_is_value_missing (var, value, MV_ANY))
+    return false;
+
+  if (var_is_numeric (var)
+      && ((chart->x_min != SYSMIS && value->f < chart->x_min)
+          || (chart->x_max != SYSMIS && value->f > chart->x_max)))
+    return false;
+
+  return true;
+}
 
 /* Create a gsl_histogram from a freq_tab */
-gsl_histogram *
-freq_tab_to_hist(const struct freq_tab *ft, const struct variable *var)
+struct histogram *
+freq_tab_to_hist (const struct freq_tab *ft, const struct variable *var)
 {
+  double x_min, x_max, valid_freq;
   int i;
-  double x_min = DBL_MAX;
-  double x_max = -DBL_MAX;
 
-  gsl_histogram *hist;
-  const double bins = 11;
+  struct histogram *histogram;
+  double iqr;
+  int bins;
 
-  struct hsh_iterator hi;
-  struct hsh_table *fh = ft->data;
-  struct freq *frq;
-
-  /* Find out the extremes of the x value */
-  for ( frq = hsh_first(fh, &hi); frq != 0; frq = hsh_next(fh, &hi) ) 
+  /* Find out the extremes of the x value, within the range to be included in
+     the histogram, and sum the total frequency of those values. */
+  x_min = DBL_MAX;
+  x_max = -DBL_MAX;
+  valid_freq = 0;
+  for (i = 0; i < ft->n_valid; i++)
     {
-      if (var_is_value_missing(var, frq->value, MV_ANY))
-	continue;
-
-      if ( frq->value[0].f < x_min ) x_min = frq->value[0].f ;
-      if ( frq->value[0].f > x_max ) x_max = frq->value[0].f ;
+      const struct freq *frq = &ft->valid[i];
+      if (chart_includes_value (&hist, var, &frq->value))
+        {
+          x_min = MIN (x_min, frq->value.f);
+          x_max = MAX (x_max, frq->value.f);
+          valid_freq += frq->count;
+        }
     }
 
-  hist = histogram_create(bins, x_min, x_max);
+  /* Freedman-Diaconis' choice of bin width. */
+  iqr = calculate_iqr ();
+  if (iqr != SYSMIS)
+    {
+      double bin_width = 2 * iqr / pow (valid_freq, 1.0 / 3.0);
+      bins = (x_max - x_min) / bin_width;
+      if (bins < 5)
+        bins = 5;
+      else if (bins > 400)
+        bins = 400;
+    }
+  else
+    bins = 5;
 
-  for( i = 0 ; i < ft->n_valid ; ++i ) 
+  histogram = histogram_create (bins, x_min, x_max);
+  for (i = 0; i < ft->n_valid; i++)
     {
-      frq = &ft->valid[i];
-      gsl_histogram_accumulate(hist, frq->value[0].f, frq->count);
+      const struct freq *frq = &ft->valid[i];
+      if (chart_includes_value (&hist, var, &frq->value))
+        histogram_add (histogram, frq->value.f, frq->count);
     }
 
-  return hist;
+  return histogram;
 }
 
-
-static struct slice *
-freq_tab_to_slice_array(const struct freq_tab *frq_tab, 
-			const struct variable *var,
-			int *n_slices);
-
+static int
+add_slice (const struct freq *freq, const struct variable *var,
+           struct slice *slice)
+{
+  if (chart_includes_value (&pie, var, &freq->value))
+    {
+      ds_init_empty (&slice->label);
+      var_append_value_name (var, &freq->value, &slice->label);
+      slice->magnitude = freq->count;
+      return 1;
+    }
+  else
+    return 0;
+}
 
 /* Allocate an array of slices and fill them from the data in frq_tab
    n_slices will contain the number of slices allocated.
    The caller is responsible for freeing slices
 */
 static struct slice *
-freq_tab_to_slice_array(const struct freq_tab *frq_tab, 
+freq_tab_to_slice_array(const struct freq_tab *frq_tab,
 			const struct variable *var,
-			int *n_slices)
+			int *n_slicesp)
 {
-  int i;
   struct slice *slices;
+  int n_slices;
+  int i;
 
-  *n_slices = frq_tab->n_valid;
-  
-  slices = xnmalloc (*n_slices, sizeof *slices);
-
-  for (i = 0 ; i < *n_slices ; ++i ) 
-    {
-      const struct freq *frq = &frq_tab->valid[i];
+  slices = xnmalloc (frq_tab->n_valid + frq_tab->n_missing, sizeof *slices);
+  n_slices = 0;
 
-      slices[i].label = var_get_value_name (var, frq->value);
-      slices[i].magnetude = frq->count;
-    }
+  for (i = 0; i < frq_tab->n_valid; i++)
+    n_slices += add_slice (&frq_tab->valid[i], var, &slices[n_slices]);
+  for (i = 0; i < frq_tab->n_missing; i++)
+    n_slices += add_slice (&frq_tab->missing[i], var, &slices[n_slices]);
 
+  *n_slicesp = n_slices;
   return slices;
 }
 
@@ -1639,17 +1328,26 @@ static void
 do_piechart(const struct variable *var, const struct freq_tab *frq_tab)
 {
   struct slice *slices;
-  int n_slices;
+  int n_slices, i;
 
-  slices = freq_tab_to_slice_array(frq_tab, var, &n_slices);
+  slices = freq_tab_to_slice_array (frq_tab, var, &n_slices);
 
-  piechart_plot(var_to_string(var), slices, n_slices);
+  if (n_slices < 2)
+    msg (SW, _("Omitting pie chart for %s, which has only %d unique values."),
+         var_get_name (var), n_slices);
+  else if (n_slices > 50)
+    msg (SW, _("Omitting pie chart for %s, which has over 50 unique values."),
+         var_get_name (var));
+  else
+    chart_item_submit (piechart_create (var_to_string(var), slices, n_slices));
 
-  free(slices);
+  for (i = 0; i < n_slices; i++)
+    ds_destroy (&slices[i].label);
+  free (slices);
 }
 
 
-/* 
+/*
    Local Variables:
    mode: c
    End: