X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=src%2Ffactor_stats.c;h=e090517bddc494e2a60279664d4ad53d96b26062;hb=64f58bbdab17e4a09b725e713f4f82f567f44076;hp=cb2197ada76e1b32bd365def88d9e73c165dbb39;hpb=9c01f251cf0e5b5eb3899fc7c62cc595f3d48511;p=pspp-builds.git diff --git a/src/factor_stats.c b/src/factor_stats.c index cb2197ad..e090517b 100644 --- a/src/factor_stats.c +++ b/src/factor_stats.c @@ -15,50 +15,67 @@ General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software -Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA -02111-1307, USA. */ +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +02110-1301, USA. */ +#include #include "factor_stats.h" -#include "config.h" #include "val.h" #include "hash.h" #include "algorithm.h" #include "alloc.h" +#include "moments.h" +#include "percentiles.h" #include #include #include #include - +#include void -metrics_precalc(struct metrics *fs) +metrics_precalc(struct metrics *m) { - fs->n = 0; - fs->ssq = 0; - fs->sum = 0; - fs->min = DBL_MAX; - fs->max = -DBL_MAX; + assert (m) ; + + m->n_missing = 0; + + m->min = DBL_MAX; + m->max = -DBL_MAX; - fs->ordered_data = hsh_create(20, + m->histogram = 0; + + m->moments = moments1_create(MOMENT_KURTOSIS); + + m->ordered_data = hsh_create(20, (hsh_compare_func *) compare_values, (hsh_hash_func *) hash_value, - 0, + (hsh_free_func *) weighted_value_free, (void *) 0); } + +/* Include val in the calculation for the metrics. + If val is null, then treat it as MISSING +*/ void -metrics_calc(struct metrics *fs, const union value *val, double weight) +metrics_calc(struct metrics *fs, const union value *val, + double weight, int case_no) { - - struct weighted_value **wv; - const double x = val->f; + double x; - fs->n += weight; - fs->ssq += x * x * weight; - fs->sum += x * weight; + if ( ! val ) + { + fs->n_missing += weight; + return ; + } + + x = val->f; + + moments1_add(fs->moments, x, weight); + if ( x < fs->min) fs->min = x; if ( x > fs->max) fs->max = x; @@ -69,121 +86,250 @@ metrics_calc(struct metrics *fs, const union value *val, double weight) if ( *wv ) { /* If this value has already been seen, then simply - increase its weight */ + increase its weight and push a new case number */ + + struct case_node *cn; assert( (*wv)->v.f == val->f ); (*wv)->w += weight; + + cn = xmalloc( sizeof (struct case_node) ) ; + cn->next = (*wv)->case_nos ; + cn->num = case_no; + + (*wv)->case_nos = cn; } else { - *wv = xmalloc( sizeof (struct weighted_value) ); + struct case_node *cn; + + *wv = weighted_value_create(); (*wv)->v = *val; (*wv)->w = weight; - hsh_insert(fs->ordered_data,(void *) *wv); + + cn = xmalloc( sizeof (struct case_node) ) ; + cn->next=0; + cn->num = case_no; + (*wv)->case_nos = cn; + } } void -metrics_postcalc(struct metrics *fs) +metrics_postcalc(struct metrics *m) { - double sample_var; double cc = 0.0; double tc ; int k1, k2 ; int i; int j = 1; - struct weighted_value **data; - - - int n_data; - - fs->mean = fs->sum / fs->n; + moments1_calculate (m->moments, &m->n, &m->mean, &m->var, + &m->skewness, &m->kurtosis); - sample_var = ( fs->ssq / fs->n - fs->mean * fs->mean ); + moments1_destroy (m->moments); - fs->var = fs->n * sample_var / ( fs->n - 1) ; - fs->stddev = sqrt(fs->var); + m->stddev = sqrt(m->var); /* FIXME: Check this is correct ??? Shouldn't we use the sample variance ??? */ - fs->stderr = sqrt (fs->var / fs->n) ; + m->se_mean = sqrt (m->var / m->n) ; - data = (struct weighted_value **) hsh_data(fs->ordered_data); - n_data = hsh_count(fs->ordered_data); - fs->wv = xmalloc ( sizeof (struct weighted_value) * n_data); - for ( i = 0 ; i < n_data ; ++i ) - fs->wv[i] = *(data[i]); + m->wvp = (struct weighted_value **) hsh_sort(m->ordered_data); + m->n_data = hsh_count(m->ordered_data); - sort (fs->wv, n_data, sizeof (struct weighted_value) , - (algo_compare_func *) compare_values, 0); + /* Trimmed mean calculation */ + if ( m->n_data <= 1 ) + { + m->trimmed_mean = m->mean; + return; + } + m->histogram = histogram_create(10, m->min, m->max); - - tc = fs->n * 0.05 ; + for ( i = 0 ; i < m->n_data ; ++i ) + { + struct weighted_value **wv = (m->wvp) ; + gsl_histogram_accumulate(m->histogram, wv[i]->v.f, wv[i]->w); + } + + tc = m->n * 0.05 ; k1 = -1; k2 = -1; - - for ( i = 0 ; i < n_data ; ++i ) + for ( i = 0 ; i < m->n_data ; ++i ) { - cc += fs->wv[i].w; - fs->wv[i].cc = cc; + cc += m->wvp[i]->w; + m->wvp[i]->cc = cc; - fs->wv[i].rank = j + (fs->wv[i].w - 1) / 2.0 ; + m->wvp[i]->rank = j + (m->wvp[i]->w - 1) / 2.0 ; - j += fs->wv[i].w; + j += m->wvp[i]->w; if ( cc < tc ) k1 = i; - } - k2 = n_data; - for ( i = n_data -1 ; i >= 0; --i ) + + + k2 = m->n_data; + for ( i = m->n_data -1 ; i >= 0; --i ) { - if ( tc > fs->n - fs->wv[i].cc) + if ( tc > m->n - m->wvp[i]->cc) k2 = i; } - fs->trimmed_mean = 0; + /* Calculate the percentiles */ + ptiles(m->ptile_hash, m->wvp, m->n_data, m->n, m->ptile_alg); + + tukey_hinges(m->wvp, m->n_data, m->n, m->hinge); + + /* Special case here */ + if ( k1 + 1 == k2 ) + { + m->trimmed_mean = m->wvp[k2]->v.f; + return; + } + + m->trimmed_mean = 0; for ( i = k1 + 2 ; i <= k2 - 1 ; ++i ) { - fs->trimmed_mean += fs->wv[i].v.f * fs->wv[i].w; + m->trimmed_mean += m->wvp[i]->v.f * m->wvp[i]->w; } - fs->trimmed_mean += (fs->n - fs->wv[k2 - 1].cc - tc) * fs->wv[k2].v.f ; - fs->trimmed_mean += (fs->wv[k1 + 1].cc - tc) * fs->wv[k1 + 1].v.f ; - fs->trimmed_mean /= 0.9 * fs->n ; + m->trimmed_mean += (m->n - m->wvp[k2 - 1]->cc - tc) * m->wvp[k2]->v.f ; + m->trimmed_mean += (m->wvp[k1 + 1]->cc - tc) * m->wvp[k1 + 1]->v.f ; + m->trimmed_mean /= 0.9 * m->n ; + } -/* Functions for hashes */ +struct weighted_value * +weighted_value_create(void) +{ + struct weighted_value *wv; + wv = xmalloc (sizeof (struct weighted_value )); + + wv->cc = 0; + wv->case_nos = 0; + + return wv; +} void -free_factor_stats(struct factor_statistics *f, int width UNUSED) +weighted_value_free(struct weighted_value *wv) { - free (f); + struct case_node *cn ; + + if ( !wv ) + return ; + + cn = wv->case_nos; + + while(cn) + { + struct case_node *next = cn->next; + + free(cn); + cn = next; + } + + free(wv); + } -int -compare_indep_values(const struct factor_statistics *f1, - const struct factor_statistics *f2, - int width) + + + + +/* Create a factor statistics object with for N dependent vars + and ID as the value of the independent variable */ +struct factor_statistics * +create_factor_statistics (int n, union value *id0, union value *id1) { - return compare_values(f1->id, f2->id, width); + struct factor_statistics *f; + + f = xmalloc( sizeof ( struct factor_statistics )); + + f->id[0] = *id0; + f->id[1] = *id1; + f->m = xmalloc( sizeof ( struct metrics ) * n ) ; + memset (f->m, 0, sizeof(struct metrics) * n); + f->n_var = n; + + return f; } -unsigned -hash_indep_value(const struct factor_statistics *f, int width) +void +metrics_destroy(struct metrics *m) { - return hash_value(f->id, width); + hsh_destroy(m->ordered_data); + hsh_destroy(m->ptile_hash); + if ( m-> histogram ) + gsl_histogram_free(m->histogram); +} + +void +factor_statistics_free(struct factor_statistics *f) +{ + + int i; + for ( i = 0 ; i < f->n_var; ++i ) + metrics_destroy(&f->m[i]); + free(f->m) ; + free(f); +} + + + + +int +factor_statistics_compare(const struct factor_statistics *f0, + const struct factor_statistics *f1, int width) +{ + + int cmp0; + + assert(f0); + assert(f1); + + cmp0 = compare_values(&f0->id[0], &f1->id[0], width); + + if ( cmp0 != 0 ) + return cmp0; + + + if ( ( f0->id[1].f == SYSMIS ) && (f1->id[1].f != SYSMIS) ) + return 1; + + if ( ( f0->id[1].f != SYSMIS ) && (f1->id[1].f == SYSMIS) ) + return -1; + + return compare_values(&f0->id[1], &f1->id[1], width); + +} + +unsigned int +factor_statistics_hash(const struct factor_statistics *f, int width) +{ + + unsigned int h; + + h = hash_value(&f->id[0], width); + + if ( f->id[1].f != SYSMIS ) + h += hash_value(&f->id[1], width); + + + return h; + } +