X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;ds=inline;f=src%2Fdata%2Fcasereader-translator.c;h=5c8b9f4209929f335c702f47e6f96312d8f1fabe;hb=df1d194af32e35739355981252785e768f21ebb0;hp=14fcd9ab70b6e3be60df6a585cfda094e48c844a;hpb=dcdf58a8089283c5eee2ea642e03dfacc60543e5;p=pspp
diff --git a/src/data/casereader-translator.c b/src/data/casereader-translator.c
index 14fcd9ab70..5c8b9f4209 100644
--- a/src/data/casereader-translator.c
+++ b/src/data/casereader-translator.c
@@ -1,5 +1,5 @@
/* PSPP - a program for statistical analysis.
- Copyright (C) 2007 Free Software Foundation, Inc.
+ Copyright (C) 2007, 2009, 2011 Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -15,15 +15,16 @@
along with this program. If not, see . */
#include
-#include
-#include
-#include
+
#include
-#include
-#include
+#include "data/casereader-provider.h"
+#include "data/casereader.h"
+#include "data/val-type.h"
+#include "data/variable.h"
+#include "libpspp/taint.h"
-#include "xalloc.h"
+#include "gl/xalloc.h"
/* Casereader that applies a user-supplied function to translate
each case into another in an arbitrary fashion. */
@@ -32,9 +33,7 @@
struct casereader_translator
{
struct casereader *subreader; /* Source of input cases. */
-
- void (*translate) (struct ccase *input, struct ccase *output, void *aux);
- bool (*destroy) (void *aux);
+ const struct casereader_translator_class *class;
void *aux;
};
@@ -42,9 +41,16 @@ static const struct casereader_class casereader_translator_class;
/* Creates and returns a new casereader whose cases are produced
by reading from SUBREADER and passing through TRANSLATE, which
- must create case OUTPUT, with OUTPUT_VALUE_CNT values, and
- populate it based on INPUT and auxiliary data AUX. TRANSLATE
- must also destroy INPUT.
+ must return the translated case, and populate it based on
+ INPUT and auxiliary data AUX. TRANSLATE must destroy its
+ input case.
+
+ TRANSLATE may be stateful, that is, the output for a given
+ case may depend on previous cases. If TRANSLATE is stateless,
+ then you may want to use casereader_translate_stateless
+ instead, since it sometimes performs better.
+
+ The cases returned by TRANSLATE must match OUTPUT_PROTO.
When the translating casereader is destroyed, DESTROY will be
called to allow any state maintained by TRANSLATE to be freed.
@@ -54,21 +60,19 @@ static const struct casereader_class casereader_translator_class;
when the translating casereader is destroyed. */
struct casereader *
casereader_create_translator (struct casereader *subreader,
- size_t output_value_cnt,
- void (*translate) (struct ccase *input,
- struct ccase *output,
- void *aux),
- bool (*destroy) (void *aux),
+ const struct caseproto *output_proto,
+ const struct casereader_translator_class *class,
void *aux)
{
struct casereader_translator *ct = xmalloc (sizeof *ct);
- struct casereader *reader;
- ct->subreader = casereader_rename (subreader);
- ct->translate = translate;
- ct->destroy = destroy;
- ct->aux = aux;
- reader = casereader_create_sequential (
- NULL, output_value_cnt, casereader_get_case_cnt (ct->subreader),
+ *ct = (struct casereader_translator) {
+ .subreader = casereader_rename (subreader),
+ .class = class,
+ .aux = aux,
+ };
+
+ struct casereader *reader = casereader_create_sequential (
+ NULL, output_proto, casereader_get_n_cases (ct->subreader),
&casereader_translator_class, ct);
taint_propagate (casereader_get_taint (ct->subreader),
casereader_get_taint (reader));
@@ -76,20 +80,15 @@ casereader_create_translator (struct casereader *subreader,
}
/* Internal read function for translating casereader. */
-static bool
+static struct ccase *
casereader_translator_read (struct casereader *reader UNUSED,
- void *ct_, struct ccase *c)
+ void *ct_)
{
struct casereader_translator *ct = ct_;
- struct ccase tmp;
-
- if (casereader_read (ct->subreader, &tmp))
- {
- ct->translate (&tmp, c, ct->aux);
- return true;
- }
- else
- return false;
+ struct ccase *tmp = casereader_read (ct->subreader);
+ if (tmp)
+ tmp = ct->class->translate (tmp, ct->aux);
+ return tmp;
}
/* Internal destroy function for translating casereader. */
@@ -98,7 +97,7 @@ casereader_translator_destroy (struct casereader *reader UNUSED, void *ct_)
{
struct casereader_translator *ct = ct_;
casereader_destroy (ct->subreader);
- ct->destroy (ct->aux);
+ ct->class->destroy (ct->aux);
free (ct);
}
@@ -110,12 +109,107 @@ static const struct casereader_class casereader_translator_class =
NULL,
NULL,
};
+
+/* Casereader that applies a user-supplied function to translate
+ each case into another in a stateless fashion. */
+/* A statelessly translating casereader. */
+struct casereader_stateless_translator
+ {
+ struct casereader *subreader; /* Source of input cases. */
+
+ casenumber case_offset;
+ const struct casereader_translator_class *class;
+ void *aux;
+ };
+
+static const struct casereader_random_class
+casereader_stateless_translator_class;
+
+/* Creates and returns a new casereader whose cases are produced by reading
+ from SUBREADER and passing through the TRANSLATE function. TRANSLATE must
+ takes ownership of its input case and returns a translated case, populating
+ the translated case based on INPUT and auxiliary data AUX.
+
+ TRANSLATE must be stateless, that is, the output for a given case must not
+ depend on previous cases. This is because cases may be retrieved in
+ arbitrary order, and some cases may be retrieved multiple times, and some
+ cases may be skipped and never retrieved at all. If TRANSLATE is stateful,
+ use casereader_create_translator instead.
+
+ The cases returned by TRANSLATE must match OUTPUT_PROTO.
+
+ When the stateless translating casereader is destroyed, DESTROY will be
+ called to allow any auxiliary data maintained by TRANSLATE to be freed.
+
+ After this function is called, SUBREADER must not ever again be referenced
+ directly. It will be destroyed automatically when the translating
+ casereader is destroyed. */
+struct casereader *
+casereader_translate_stateless (
+ struct casereader *subreader,
+ const struct caseproto *output_proto,
+ const struct casereader_translator_class *class,
+ void *aux)
+{
+ struct casereader_stateless_translator *cst = xmalloc (sizeof *cst);
+ *cst = (struct casereader_stateless_translator) {
+ .subreader = casereader_rename (subreader),
+ .class = class,
+ .aux = aux,
+ };
+ struct casereader *reader = casereader_create_random (
+ output_proto, casereader_get_n_cases (cst->subreader),
+ &casereader_stateless_translator_class, cst);
+ taint_propagate (casereader_get_taint (cst->subreader),
+ casereader_get_taint (reader));
+ return reader;
+}
+
+/* Internal read function for stateless translating casereader. */
+static struct ccase *
+casereader_stateless_translator_read (struct casereader *reader UNUSED,
+ void *cst_, casenumber idx)
+{
+ struct casereader_stateless_translator *cst = cst_;
+ struct ccase *tmp = casereader_peek (cst->subreader, idx);
+ if (tmp != NULL)
+ tmp = cst->class->translate (tmp, cst->aux);
+ return tmp;
+}
+
+/* Internal destroy function for translating casereader. */
+static void
+casereader_stateless_translator_destroy (struct casereader *reader UNUSED,
+ void *cst_)
+{
+ struct casereader_stateless_translator *cst = cst_;
+ casereader_destroy (cst->subreader);
+ cst->class->destroy (cst->aux);
+ free (cst);
+}
+
+static void
+casereader_stateless_translator_advance (struct casereader *reader UNUSED,
+ void *cst_, casenumber n)
+{
+ struct casereader_stateless_translator *cst = cst_;
+ cst->case_offset += casereader_advance (cst->subreader, n);
+}
+
+/* Casereader class for stateless translating casereader. */
+static const struct casereader_random_class
+casereader_stateless_translator_class =
+ {
+ casereader_stateless_translator_read,
+ casereader_stateless_translator_destroy,
+ casereader_stateless_translator_advance,
+ };
struct casereader_append_numeric
{
- int value_ofs;
+ struct caseproto *proto;
casenumber n;
new_value_func *func;
void *aux;
@@ -124,14 +218,13 @@ struct casereader_append_numeric
static bool can_destroy (void *can_);
-static void can_translate (struct ccase *input, struct ccase *output,
- void *can_);
+static struct ccase *can_translate (struct ccase *, void *can_);
/* Creates and returns a new casereader whose cases are produced
by reading from SUBREADER and appending an additional value,
generated by FUNC. AUX is an optional parameter which
gets passed to FUNC. FUNC will also receive N as it, which is
- the ordinal number of the case in the reader. DESTROY is an
+ the ordinal number of the case in the reader. DESTROY is an
optional parameter used to destroy AUX.
After this function is called, SUBREADER must not ever again
@@ -143,25 +236,28 @@ casereader_create_append_numeric (struct casereader *subreader,
void (*destroy) (void *aux))
{
struct casereader_append_numeric *can = xmalloc (sizeof *can);
- can->value_ofs = casereader_get_value_cnt (subreader);
+ can->proto = caseproto_ref (casereader_get_proto (subreader));
+ can->proto = caseproto_add_width (can->proto, 0);
can->n = 0;
can->aux = aux;
can->func = func;
can->destroy = destroy;
- return casereader_create_translator (subreader, can->value_ofs + 1,
- can_translate, can_destroy, can);
+
+ static const struct casereader_translator_class class = {
+ can_translate, can_destroy,
+ };
+ return casereader_create_translator (subreader, can->proto, &class, can);
}
-static void
-can_translate (struct ccase *input, struct ccase *output, void *can_)
+static struct ccase *
+can_translate (struct ccase *c, void *can_)
{
struct casereader_append_numeric *can = can_;
- double new_value = can->func (input, can->n++, can->aux);
- case_nullify (output);
- case_move (output, input);
- case_resize (output, can->value_ofs + 1);
- case_data_rw_idx (output, can->value_ofs)->f = new_value;
+ double new_value = can->func (c, can->n++, can->aux);
+ c = case_unshare_and_resize (c, can->proto);
+ *case_num_rw_idx (c, caseproto_get_n_widths (can->proto) - 1) = new_value;
+ return c;
}
static bool
@@ -170,6 +266,7 @@ can_destroy (void *can_)
struct casereader_append_numeric *can = can_;
if (can->destroy)
can->destroy (can->aux);
+ caseproto_unref (can->proto);
free (can);
return true;
}
@@ -204,7 +301,7 @@ struct casereader *
casereader_create_arithmetic_sequence (struct casereader *subreader,
double first, double increment)
{
- struct arithmetic_sequence *as = xzalloc (sizeof *as);
+ struct arithmetic_sequence *as = XZALLOC (struct arithmetic_sequence);
as->first = first;
as->increment = increment;
return casereader_create_append_numeric (subreader, next_arithmetic,
@@ -220,28 +317,39 @@ struct casereader_append_rank
casenumber n;
const struct variable *var;
const struct variable *weight;
- int value_ofs;
+ struct caseproto *proto;
casenumber n_common;
double mean_rank;
double cc;
distinct_func *distinct;
void *aux;
+ enum rank_error *err;
+ double prev_value;
};
static bool car_destroy (void *car_);
-static void car_translate (struct ccase *input, struct ccase *output,
- void *car_);
+static struct ccase *car_translate (struct ccase *input, void *car_);
/* Creates and returns a new casereader whose cases are produced
by reading from SUBREADER and appending an additional value,
- which is the rank of the observation. SUBREADER must be sorted
- on V. W is the weight variable of the dictionary containing V,
- or NULL if there is no weight variable.
+ which is the rank of the observation. W is the weight variable
+ of the dictionary containing V, or NULL if there is no weight
+ variable.
+
+ The following preconditions must be met:
+
+ 1. SUBREADER must be sorted on V.
+
+ 2. The weight variables, must be non-negative.
+
+ If either of these preconditions are not satisfied, then the rank
+ variables may not be correct. In this case, if ERR is non-null,
+ it will be set according to the erroneous conditions encountered.
If DISTINCT_CALLBACK is non-null, then it will be called exactly
once for every case containing a distinct value of V. AUX is
- an auxilliary pointer passed to DISTINCT_CALLBACK.
+ an auxiliary pointer passed to DISTINCT_CALLBACK.
After this function is called, SUBREADER must not ever again
be referenced directly. It will be destroyed automatically
@@ -250,12 +358,14 @@ struct casereader *
casereader_create_append_rank (struct casereader *subreader,
const struct variable *v,
const struct variable *w,
+ enum rank_error *err,
distinct_func *distinct_callback,
void *aux
- )
+ )
{
struct casereader_append_rank *car = xmalloc (sizeof *car);
- car->value_ofs = casereader_get_value_cnt (subreader);
+ car->proto = caseproto_ref (casereader_get_proto (subreader));
+ car->proto = caseproto_add_width (car->proto, 0);
car->weight = w;
car->var = v;
car->n = 0;
@@ -264,9 +374,13 @@ casereader_create_append_rank (struct casereader *subreader,
car->clone = casereader_clone (subreader);
car->distinct = distinct_callback;
car->aux = aux;
+ car->err = err;
+ car->prev_value = SYSMIS;
- return casereader_create_translator (subreader, car->value_ofs + 1,
- car_translate, car_destroy, car);
+ static const struct casereader_translator_class class = {
+ car_translate, car_destroy
+ };
+ return casereader_create_translator (subreader, car->proto, &class, car);
}
@@ -275,42 +389,59 @@ car_destroy (void *car_)
{
struct casereader_append_rank *car = car_;
casereader_destroy (car->clone);
+ caseproto_unref (car->proto);
free (car);
return true;
}
-
-static void
-car_translate (struct ccase *input, struct ccase *output, void *car_)
+static struct ccase *
+car_translate (struct ccase *input, void *car_)
{
struct casereader_append_rank *car = car_;
- double value = case_data (input, car->var)->f;
+ const double value = case_num (input, car->var);
- if ( car->n_common == 1)
+ if (car->prev_value != SYSMIS)
+ {
+ if (car->err && value < car->prev_value)
+ *car->err |= RANK_ERR_UNSORTED;
+ }
+
+ if (car->n_common == 1)
{
double vxx = SYSMIS;
casenumber k = 0;
double weight = 1.0;
if (car->weight)
- weight = case_data (input, car->weight)->f;
+ {
+ weight = case_num (input, car->weight);
+ if (car->err && weight < 0)
+ *car->err |= RANK_ERR_NEGATIVE_WEIGHT;
+ }
do
{
- struct ccase c;
- if ( ! casereader_peek (car->clone, car->n + ++k, &c))
+ struct ccase *c = casereader_peek (car->clone, car->n + ++k);
+ if (c == NULL)
break;
- vxx = case_data (&c, car->var)->f;
+ vxx = case_num (c, car->var);
- if ( vxx == value)
+ if (vxx == value)
{
if (car->weight)
- weight += case_data (&c, car->weight)->f;
+ {
+ double w = case_num (c, car->weight);
+
+ if (car->err && w < 0)
+ *car->err |= RANK_ERR_NEGATIVE_WEIGHT;
+
+ weight += w;
+ }
else
weight += 1.0;
car->n_common++;
}
- case_destroy (&c);
+ case_unref (c);
}
while (vxx == value);
car->mean_rank = car->cc + (weight + 1) / 2.0;
@@ -324,10 +455,142 @@ car_translate (struct ccase *input, struct ccase *output, void *car_)
car->n++;
- case_nullify (output);
- case_move (output, input);
- case_resize (output, car->value_ofs + 1);
- case_data_rw_idx (output, car->value_ofs)->f = car->mean_rank ;
+ input = case_unshare_and_resize (input, car->proto);
+ *case_num_rw_idx (input, caseproto_get_n_widths (car->proto) - 1)
+ = car->mean_rank;
+ car->prev_value = value;
+ return input;
+}
+
+
+
+
+struct consolidator
+{
+ const struct variable *key;
+ const struct variable *weight;
+ double cc;
+ double prev_cc;
+
+ casenumber n;
+ struct casereader *clone;
+ struct caseproto *proto;
+ int direction;
+};
+
+static bool
+uniquify (const struct ccase *c, void *aux)
+{
+ struct consolidator *cdr = aux;
+ const union value *current_value = case_data (c, cdr->key);
+ const int key_width = var_get_width (cdr->key);
+ const double weight = cdr->weight ? case_num (c, cdr->weight) : 1.0;
+ struct ccase *next_case = casereader_peek (cdr->clone, cdr->n + 1);
+ int dir = 0;
+
+ cdr->n ++;
+ cdr->cc += weight;
+
+ if (NULL == next_case)
+ goto end;
+
+ dir = value_compare_3way (case_data (next_case, cdr->key),
+ current_value, key_width);
+ if (dir > 0)
+ dir = 1;
+ if (dir < 0)
+ dir = -1;
+
+ case_unref (next_case);
+ if (dir != 0)
+ {
+ /* Insist that the data are sorted */
+ assert (cdr->direction == 0 || dir == cdr->direction);
+ cdr->direction = dir;
+ goto end;
+ }
+
+ return false;
+
+ end:
+ cdr->prev_cc = cdr->cc;
+ cdr->cc = 0;
+ return true;
+}
+
+
+
+static struct ccase *
+consolodate_weight (struct ccase *input, void *aux)
+{
+ struct consolidator *cdr = aux;
+ struct ccase *c;
+
+ if (cdr->weight)
+ {
+ c = case_unshare (input);
+ *case_num_rw (c, cdr->weight) = cdr->prev_cc;
+ }
+ else
+ {
+ c = case_unshare_and_resize (input, cdr->proto);
+ *case_num_rw_idx (c, caseproto_get_n_widths (cdr->proto) - 1) = cdr->prev_cc;
+ }
+
+ return c;
+}
+
+
+static bool
+uniquify_destroy (void *aux)
+{
+ struct consolidator *cdr = aux;
+
+ casereader_destroy (cdr->clone);
+ caseproto_unref (cdr->proto);
+ free (cdr);
+
+ return true;
}
+
+/* Returns a new casereader which is based upon INPUT, but which contains a maximum
+ of one case for each distinct value of KEY.
+ If WEIGHT is non-null, then the new casereader's values for this variable
+ will be the sum of all values matching KEY.
+ IF WEIGHT is null, then the new casereader will have an additional numeric
+ value appended, which will contain the total number of cases containing
+ KEY.
+ INPUT must be sorted on KEY
+*/
+struct casereader *
+casereader_create_distinct (struct casereader *input,
+ const struct variable *key,
+ const struct variable *weight)
+{
+ struct casereader *u ;
+ struct caseproto *output_proto = caseproto_ref (casereader_get_proto (input));
+
+ struct consolidator *cdr = xmalloc (sizeof (*cdr));
+ cdr->n = 0;
+ cdr->key = key;
+ cdr->weight = weight;
+ cdr->cc = 0;
+ cdr->clone = casereader_clone (input);
+ cdr->direction = 0;
+
+ if (NULL == cdr->weight)
+ output_proto = caseproto_add_width (output_proto, 0);
+
+ cdr->proto = output_proto;
+
+ u = casereader_create_filter_func (input, uniquify,
+ NULL, cdr, NULL);
+
+ static const struct casereader_translator_class class = {
+ consolodate_weight, uniquify_destroy,
+ };
+ return casereader_create_translator (u, output_proto, &class, cdr);
+}
+