From 9703148a4b6ae39bd7ff79741020dac46018a7f7 Mon Sep 17 00:00:00 2001 From: Jason Stover Date: Sat, 17 Sep 2005 15:55:56 +0000 Subject: [PATCH] Initial version --- lib/linreg/Makefile.am | 17 ++ lib/linreg/linreg.c | 342 +++++++++++++++++++++++++++++++++++++++ lib/linreg/pspp_linreg.h | 161 ++++++++++++++++++ lib/linreg/sweep.c | 155 ++++++++++++++++++ 4 files changed, 675 insertions(+) create mode 100644 lib/linreg/Makefile.am create mode 100644 lib/linreg/linreg.c create mode 100644 lib/linreg/pspp_linreg.h create mode 100644 lib/linreg/sweep.c diff --git a/lib/linreg/Makefile.am b/lib/linreg/Makefile.am new file mode 100644 index 0000000000..5fa2f5596a --- /dev/null +++ b/lib/linreg/Makefile.am @@ -0,0 +1,17 @@ +## Process this file with automake to produce Makefile.in -*- makefile -*- + +noinst_LIBRARIES = liblinreg.a + +AM_CPPFLAGS = -I$(top_srcdir) + +AM_CFLAGS= + +if cc_is_gcc +AM_CFLAGS+=-Wall -W -Wwrite-strings -Wstrict-prototypes \ +-Wpointer-arith -Wno-sign-compare -Wmissing-prototypes \ +-ansi +endif + +liblinreg_a_SOURCES = sweep.c linreg.c pspp_linreg.h + +EXTRA_DIST = README diff --git a/lib/linreg/linreg.c b/lib/linreg/linreg.c new file mode 100644 index 0000000000..64f782d3f6 --- /dev/null +++ b/lib/linreg/linreg.c @@ -0,0 +1,342 @@ +/* lib/linreg/linreg.c + + Copyright (C) 2005 Free Software Foundation, Inc. + Written by Jason H. Stover. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or (at + your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02111-1307, USA. +*/ + +/* + Find the least-squares estimate of b for the linear model: + + Y = Xb + Z + + where Y is an n-by-1 column vector, X is an n-by-p matrix of + independent variables, b is a p-by-1 vector of regression coefficients, + and Z is an n-by-1 normally-distributed random vector with independent + identically distributed components with mean 0. + + This estimate is found via the sweep operator or singular-value + decomposition with gsl. + + + References: + + 1. Matrix Computations, third edition. GH Golub and CF Van Loan. + The Johns Hopkins University Press. 1996. ISBN 0-8018-5414-8. + + 2. Numerical Analysis for Statisticians. K Lange. Springer. 1999. + ISBN 0-387-94979-8. + + 3. Numerical Linear Algebra for Applications in Statistics. JE Gentle. + Springer. 1998. ISBN 0-387-98542-5. +*/ + +#include "pspp_linreg.h" +#include +/* + Get the mean and standard deviation of a vector + of doubles via a form of the Kalman filter as + described on page 32 of [3]. + */ +static int +linreg_mean_std (gsl_vector_const_view v, double *mp, double *sp, double *ssp) +{ + size_t i; + double j = 0.0; + double d; + double tmp; + double mean; + double variance; + + mean = gsl_vector_get (&v.vector, 0); + variance = 0; + for (i = 1; i < v.vector.size; i++) + { + j = (double) i + 1.0; + tmp = gsl_vector_get (&v.vector, i); + d = (tmp - mean) / j; + mean += d; + variance += j * (j - 1.0) * d * d; + } + *mp = mean; + *sp = sqrt (variance / (j - 1.0)); + *ssp = variance; + + return GSL_SUCCESS; +} + +/* + Allocate a pspp_linreg_cache and return a pointer + to it. n is the number of cases, p is the number of + independent variables. + */ +pspp_linreg_cache * +pspp_linreg_cache_alloc (size_t n, size_t p) +{ + pspp_linreg_cache *cache; + + cache = (pspp_linreg_cache *) malloc (sizeof (pspp_linreg_cache)); + cache->param_estimates = gsl_vector_alloc (p + 1); + cache->indep_means = gsl_vector_alloc (p); + cache->indep_std = gsl_vector_alloc (p); + cache->ssx = gsl_vector_alloc (p); /* Sums of squares for the independent + variables. + */ + cache->ss_indeps = gsl_vector_alloc (p); /* Sums of squares for the model + parameters. + */ + cache->cov = gsl_matrix_alloc (p + 1, p + 1); /* Covariance matrix. */ + cache->n_obs = n; + cache->n_indeps = p; + /* + Default settings. + */ + cache->method = PSPP_LINREG_SWEEP; + + return cache; +} + +void +pspp_linreg_cache_free (pspp_linreg_cache * cache) +{ + gsl_vector_free (cache->param_estimates); + gsl_vector_free (cache->indep_means); + gsl_vector_free (cache->indep_std); + gsl_vector_free (cache->ss_indeps); + gsl_matrix_free (cache->cov); + free (cache); +} + +/* + Fit the linear model via least squares. All pointers passed to pspp_linreg + are assumed to be allocated to the correct size and initialized to the + values as indicated by opts. + */ +int +pspp_linreg (const gsl_vector * Y, const gsl_matrix * X, + const pspp_linreg_opts * opts, pspp_linreg_cache * cache) +{ + int rc; + gsl_matrix *design; + gsl_matrix_view xtx; + gsl_matrix_view xm; + gsl_matrix_view xmxtx; + gsl_vector_view xty; + gsl_vector_view xi; + gsl_vector_view xj; + + size_t i; + size_t j; + double tmp; + double m; + double s; + double ss; + double mse; + + if (cache == NULL) + { + return GSL_EFAULT; + } + if (opts->get_depvar_mean_std) + { + linreg_mean_std (gsl_vector_const_subvector (Y, 0, Y->size), + &m, &s, &ss); + cache->depvar_mean = m; + cache->depvar_std = s; + cache->sst = ss; + } + for (i = 0; i < cache->n_indeps; i++) + { + if (opts->get_indep_mean_std[i]) + { + linreg_mean_std (gsl_matrix_const_column (X, i), &m, &s, &ss); + gsl_vector_set (cache->indep_means, i, m); + gsl_vector_set (cache->indep_std, i, s); + gsl_vector_set (cache->ssx, i, ss); + } + } + cache->dft = cache->n_obs - 1; + cache->dfm = cache->n_indeps; + cache->dfe = cache->dft - cache->dfm; + if (cache->method == PSPP_LINREG_SWEEP) + { + gsl_matrix *sw; + /* + Subtract the means to improve the condition of the design + matrix. This requires copying X and Y. We do not divide by the + standard deviations of the independent variables here since doing + so would cause a miscalculation of the residual sums of + squares. Dividing by the standard deviation is done GSL's linear + regression functions, so if the design matrix has a very poor + condition, use QR decomposition. + * + The design matrix here does not include a column for the intercept + (i.e., a column of 1's). If using PSPP_LINREG_QR, we need that column, + so design is allocated here when sweeping, or below if using QR. + */ + design = gsl_matrix_alloc (X->size1, X->size2); + for (i = 0; i < X->size2; i++) + { + m = gsl_vector_get (cache->indep_means, i); + for (j = 0; j < X->size1; j++) + { + tmp = (gsl_matrix_get (X, j, i) - m); + gsl_matrix_set (design, j, i, tmp); + } + } + sw = gsl_matrix_calloc (cache->n_indeps + 1, cache->n_indeps + 1); + xtx = gsl_matrix_submatrix (sw, 0, 0, cache->n_indeps, cache->n_indeps); + + for (i = 0; i < xtx.matrix.size1; i++) + { + tmp = gsl_vector_get (cache->ssx, i); + gsl_matrix_set (&(xtx.matrix), i, i, tmp); + xi = gsl_matrix_column (design, i); + for (j = (i + 1); j < xtx.matrix.size2; j++) + { + xj = gsl_matrix_column (design, j); + gsl_blas_ddot (&(xi.vector), &(xj.vector), &tmp); + gsl_matrix_set (&(xtx.matrix), i, j, tmp); + } + } + + gsl_matrix_set (sw, cache->n_indeps, cache->n_indeps, cache->sst); + xty = gsl_matrix_column (sw, cache->n_indeps); + /* + This loop starts at 1, with i=0 outside the loop, so we can get + the model sum of squares due to the first independent variable. + */ + xi = gsl_matrix_column (design, 0); + gsl_blas_ddot (&(xi.vector), Y, &tmp); + gsl_vector_set (&(xty.vector), 0, tmp); + tmp *= tmp / gsl_vector_get (cache->ssx, 0); + gsl_vector_set (cache->ss_indeps, 0, tmp); + for (i = 1; i < cache->n_indeps; i++) + { + xi = gsl_matrix_column (design, i); + gsl_blas_ddot (&(xi.vector), Y, &tmp); + gsl_vector_set (&(xty.vector), i, tmp); + } + + /* + Sweep on the matrix sw, which contains XtX, XtY and YtY. + */ + pspp_reg_sweep (sw); + cache->sse = gsl_matrix_get (sw, cache->n_indeps, cache->n_indeps); + cache->mse = cache->sse / cache->dfe; + /* + Get the intercept. + */ + m = cache->depvar_mean; + for (i = 0; i < cache->n_indeps; i++) + { + tmp = gsl_matrix_get (sw, i, cache->n_indeps); + gsl_vector_set (cache->param_estimates, i + 1, tmp); + m -= tmp * gsl_vector_get (cache->indep_means, i); + } + /* + Get the covariance matrix of the parameter estimates. + Only the upper triangle is necessary. + */ + + /* + The loops below do not compute the entries related + to the estimated intercept. + */ + for (i = 0; i < cache->n_indeps; i++) + for (j = i; j < cache->n_indeps; j++) + { + tmp = -1.0 * cache->mse * gsl_matrix_get (sw, i, j); + gsl_matrix_set (cache->cov, i + 1, j + 1, tmp); + } + /* + Get the covariances related to the intercept. + */ + xtx = gsl_matrix_submatrix (sw, 0, 0, cache->n_indeps, cache->n_indeps); + xmxtx = gsl_matrix_submatrix (cache->cov, 0, 1, 1, cache->n_indeps); + xm = gsl_matrix_view_vector (cache->indep_means, 1, cache->n_indeps); + rc = gsl_blas_dsymm (CblasRight, CblasUpper, cache->mse, + &xtx.matrix, &xm.matrix, 0.0, &xmxtx.matrix); + if (rc == GSL_SUCCESS) + { + tmp = cache->mse / cache->n_obs; + for (i = 1; i < 1 + cache->n_indeps; i++) + { + tmp -= gsl_matrix_get (cache->cov, 0, i) + * gsl_vector_get (cache->indep_means, i - 1); + } + gsl_matrix_set (cache->cov, 0, 0, tmp); + + gsl_vector_set (cache->param_estimates, 0, m); + } + else + { + fprintf (stderr, "%s:%d:gsl_blas_dsymm: %s\n", + __FILE__, __LINE__, gsl_strerror (rc)); + exit (rc); + } + gsl_matrix_free (sw); + } + else + { + /* + Use QR decomposition via GSL. This section has not been tested. + */ + design = gsl_matrix_alloc (X->size1, 1 + X->size2); + + for (j = 0; j < X->size1; j++) + { + gsl_matrix_set (design, j, 0, 1.0); + for (i = 0; i < X->size2; i++) + { + tmp = gsl_matrix_get (X, j, i); + gsl_matrix_set (design, j, i + 1, tmp); + } + } + gsl_multifit_linear_workspace *wk = + gsl_multifit_linear_alloc (design->size1, design->size2); + rc = gsl_multifit_linear (design, Y, cache->param_estimates, + cache->cov, &(cache->sse), wk); + if (rc == GSL_SUCCESS) + { + gsl_multifit_linear_free (wk); + } + else + { + fprintf (stderr, "%s:%d: gsl_multifit_linear returned %d\n", + __FILE__, __LINE__, rc); + } + } + + + cache->ssm = cache->sst - cache->sse; + /* + Get the remaining sums of squares for the independent + variables. + */ + m = 0; + for (i = 1; i < cache->n_indeps; i++) + { + j = i - 1; + m += gsl_vector_get (cache->ss_indeps, j); + tmp = cache->ssm - m; + gsl_vector_set (cache->ss_indeps, i, tmp); + } + + gsl_matrix_free (design); + return GSL_SUCCESS; +} diff --git a/lib/linreg/pspp_linreg.h b/lib/linreg/pspp_linreg.h new file mode 100644 index 0000000000..209ee0b8da --- /dev/null +++ b/lib/linreg/pspp_linreg.h @@ -0,0 +1,161 @@ +/* lib/linreg/linreg.c + + Copyright (C) 2005 Free Software Foundation, Inc. + Written by Jason H Stover. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or (at + your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02111-1307, USA. + */ + +/* + Find the least-squares estimate of b for the linear model: + + Y = Xb + Z + + where Y is an n-by-1 column vector, X is an n-by-p matrix of + independent variables, b is a p-by-1 vector of regression coefficients, + and Z is an n-by-1 normally-distributed random vector with independent + identically distributed components with mean 0. + + This estimate is found via the sweep operator or singular-value + decomposition. + + + References: + + Matrix Computations, third edition. GH Golub and CF Van Loan. + The Johns Hopkins University Press. 1996. ISBN 0-8018-5414-8. + + Numerical Analysis for Statisticians. K Lange. Springer. 1999. + ISBN 0-387-94979-8. + + Numerical Linear Algebra for Applications in Statistics. JE Gentle. + Springer. 1998. ISBN 0-387-98542-5. + */ +#ifndef PSPP_LINREG_H +#define PSPP_LINREG_H +#include +#include +#include +#include +#include +#include +#include +#include +enum +{ + PSPP_LINREG_SWEEP, + PSPP_LINREG_SVD +}; + +/* + Cache for the relevant data from the model. There are several + members which the caller may not use, and which could use a lot of + storage. Therefore non-essential members of the struct will be + allocated only when requested. + */ +struct pspp_linreg_cache_struct +{ + int n_obs; /* Number of observations. */ + int n_indeps; /* Number of independent variables. */ + gsl_vector *depvar; + gsl_matrix *indepvar; + gsl_vector *residuals; + gsl_vector *param_estimates; + int method; /* Method to use to estimate parameters. */ + /* + Means and standard deviations of the variables. + If these pointers are null when pspp_linreg() is + called, pspp_linreg() will compute their values. + + Entry i of indep_means is the mean of independent + variable i, whose observations are stored in column i + of indepvar. + */ + double depvar_mean; + double depvar_std; + gsl_vector *indep_means; + gsl_vector *indep_std; + + /* + Sums of squares. + */ + double ssm; /* Sums of squares for the overall model. */ + gsl_vector *ss_indeps; /* Sums of squares from each + independent variable. + */ + double sst; /* Sum of squares total. */ + double sse; /* Sum of squares error. */ + double mse; /* Mean squared error. This is just sse / dfe, but + since it is the best unbiased estimate of the population + variance, it has its own entry here. + */ + gsl_vector *ssx; /* Centered sums of squares for independent variables, + i.e. \sum (x[i] - mean(x))^2. + */ + double ssy; /* Centered sums of squares for dependent variable. */ + /* + Covariance matrix of the parameter estimates. + */ + gsl_matrix *cov; + /* + Degrees of freedom. + */ + double dft; + double dfe; + double dfm; + + /* + 'Hat' or Hessian matrix, i.e. (X'X)^{-1}, where X is our + design matrix. + */ + gsl_matrix *hat; +}; +typedef struct pspp_linreg_cache_struct pspp_linreg_cache; + +/* + Options describing what special values should be computed. + */ +struct pspp_linreg_opts_struct +{ + int resid; /* Should the residuals be returned? */ + + int get_depvar_mean_std; + int *get_indep_mean_std; /* Array of booleans dictating which + independent variables need their means + and standard deviations computed within + pspp_linreg. This array MUST be of + length n_indeps. If element i is 1, + pspp_linreg will compute the mean and + variance of indpendent variable i. If + element i is 0, it will not compute the + mean and standard deviation, and assume + the values are stored. + cache->indep_mean[i] is the mean and + cache->indep_std[i] is the sample + standard deviation. + */ +}; +typedef struct pspp_linreg_opts_struct pspp_linreg_opts; + +int pspp_reg_sweep (gsl_matrix * A); + +pspp_linreg_cache *pspp_linreg_cache_alloc (size_t n, size_t p); + +void pspp_linreg_cache_free (pspp_linreg_cache * cache); + +int pspp_linreg (const gsl_vector * Y, const gsl_matrix * X, + const pspp_linreg_opts * opts, pspp_linreg_cache * cache); +#endif diff --git a/lib/linreg/sweep.c b/lib/linreg/sweep.c new file mode 100644 index 0000000000..6e114266c7 --- /dev/null +++ b/lib/linreg/sweep.c @@ -0,0 +1,155 @@ +/* lib/linreg/sweep.c + + Copyright (C) 2005 Free Software Foundation, Inc. + Written by Jason H Stover. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or (at + your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02111-1307, USA. + */ + +/* + Find the least-squares estimate of b for the linear model: + + Y = Xb + Z + + where Y is an n-by-1 column vector, X is an n-by-p matrix of + independent variables, b is a p-by-1 vector of regression coefficients, + and Z is an n-by-1 normally-distributed random vector with independent + identically distributed components with mean 0. + + This estimate is found via the sweep operator, which is a modification + of Gauss-Jordan pivoting. + + + References: + + Matrix Computations, third edition. GH Golub and CF Van Loan. + The Johns Hopkins University Press. 1996. ISBN 0-8018-5414-8. + + Numerical Analysis for Statisticians. K Lange. Springer. 1999. + ISBN 0-387-94979-8. + + Numerical Linear Algebra for Applications in Statistics. JE Gentle. + Springer. 1998. ISBN 0-387-98542-5. + */ + +#include "pspp_linreg.h" + +/* + The matrix A will be overwritten. In ordinary uses of the sweep + operator, A will be the matrix + + __ __ + |X'X X'Y| + | | + |Y'X Y'Y| + -- -- + + X refers to the design matrix and Y to the vector of dependent + observations. pspp_reg_sweep sweeps on the diagonal elements of + X'X. + + The matrix A is assumed to be symmetric, so the sweep operation is + performed only for the upper triangle of A. + */ + +int +pspp_reg_sweep (gsl_matrix * A) +{ + double sweep_element; + double tmp; + int i; + int j; + int k; + gsl_matrix *B; + + if (A != NULL) + { + if (A->size1 == A->size2) + { + B = gsl_matrix_alloc (A->size1, A->size2); + for (k = 0; k < (A->size1 - 1); k++) + { + sweep_element = gsl_matrix_get (A, k, k); + if (fabs (sweep_element) > GSL_DBL_MIN) + { + tmp = -1.0 / sweep_element; + gsl_matrix_set (B, k, k, tmp); + /* + Rows before current row k. + */ + for (i = 0; i < k; i++) + { + for (j = i; j < A->size2; j++) + { + /* + Use only the upper triangle of A. + */ + if (j < k) + { + tmp = gsl_matrix_get (A, i, j) - + gsl_matrix_get (A, i, k) + * gsl_matrix_get (A, j, k) / sweep_element; + gsl_matrix_set (B, i, j, tmp); + } + else if (j > k) + { + tmp = gsl_matrix_get (A, i, j) - + gsl_matrix_get (A, i, k) + * gsl_matrix_get (A, k, j) / sweep_element; + gsl_matrix_set (B, i, j, tmp); + } + else + { + tmp = gsl_matrix_get (A, i, k) / sweep_element; + gsl_matrix_set (B, i, j, tmp); + } + } + } + /* + Current row k. + */ + for (j = k + 1; j < A->size1; j++) + { + tmp = gsl_matrix_get (A, k, j) / sweep_element; + gsl_matrix_set (B, k, j, tmp); + } + /* + Rows after the current row k. + */ + for (i = k + 1; i < A->size1; i++) + { + for (j = i; j < A->size2; j++) + { + tmp = gsl_matrix_get (A, i, j) - + gsl_matrix_get (A, k, i) + * gsl_matrix_get (A, k, j) / sweep_element; + gsl_matrix_set (B, i, j, tmp); + } + } + } + for (i = 0; i < A->size1; i++) + for (j = i; j < A->size2; j++) + { + gsl_matrix_set (A, i, j, gsl_matrix_get (B, i, j)); + } + } + gsl_matrix_free (B); + return GSL_SUCCESS; + } + return GSL_ENOTSQR; + } + return GSL_EFAULT; +} -- 2.30.2