variables parsed

[pspp] / rust / src / cooked.rs

use std::{cell::RefCell, ops::Range, rc::Rc, collections::HashMap};

use crate::{
    dictionary::{Dictionary, VarWidth, Variable},
    encoding::Error as EncodingError,
    endian::Endian,
    format::{Error as FormatError, Spec, UncheckedSpec},
    identifier::{Error as IdError, Identifier},
    raw::{
        self, Cases, DecodedRecord, DocumentRecord, EncodingRecord, Extension, FileAttributeRecord,
        FloatInfoRecord, HeaderRecord, IntegerInfoRecord, LongNamesRecord,
        LongStringMissingValueRecord, LongStringValueLabelRecord, MultipleResponseRecord,
        NumberOfCasesRecord, ProductInfoRecord, RawStr, ValueLabelRecord, VarDisplayRecord,
        VariableAttributeRecord, VariableRecord, VariableSetRecord, VeryLongStringsRecord, ZHeader,
        ZTrailer,
    },
};
use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
use encoding_rs::Encoding;
use num::Integer;
use thiserror::Error as ThisError;

pub use crate::raw::{CategoryLabels, Compression};

#[derive(ThisError, Debug)]
pub enum Error {
    #[error("Missing header record")]
    MissingHeaderRecord,

    // XXX this is an internal error
    #[error("More than one file header record")]
    DuplicateHeaderRecord,

    #[error("{0}")]
    EncodingError(EncodingError),

    #[error("Using default encoding {0}.")]
    UsingDefaultEncoding(String),

    #[error("Variable record from offset {:x} to {:x} specifies width {width} not in valid range [-1,255).", offsets.start, offsets.end)]
    InvalidVariableWidth { offsets: Range<u64>, width: i32 },

    #[error("This file has corrupted metadata written by a buggy version of PSPP.  To ensure that other software can read it correctly, save a new copy of the file.")]
    InvalidLongMissingValueFormat,

    #[error("File creation date {creation_date} is not in the expected format \"DD MMM YY\" format.  Using 01 Jan 1970.")]
    InvalidCreationDate { creation_date: String },

    #[error("File creation time {creation_time} is not in the expected format \"HH:MM:SS\" format.  Using midnight.")]
    InvalidCreationTime { creation_time: String },

    #[error("{id_error}  Renaming variable to {new_name}.")]
    InvalidVariableName {
        id_error: IdError,
        new_name: Identifier,
    },

    #[error(
        "Substituting {new_spec} for invalid print format on variable {variable}.  {format_error}"
    )]
    InvalidPrintFormat {
        new_spec: Spec,
        variable: Identifier,
        format_error: FormatError,
    },

    #[error(
        "Substituting {new_spec} for invalid write format on variable {variable}.  {format_error}"
    )]
    InvalidWriteFormat {
        new_spec: Spec,
        variable: Identifier,
        format_error: FormatError,
    },

    #[error("Renaming variable with duplicate name {duplicate_name} to {new_name}.")]
    DuplicateVariableName {
        duplicate_name: Identifier,
        new_name: Identifier,
    },

    #[error("Dictionary index {dict_index} is outside valid range [1,{max_index}].")]
    InvalidDictIndex { dict_index: usize, max_index: usize },

    #[error("Dictionary index {0} refers to a long string continuation.")]
    DictIndexIsContinuation(usize),

    #[error("Variables associated with value label are not all of identical type.  Variable {numeric_var} is numeric, but variable {string_var} is string.")]
    ValueLabelsDifferentTypes {
        numeric_var: Identifier,
        string_var: Identifier,
    },

    #[error(
        "Value labels may not be added to long string variable {0} using record types 3 or 4."
    )]
    InvalidLongStringValueLabel(Identifier),

    #[error("Invalid multiple response set name.  {0}")]
    InvalidMrSetName(IdError),

    #[error("Multiple response set {mr_set} includes unknown variable {short_name}.")]
    UnknownMrSetVariable {
        mr_set: Identifier,
        short_name: Identifier,
    },

    #[error("Multiple response set {0} has no variables.")]
    EmptyMrSet(Identifier),

    #[error("Multiple response set {0} has only one variable.")]
    OneVarMrSet(Identifier),

    #[error("Multiple response set {0} contains both string and numeric variables.")]
    MixedMrSet(Identifier),

    #[error(
        "Invalid numeric format for counted value {number} in multiple response set {mr_set}."
    )]
    InvalidMDGroupCountedValue { mr_set: Identifier, number: String },

    #[error("Counted value {value} has width {width}, but it must be no wider than {max_width}, the width of the narrowest variable in multiple response set {mr_set}.")]
    TooWideMDGroupCountedValue {
        mr_set: Identifier,
        value: String,
        width: usize,
        max_width: u16,
    },

    #[error("Long string value label for variable {name} has width {width}, which is not in the valid range [{min_width},{max_width}].")]
    InvalidLongValueLabelWidth {
        name: Identifier,
        width: u32,
        min_width: u16,
        max_width: u16,
    },

    #[error("Invalid attribute name.  {0}")]
    InvalidAttributeName(IdError),

    #[error("Invalid short name in long variable name record.  {0}")]
    InvalidShortName(IdError),

    #[error("Invalid name in long variable name record.  {0}")]
    InvalidLongName(IdError),

    #[error("Invalid variable name in very long string record.  {0}")]
    InvalidLongStringName(IdError),

    #[error("Invalid variable name in long string value label record.  {0}")]
    InvalidLongStringValueLabelName(IdError),

    #[error("Invalid variable name in attribute record.  {0}")]
    InvalidAttributeVariableName(IdError),

    // XXX This is risky because `text` might be arbitarily long.
    #[error("Text string contains invalid bytes for {encoding} encoding: {text}")]
    MalformedString { encoding: String, text: String },

    #[error("Details TBD")]
    TBD,
}

type DictIndex = usize;

#[derive(Clone, Debug)]
pub struct Headers {
    pub header: HeaderRecord<String>,
    pub variable: Vec<VariableRecord<String, String>>,
    pub value_label: Vec<ValueLabelRecord<RawStr<8>, String>>,
    pub document: Vec<DocumentRecord<String>>,
    pub integer_info: Option<IntegerInfoRecord>,
    pub float_info: Option<FloatInfoRecord>,
    pub var_display: Option<VarDisplayRecord>,
    pub multiple_response: Vec<MultipleResponseRecord<Identifier, String>>,
    pub long_string_value_labels: Vec<LongStringValueLabelRecord<Identifier, String>>,
    pub long_string_missing_values: Vec<LongStringMissingValueRecord<Identifier, String>>,
    pub encoding: Option<EncodingRecord>,
    pub number_of_cases: Option<NumberOfCasesRecord>,
    pub variable_sets: Vec<VariableSetRecord>,
    pub product_info: Option<ProductInfoRecord>,
    pub long_names: Vec<LongNamesRecord>,
    pub very_long_strings: Vec<VeryLongStringsRecord>,
    pub file_attributes: Vec<FileAttributeRecord>,
    pub variable_attributes: Vec<VariableAttributeRecord>,
    pub other_extension: Vec<Extension>,
    pub end_of_headers: Option<u32>,
    pub z_header: Option<ZHeader>,
    pub z_trailer: Option<ZTrailer>,
    pub cases: Option<Rc<RefCell<Cases>>>,
}

fn take_first<T, F>(mut vec: Vec<T>, more_than_one: F) -> Option<T>
where
    F: FnOnce(),
{
    if vec.len() > 1 {
        more_than_one();
    }
    vec.drain(..).next()
}

impl Headers {
    pub fn new(headers: Vec<raw::DecodedRecord>, warn: &impl Fn(Error)) -> Result<Headers, Error> {
        let mut file_header = Vec::new();
        let mut variable = Vec::new();
        let mut value_label = Vec::new();
        let mut document = Vec::new();
        let mut integer_info = Vec::new();
        let mut float_info = Vec::new();
        let mut var_display = Vec::new();
        let mut multiple_response = Vec::new();
        let mut long_string_value_labels = Vec::new();
        let mut long_string_missing_values = Vec::new();
        let mut encoding = Vec::new();
        let mut number_of_cases = Vec::new();
        let mut variable_sets = Vec::new();
        let mut product_info = Vec::new();
        let mut long_names = Vec::new();
        let mut very_long_strings = Vec::new();
        let mut file_attributes = Vec::new();
        let mut variable_attributes = Vec::new();
        let mut other_extension = Vec::new();
        let mut end_of_headers = Vec::new();
        let mut z_header = Vec::new();
        let mut z_trailer = Vec::new();
        let mut cases = Vec::new();

        for header in headers {
            match header {
                DecodedRecord::Header(record) => {
                    file_header.push(record);
                }
                DecodedRecord::Variable(record) => {
                    variable.push(record);
                }
                DecodedRecord::ValueLabel(record) => {
                    value_label.push(record);
                }
                DecodedRecord::Document(record) => {
                    document.push(record);
                }
                DecodedRecord::IntegerInfo(record) => {
                    integer_info.push(record);
                }
                DecodedRecord::FloatInfo(record) => {
                    float_info.push(record);
                }
                DecodedRecord::VariableSets(record) => {
                    variable_sets.push(record);
                }
                DecodedRecord::VarDisplay(record) => {
                    var_display.push(record);
                }
                DecodedRecord::MultipleResponse(record) => {
                    multiple_response.push(record);
                }
                DecodedRecord::LongStringValueLabels(record) => {
                    long_string_value_labels.push(record)
                }
                DecodedRecord::LongStringMissingValues(record) => {
                    long_string_missing_values.push(record);
                }
                DecodedRecord::Encoding(record) => {
                    encoding.push(record);
                }
                DecodedRecord::NumberOfCases(record) => {
                    number_of_cases.push(record);
                }
                DecodedRecord::ProductInfo(record) => {
                    product_info.push(record);
                }
                DecodedRecord::LongNames(record) => {
                    long_names.push(record);
                }
                DecodedRecord::VeryLongStrings(record) => {
                    very_long_strings.push(record);
                }
                DecodedRecord::FileAttributes(record) => {
                    file_attributes.push(record);
                }
                DecodedRecord::VariableAttributes(record) => {
                    variable_attributes.push(record);
                }
                DecodedRecord::OtherExtension(record) => {
                    other_extension.push(record);
                }
                DecodedRecord::EndOfHeaders(record) => {
                    end_of_headers.push(record);
                }
                DecodedRecord::ZHeader(record) => {
                    z_header.push(record);
                }
                DecodedRecord::ZTrailer(record) => {
                    z_trailer.push(record);
                }
                DecodedRecord::Cases(record) => {
                    cases.push(record);
                }
            }
        }

        let Some(file_header) = take_first(file_header, || warn(Error::DuplicateHeaderRecord))
        else {
            return Err(Error::MissingHeaderRecord);
        };

        Ok(Headers {
            header: file_header,
            variable,
            value_label,
            document,
            integer_info: take_first(integer_info, || warn(Error::TBD)),
            float_info: take_first(float_info, || warn(Error::TBD)),
            var_display: take_first(var_display, || warn(Error::TBD)),
            multiple_response,
            long_string_value_labels,
            long_string_missing_values,
            encoding: take_first(encoding, || warn(Error::TBD)),
            number_of_cases: take_first(number_of_cases, || warn(Error::TBD)),
            variable_sets,
            product_info: take_first(product_info, || warn(Error::TBD)),
            long_names,
            very_long_strings,
            file_attributes,
            variable_attributes,
            other_extension,
            end_of_headers: take_first(end_of_headers, || warn(Error::TBD)),
            z_header: take_first(z_header, || warn(Error::TBD)),
            z_trailer: take_first(z_trailer, || warn(Error::TBD)),
            cases: take_first(cases, || warn(Error::TBD)),
        })
    }
}

pub struct Metadata {
    creation: NaiveDateTime,
    endian: Endian,
    compression: Option<Compression>,
    n_cases: Option<u64>,
    product: String,
    product_ext: Option<String>,
    version: Option<(i32, i32, i32)>,
}

impl Metadata {
    fn decode(headers: &Headers, warn: impl Fn(Error)) -> Self {
        let header = &headers.header;
        let creation_date = NaiveDate::parse_from_str(&header.creation_date, "%e %b %Y")
            .unwrap_or_else(|_| {
                warn(Error::InvalidCreationDate {
                    creation_date: header.creation_date.to_string(),
                });
                Default::default()
            });
        let creation_time = NaiveTime::parse_from_str(&header.creation_time, "%H:%M:%S")
            .unwrap_or_else(|_| {
                warn(Error::InvalidCreationTime {
                    creation_time: header.creation_time.to_string(),
                });
                Default::default()
            });
        let creation = NaiveDateTime::new(creation_date, creation_time);

        let product = header
            .eye_catcher
            .trim_start_matches("@(#) SPSS DATA FILE")
            .trim_end()
            .to_string();

        Self {
            creation,
            endian: header.endian,
            compression: header.compression,
            n_cases: header.n_cases.map(|n| n as u64),
            product,
            product_ext: headers.product_info.as_ref().map(|pe| fix_line_ends(&pe.0)),
            version: headers.integer_info.as_ref().map(|ii| ii.version),
        }
    }
}

struct Decoder {
    //pub raw: raw::Decoder,
    pub encoding: &'static Encoding,
    //pub variables: HashMap<DictIndex, Variable>,
    //pub var_names: HashMap<Identifier, DictIndex>,
    //pub dictionary: Dictionary,
    //n_dict_indexes: usize,
    n_generated_names: usize,
}

impl Decoder {
    fn generate_name(&mut self, dictionary: &Dictionary) -> Identifier {
        loop {
            self.n_generated_names += 1;
            let name = Identifier::new(&format!("VAR{:03}", self.n_generated_names), self.encoding)
                .unwrap();
            if !dictionary.variables.contains(&name) {
                return name;
            }
            assert!(self.n_generated_names < usize::MAX);
        }
    }
}

pub fn decode(
    mut headers: Headers,
    encoding: &'static Encoding,
    warn: impl Fn(Error),
) -> Result<(Dictionary, Metadata), Error> {
    let mut dictionary = Dictionary::new(encoding);

    let file_label = fix_line_ends(headers.header.file_label.trim_end_matches(' '));
    if !file_label.is_empty() {
        dictionary.file_label = Some(file_label);
    }

    for attributes in headers.file_attributes.drain(..) {
        dictionary.attributes.extend(attributes.0 .0.into_iter())
    }

    // Concatenate all the document records (really there should only be one)
    // and trim off the trailing spaces that pad them to 80 bytes.
    dictionary.documents = headers
        .document
        .drain(..)
        .flat_map(|record| record.lines)
        .map(trim_end_spaces)
        .collect();

    // XXX warn for weird integer format
    // XXX warn for weird floating-point format, etc.

    let mut decoder = Decoder {
        encoding,
        n_generated_names: 0,
    };

    let mut header_vars = headers.variable.iter().enumerate();
    let mut var_index_map = HashMap::new();
    while let Some((value_index, input)) = header_vars.next() {
        let name = trim_end_spaces(input.name.to_string());
        let name = match Identifier::new(&name, encoding) {
            Ok(name) => {
                if !dictionary.variables.contains(&name) {
                    name
                } else {
                    let new_name = decoder.generate_name(&dictionary);
                    warn(Error::DuplicateVariableName {
                        duplicate_name: name.clone(),
                        new_name: new_name.clone(),
                    });
                    new_name
                }
            }
            Err(id_error) => {
                let new_name = decoder.generate_name(&dictionary);
                warn(Error::InvalidVariableName {
                    id_error,
                    new_name: new_name.clone(),
                });
                new_name
            }
        };
        let mut variable = Variable::new(name.clone(), VarWidth::from_raw(input.width).unwrap());

        // Set the short name the same as the long name (even if we renamed it).
        variable.short_names = vec![name];

        variable.label = input.label.clone();

        variable.missing_values = input.missing_values.clone();

        variable.print_format = decode_format(
            input.print_format,
            variable.width,
            |new_spec, format_error| {
                warn(Error::InvalidPrintFormat {
                    new_spec,
                    variable: variable.name.clone(),
                    format_error,
                })
            },
        );
        variable.write_format = decode_format(
            input.write_format,
            variable.width,
            |new_spec, format_error| {
                warn(Error::InvalidWriteFormat {
                    new_spec,
                    variable: variable.name.clone(),
                    format_error,
                })
            },
        );

        // Skip long string continuation records.
        if input.width > 0 {
            #[allow(unstable_name_collisions)]
            for _ in 1..input.width.div_ceil(&8) {
                if let Some((_, continuation)) = header_vars.next() {
                    if continuation.width == -1 {
                        continue;
                    }
                }
                return Err(Error::TBD);
            }
        }

        let dict_index = dictionary.add_var(variable).unwrap();
        assert_eq!(var_index_map.insert(value_index, dict_index), None);
    }

    let metadata = Metadata::decode(&headers, warn);
    Ok((dictionary, metadata))
}

fn trim_end_spaces(mut s: String) -> String {
    s.truncate(s.trim_end_matches(' ').len());
    s
}

/// Returns a copy of `s` in which all lone CR and CR LF pairs have been
/// replaced by LF.
///
/// (A product that identifies itself as VOXCO INTERVIEWER 4.3 produces system
/// files that use CR-only line ends in the file label and extra product
/// info.) */
fn fix_line_ends(s: &str) -> String {
    let mut out = String::with_capacity(s.len());
    let mut s = s.chars().peekable();
    while let Some(c) = s.next() {
        match c {
            '\r' => {
                s.next_if_eq(&'\n');
                out.push('\n')
            }
            c => out.push(c),
        }
    }
    out
}

fn decode_format(raw: raw::Spec, width: VarWidth, warn: impl Fn(Spec, FormatError)) -> Spec {
    UncheckedSpec::try_from(raw)
        .and_then(Spec::try_from)
        .and_then(|x| x.check_width_compatibility(width))
        .unwrap_or_else(|error| {
            let new_format = Spec::default_for_width(width);
            warn(new_format, error);
            new_format
        })
}

/*
impl Decoder {
    fn generate_name(&mut self) -> Identifier {
        loop {
            self.n_generated_names += 1;
            let name = Identifier::new(&format!("VAR{:03}", self.n_generated_names), self.encoding)
                .unwrap();
            if !self.var_names.contains_key(&name) {
                return name;
            }
            assert!(self.n_generated_names < usize::MAX);
        }
    }
    fn decode_string_cow<'a>(&self, input: &'a [u8], warn: &impl Fn(Error)) -> Cow<'a, str> {
        let (output, malformed) = self.encoding.decode_without_bom_handling(input);
        if malformed {
            warn(Error::MalformedString {
                encoding: self.encoding.name().into(),
                text: output.clone().into(),
            });
        }
        output
    }
    fn decode_string(&self, input: &[u8], warn: &impl Fn(Error)) -> String {
        self.decode_string_cow(input, warn).into()
    }
    pub fn decode_identifier(
        &self,
        input: &[u8],
        warn: &impl Fn(Error),
    ) -> Result<Identifier, IdError> {
        let s = self.decode_string_cow(input, warn);
        Identifier::new(&s, self.encoding)
    }
    fn get_var_by_index(&self, dict_index: usize) -> Result<&Variable, Error> {
        let max_index = self.n_dict_indexes;
        if dict_index == 0 || dict_index > max_index {
            return Err(Error::InvalidDictIndex {
                dict_index,
                max_index,
            });
        }
        let Some(variable) = self.variables.get(&(dict_index - 1)) else {
            return Err(Error::DictIndexIsContinuation(dict_index));
        };
        Ok(variable)
    }

    /// Returns `input` decoded from `self.encoding` into UTF-8 such that
    /// re-encoding the result back into `self.encoding` will have exactly the
    /// same length in bytes.
    ///
    /// XXX warn about errors?
    fn decode_exact_length<'a>(&self, input: &'a [u8]) -> Cow<'a, str> {
        if let (s, false) = self.encoding.decode_without_bom_handling(input) {
            // This is the common case.  Usually there will be no errors.
            s
        } else {
            // Unusual case.  Don't bother to optimize it much.
            let mut decoder = self.encoding.new_decoder_without_bom_handling();
            let mut output = String::with_capacity(
                decoder
                    .max_utf8_buffer_length_without_replacement(input.len())
                    .unwrap(),
            );
            let mut rest = input;
            while !rest.is_empty() {
                match decoder.decode_to_string_without_replacement(rest, &mut output, true) {
                    (DecoderResult::InputEmpty, _) => break,
                    (DecoderResult::OutputFull, _) => unreachable!(),
                    (DecoderResult::Malformed(a, b), consumed) => {
                        let skipped = a as usize + b as usize;
                        output.extend(repeat('?').take(skipped));
                        rest = &rest[consumed..];
                    }
                }
            }
            assert_eq!(self.encoding.encode(&output).0.len(), input.len());
            output.into()
        }
    }
}

pub trait TryDecode: Sized {
    type Input<'a>;
    fn try_decode(
        decoder: &mut Decoder,
        input: &Self::Input<'_>,
        warn: impl Fn(Error),
    ) -> Result<Option<Self>, Error>;
}

pub trait Decode<Input>: Sized {
    fn decode(decoder: &Decoder, input: &Input, warn: impl Fn(Error)) -> Self;
}

impl<const N: usize> Decode<RawStr<N>> for String {
    fn decode(decoder: &Decoder, input: &RawStr<N>, warn: impl Fn(Error)) -> Self {
        decoder.decode_string(&input.0, &warn)
    }
}
*/
/*
#[derive(Clone, Debug)]
pub struct HeaderRecord {
    pub eye_catcher: String,
    pub weight_index: Option<usize>,
    pub n_cases: Option<u64>,
    pub creation: NaiveDateTime,
    pub file_label: String,
}

fn trim_end_spaces(mut s: String) -> String {
    s.truncate(s.trim_end_matches(' ').len());
    s
}

/// Data file info that doesn't fit in [Dictionary].
pub struct Metadata {
    creation: NaiveDateTime,
    endian: Endian,
    compression: Option<Compression>,
    n_cases: Option<u64>,
    product: String,
    product_ext: Option<String>,
    version: Option<(i32, i32, i32)>,
}

impl Metadata {
    fn decode(
        header: &crate::raw::HeaderRecord<Cow<str>>,
        integer_info: Option<&IntegerInfoRecord>,
        product_ext: Option<&ProductInfoRecord>,
        warn: impl Fn(Error),
    ) -> Self {
        let creation_date = NaiveDate::parse_from_str(&header.creation_date, "%e %b %Y")
            .unwrap_or_else(|_| {
                warn(Error::InvalidCreationDate {
                    creation_date: header.creation_date.to_string(),
                });
                Default::default()
            });
        let creation_time = NaiveTime::parse_from_str(&header.creation_time, "%H:%M:%S")
            .unwrap_or_else(|_| {
                warn(Error::InvalidCreationTime {
                    creation_time: header.creation_time.to_string(),
                });
                Default::default()
            });
        let creation = NaiveDateTime::new(creation_date, creation_time);

        let product = header
            .eye_catcher
            .trim_start_matches("@(#) SPSS DATA FILE")
            .trim_end()
            .to_string();

        Self {
            creation,
            endian: header.endian,
            compression: header.compression,
            n_cases: header.n_cases.map(|n| n as u64),
            product,
            product_ext: product_ext.map(|pe| pe.0.clone()),
            version: integer_info.map(|ii| ii.version),
        }
    }
}

impl TryDecode for HeaderRecord {
    type Input<'a> = crate::raw::HeaderRecord<Cow<'a, str>>;

    fn try_decode(
        _decoder: &mut Decoder,
        input: &Self::Input<'_>,
        warn: impl Fn(Error),
    ) -> Result<Option<Self>, Error> {
        let eye_catcher = trim_end_spaces(input.eye_catcher.to_string());
        let file_label = trim_end_spaces(input.file_label.to_string());
        let creation_date = NaiveDate::parse_from_str(&input.creation_date, "%e %b %Y")
            .unwrap_or_else(|_| {
                warn(Error::InvalidCreationDate {
                    creation_date: input.creation_date.to_string(),
                });
                Default::default()
            });
        let creation_time = NaiveTime::parse_from_str(&input.creation_time, "%H:%M:%S")
            .unwrap_or_else(|_| {
                warn(Error::InvalidCreationTime {
                    creation_time: input.creation_time.to_string(),
                });
                Default::default()
            });
        Ok(Some(HeaderRecord {
            eye_catcher,
            weight_index: input.weight_index.map(|n| n as usize),
            n_cases: input.n_cases.map(|n| n as u64),
            creation: NaiveDateTime::new(creation_date, creation_time),
            file_label,
        }))
    }
}

#[derive(Clone, Debug)]
pub struct VariableRecord {
    pub width: VarWidth,
    pub name: Identifier,
    pub print_format: Spec,
    pub write_format: Spec,
    pub missing_values: MissingValues<String>,
    pub label: Option<String>,
}


fn parse_variable_record(
    decoder: &mut Decoder,
    input: &raw::VariableRecord<Cow<str>, String>,
    warn: impl Fn(Error),
) -> Result<(), Error> {
    let width = match input.width {
        0 => VarWidth::Numeric,
        w @ 1..=255 => VarWidth::String(w as u16),
        -1 => return Ok(()),
        _ => {
            return Err(Error::InvalidVariableWidth {
                offsets: input.offsets.clone(),
                width: input.width,
            })
        }
    };
    let name = trim_end_spaces(input.name.to_string());
    let name = match Identifier::new(&name, decoder.encoding) {
        Ok(name) => {
            if !decoder.var_names.contains_key(&name) {
                name
            } else {
                let new_name = decoder.generate_name();
                warn(Error::DuplicateVariableName {
                    duplicate_name: name.clone(),
                    new_name: new_name.clone(),
                });
                new_name
            }
        }
        Err(id_error) => {
            let new_name = decoder.generate_name();
            warn(Error::InvalidVariableName {
                id_error,
                new_name: new_name.clone(),
            });
            new_name
        }
    };
    let variable = Variable {
        dict_index: decoder.n_dict_indexes,
        short_name: name.clone(),
        long_name: None,
        width,
    };
    decoder.n_dict_indexes += width.n_dict_indexes();
    assert!(decoder
        .var_names
        .insert(name.clone(), variable.dict_index)
        .is_none());
    assert!(decoder
        .variables
        .insert(variable.dict_index, variable)
        .is_none());

    let print_format = decode_format(input.print_format, width, |new_spec, format_error| {
        warn(Error::InvalidPrintFormat {
            new_spec,
            variable: name.clone(),
            format_error,
        })
    });
    let write_format = decode_format(input.write_format, width, |new_spec, format_error| {
        warn(Error::InvalidWriteFormat {
            new_spec,
            variable: name.clone(),
            format_error,
        })
    });
    let mut variable = dictionary::Variable::new(name, width);
    variable.print_format = print_format;
    variable.write_format = write_format;
    variable.missing_values = input.missing_values.clone();
    if let Some(ref label) = input.label {
        variable.label = Some(label.to_string());
    }
    decoder.dictionary.add_var(variable).unwrap();
    Ok(())
}

#[derive(Clone, Debug)]
pub struct DocumentRecord(Vec<String>);

impl TryDecode for DocumentRecord {
    type Input<'a> = crate::raw::DocumentRecord<RawDocumentLine>;

    fn try_decode(
        decoder: &mut Decoder,
        input: &Self::Input<'_>,
        warn: impl Fn(Error),
    ) -> Result<Option<Self>, Error> {
        Ok(Some(DocumentRecord(
            input
                .lines
                .iter()
                .map(|s| trim_end_spaces(decoder.decode_string(&s.0, &warn)))
                .collect(),
        )))
    }
}

trait TextRecord
where
    Self: Sized,
{
    const NAME: &'static str;
    fn parse(input: &str, warn: impl Fn(Error)) -> Result<Self, Error>;
}

#[derive(Clone, Debug)]
pub struct VariableSet {
    pub name: String,
    pub vars: Vec<String>,
}

impl VariableSet {
    fn parse(input: &str) -> Result<Self, Error> {
        let (name, input) = input.split_once('=').ok_or(Error::TBD)?;
        let vars = input.split_ascii_whitespace().map(String::from).collect();
        Ok(VariableSet {
            name: name.into(),
            vars,
        })
    }
}

trait WarnOnError<T> {
    fn warn_on_error<F: Fn(Error)>(self, warn: &F) -> Option<T>;
}
impl<T> WarnOnError<T> for Result<T, Error> {
    fn warn_on_error<F: Fn(Error)>(self, warn: &F) -> Option<T> {
        match self {
            Ok(result) => Some(result),
            Err(error) => {
                warn(error);
                None
            }
        }
    }
}

#[derive(Clone, Debug)]
pub struct ValueLabel {
    pub value: Value,
    pub label: String,
}

#[derive(Clone, Debug)]
pub struct ValueLabelRecord {
    pub var_type: VarType,
    pub labels: Vec<ValueLabel>,
    pub variables: Vec<Identifier>,
}

impl TryDecode for ValueLabelRecord {
    type Input<'a> = crate::raw::ValueLabelRecord<RawStr<8>, RawString>;
    fn try_decode(
        decoder: &mut Decoder,
        input: &Self::Input<'_>,
        warn: impl Fn(Error),
    ) -> Result<Option<ValueLabelRecord>, Error> {
        let variables: Vec<&Variable> = input
            .dict_indexes
            .iter()
            .filter_map(|&dict_index| {
                decoder
                    .get_var_by_index(dict_index as usize)
                    .warn_on_error(&warn)
            })
            .filter(|&variable| match variable.width {
                VarWidth::String(width) if width > 8 => {
                    warn(Error::InvalidLongStringValueLabel(
                        variable.short_name.clone(),
                    ));
                    false
                }
                _ => true,
            })
            .collect();
        let mut i = variables.iter();
        let Some(&first_var) = i.next() else {
            return Ok(None);
        };
        let var_type: VarType = first_var.width.into();
        for &variable in i {
            let this_type: VarType = variable.width.into();
            if var_type != this_type {
                let (numeric_var, string_var) = match var_type {
                    VarType::Numeric => (first_var, variable),
                    VarType::String => (variable, first_var),
                };
                warn(Error::ValueLabelsDifferentTypes {
                    numeric_var: numeric_var.short_name.clone(),
                    string_var: string_var.short_name.clone(),
                });
                return Ok(None);
            }
        }
        let labels = input
            .labels
            .iter()
            .map(|raw::ValueLabel { value, label }| {
                let label = decoder.decode_string(&label.0, &warn);
                let value = Value::decode(value, decoder);
                ValueLabel { value, label }
            })
            .collect();
        let variables = variables
            .iter()
            .map(|&variable| variable.short_name.clone())
            .collect();
        Ok(Some(ValueLabelRecord {
            var_type,
            labels,
            variables,
        }))
    }
}

#[derive(Clone, Debug)]
pub struct VariableSetRecord(Vec<VariableSet>);

impl TextRecord for VariableSetRecord {
    const NAME: &'static str = "variable set";
    fn parse(input: &str, warn: impl Fn(Error)) -> Result<Self, Error> {
        let mut sets = Vec::new();
        for line in input.lines() {
            if let Some(set) = VariableSet::parse(line).warn_on_error(&warn) {
                sets.push(set)
            }
        }
        Ok(VariableSetRecord(sets))
    }
}

#[derive(Clone, Debug)]
pub struct LongName {
    pub short_name: Identifier,
    pub long_name: Identifier,
}

impl LongName {
    fn new(decoder: &mut Decoder, short_name: &str, long_name: &str) -> Result<LongName, Error> {
        let short_name =
            Identifier::new(short_name, decoder.encoding).map_err(Error::InvalidShortName)?;
        let long_name =
            Identifier::new(long_name, decoder.encoding).map_err(Error::InvalidLongName)?;
        Ok(LongName {
            short_name,
            long_name,
        })
    }
}

#[derive(Clone, Debug)]
pub struct LongNameRecord(Vec<LongName>);

impl LongNameRecord {
    pub fn parse(decoder: &mut Decoder, input: &str, warn: impl Fn(Error)) -> Result<Self, Error> {
        let mut names = Vec::new();
        for pair in input.split('\t').filter(|s| !s.is_empty()) {
            if let Some((short_name, long_name)) = pair.split_once('=') {
                if let Some(long_name) =
                    LongName::new(decoder, short_name, long_name).warn_on_error(&warn)
                {
                    names.push(long_name);
                }
            } else {
                warn(Error::TBD)
            }
        }
        Ok(LongNameRecord(names))
    }
}

#[derive(Clone, Debug)]
pub struct VeryLongString {
    pub short_name: Identifier,
    pub length: u16,
}

impl VeryLongString {
    fn parse(decoder: &Decoder, input: &str) -> Result<VeryLongString, Error> {
        let Some((short_name, length)) = input.split_once('=') else {
            return Err(Error::TBD);
        };
        let short_name =
            Identifier::new(short_name, decoder.encoding).map_err(Error::InvalidLongStringName)?;
        let length: u16 = length.parse().map_err(|_| Error::TBD)?;
        if length > VarWidth::MAX_STRING {
            return Err(Error::TBD);
        }
        Ok(VeryLongString { short_name, length })
    }
}

#[derive(Clone, Debug)]
pub struct VeryLongStringRecord(Vec<VeryLongString>);

impl VeryLongStringRecord {
    pub fn parse(decoder: &Decoder, input: &str, warn: impl Fn(Error)) -> Result<Self, Error> {
        let mut very_long_strings = Vec::new();
        for tuple in input
            .split('\0')
            .map(|s| s.trim_end_matches('\t'))
            .filter(|s| !s.is_empty())
        {
            if let Some(vls) = VeryLongString::parse(decoder, tuple).warn_on_error(&warn) {
                very_long_strings.push(vls)
            }
        }
        Ok(VeryLongStringRecord(very_long_strings))
    }
}

#[derive(Clone, Debug)]
pub struct Attribute {
    pub name: Identifier,
    pub values: Vec<String>,
}

impl Attribute {
    fn parse<'a>(
        decoder: &Decoder,
        input: &'a str,
        warn: &impl Fn(Error),
    ) -> Result<(Option<Attribute>, &'a str), Error> {
        let Some((name, mut input)) = input.split_once('(') else {
            return Err(Error::TBD);
        };
        let mut values = Vec::new();
        loop {
            let Some((value, rest)) = input.split_once('\n') else {
                return Err(Error::TBD);
            };
            if let Some(stripped) = value
                .strip_prefix('\'')
                .and_then(|value| value.strip_suffix('\''))
            {
                values.push(stripped.into());
            } else {
                warn(Error::TBD);
                values.push(value.into());
            }
            if let Some(rest) = rest.strip_prefix(')') {
                let attribute = Identifier::new(name, decoder.encoding)
                    .map_err(Error::InvalidAttributeName)
                    .warn_on_error(warn)
                    .map(|name| Attribute { name, values });
                return Ok((attribute, rest));
            };
            input = rest;
        }
    }
}

#[derive(Clone, Debug)]
pub struct AttributeSet(pub Vec<Attribute>);

impl AttributeSet {
    fn parse<'a>(
        decoder: &Decoder,
        mut input: &'a str,
        sentinel: Option<char>,
        warn: &impl Fn(Error),
    ) -> Result<(AttributeSet, &'a str), Error> {
        let mut attributes = Vec::new();
        let rest = loop {
            match input.chars().next() {
                None => break input,
                c if c == sentinel => break &input[1..],
                _ => {
                    let (attribute, rest) = Attribute::parse(decoder, input, &warn)?;
                    if let Some(attribute) = attribute {
                        attributes.push(attribute);
                    }
                    input = rest;
                }
            }
        };
        Ok((AttributeSet(attributes), rest))
    }
}

#[derive(Clone, Debug)]
pub struct FileAttributeRecord(AttributeSet);

impl FileAttributeRecord {
    pub fn parse(decoder: &Decoder, input: &str, warn: impl Fn(Error)) -> Result<Self, Error> {
        let (set, rest) = AttributeSet::parse(decoder, input, None, &warn)?;
        if !rest.is_empty() {
            warn(Error::TBD);
        }
        Ok(FileAttributeRecord(set))
    }
}

#[derive(Clone, Debug)]
pub struct VarAttributeSet {
    pub long_var_name: Identifier,
    pub attributes: AttributeSet,
}

impl VarAttributeSet {
    fn parse<'a>(
        decoder: &Decoder,
        input: &'a str,
        warn: &impl Fn(Error),
    ) -> Result<(Option<VarAttributeSet>, &'a str), Error> {
        let Some((long_var_name, rest)) = input.split_once(':') else {
            return Err(Error::TBD);
        };
        let (attributes, rest) = AttributeSet::parse(decoder, rest, Some('/'), warn)?;
        let var_attribute = Identifier::new(long_var_name, decoder.encoding)
            .map_err(Error::InvalidAttributeVariableName)
            .warn_on_error(warn)
            .map(|name| VarAttributeSet {
                long_var_name: name,
                attributes,
            });
        Ok((var_attribute, rest))
    }
}

#[derive(Clone, Debug)]
pub struct VariableAttributeRecord(Vec<VarAttributeSet>);

impl VariableAttributeRecord {
    pub fn parse(decoder: &Decoder, mut input: &str, warn: impl Fn(Error)) -> Result<Self, Error> {
        let mut var_attribute_sets = Vec::new();
        while !input.is_empty() {
            let Some((var_attribute, rest)) =
                VarAttributeSet::parse(decoder, input, &warn).warn_on_error(&warn)
            else {
                break;
            };
            if let Some(var_attribute) = var_attribute {
                var_attribute_sets.push(var_attribute);
            }
            input = rest;
        }
        Ok(VariableAttributeRecord(var_attribute_sets))
    }
}

#[derive(Clone, Debug)]
pub enum MultipleResponseType {
    MultipleDichotomy {
        value: Value,
        labels: CategoryLabels,
    },
    MultipleCategory,
}

impl MultipleResponseType {
    fn decode(
        decoder: &Decoder,
        mr_set: &Identifier,
        input: &raw::MultipleResponseType,
        min_width: VarWidth,
        warn: &impl Fn(Error),
    ) -> Result<Self, Error> {
        let mr_type = match input {
            raw::MultipleResponseType::MultipleDichotomy { value, labels } => {
                let value = decoder.decode_string_cow(&value.0, warn);
                let value = match min_width {
                    VarWidth::Numeric => {
                        let number: f64 = value.trim().parse().map_err(|_| {
                            Error::InvalidMDGroupCountedValue {
                                mr_set: mr_set.clone(),
                                number: value.into(),
                            }
                        })?;
                        Value::Number(Some(number.into()))
                    }
                    VarWidth::String(max_width) => {
                        let value = value.trim_end_matches(' ');
                        let width = value.len();
                        if width > max_width as usize {
                            return Err(Error::TooWideMDGroupCountedValue {
                                mr_set: mr_set.clone(),
                                value: value.into(),
                                width,
                                max_width,
                            });
                        };
                        Value::String(value.into())
                    }
                };
                MultipleResponseType::MultipleDichotomy {
                    value,
                    labels: *labels,
                }
            }
            raw::MultipleResponseType::MultipleCategory => MultipleResponseType::MultipleCategory,
        };
        Ok(mr_type)
    }
}

#[derive(Clone, Debug)]
pub struct MultipleResponseSet {
    pub name: Identifier,
    pub min_width: VarWidth,
    pub max_width: VarWidth,
    pub label: String,
    pub mr_type: MultipleResponseType,
    pub dict_indexes: Vec<DictIndex>,
}

impl MultipleResponseSet {
    fn decode(
        decoder: &Decoder,
        input: &raw::MultipleResponseSet<Identifier, Cow<str>>,
        warn: &impl Fn(Error),
    ) -> Result<Self, Error> {
        let mr_set_name = input.name.clone();
        let mut dict_indexes = Vec::with_capacity(input.short_names.len());
        for short_name in input.short_names.iter() {
            let Some(&dict_index) = decoder.var_names.get(&short_name) else {
                warn(Error::UnknownMrSetVariable {
                    mr_set: mr_set_name.clone(),
                    short_name: short_name.clone(),
                });
                continue;
            };
            dict_indexes.push(dict_index);
        }

        match dict_indexes.len() {
            0 => return Err(Error::EmptyMrSet(mr_set_name)),
            1 => return Err(Error::OneVarMrSet(mr_set_name)),
            _ => (),
        }

        let Some((Some(min_width), Some(max_width))) = dict_indexes
            .iter()
            .map(|dict_index| decoder.variables[dict_index].width)
            .map(|w| (Some(w), Some(w)))
            .reduce(|(na, wa), (nb, wb)| (VarWidth::narrower(na, nb), VarWidth::wider(wa, wb)))
        else {
            return Err(Error::MixedMrSet(mr_set_name));
        };

        let mr_type =
            MultipleResponseType::decode(decoder, &mr_set_name, &input.mr_type, min_width, warn)?;

        Ok(MultipleResponseSet {
            name: mr_set_name,
            min_width,
            max_width,
            label: input.label.to_string(),
            mr_type,
            dict_indexes,
        })
    }
}

#[derive(Clone, Debug)]
pub struct MultipleResponseRecord(pub Vec<MultipleResponseSet>);

impl TryDecode for MultipleResponseRecord {
    type Input<'a> = raw::MultipleResponseRecord<Identifier, Cow<'a, str>>;

    fn try_decode(
        decoder: &mut Decoder,
        input: &Self::Input<'_>,
        warn: impl Fn(Error),
    ) -> Result<Option<Self>, Error> {
        let mut sets = Vec::with_capacity(input.0.len());
        for set in &input.0 {
            match MultipleResponseSet::decode(decoder, set, &warn) {
                Ok(set) => sets.push(set),
                Err(error) => warn(error),
            }
        }
        Ok(Some(MultipleResponseRecord(sets)))
    }
}

#[derive(Clone, Debug)]
pub struct LongStringValueLabels {
    pub var_name: Identifier,
    pub width: VarWidth,
    pub labels: Vec<ValueLabel>,
}

impl LongStringValueLabels {
    fn decode(
        decoder: &Decoder,
        input: &raw::LongStringValueLabels<RawString>,
        warn: &impl Fn(Error),
    ) -> Result<Self, Error> {
        let var_name = decoder.decode_string(&input.var_name.0, warn);
        let var_name = Identifier::new(var_name.trim_end(), decoder.encoding)
            .map_err(Error::InvalidLongStringValueLabelName)?;

        let min_width = 9;
        let max_width = VarWidth::MAX_STRING;
        if input.width < 9 || input.width > max_width as u32 {
            return Err(Error::InvalidLongValueLabelWidth {
                name: var_name,
                width: input.width,
                min_width,
                max_width,
            });
        }
        let width = input.width as u16;

        let mut labels = Vec::with_capacity(input.labels.len());
        for (value, label) in input.labels.iter() {
            let value = Value::String(decoder.decode_exact_length(&value.0).into());
            let label = decoder.decode_string(&label.0, warn);
            labels.push(ValueLabel { value, label });
        }

        Ok(LongStringValueLabels {
            var_name,
            width: VarWidth::String(width),
            labels,
        })
    }
}

#[derive(Clone, Debug)]
pub struct LongStringValueLabelRecord(pub Vec<LongStringValueLabels>);

impl TryDecode for LongStringValueLabelRecord {
    type Input<'a> = raw::LongStringValueLabelRecord<RawString>;

    fn try_decode(
        decoder: &mut Decoder,
        input: &Self::Input<'_>,
        warn: impl Fn(Error),
    ) -> Result<Option<Self>, Error> {
        let mut labels = Vec::with_capacity(input.0.len());
        for label in &input.0 {
            match LongStringValueLabels::decode(decoder, label, &warn) {
                Ok(set) => labels.push(set),
                Err(error) => warn(error),
            }
        }
        Ok(Some(LongStringValueLabelRecord(labels)))
    }
}

#[cfg(test)]
mod test {
    use encoding_rs::WINDOWS_1252;

    #[test]
    fn test() {
        let mut s = String::new();
        s.push(char::REPLACEMENT_CHARACTER);
        let encoded = WINDOWS_1252.encode(&s).0;
        let decoded = WINDOWS_1252.decode(&encoded[..]).0;
        println!("{:?}", decoded);
    }

    #[test]
    fn test2() {
        let charset: Vec<u8> = (0..=255).collect();
        println!("{}", charset.len());
        let decoded = WINDOWS_1252.decode(&charset[..]).0;
        println!("{}", decoded.len());
        let encoded = WINDOWS_1252.encode(&decoded[..]).0;
        println!("{}", encoded.len());
        assert_eq!(&charset[..], &encoded[..]);
    }
}
*/