1 use std::{borrow::Cow, cmp::Ordering, collections::HashMap, iter::repeat, ops::Range};
4 encoding::{default_encoding, get_encoding, Error as EncodingError},
6 format::{Error as FormatError, Spec, UncheckedSpec},
7 identifier::{Error as IdError, Identifier},
8 raw::{self, UnencodedStr, VarType},
10 use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
11 use encoding_rs::{DecoderResult, Encoding};
12 use num::integer::div_ceil;
13 use ordered_float::OrderedFloat;
14 use thiserror::Error as ThisError;
16 pub use crate::raw::{CategoryLabels, Compression};
18 #[derive(ThisError, Debug)]
20 // XXX this is really an internal error and maybe we should change the
21 // interfaces to make it impossible
22 #[error("Missing header record")]
26 EncodingError(EncodingError),
28 #[error("Using default encoding {0}.")]
29 UsingDefaultEncoding(String),
31 #[error("Variable record from offset {:x} to {:x} specifies width {width} not in valid range [-1,255).", offsets.start, offsets.end)]
32 InvalidVariableWidth { offsets: Range<u64>, width: i32 },
34 #[error("This file has corrupted metadata written by a buggy version of PSPP. To ensure that other software can read it correctly, save a new copy of the file.")]
35 InvalidLongMissingValueFormat,
37 #[error("File creation date {creation_date} is not in the expected format \"DD MMM YY\" format. Using 01 Jan 1970.")]
38 InvalidCreationDate { creation_date: String },
40 #[error("File creation time {creation_time} is not in the expected format \"HH:MM:SS\" format. Using midnight.")]
41 InvalidCreationTime { creation_time: String },
43 #[error("{id_error} Renaming variable to {new_name}.")]
50 "Substituting {new_spec} for invalid print format on variable {variable}. {format_error}"
55 format_error: FormatError,
59 "Substituting {new_spec} for invalid write format on variable {variable}. {format_error}"
64 format_error: FormatError,
67 #[error("Renaming variable with duplicate name {duplicate_name} to {new_name}.")]
68 DuplicateVariableName {
69 duplicate_name: Identifier,
73 #[error("Dictionary index {dict_index} is outside valid range [1,{max_index}].")]
74 InvalidDictIndex { dict_index: usize, max_index: usize },
76 #[error("Dictionary index {0} refers to a long string continuation.")]
77 DictIndexIsContinuation(usize),
79 #[error("Variables associated with value label are not all of identical type. Variable {numeric_var} is numeric, but variable {string_var} is string.")]
80 ValueLabelsDifferentTypes {
81 numeric_var: Identifier,
82 string_var: Identifier,
86 "Value labels may not be added to long string variable {0} using record types 3 or 4."
88 InvalidLongStringValueLabel(Identifier),
90 #[error("Invalid multiple response set name. {0}")]
91 InvalidMrSetName(IdError),
93 #[error("Multiple response set {mr_set} includes unknown variable {short_name}.")]
94 UnknownMrSetVariable {
96 short_name: Identifier,
99 #[error("Multiple response set {0} has no variables.")]
100 EmptyMrSet(Identifier),
102 #[error("Multiple response set {0} has only one variable.")]
103 OneVarMrSet(Identifier),
105 #[error("Multiple response set {0} contains both string and numeric variables.")]
106 MixedMrSet(Identifier),
109 "Invalid numeric format for counted value {number} in multiple response set {mr_set}."
111 InvalidMDGroupCountedValue { mr_set: Identifier, number: String },
113 #[error("Counted value {value} has width {width}, but it must be no wider than {max_width}, the width of the narrowest variable in multiple response set {mr_set}.")]
114 TooWideMDGroupCountedValue {
121 #[error("Long string value label for variable {name} has width {width}, which is not in the valid range [{min_width},{max_width}].")]
122 InvalidLongValueLabelWidth {
129 #[error("Invalid attribute name. {0}")]
130 InvalidAttributeName(IdError),
132 #[error("Invalid short name in long variable name record. {0}")]
133 InvalidShortName(IdError),
135 #[error("Invalid name in long variable name record. {0}")]
136 InvalidLongName(IdError),
138 #[error("Invalid variable name in very long string record. {0}")]
139 InvalidLongStringName(IdError),
141 #[error("Invalid variable name in long string value label record. {0}")]
142 InvalidLongStringValueLabelName(IdError),
144 #[error("Invalid variable name in attribute record. {0}")]
145 InvalidAttributeVariableName(IdError),
147 // XXX This is risky because `text` might be arbitarily long.
148 #[error("Text string contains invalid bytes for {encoding} encoding: {text}")]
149 MalformedString { encoding: String, text: String },
151 #[error("Invalid variable measurement level value {0}")]
152 InvalidMeasurement(u32),
154 #[error("Invalid variable display alignment value {0}")]
155 InvalidAlignment(u32),
157 #[error("Details TBD")]
161 #[derive(Clone, Debug)]
163 Header(HeaderRecord),
164 Variable(VariableRecord),
165 ValueLabel(ValueLabelRecord),
166 Document(DocumentRecord),
167 IntegerInfo(IntegerInfoRecord),
168 FloatInfo(FloatInfoRecord),
169 VariableSets(VariableSetRecord),
170 VarDisplay(VarDisplayRecord),
171 MultipleResponse(MultipleResponseRecord),
172 LongStringMissingValues(LongStringMissingValuesRecord),
173 LongStringValueLabels(LongStringValueLabelRecord),
174 Encoding(EncodingRecord),
175 NumberOfCases(NumberOfCasesRecord),
176 ProductInfo(ProductInfoRecord),
177 LongNames(LongNameRecord),
178 VeryLongStrings(VeryLongStringRecord),
179 FileAttributes(FileAttributeRecord),
180 VariableAttributes(VariableAttributeRecord),
181 OtherExtension(Extension),
185 pub use crate::raw::EncodingRecord;
186 pub use crate::raw::Extension;
187 pub use crate::raw::FloatInfoRecord;
188 pub use crate::raw::IntegerInfoRecord;
189 pub use crate::raw::NumberOfCasesRecord;
191 type DictIndex = usize;
193 pub struct Variable {
194 pub dict_index: DictIndex,
195 pub short_name: Identifier,
196 pub long_name: Option<Identifier>,
201 pub compression: Option<Compression>,
203 pub encoding: &'static Encoding,
204 pub variables: HashMap<DictIndex, Variable>,
205 pub var_names: HashMap<Identifier, DictIndex>,
206 n_dict_indexes: usize,
207 n_generated_names: usize,
211 headers: Vec<raw::Record>,
212 encoding: Option<&'static Encoding>,
213 warn: &impl Fn(Error),
214 ) -> Result<Vec<Record>, Error> {
215 let Some(header_record) = headers.iter().find_map(|rec| {
216 if let raw::Record::Header(header) = rec {
222 return Err(Error::MissingHeaderRecord);
224 let encoding = match encoding {
225 Some(encoding) => encoding,
227 let encoding = headers.iter().find_map(|rec| {
228 if let raw::Record::Encoding(ref e) = rec {
234 let character_code = headers.iter().find_map(|rec| {
235 if let raw::Record::IntegerInfo(ref r) = rec {
236 Some(r.character_code)
241 match get_encoding(encoding, character_code) {
242 Ok(encoding) => encoding,
243 Err(err @ EncodingError::Ebcdic) => return Err(Error::EncodingError(err)),
245 warn(Error::EncodingError(err));
246 // Warn that we're using the default encoding.
253 let mut decoder = Decoder {
254 compression: header_record.compression,
255 endian: header_record.endian,
257 variables: HashMap::new(),
258 var_names: HashMap::new(),
260 n_generated_names: 0,
263 let mut output = Vec::with_capacity(headers.len());
264 for header in &headers {
266 raw::Record::Header(ref input) => {
267 if let Some(header) = HeaderRecord::try_decode(&mut decoder, input, warn)? {
268 output.push(Record::Header(header))
271 raw::Record::Variable(ref input) => {
272 if let Some(variable) = VariableRecord::try_decode(&mut decoder, input, warn)? {
273 output.push(Record::Variable(variable));
276 raw::Record::ValueLabel(ref input) => {
277 if let Some(value_label) = ValueLabelRecord::try_decode(&mut decoder, input, warn)?
279 output.push(Record::ValueLabel(value_label));
282 raw::Record::Document(ref input) => {
283 if let Some(document) = DocumentRecord::try_decode(&mut decoder, input, warn)? {
284 output.push(Record::Document(document))
287 raw::Record::IntegerInfo(ref input) => output.push(Record::IntegerInfo(input.clone())),
288 raw::Record::FloatInfo(ref input) => output.push(Record::FloatInfo(input.clone())),
289 raw::Record::VariableSets(ref input) => {
290 let s = decoder.decode_string_cow(&input.text.0, warn);
291 output.push(Record::VariableSets(VariableSetRecord::parse(&s, warn)?));
293 raw::Record::VarDisplay(ref input) => {
294 if let Some(vdr) = VarDisplayRecord::try_decode(&mut decoder, input, warn)? {
295 output.push(Record::VarDisplay(vdr))
298 raw::Record::MultipleResponse(ref input) => {
299 if let Some(mrr) = MultipleResponseRecord::try_decode(&mut decoder, input, warn)? {
300 output.push(Record::MultipleResponse(mrr))
303 raw::Record::LongStringMissingValues(ref input) => {
304 if let Some(mrr) = LongStringMissingValuesRecord::try_decode(&mut decoder, input, warn)? {
305 output.push(Record::LongStringMissingValues(mrr))
308 raw::Record::LongStringValueLabels(ref input) => {
310 LongStringValueLabelRecord::try_decode(&mut decoder, input, warn)?
312 output.push(Record::LongStringValueLabels(mrr))
315 raw::Record::Encoding(ref input) => output.push(Record::Encoding(input.clone())),
316 raw::Record::NumberOfCases(ref input) => {
317 output.push(Record::NumberOfCases(input.clone()))
319 raw::Record::ProductInfo(ref input) => {
320 let s = decoder.decode_string_cow(&input.text.0, warn);
321 output.push(Record::ProductInfo(ProductInfoRecord::parse(&s, warn)?));
323 raw::Record::LongNames(ref input) => {
324 let s = decoder.decode_string_cow(&input.text.0, warn);
325 output.push(Record::LongNames(LongNameRecord::parse(
331 raw::Record::VeryLongStrings(ref input) => {
332 let s = decoder.decode_string_cow(&input.text.0, warn);
333 output.push(Record::VeryLongStrings(VeryLongStringRecord::parse(
337 raw::Record::FileAttributes(ref input) => {
338 let s = decoder.decode_string_cow(&input.text.0, warn);
339 output.push(Record::FileAttributes(FileAttributeRecord::parse(
343 raw::Record::VariableAttributes(ref input) => {
344 let s = decoder.decode_string_cow(&input.text.0, warn);
345 output.push(Record::VariableAttributes(VariableAttributeRecord::parse(
349 raw::Record::OtherExtension(ref input) => {
350 output.push(Record::OtherExtension(input.clone()))
352 raw::Record::EndOfHeaders(_) => (),
353 raw::Record::ZHeader(_) => (),
354 raw::Record::ZTrailer(_) => (),
355 raw::Record::Cases(_) => (),
362 fn generate_name(&mut self) -> Identifier {
364 self.n_generated_names += 1;
365 let name = Identifier::new(&format!("VAR{:03}", self.n_generated_names), self.encoding)
367 if !self.var_names.contains_key(&name) {
370 assert!(self.n_generated_names < usize::MAX);
373 fn decode_string_cow<'a>(&self, input: &'a [u8], warn: &impl Fn(Error)) -> Cow<'a, str> {
374 let (output, malformed) = self.encoding.decode_without_bom_handling(input);
376 warn(Error::MalformedString {
377 encoding: self.encoding.name().into(),
378 text: output.clone().into(),
383 fn decode_string(&self, input: &[u8], warn: &impl Fn(Error)) -> String {
384 self.decode_string_cow(input, warn).into()
386 pub fn decode_identifier(
389 warn: &impl Fn(Error),
390 ) -> Result<Identifier, IdError> {
391 let s = self.decode_string_cow(input, warn);
392 Identifier::new(&s, self.encoding)
394 fn get_var_by_index(&self, dict_index: usize) -> Result<&Variable, Error> {
395 let max_index = self.n_dict_indexes;
396 if dict_index == 0 || dict_index > max_index {
397 return Err(Error::InvalidDictIndex {
402 let Some(variable) = self.variables.get(&(dict_index - 1)) else {
403 return Err(Error::DictIndexIsContinuation(dict_index));
408 /// Returns `input` decoded from `self.encoding` into UTF-8 such that
409 /// re-encoding the result back into `self.encoding` will have exactly the
410 /// same length in bytes.
412 /// XXX warn about errors?
413 fn decode_exact_length<'a>(&self, input: &'a [u8]) -> Cow<'a, str> {
414 if let (s, false) = self.encoding.decode_without_bom_handling(input) {
415 // This is the common case. Usually there will be no errors.
418 // Unusual case. Don't bother to optimize it much.
419 let mut decoder = self.encoding.new_decoder_without_bom_handling();
420 let mut output = String::with_capacity(
422 .max_utf8_buffer_length_without_replacement(input.len())
425 let mut rest = input;
426 while !rest.is_empty() {
427 match decoder.decode_to_string_without_replacement(rest, &mut output, true) {
428 (DecoderResult::InputEmpty, _) => break,
429 (DecoderResult::OutputFull, _) => unreachable!(),
430 (DecoderResult::Malformed(a, b), consumed) => {
431 let skipped = a as usize + b as usize;
432 output.extend(repeat('?').take(skipped));
433 rest = &rest[consumed..];
437 assert_eq!(self.encoding.encode(&output).0.len(), input.len());
443 pub trait TryDecode: Sized {
446 decoder: &mut Decoder,
448 warn: impl Fn(Error),
449 ) -> Result<Option<Self>, Error>;
452 pub trait Decode<Input>: Sized {
453 fn decode(decoder: &Decoder, input: &Input, warn: impl Fn(Error)) -> Self;
456 impl<const N: usize> Decode<UnencodedStr<N>> for String {
457 fn decode(decoder: &Decoder, input: &UnencodedStr<N>, warn: impl Fn(Error)) -> Self {
458 decoder.decode_string(&input.0, &warn)
462 #[derive(Clone, Debug)]
463 pub struct HeaderRecord {
464 pub eye_catcher: String,
465 pub weight_index: Option<usize>,
466 pub n_cases: Option<u64>,
467 pub creation: NaiveDateTime,
468 pub file_label: String,
471 fn trim_end_spaces(mut s: String) -> String {
472 s.truncate(s.trim_end_matches(' ').len());
476 impl TryDecode for HeaderRecord {
477 type Input = crate::raw::HeaderRecord;
480 decoder: &mut Decoder,
482 warn: impl Fn(Error),
483 ) -> Result<Option<Self>, Error> {
484 let eye_catcher = trim_end_spaces(decoder.decode_string(&input.eye_catcher.0, &warn));
485 let file_label = trim_end_spaces(decoder.decode_string(&input.file_label.0, &warn));
486 let creation_date = decoder.decode_string_cow(&input.creation_date.0, &warn);
488 NaiveDate::parse_from_str(&creation_date, "%e %b %Y").unwrap_or_else(|_| {
489 warn(Error::InvalidCreationDate {
490 creation_date: creation_date.into(),
494 let creation_time = decoder.decode_string_cow(&input.creation_time.0, &warn);
496 NaiveTime::parse_from_str(&creation_time, "%H:%M:%S").unwrap_or_else(|_| {
497 warn(Error::InvalidCreationTime {
498 creation_time: creation_time.into(),
502 Ok(Some(HeaderRecord {
504 weight_index: input.weight_index.map(|n| n as usize),
505 n_cases: input.n_cases.map(|n| n as u64),
506 creation: NaiveDateTime::new(creation_date, creation_time),
512 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
518 impl PartialOrd for VarWidth {
519 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
520 match (self, other) {
521 (VarWidth::Numeric, VarWidth::Numeric) => Some(Ordering::Equal),
522 (VarWidth::String(a), VarWidth::String(b)) => Some(a.cmp(b)),
529 const MAX_STRING: u16 = 32767;
531 fn n_dict_indexes(self) -> usize {
533 VarWidth::Numeric => 1,
534 VarWidth::String(w) => div_ceil(w as usize, 8),
541 f: impl Fn(u16, u16) -> u16,
542 ) -> Option<VarWidth> {
544 (Some(VarWidth::Numeric), Some(VarWidth::Numeric)) => Some(VarWidth::Numeric),
545 (Some(VarWidth::String(a)), Some(VarWidth::String(b))) => {
546 Some(VarWidth::String(f(a, b)))
552 /// Returns the wider of `self` and `other`:
553 /// - Numerical variable widths are equally wide.
554 /// - Longer strings are wider than shorter strings.
555 /// - Numerical and string types are incomparable, so result in `None`.
556 /// - Any `None` in the input yields `None` in the output.
557 pub fn wider(a: Option<VarWidth>, b: Option<VarWidth>) -> Option<VarWidth> {
558 Self::width_predicate(a, b, |a, b| a.max(b))
561 /// Returns the narrower of `self` and `other` (see [`Self::wider`]).
562 pub fn narrower(a: Option<VarWidth>, b: Option<VarWidth>) -> Option<VarWidth> {
563 Self::width_predicate(a, b, |a, b| a.min(b))
567 impl From<VarWidth> for VarType {
568 fn from(source: VarWidth) -> Self {
570 VarWidth::Numeric => VarType::Numeric,
571 VarWidth::String(_) => VarType::String,
576 #[derive(Clone, Debug)]
577 pub struct VariableRecord {
579 pub name: Identifier,
580 pub print_format: Spec,
581 pub write_format: Spec,
582 pub missing_values: MissingValues,
583 pub label: Option<String>,
586 #[derive(Clone, Debug)]
587 pub struct MissingValues {
588 /// Individual missing values, up to 3 of them.
589 pub values: Vec<Value>,
591 /// Optional range of missing values.
592 pub range: Option<(Value, Value)>,
595 impl Decode<raw::MissingValues> for MissingValues {
596 fn decode(decoder: &Decoder, input: &raw::MissingValues, _warn: impl Fn(Error)) -> Self {
601 .map(|value| Value::decode(value, decoder))
606 .map(|(low, high)| (Value::decode(low, decoder), Value::decode(high, decoder))),
611 fn decode_format(raw: raw::Spec, width: VarWidth, warn: impl Fn(Spec, FormatError)) -> Spec {
612 UncheckedSpec::try_from(raw)
613 .and_then(Spec::try_from)
614 .and_then(|x| x.check_width_compatibility(width))
615 .unwrap_or_else(|error| {
616 let new_format = Spec::default_for_width(width);
617 warn(new_format, error);
622 impl TryDecode for VariableRecord {
623 type Input = raw::VariableRecord;
626 decoder: &mut Decoder,
627 input: &crate::raw::VariableRecord,
628 warn: impl Fn(Error),
629 ) -> Result<Option<VariableRecord>, Error> {
630 let width = match input.width {
631 0 => VarWidth::Numeric,
632 w @ 1..=255 => VarWidth::String(w as u16),
633 -1 => return Ok(None),
635 return Err(Error::InvalidVariableWidth {
636 offsets: input.offsets.clone(),
641 let name = trim_end_spaces(decoder.decode_string(&input.name.0, &warn));
642 let name = match Identifier::new(&name, decoder.encoding) {
644 if !decoder.var_names.contains_key(&name) {
647 let new_name = decoder.generate_name();
648 warn(Error::DuplicateVariableName {
649 duplicate_name: name.clone(),
650 new_name: new_name.clone(),
656 let new_name = decoder.generate_name();
657 warn(Error::InvalidVariableName {
659 new_name: new_name.clone(),
664 let variable = Variable {
665 dict_index: decoder.n_dict_indexes,
666 short_name: name.clone(),
670 decoder.n_dict_indexes += width.n_dict_indexes();
673 .insert(name.clone(), variable.dict_index)
677 .insert(variable.dict_index, variable)
680 let print_format = decode_format(input.print_format, width, |new_spec, format_error| {
681 warn(Error::InvalidPrintFormat {
683 variable: name.clone(),
687 let write_format = decode_format(input.write_format, width, |new_spec, format_error| {
688 warn(Error::InvalidWriteFormat {
690 variable: name.clone(),
697 .map(|label| decoder.decode_string(&label.0, &warn));
698 Ok(Some(VariableRecord {
703 missing_values: MissingValues::decode(decoder, &input.missing_values, warn),
709 #[derive(Clone, Debug)]
710 pub struct DocumentRecord(Vec<String>);
712 impl TryDecode for DocumentRecord {
713 type Input = crate::raw::DocumentRecord;
716 decoder: &mut Decoder,
718 warn: impl Fn(Error),
719 ) -> Result<Option<Self>, Error> {
720 Ok(Some(DocumentRecord(
724 .map(|s| trim_end_spaces(decoder.decode_string(&s.0, &warn)))
734 const NAME: &'static str;
735 fn parse(input: &str, warn: impl Fn(Error)) -> Result<Self, Error>;
738 #[derive(Clone, Debug)]
739 pub struct VariableSet {
741 pub vars: Vec<String>,
745 fn parse(input: &str) -> Result<Self, Error> {
746 let (name, input) = input.split_once('=').ok_or(Error::TBD)?;
747 let vars = input.split_ascii_whitespace().map(String::from).collect();
755 trait WarnOnError<T> {
756 fn warn_on_error<F: Fn(Error)>(self, warn: &F) -> Option<T>;
758 impl<T> WarnOnError<T> for Result<T, Error> {
759 fn warn_on_error<F: Fn(Error)>(self, warn: &F) -> Option<T> {
761 Ok(result) => Some(result),
770 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
772 Number(Option<OrderedFloat<f64>>),
777 pub fn decode(raw: &raw::Value, decoder: &Decoder) -> Self {
779 raw::Value::Number(x) => Value::Number(x.map(|x| x.into())),
780 raw::Value::String(s) => Value::String(decoder.decode_exact_length(&s.0).into()),
785 #[derive(Clone, Debug)]
786 pub struct ValueLabel {
791 #[derive(Clone, Debug)]
792 pub struct ValueLabelRecord {
793 pub var_type: VarType,
794 pub labels: Vec<ValueLabel>,
795 pub variables: Vec<Identifier>,
798 impl TryDecode for ValueLabelRecord {
799 type Input = crate::raw::ValueLabelRecord;
801 decoder: &mut Decoder,
803 warn: impl Fn(Error),
804 ) -> Result<Option<ValueLabelRecord>, Error> {
805 let variables: Vec<&Variable> = input
808 .filter_map(|&dict_index| {
810 .get_var_by_index(dict_index as usize)
811 .warn_on_error(&warn)
813 .filter(|&variable| match variable.width {
814 VarWidth::String(width) if width > 8 => {
815 warn(Error::InvalidLongStringValueLabel(
816 variable.short_name.clone(),
823 let mut i = variables.iter();
824 let Some(&first_var) = i.next() else {
827 let var_type: VarType = first_var.width.into();
829 let this_type: VarType = variable.width.into();
830 if var_type != this_type {
831 let (numeric_var, string_var) = match var_type {
832 VarType::Numeric => (first_var, variable),
833 VarType::String => (variable, first_var),
835 warn(Error::ValueLabelsDifferentTypes {
836 numeric_var: numeric_var.short_name.clone(),
837 string_var: string_var.short_name.clone(),
845 .map(|(value, label)| {
846 let label = decoder.decode_string(&label.0, &warn);
847 let value = Value::decode(
848 &raw::Value::from_raw(value, var_type, decoder.endian),
851 ValueLabel { value, label }
854 let variables = variables
856 .map(|&variable| variable.short_name.clone())
858 Ok(Some(ValueLabelRecord {
866 #[derive(Clone, Debug)]
867 pub struct VariableSetRecord(Vec<VariableSet>);
869 impl TextRecord for VariableSetRecord {
870 const NAME: &'static str = "variable set";
871 fn parse(input: &str, warn: impl Fn(Error)) -> Result<Self, Error> {
872 let mut sets = Vec::new();
873 for line in input.lines() {
874 if let Some(set) = VariableSet::parse(line).warn_on_error(&warn) {
878 Ok(VariableSetRecord(sets))
882 #[derive(Clone, Debug)]
883 pub struct ProductInfoRecord(pub String);
885 impl TextRecord for ProductInfoRecord {
886 const NAME: &'static str = "extra product info";
887 fn parse(input: &str, _warn: impl Fn(Error)) -> Result<Self, Error> {
888 Ok(ProductInfoRecord(input.into()))
892 #[derive(Clone, Debug)]
893 pub struct LongName {
894 pub short_name: Identifier,
895 pub long_name: Identifier,
899 fn new(decoder: &mut Decoder, short_name: &str, long_name: &str) -> Result<LongName, Error> {
901 Identifier::new(short_name, decoder.encoding).map_err(Error::InvalidShortName)?;
903 Identifier::new(long_name, decoder.encoding).map_err(Error::InvalidLongName)?;
911 #[derive(Clone, Debug)]
912 pub struct LongNameRecord(Vec<LongName>);
914 impl LongNameRecord {
915 pub fn parse(decoder: &mut Decoder, input: &str, warn: impl Fn(Error)) -> Result<Self, Error> {
916 let mut names = Vec::new();
917 for pair in input.split('\t').filter(|s| !s.is_empty()) {
918 if let Some((short_name, long_name)) = pair.split_once('=') {
919 if let Some(long_name) =
920 LongName::new(decoder, short_name, long_name).warn_on_error(&warn)
922 names.push(long_name);
928 Ok(LongNameRecord(names))
932 #[derive(Clone, Debug)]
933 pub struct VeryLongString {
934 pub short_name: Identifier,
938 impl VeryLongString {
939 fn parse(decoder: &Decoder, input: &str) -> Result<VeryLongString, Error> {
940 let Some((short_name, length)) = input.split_once('=') else {
941 return Err(Error::TBD);
944 Identifier::new(short_name, decoder.encoding).map_err(Error::InvalidLongStringName)?;
945 let length: u16 = length.parse().map_err(|_| Error::TBD)?;
946 if length > VarWidth::MAX_STRING {
947 return Err(Error::TBD);
949 Ok(VeryLongString { short_name, length })
953 #[derive(Clone, Debug)]
954 pub struct VeryLongStringRecord(Vec<VeryLongString>);
956 impl VeryLongStringRecord {
957 pub fn parse(decoder: &Decoder, input: &str, warn: impl Fn(Error)) -> Result<Self, Error> {
958 let mut very_long_strings = Vec::new();
961 .map(|s| s.trim_end_matches('\t'))
962 .filter(|s| !s.is_empty())
964 if let Some(vls) = VeryLongString::parse(decoder, tuple).warn_on_error(&warn) {
965 very_long_strings.push(vls)
968 Ok(VeryLongStringRecord(very_long_strings))
972 #[derive(Clone, Debug)]
973 pub struct Attribute {
974 pub name: Identifier,
975 pub values: Vec<String>,
982 warn: &impl Fn(Error),
983 ) -> Result<(Option<Attribute>, &'a str), Error> {
984 let Some((name, mut input)) = input.split_once('(') else {
985 return Err(Error::TBD);
987 let mut values = Vec::new();
989 let Some((value, rest)) = input.split_once('\n') else {
990 return Err(Error::TBD);
992 if let Some(stripped) = value
994 .and_then(|value| value.strip_suffix('\''))
996 values.push(stripped.into());
999 values.push(value.into());
1001 if let Some(rest) = rest.strip_prefix(')') {
1002 let attribute = Identifier::new(name, decoder.encoding)
1003 .map_err(Error::InvalidAttributeName)
1004 .warn_on_error(warn)
1005 .map(|name| Attribute { name, values });
1006 return Ok((attribute, rest));
1013 #[derive(Clone, Debug)]
1014 pub struct AttributeSet(pub Vec<Attribute>);
1020 sentinel: Option<char>,
1021 warn: &impl Fn(Error),
1022 ) -> Result<(AttributeSet, &'a str), Error> {
1023 let mut attributes = Vec::new();
1025 match input.chars().next() {
1026 None => break input,
1027 c if c == sentinel => break &input[1..],
1029 let (attribute, rest) = Attribute::parse(decoder, input, &warn)?;
1030 if let Some(attribute) = attribute {
1031 attributes.push(attribute);
1037 Ok((AttributeSet(attributes), rest))
1041 #[derive(Clone, Debug)]
1042 pub struct FileAttributeRecord(AttributeSet);
1044 impl FileAttributeRecord {
1045 pub fn parse(decoder: &Decoder, input: &str, warn: impl Fn(Error)) -> Result<Self, Error> {
1046 let (set, rest) = AttributeSet::parse(decoder, input, None, &warn)?;
1047 if !rest.is_empty() {
1050 Ok(FileAttributeRecord(set))
1054 #[derive(Clone, Debug)]
1055 pub struct VarAttributeSet {
1056 pub long_var_name: Identifier,
1057 pub attributes: AttributeSet,
1060 impl VarAttributeSet {
1064 warn: &impl Fn(Error),
1065 ) -> Result<(Option<VarAttributeSet>, &'a str), Error> {
1066 let Some((long_var_name, rest)) = input.split_once(':') else {
1067 return Err(Error::TBD);
1069 let (attributes, rest) = AttributeSet::parse(decoder, rest, Some('/'), warn)?;
1070 let var_attribute = Identifier::new(long_var_name, decoder.encoding)
1071 .map_err(Error::InvalidAttributeVariableName)
1072 .warn_on_error(warn)
1073 .map(|name| VarAttributeSet {
1074 long_var_name: name,
1077 Ok((var_attribute, rest))
1081 #[derive(Clone, Debug)]
1082 pub struct VariableAttributeRecord(Vec<VarAttributeSet>);
1084 impl VariableAttributeRecord {
1085 pub fn parse(decoder: &Decoder, mut input: &str, warn: impl Fn(Error)) -> Result<Self, Error> {
1086 let mut var_attribute_sets = Vec::new();
1087 while !input.is_empty() {
1088 let Some((var_attribute, rest)) =
1089 VarAttributeSet::parse(decoder, input, &warn).warn_on_error(&warn)
1093 if let Some(var_attribute) = var_attribute {
1094 var_attribute_sets.push(var_attribute);
1098 Ok(VariableAttributeRecord(var_attribute_sets))
1102 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1110 fn try_decode(source: u32) -> Result<Option<Measure>, Error> {
1113 1 => Ok(Some(Measure::Nominal)),
1114 2 => Ok(Some(Measure::Ordinal)),
1115 3 => Ok(Some(Measure::Scale)),
1116 _ => Err(Error::InvalidMeasurement(source)),
1121 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
1122 pub enum Alignment {
1129 fn try_decode(source: u32) -> Result<Option<Alignment>, Error> {
1132 1 => Ok(Some(Alignment::Left)),
1133 2 => Ok(Some(Alignment::Right)),
1134 3 => Ok(Some(Alignment::Center)),
1135 _ => Err(Error::InvalidAlignment(source)),
1140 #[derive(Clone, Debug)]
1141 pub struct VarDisplay {
1142 pub measure: Option<Measure>,
1143 pub width: Option<u32>,
1144 pub alignment: Option<Alignment>,
1147 #[derive(Clone, Debug)]
1148 pub struct VarDisplayRecord(pub Vec<VarDisplay>);
1150 impl TryDecode for VarDisplayRecord {
1151 type Input = raw::VarDisplayRecord;
1153 decoder: &mut Decoder,
1154 input: &Self::Input,
1155 warn: impl Fn(Error),
1156 ) -> Result<Option<Self>, Error> {
1157 let n_vars = decoder.variables.len();
1158 let n_per_var = if input.0.len() == 3 * n_vars {
1160 } else if input.0.len() == 2 * n_vars {
1163 return Err(Error::TBD);
1166 let var_displays = input
1170 let (measure, width, alignment) = match n_per_var == 3 {
1171 true => (chunk[0], Some(chunk[1]), chunk[2]),
1172 false => (chunk[0], None, chunk[1]),
1174 let measure = Measure::try_decode(measure).warn_on_error(&warn).flatten();
1175 let alignment = Alignment::try_decode(alignment)
1176 .warn_on_error(&warn)
1185 Ok(Some(VarDisplayRecord(var_displays)))
1189 #[derive(Clone, Debug)]
1190 pub enum MultipleResponseType {
1193 labels: CategoryLabels,
1198 impl MultipleResponseType {
1201 mr_set: &Identifier,
1202 input: &raw::MultipleResponseType,
1203 min_width: VarWidth,
1204 warn: &impl Fn(Error),
1205 ) -> Result<Self, Error> {
1206 let mr_type = match input {
1207 raw::MultipleResponseType::MultipleDichotomy { value, labels } => {
1208 let value = decoder.decode_string_cow(&value.0, warn);
1209 let value = match min_width {
1210 VarWidth::Numeric => {
1211 let number: f64 = value.trim().parse().map_err(|_| {
1212 Error::InvalidMDGroupCountedValue {
1213 mr_set: mr_set.clone(),
1214 number: value.into(),
1217 Value::Number(Some(number.into()))
1219 VarWidth::String(max_width) => {
1220 let value = value.trim_end_matches(' ');
1221 let width = value.len();
1222 if width > max_width as usize {
1223 return Err(Error::TooWideMDGroupCountedValue {
1224 mr_set: mr_set.clone(),
1225 value: value.into(),
1230 Value::String(value.into())
1233 MultipleResponseType::MultipleDichotomy {
1238 raw::MultipleResponseType::MultipleCategory => MultipleResponseType::MultipleCategory,
1244 #[derive(Clone, Debug)]
1245 pub struct MultipleResponseSet {
1246 pub name: Identifier,
1247 pub min_width: VarWidth,
1248 pub max_width: VarWidth,
1250 pub mr_type: MultipleResponseType,
1251 pub dict_indexes: Vec<DictIndex>,
1254 impl MultipleResponseSet {
1257 input: &raw::MultipleResponseSet,
1258 warn: &impl Fn(Error),
1259 ) -> Result<Self, Error> {
1260 let mr_set_name = decoder
1261 .decode_identifier(&input.name.0, warn)
1262 .map_err(Error::InvalidMrSetName)?;
1264 let label = decoder.decode_string(&input.label.0, warn);
1266 let mut dict_indexes = Vec::with_capacity(input.short_names.len());
1267 for short_name in input.short_names.iter() {
1268 let short_name = match decoder.decode_identifier(&short_name.0, warn) {
1271 warn(Error::InvalidMrSetName(error));
1275 let Some(&dict_index) = decoder.var_names.get(&short_name) else {
1276 warn(Error::UnknownMrSetVariable {
1277 mr_set: mr_set_name.clone(),
1278 short_name: short_name.clone(),
1282 dict_indexes.push(dict_index);
1285 match dict_indexes.len() {
1286 0 => return Err(Error::EmptyMrSet(mr_set_name)),
1287 1 => return Err(Error::OneVarMrSet(mr_set_name)),
1291 let Some((Some(min_width), Some(max_width))) = dict_indexes
1293 .map(|dict_index| decoder.variables[dict_index].width)
1294 .map(|w| (Some(w), Some(w)))
1295 .reduce(|(na, wa), (nb, wb)| (VarWidth::narrower(na, nb), VarWidth::wider(wa, wb)))
1297 return Err(Error::MixedMrSet(mr_set_name));
1301 MultipleResponseType::decode(decoder, &mr_set_name, &input.mr_type, min_width, warn)?;
1303 Ok(MultipleResponseSet {
1314 #[derive(Clone, Debug)]
1315 pub struct MultipleResponseRecord(pub Vec<MultipleResponseSet>);
1317 impl TryDecode for MultipleResponseRecord {
1318 type Input = raw::MultipleResponseRecord;
1321 decoder: &mut Decoder,
1322 input: &Self::Input,
1323 warn: impl Fn(Error),
1324 ) -> Result<Option<Self>, Error> {
1325 let mut sets = Vec::with_capacity(input.0.len());
1326 for set in &input.0 {
1327 match MultipleResponseSet::decode(decoder, set, &warn) {
1328 Ok(set) => sets.push(set),
1329 Err(error) => warn(error),
1332 Ok(Some(MultipleResponseRecord(sets)))
1336 #[derive(Clone, Debug)]
1337 pub struct LongStringMissingValues {
1339 pub var_name: Identifier,
1342 pub missing_values: MissingValues,
1345 impl LongStringMissingValues {
1348 input: &raw::LongStringMissingValues,
1349 warn: &impl Fn(Error),
1350 ) -> Result<Self, Error> {
1351 let var_name = decoder.decode_string(&input.var_name.0, warn);
1352 let var_name = Identifier::new(var_name.trim_end(), decoder.encoding)
1353 .map_err(Error::InvalidLongStringValueLabelName)?;
1355 let missing_values = MissingValues::decode(decoder, &input.missing_values, warn);
1357 Ok(LongStringMissingValues {
1364 #[derive(Clone, Debug)]
1365 pub struct LongStringMissingValuesRecord(Vec<LongStringMissingValues>);
1367 impl TryDecode for LongStringMissingValuesRecord {
1368 type Input = raw::LongStringMissingValueSet;
1371 decoder: &mut Decoder,
1372 input: &Self::Input,
1373 warn: impl Fn(Error),
1374 ) -> Result<Option<Self>, Error> {
1375 let mut labels = Vec::with_capacity(input.0.len());
1376 for label in &input.0 {
1377 match LongStringMissingValues::decode(decoder, label, &warn) {
1378 Ok(set) => labels.push(set),
1379 Err(error) => warn(error),
1382 Ok(Some(LongStringMissingValuesRecord(labels)))
1386 #[derive(Clone, Debug)]
1387 pub struct LongStringValueLabels {
1388 pub var_name: Identifier,
1389 pub width: VarWidth,
1390 pub labels: Vec<ValueLabel>,
1393 impl LongStringValueLabels {
1396 input: &raw::LongStringValueLabels,
1397 warn: &impl Fn(Error),
1398 ) -> Result<Self, Error> {
1399 let var_name = decoder.decode_string(&input.var_name.0, warn);
1400 let var_name = Identifier::new(var_name.trim_end(), decoder.encoding)
1401 .map_err(Error::InvalidLongStringValueLabelName)?;
1404 let max_width = VarWidth::MAX_STRING;
1405 if input.width < 9 || input.width > max_width as u32 {
1406 return Err(Error::InvalidLongValueLabelWidth {
1413 let width = input.width as u16;
1415 let mut labels = Vec::with_capacity(input.labels.len());
1416 for (value, label) in input.labels.iter() {
1417 let value = Value::String(decoder.decode_exact_length(&value.0).into());
1418 let label = decoder.decode_string(&label.0, warn);
1419 labels.push(ValueLabel { value, label });
1422 Ok(LongStringValueLabels {
1424 width: VarWidth::String(width),
1430 #[derive(Clone, Debug)]
1431 pub struct LongStringValueLabelRecord(pub Vec<LongStringValueLabels>);
1433 impl TryDecode for LongStringValueLabelRecord {
1434 type Input = raw::LongStringValueLabelRecord;
1437 decoder: &mut Decoder,
1438 input: &Self::Input,
1439 warn: impl Fn(Error),
1440 ) -> Result<Option<Self>, Error> {
1441 let mut labels = Vec::with_capacity(input.0.len());
1442 for label in &input.0 {
1443 match LongStringValueLabels::decode(decoder, label, &warn) {
1444 Ok(set) => labels.push(set),
1445 Err(error) => warn(error),
1448 Ok(Some(LongStringValueLabelRecord(labels)))
1454 use encoding_rs::WINDOWS_1252;
1458 let mut s = String::new();
1459 s.push(char::REPLACEMENT_CHARACTER);
1460 let encoded = WINDOWS_1252.encode(&s).0;
1461 let decoded = WINDOWS_1252.decode(&encoded[..]).0;
1462 println!("{:?}", decoded);
1467 let charset: Vec<u8> = (0..=255).collect();
1468 println!("{}", charset.len());
1469 let decoded = WINDOWS_1252.decode(&charset[..]).0;
1470 println!("{}", decoded.len());
1471 let encoded = WINDOWS_1252.encode(&decoded[..]).0;
1472 println!("{}", encoded.len());
1473 assert_eq!(&charset[..], &encoded[..]);