1 #![allow(unused_variables)]
2 use endian::{Endian, Parse};
4 use num_derive::FromPrimitive;
5 use std::io::{BufReader, Error as IoError, Read, Seek};
10 #[derive(Error, Debug)]
12 #[error("Not an SPSS system file")]
15 #[error("Invalid magic number {0:?}")]
18 #[error("I/O error ({source})")]
24 #[error("Invalid SAV compression code {0}")]
25 InvalidSavCompression(u32),
27 #[error("Invalid ZSAV compression code {0}")]
28 InvalidZsavCompression(u32),
30 #[error("Misplaced type 4 record near offset {0:#x}.")]
31 MisplacedType4Record(u64),
33 #[error("Document record at offset {offset:#x} has document line count ({n}) greater than the maximum number {max}.")]
34 BadDocumentLength { offset: u64, n: u32, max: u32 },
36 #[error("At offset {offset:#x}, Unrecognized record type {rec_type}.")]
37 BadRecordType { offset: u64, rec_type: u32 },
39 #[error("At offset {offset:#x}, variable label code ({code}) is not 0 or 1.")]
40 BadVariableLabelCode { offset: u64, code: u32 },
43 "At offset {offset:#x}, numeric missing value code ({code}) is not -3, -2, 0, 1, 2, or 3."
45 BadNumericMissingValueCode { offset: u64, code: i32 },
47 #[error("At offset {offset:#x}, string missing value code ({code}) is not 0, 1, 2, or 3.")]
48 BadStringMissingValueCode { offset: u64, code: i32 },
50 #[error("At offset {offset:#x}, number of value labels ({n}) is greater than the maximum number {max}.")]
51 BadNumberOfValueLabels { offset: u64, n: u32, max: u32 },
53 #[error("At offset {offset:#x}, variable index record (type 4) does not immediately follow value label record (type 3) as it should.")]
54 MissingVariableIndexRecord { offset: u64 },
56 #[error("At offset {offset:#x}, number of variables indexes ({n}) is greater than the maximum number ({max}).")]
57 BadNumberOfVarIndexes { offset: u64, n: u32, max: u32 },
59 #[error("At offset {offset:#x}, record type 7 subtype {subtype} is too large with element size {size} and {count} elements.")]
60 ExtensionRecordTooLarge {
67 #[error("Wrong ZLIB data header offset {zheader_offset:#x} (expected {offset:#x}).")]
68 BadZlibHeaderOffset { offset: u64, zheader_offset: u64 },
70 #[error("At offset {offset:#x}, impossible ZLIB trailer offset {ztrailer_offset:#x}.")]
71 BadZlibTrailerOffset { offset: u64, ztrailer_offset: u64 },
73 #[error("At offset {offset:#x}, impossible ZLIB trailer length {ztrailer_len}.")]
74 BadZlibTrailerLen { offset: u64, ztrailer_len: u64 },
77 #[derive(Error, Debug)]
79 #[error("Unexpected floating-point bias {0} or unrecognized floating-point format.")]
82 #[error("Duplicate type 6 (document) record.")]
83 DuplicateDocumentRecord,
86 #[derive(Copy, Clone, Debug)]
87 pub enum Compression {
96 ValueLabel(ValueLabel),
97 VarIndexes(VarIndexes),
106 /// Endianness of the data in the file header.
107 pub endianness: Endian,
109 /// 0-based variable index of the weight variable, or `None` if the file is
111 pub weight_index: Option<u32>,
113 /// Number of variable positions, or `None` if the value in the file is
114 /// questionably trustworthy.
115 pub nominal_case_size: Option<u32>,
117 /// `dd mmm yy` in the file's encoding.
118 pub creation_date: [u8; 9],
120 /// `HH:MM:SS` in the file's encoding.
121 pub creation_time: [u8; 8],
123 /// Eye-catcher string, then product name, in the file's encoding. Padded
124 /// on the right with spaces.
125 pub eye_catcher: [u8; 60],
127 /// File label, in the file's encoding. Padded on the right with spaces.
128 pub file_label: [u8; 64],
131 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
132 pub struct Magic([u8; 4]);
135 /// Magic number for a regular system file.
136 pub const SAV: Magic = Magic(*b"$FL2");
138 /// Magic number for a system file that contains zlib-compressed data.
139 pub const ZSAV: Magic = Magic(*b"$FL3");
141 /// Magic number for an EBDIC-encoded system file. This is `$FL2` encoded
143 pub const EBCDIC: Magic = Magic([0x5b, 0xc6, 0xd3, 0xf2]);
146 impl TryFrom<[u8; 4]> for Magic {
149 fn try_from(value: [u8; 4]) -> Result<Self, Self::Error> {
150 let magic = Magic(value);
152 Magic::SAV | Magic::ZSAV | Magic::EBCDIC => Ok(magic),
153 _ => Err(Error::BadMagic(value)),
158 pub struct Reader<R: Read> {
161 endianness: Option<Endian>,
164 impl<R: Read + Seek> Reader<R> {
165 pub fn new(r: R) -> Result<Reader<R>, Error> {
167 r: BufReader::new(r),
173 pub fn read(&mut self) -> Result<Option<Record>, Error> {
174 let retval = self.do_read();
177 self.state = u32::MAX;
180 Ok(Some((record, next_state))) => {
181 self.state = next_state;
184 Err(error) => Err(error),
188 pub fn do_read(&mut self) -> Result<Option<(Record, u32)>, Error> {
191 let header = read_header(&mut self.r)?;
192 self.endianness = Some(header.endianness);
193 Ok(Some((Record::Header(header), 1)))
196 let e = self.endianness.unwrap();
197 let rec_type: u32 = e.parse(read_bytes(&mut self.r)?);
198 let record = match rec_type {
199 2 => Record::Variable(read_variable_record(&mut self.r, e)?),
200 3 => Record::ValueLabel(read_value_label_record(&mut self.r, e)?),
201 4 => Record::VarIndexes(read_var_indexes_record(&mut self.r, e)?),
202 6 => Record::Document(read_document_record(&mut self.r, e)?),
203 7 => Record::Extension(read_extension_record(&mut self.r, e)?),
204 999 => Record::EndOfHeaders,
206 return Err(Error::BadRecordType {
207 offset: self.r.stream_position()?,
212 Ok(Some((record, 1)))
218 fn read_header<R: Read>(r: &mut R) -> Result<Header, Error> {
219 let magic: [u8; 4] = read_bytes(r)?;
220 let magic: Magic = magic.try_into().map_err(|_| Error::NotASystemFile)?;
222 let eye_catcher: [u8; 60] = read_bytes(r)?;
223 let layout_code: [u8; 4] = read_bytes(r)?;
224 let endianness = Endian::identify_u32(2, layout_code)
225 .or_else(|| Endian::identify_u32(2, layout_code))
226 .ok_or_else(|| Error::NotASystemFile)?;
228 let nominal_case_size: u32 = endianness.parse(read_bytes(r)?);
229 let nominal_case_size =
230 (nominal_case_size <= i32::MAX as u32 / 16).then_some(nominal_case_size);
232 let compression_code: u32 = endianness.parse(read_bytes(r)?);
233 let compression = match (magic, compression_code) {
234 (Magic::ZSAV, 2) => Some(Compression::ZLib),
235 (Magic::ZSAV, code) => return Err(Error::InvalidZsavCompression(code)),
237 (_, 1) => Some(Compression::Simple),
238 (_, code) => return Err(Error::InvalidSavCompression(code)),
241 let weight_index: u32 = endianness.parse(read_bytes(r)?);
242 let weight_index = (weight_index > 0).then_some(weight_index - 1);
244 let n_cases: u32 = endianness.parse(read_bytes(r)?);
245 let n_cases = (n_cases < i32::MAX as u32 / 2).then_some(n_cases);
247 let bias: f64 = endianness.parse(read_bytes(r)?);
249 let creation_date: [u8; 9] = read_bytes(r)?;
250 let creation_time: [u8; 8] = read_bytes(r)?;
251 let file_label: [u8; 64] = read_bytes(r)?;
252 let _: [u8; 3] = read_bytes(r)?;
266 pub struct Variable {
267 /// Offset from the start of the file to the start of the record.
270 /// Variable width, in the range -1..=255.
273 /// Variable name, padded on the right with spaces.
277 pub print_format: u32,
280 pub write_format: u32,
282 /// Missing value code, one of -3, -2, 0, 1, 2, or 3.
283 pub missing_value_code: i32,
285 /// Raw missing values, up to 3 of them.
286 pub missing: Vec<[u8; 8]>,
288 /// Optional variable label.
289 pub label: Option<Vec<u8>>,
292 fn read_variable_record<R: Read + Seek>(
293 r: &mut BufReader<R>,
295 ) -> Result<Variable, Error> {
296 let offset = r.stream_position()?;
297 let width: i32 = e.parse(read_bytes(r)?);
298 let has_variable_label: u32 = e.parse(read_bytes(r)?);
299 let missing_value_code: i32 = e.parse(read_bytes(r)?);
300 let print_format: u32 = e.parse(read_bytes(r)?);
301 let write_format: u32 = e.parse(read_bytes(r)?);
302 let name: [u8; 8] = read_bytes(r)?;
304 let label = match has_variable_label {
307 let len: u32 = e.parse(read_bytes(r)?);
308 let read_len = len.min(65535) as usize;
309 let label = Some(read_vec(r, read_len)?);
311 let padding_bytes = Integer::next_multiple_of(&len, &4) - len;
312 let _ = read_vec(r, padding_bytes as usize)?;
317 return Err(Error::BadVariableLabelCode {
319 code: has_variable_label,
324 let mut missing = Vec::new();
325 if missing_value_code != 0 {
326 match (width, missing_value_code) {
327 (0, -3 | -2 | 1 | 2 | 3) => (),
329 return Err(Error::BadNumericMissingValueCode {
331 code: missing_value_code,
336 return Err(Error::BadStringMissingValueCode {
338 code: missing_value_code,
343 for _ in 0..missing_value_code.abs() {
344 missing.push(read_bytes(r)?);
360 pub struct ValueLabel {
361 /// Offset from the start of the file to the start of the record.
365 pub labels: Vec<([u8; 8], Vec<u8>)>,
369 /// Maximum number of value labels in a record.
370 pub const MAX: u32 = u32::MAX / 8;
373 fn read_value_label_record<R: Read + Seek>(
374 r: &mut BufReader<R>,
376 ) -> Result<ValueLabel, Error> {
377 let offset = r.stream_position()?;
378 let n: u32 = e.parse(read_bytes(r)?);
379 if n > ValueLabel::MAX {
380 return Err(Error::BadNumberOfValueLabels {
383 max: ValueLabel::MAX,
387 let mut labels = Vec::new();
389 let value: [u8; 8] = read_bytes(r)?;
390 let label_len: u8 = e.parse(read_bytes(r)?);
391 let label_len = label_len as usize;
392 let padded_len = Integer::next_multiple_of(&(label_len + 1), &8);
394 let mut label = read_vec(r, padded_len)?;
395 label.truncate(label_len);
396 labels.push((value, label));
398 Ok(ValueLabel { offset, labels })
401 pub struct VarIndexes {
402 /// Offset from the start of the file to the start of the record.
405 /// The 0-based indexes of the variable indexes.
406 pub var_indexes: Vec<u32>,
410 /// Maximum number of variable indexes in a record.
411 pub const MAX: u32 = u32::MAX / 8;
414 fn read_var_indexes_record<R: Read + Seek>(
415 r: &mut BufReader<R>,
417 ) -> Result<VarIndexes, Error> {
418 let offset = r.stream_position()?;
419 let n: u32 = e.parse(read_bytes(r)?);
420 if n > VarIndexes::MAX {
421 return Err(Error::BadNumberOfVarIndexes {
424 max: VarIndexes::MAX,
427 let mut var_indexes = Vec::with_capacity(n as usize);
429 var_indexes.push(e.parse(read_bytes(r)?));
438 pub const DOC_LINE_LEN: u32 = 80;
439 pub const DOC_MAX_LINES: u32 = i32::MAX as u32 / DOC_LINE_LEN;
441 pub struct Document {
442 /// Offset from the start of the file to the start of the record.
445 /// The document, as an array of 80-byte lines.
446 pub lines: Vec<[u8; DOC_LINE_LEN as usize]>,
449 fn read_document_record<R: Read + Seek>(
450 r: &mut BufReader<R>,
452 ) -> Result<Document, Error> {
453 let offset = r.stream_position()?;
454 let n: u32 = e.parse(read_bytes(r)?);
456 0..=DOC_MAX_LINES => {
457 let pos = r.stream_position()?;
458 let mut lines = Vec::with_capacity(n as usize);
460 let line: [u8; 80] = read_bytes(r)?;
463 Ok(Document { pos, lines })
465 _ => Err(Error::BadDocumentLength {
473 #[derive(FromPrimitive)]
475 /// Machine integer info.
477 /// Machine floating-point info.
483 /// Multiple response sets.
487 /// Extra product info text.
489 /// Variable display parameters.
491 /// Long variable names.
495 /// Extended number of cases.
497 /// Data file attributes.
499 /// Variable attributes.
501 /// Multiple response sets (extended).
503 /// Character encoding.
505 /// Value labels for long strings.
507 /// Missing values for long strings.
509 /// "Format properties in dataview table".
513 pub struct Extension {
514 /// Offset from the start of the file to the start of the record.
520 /// Size of each data element.
523 /// Number of data elements.
526 /// `size * count` bytes of data.
530 fn extension_record_size_requirements(extension: ExtensionType) -> (u32, u32) {
532 /* Implemented record types. */
533 ExtensionType::Integer => (4, 8),
534 ExtensionType::Float => (8, 3),
535 ExtensionType::VarSets => (1, 0),
536 ExtensionType::Mrsets => (1, 0),
537 ExtensionType::ProductInfo => (1, 0),
538 ExtensionType::Display => (4, 0),
539 ExtensionType::LongNames => (1, 0),
540 ExtensionType::LongStrings => (1, 0),
541 ExtensionType::Ncases => (8, 2),
542 ExtensionType::FileAttrs => (1, 0),
543 ExtensionType::VarAttrs => (1, 0),
544 ExtensionType::Mrsets2 => (1, 0),
545 ExtensionType::Encoding => (1, 0),
546 ExtensionType::LongLabels => (1, 0),
547 ExtensionType::LongMissing => (1, 0),
549 /* Ignored record types. */
550 ExtensionType::Date => (0, 0),
551 ExtensionType::DataEntry => (0, 0),
552 ExtensionType::Dataview => (0, 0),
556 fn read_extension_record<R: Read + Seek>(
557 r: &mut BufReader<R>,
559 ) -> Result<Extension, Error> {
560 let subtype = e.parse(read_bytes(r)?);
561 let offset = r.stream_position()?;
562 let size: u32 = e.parse(read_bytes(r)?);
563 let count = e.parse(read_bytes(r)?);
564 let Some(product) = size.checked_mul(count) else {
565 return Err(Error::ExtensionRecordTooLarge {
572 let offset = r.stream_position()?;
573 let data = read_vec(r, product as usize)?;
584 /// File offset to the start of the record.
587 /// File offset to the ZLIB data header.
590 /// File offset to the ZLIB trailer.
591 ztrailer_offset: u64,
593 /// Length of the ZLIB trailer in bytes.
597 fn read_zheader<R: Read + Seek>(r: &mut BufReader<R>, e: Endian) -> Result<ZHeader, Error> {
598 let offset = r.stream_position()?;
599 let zheader_offset: u64 = e.parse(read_bytes(r)?);
600 let ztrailer_offset: u64 = e.parse(read_bytes(r)?);
601 let ztrailer_len: u64 = e.parse(read_bytes(r)?);
603 if zheader_offset != offset {
604 return Err(Error::BadZlibHeaderOffset {
609 if ztrailer_offset < offset {
610 return Err(Error::BadZlibTrailerOffset {
615 if ztrailer_len < 24 || ztrailer_len % 24 != 0 {
616 return Err(Error::BadZlibTrailerLen {
630 fn read_bytes<const N: usize, R: Read>(r: &mut R) -> Result<[u8; N], IoError> {
631 let mut buf = [0; N];
632 r.read_exact(&mut buf)?;
636 fn read_vec<R: Read>(r: &mut BufReader<R>, n: usize) -> Result<Vec<u8>, IoError> {
637 let mut vec = vec![0; n];
638 r.read_exact(&mut vec)?;
643 fn trim_end(mut s: Vec<u8>, c: u8) -> Vec<u8> {
644 while s.last() == Some(&c) {
650 fn skip_bytes<R: Read>(r: &mut R, mut n: u64) -> Result<(), IoError> {
651 let mut buf = [0; 1024];
653 let chunk = u64::min(n, buf.len() as u64);
654 r.read_exact(&mut buf[0..chunk as usize])?;