1 #![allow(unused_variables)]
2 use endian::{Endian, Parse};
4 use num_derive::FromPrimitive;
5 use std::io::{BufReader, Error as IoError, ErrorKind, Read, Seek};
10 #[derive(Error, Debug)]
12 #[error("Not an SPSS system file")]
15 #[error("Invalid magic number {0:?}")]
18 #[error("I/O error ({0})")]
24 #[error("Invalid SAV compression code {0}")]
25 InvalidSavCompression(u32),
27 #[error("Invalid ZSAV compression code {0}")]
28 InvalidZsavCompression(u32),
30 #[error("Variable record at offset {offset:#x} specifies width {width} not in valid range [-1,255).")]
31 BadVariableWidth { offset: u64, width: i32 },
33 #[error("Misplaced type 4 record near offset {0:#x}.")]
34 MisplacedType4Record(u64),
36 #[error("Document record at offset {offset:#x} has document line count ({n}) greater than the maximum number {max}.")]
37 BadDocumentLength { offset: u64, n: u32, max: u32 },
39 #[error("At offset {offset:#x}, Unrecognized record type {rec_type}.")]
40 BadRecordType { offset: u64, rec_type: u32 },
42 #[error("At offset {offset:#x}, variable label code ({code}) is not 0 or 1.")]
43 BadVariableLabelCode { offset: u64, code: u32 },
46 "At offset {offset:#x}, numeric missing value code ({code}) is not -3, -2, 0, 1, 2, or 3."
48 BadNumericMissingValueCode { offset: u64, code: i32 },
50 #[error("At offset {offset:#x}, string missing value code ({code}) is not 0, 1, 2, or 3.")]
51 BadStringMissingValueCode { offset: u64, code: i32 },
53 #[error("At offset {offset:#x}, number of value labels ({n}) is greater than the maximum number {max}.")]
54 BadNumberOfValueLabels { offset: u64, n: u32, max: u32 },
56 #[error("At offset {offset:#x}, variable index record (type 4) does not immediately follow value label record (type 3) as it should.")]
57 MissingVariableIndexRecord { offset: u64 },
59 #[error("At offset {offset:#x}, number of variables indexes ({n}) is greater than the maximum number ({max}).")]
60 BadNumberOfVarIndexes { offset: u64, n: u32, max: u32 },
62 #[error("At offset {offset:#x}, record type 7 subtype {subtype} is too large with element size {size} and {count} elements.")]
63 ExtensionRecordTooLarge {
70 #[error("Wrong ZLIB data header offset {zheader_offset:#x} (expected {offset:#x}).")]
71 BadZlibHeaderOffset { offset: u64, zheader_offset: u64 },
73 #[error("At offset {offset:#x}, impossible ZLIB trailer offset {ztrailer_offset:#x}.")]
74 BadZlibTrailerOffset { offset: u64, ztrailer_offset: u64 },
76 #[error("At offset {offset:#x}, impossible ZLIB trailer length {ztrailer_len}.")]
77 BadZlibTrailerLen { offset: u64, ztrailer_len: u64 },
80 #[derive(Error, Debug)]
82 #[error("Unexpected floating-point bias {0} or unrecognized floating-point format.")]
85 #[error("Duplicate type 6 (document) record.")]
86 DuplicateDocumentRecord,
89 #[derive(Copy, Clone, Debug)]
90 pub enum Compression {
99 ValueLabel(ValueLabel),
100 VarIndexes(VarIndexes),
101 Extension(Extension),
110 /// Eye-catcher string, product name, in the file's encoding. Padded
111 /// on the right with spaces.
112 pub eye_catcher: [u8; 60],
114 /// Layout code, normally either 2 or 3.
115 pub layout_code: u32,
117 /// Number of variable positions, or `None` if the value in the file is
118 /// questionably trustworthy.
119 pub nominal_case_size: Option<u32>,
121 /// Compression type, if any,
122 pub compression: Option<Compression>,
124 /// 0-based variable index of the weight variable, or `None` if the file is
126 pub weight_index: Option<u32>,
128 /// Claimed number of cases, if known.
129 pub n_cases: Option<u32>,
131 /// Compression bias, usually 100.0.
134 /// `dd mmm yy` in the file's encoding.
135 pub creation_date: [u8; 9],
137 /// `HH:MM:SS` in the file's encoding.
138 pub creation_time: [u8; 8],
140 /// File label, in the file's encoding. Padded on the right with spaces.
141 pub file_label: [u8; 64],
143 /// Endianness of the data in the file header.
144 pub endianness: Endian,
147 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
148 pub struct Magic([u8; 4]);
151 /// Magic number for a regular system file.
152 pub const SAV: Magic = Magic(*b"$FL2");
154 /// Magic number for a system file that contains zlib-compressed data.
155 pub const ZSAV: Magic = Magic(*b"$FL3");
157 /// Magic number for an EBDIC-encoded system file. This is `$FL2` encoded
159 pub const EBCDIC: Magic = Magic([0x5b, 0xc6, 0xd3, 0xf2]);
162 impl TryFrom<[u8; 4]> for Magic {
165 fn try_from(value: [u8; 4]) -> Result<Self, Self::Error> {
166 let magic = Magic(value);
168 Magic::SAV | Magic::ZSAV | Magic::EBCDIC => Ok(magic),
169 _ => Err(Error::BadMagic(value)),
174 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
181 fn from_width(width: i32) -> VarType {
183 0 => VarType::Number,
184 _ => VarType::String,
189 pub struct Reader<R: Read> {
191 var_types: Vec<VarType>,
197 Headers(Endian, Option<Compression>),
202 #[derive(Copy, Clone)]
209 pub fn from_raw(var_type: VarType, raw: [u8; 8], endian: Endian) -> Value {
211 VarType::String => Value::String(raw),
213 let number: f64 = endian.parse(raw);
214 Value::Number((number != -f64::MAX).then_some(number))
220 impl<R: Read + Seek> Reader<R> {
221 pub fn new(r: R) -> Result<Reader<R>, Error> {
223 r: BufReader::new(r),
224 var_types: Vec::new(),
225 state: ReaderState::Start,
228 fn _next(&mut self) -> Result<Option<(Record, ReaderState)>, Error> {
230 ReaderState::Start => {
231 let header = read_header(&mut self.r)?;
232 let next_state = ReaderState::Headers(header.endianness, header.compression);
233 Ok(Some((Record::Header(header), next_state)))
235 ReaderState::Headers(endian, compression) => {
236 let rec_type: u32 = endian.parse(read_bytes(&mut self.r)?);
237 let record = match rec_type {
239 let variable = read_variable_record(&mut self.r, endian)?;
240 self.var_types.push(VarType::from_width(variable.width));
241 Record::Variable(variable)
243 3 => Record::ValueLabel(read_value_label_record(&mut self.r, endian)?),
244 4 => Record::VarIndexes(read_var_indexes_record(&mut self.r, endian)?),
245 6 => Record::Document(read_document_record(&mut self.r, endian)?),
246 7 => Record::Extension(read_extension_record(&mut self.r, endian)?),
248 let _: [u8; 4] = read_bytes(&mut self.r)?;
249 let next_state = match compression {
250 None => ReaderState::Data(endian),
251 _ => ReaderState::End,
253 return Ok(Some((Record::EndOfHeaders, next_state)));
256 return Err(Error::BadRecordType {
257 offset: self.r.stream_position()?,
262 Ok(Some((record, ReaderState::Headers(endian, compression))))
264 ReaderState::Data(endian) => {
265 let mut values = Vec::with_capacity(self.var_types.len());
266 for (i, &var_type) in self.var_types.iter().enumerate() {
267 let raw = match read_bytes(&mut self.r) {
270 if i == 0 && err.kind() == ErrorKind::UnexpectedEof {
273 return Err(Error::Io(err));
277 values.push(Value::from_raw(var_type, raw, endian));
279 Ok(Some((Record::Case(values), ReaderState::Data(endian))))
281 ReaderState::End => Ok(None),
286 impl<R: Read + Seek> Iterator for Reader<R> {
287 type Item = Result<Record, Error>;
289 fn next(&mut self) -> Option<Self::Item> {
290 let retval = self._next();
293 self.state = ReaderState::End;
296 Ok(Some((record, next_state))) => {
297 self.state = next_state;
301 self.state = ReaderState::End;
308 fn read_header<R: Read>(r: &mut R) -> Result<Header, Error> {
309 let magic: [u8; 4] = read_bytes(r)?;
310 let magic: Magic = magic.try_into().map_err(|_| Error::NotASystemFile)?;
312 let eye_catcher: [u8; 60] = read_bytes(r)?;
313 let layout_code: [u8; 4] = read_bytes(r)?;
314 let endianness = Endian::identify_u32(2, layout_code)
315 .or_else(|| Endian::identify_u32(2, layout_code))
316 .ok_or_else(|| Error::NotASystemFile)?;
317 let layout_code = endianness.parse(layout_code);
319 let nominal_case_size: u32 = endianness.parse(read_bytes(r)?);
320 let nominal_case_size =
321 (nominal_case_size <= i32::MAX as u32 / 16).then_some(nominal_case_size);
323 let compression_code: u32 = endianness.parse(read_bytes(r)?);
324 let compression = match (magic, compression_code) {
325 (Magic::ZSAV, 2) => Some(Compression::ZLib),
326 (Magic::ZSAV, code) => return Err(Error::InvalidZsavCompression(code)),
328 (_, 1) => Some(Compression::Simple),
329 (_, code) => return Err(Error::InvalidSavCompression(code)),
332 let weight_index: u32 = endianness.parse(read_bytes(r)?);
333 let weight_index = (weight_index > 0).then_some(weight_index - 1);
335 let n_cases: u32 = endianness.parse(read_bytes(r)?);
336 let n_cases = (n_cases < i32::MAX as u32 / 2).then_some(n_cases);
338 let bias: f64 = endianness.parse(read_bytes(r)?);
340 let creation_date: [u8; 9] = read_bytes(r)?;
341 let creation_time: [u8; 8] = read_bytes(r)?;
342 let file_label: [u8; 64] = read_bytes(r)?;
343 let _: [u8; 3] = read_bytes(r)?;
361 pub struct Variable {
362 /// Offset from the start of the file to the start of the record.
365 /// Variable width, in the range -1..=255.
368 /// Variable name, padded on the right with spaces.
372 pub print_format: u32,
375 pub write_format: u32,
377 /// Missing value code, one of -3, -2, 0, 1, 2, or 3.
378 pub missing_value_code: i32,
380 /// Raw missing values, up to 3 of them.
381 pub missing: Vec<[u8; 8]>,
383 /// Optional variable label.
384 pub label: Option<Vec<u8>>,
387 fn read_variable_record<R: Read + Seek>(
388 r: &mut BufReader<R>,
390 ) -> Result<Variable, Error> {
391 let offset = r.stream_position()?;
392 let width: i32 = e.parse(read_bytes(r)?);
393 let has_variable_label: u32 = e.parse(read_bytes(r)?);
394 let missing_value_code: i32 = e.parse(read_bytes(r)?);
395 let print_format: u32 = e.parse(read_bytes(r)?);
396 let write_format: u32 = e.parse(read_bytes(r)?);
397 let name: [u8; 8] = read_bytes(r)?;
399 let label = match has_variable_label {
402 let len: u32 = e.parse(read_bytes(r)?);
403 let read_len = len.min(65535) as usize;
404 let label = Some(read_vec(r, read_len)?);
406 let padding_bytes = Integer::next_multiple_of(&len, &4) - len;
407 let _ = read_vec(r, padding_bytes as usize)?;
412 return Err(Error::BadVariableLabelCode {
414 code: has_variable_label,
419 let mut missing = Vec::new();
420 if missing_value_code != 0 {
421 match (width, missing_value_code) {
422 (0, -3 | -2 | 1 | 2 | 3) => (),
424 return Err(Error::BadNumericMissingValueCode {
426 code: missing_value_code,
431 return Err(Error::BadStringMissingValueCode {
433 code: missing_value_code,
438 for _ in 0..missing_value_code.abs() {
439 missing.push(read_bytes(r)?);
455 pub struct ValueLabel {
456 /// Offset from the start of the file to the start of the record.
460 pub labels: Vec<([u8; 8], Vec<u8>)>,
464 /// Maximum number of value labels in a record.
465 pub const MAX: u32 = u32::MAX / 8;
468 fn read_value_label_record<R: Read + Seek>(
469 r: &mut BufReader<R>,
471 ) -> Result<ValueLabel, Error> {
472 let offset = r.stream_position()?;
473 let n: u32 = e.parse(read_bytes(r)?);
474 if n > ValueLabel::MAX {
475 return Err(Error::BadNumberOfValueLabels {
478 max: ValueLabel::MAX,
482 let mut labels = Vec::new();
484 let value: [u8; 8] = read_bytes(r)?;
485 let label_len: u8 = e.parse(read_bytes(r)?);
486 let label_len = label_len as usize;
487 let padded_len = Integer::next_multiple_of(&(label_len + 1), &8);
489 let mut label = read_vec(r, padded_len)?;
490 label.truncate(label_len);
491 labels.push((value, label));
493 Ok(ValueLabel { offset, labels })
496 pub struct VarIndexes {
497 /// Offset from the start of the file to the start of the record.
500 /// The 0-based indexes of the variable indexes.
501 pub var_indexes: Vec<u32>,
505 /// Maximum number of variable indexes in a record.
506 pub const MAX: u32 = u32::MAX / 8;
509 fn read_var_indexes_record<R: Read + Seek>(
510 r: &mut BufReader<R>,
512 ) -> Result<VarIndexes, Error> {
513 let offset = r.stream_position()?;
514 let n: u32 = e.parse(read_bytes(r)?);
515 if n > VarIndexes::MAX {
516 return Err(Error::BadNumberOfVarIndexes {
519 max: VarIndexes::MAX,
522 let mut var_indexes = Vec::with_capacity(n as usize);
524 var_indexes.push(e.parse(read_bytes(r)?));
533 pub const DOC_LINE_LEN: u32 = 80;
534 pub const DOC_MAX_LINES: u32 = i32::MAX as u32 / DOC_LINE_LEN;
536 pub struct Document {
537 /// Offset from the start of the file to the start of the record.
540 /// The document, as an array of 80-byte lines.
541 pub lines: Vec<[u8; DOC_LINE_LEN as usize]>,
544 fn read_document_record<R: Read + Seek>(
545 r: &mut BufReader<R>,
547 ) -> Result<Document, Error> {
548 let offset = r.stream_position()?;
549 let n: u32 = e.parse(read_bytes(r)?);
551 0..=DOC_MAX_LINES => {
552 let pos = r.stream_position()?;
553 let mut lines = Vec::with_capacity(n as usize);
555 let line: [u8; 80] = read_bytes(r)?;
558 Ok(Document { pos, lines })
560 _ => Err(Error::BadDocumentLength {
568 #[derive(FromPrimitive)]
570 /// Machine integer info.
572 /// Machine floating-point info.
578 /// Multiple response sets.
582 /// Extra product info text.
584 /// Variable display parameters.
586 /// Long variable names.
590 /// Extended number of cases.
592 /// Data file attributes.
594 /// Variable attributes.
596 /// Multiple response sets (extended).
598 /// Character encoding.
600 /// Value labels for long strings.
602 /// Missing values for long strings.
604 /// "Format properties in dataview table".
608 pub struct Extension {
609 /// Offset from the start of the file to the start of the record.
615 /// Size of each data element.
618 /// Number of data elements.
621 /// `size * count` bytes of data.
625 fn extension_record_size_requirements(extension: ExtensionType) -> (u32, u32) {
627 /* Implemented record types. */
628 ExtensionType::Integer => (4, 8),
629 ExtensionType::Float => (8, 3),
630 ExtensionType::VarSets => (1, 0),
631 ExtensionType::Mrsets => (1, 0),
632 ExtensionType::ProductInfo => (1, 0),
633 ExtensionType::Display => (4, 0),
634 ExtensionType::LongNames => (1, 0),
635 ExtensionType::LongStrings => (1, 0),
636 ExtensionType::Ncases => (8, 2),
637 ExtensionType::FileAttrs => (1, 0),
638 ExtensionType::VarAttrs => (1, 0),
639 ExtensionType::Mrsets2 => (1, 0),
640 ExtensionType::Encoding => (1, 0),
641 ExtensionType::LongLabels => (1, 0),
642 ExtensionType::LongMissing => (1, 0),
644 /* Ignored record types. */
645 ExtensionType::Date => (0, 0),
646 ExtensionType::DataEntry => (0, 0),
647 ExtensionType::Dataview => (0, 0),
651 fn read_extension_record<R: Read + Seek>(
652 r: &mut BufReader<R>,
654 ) -> Result<Extension, Error> {
655 let subtype = e.parse(read_bytes(r)?);
656 let offset = r.stream_position()?;
657 let size: u32 = e.parse(read_bytes(r)?);
658 let count = e.parse(read_bytes(r)?);
659 let Some(product) = size.checked_mul(count) else {
660 return Err(Error::ExtensionRecordTooLarge {
667 let offset = r.stream_position()?;
668 let data = read_vec(r, product as usize)?;
679 /// File offset to the start of the record.
682 /// File offset to the ZLIB data header.
685 /// File offset to the ZLIB trailer.
686 ztrailer_offset: u64,
688 /// Length of the ZLIB trailer in bytes.
692 fn read_zheader<R: Read + Seek>(r: &mut BufReader<R>, e: Endian) -> Result<ZHeader, Error> {
693 let offset = r.stream_position()?;
694 let zheader_offset: u64 = e.parse(read_bytes(r)?);
695 let ztrailer_offset: u64 = e.parse(read_bytes(r)?);
696 let ztrailer_len: u64 = e.parse(read_bytes(r)?);
706 fn read_bytes<const N: usize, R: Read>(r: &mut R) -> Result<[u8; N], IoError> {
707 let mut buf = [0; N];
708 r.read_exact(&mut buf)?;
712 fn read_vec<R: Read>(r: &mut BufReader<R>, n: usize) -> Result<Vec<u8>, IoError> {
713 let mut vec = vec![0; n];
714 r.read_exact(&mut vec)?;
719 fn trim_end(mut s: Vec<u8>, c: u8) -> Vec<u8> {
720 while s.last() == Some(&c) {
726 fn skip_bytes<R: Read>(r: &mut R, mut n: u64) -> Result<(), IoError> {
727 let mut buf = [0; 1024];
729 let chunk = u64::min(n, buf.len() as u64);
730 r.read_exact(&mut buf[0..chunk as usize])?;