From c772b6354cf5475fdb2d94d8d552e66cf99528c8 Mon Sep 17 00:00:00 2001 From: Ben Pfaff Date: Sun, 6 Jul 2025 15:23:19 -0700 Subject: [PATCH] completed all the tests! --- rust/doc/src/system-file.md | 6 +- rust/pspp/src/format/display/mod.rs | 2 +- rust/pspp/src/format/mod.rs | 2 +- rust/pspp/src/sys/raw.rs | 229 +++++++++++++++--- .../partial_compressed_data_record.expected | 31 +++ ...ial_data_record_between_variables.expected | 27 +++ ...al_data_record_within_long_string.expected | 26 ++ ...fied_number_of_variable_positions.expected | 6 + ...ariable_labels_and_missing_values.expected | 6 + .../wrong_variable_positions_but_v13.expected | 6 + .../sys/testdata/zcompressed_data.expected | 8 + .../zcompressed_data_bad_zheader_ofs.expected | 1 + .../zcompressed_data_bad_zheader_ofs.sack | 2 +- ...zcompressed_data_bad_ztrailer_ofs.expected | 1 + .../zcompressed_data_bad_ztrailer_ofs.sack | 2 +- ...ata_compressed_sizes_don_t_add_up.expected | 1 + ...ed_data_compressed_sizes_don_t_add_up.sack | 2 +- ...data_compressed_sizes_dont_add_up.expected | 1 + ...compression_expands_data_too_much.expected | 1 + ...ata_compression_expands_data_too_much.sack | 4 +- ...pressed_data_invalid_ztrailer_len.expected | 1 + ...zcompressed_data_invalid_ztrailer_len.sack | 2 +- ...data_uncompressed_size_block_size.expected | 33 +++ ...sed_data_uncompressed_size_block_size.sack | 2 +- ...zcompressed_data_wrong_block_size.expected | 1 + .../zcompressed_data_wrong_block_size.sack | 2 +- ...pressed_data_wrong_compressed_ofs.expected | 1 + ...zcompressed_data_wrong_compressed_ofs.sack | 2 +- .../zcompressed_data_wrong_n_blocks.expected | 1 + .../zcompressed_data_wrong_n_blocks.sack | 2 +- ...essed_data_wrong_uncompressed_ofs.expected | 1 + ...ompressed_data_wrong_uncompressed_ofs.sack | 2 +- ...mpressed_data_wrong_ztrailer_bias.expected | 1 + .../zcompressed_data_wrong_ztrailer_bias.sack | 2 +- ...ompressed_data_wrong_ztrailer_len.expected | 1 + .../zcompressed_data_wrong_ztrailer_len.sack | 2 +- ...mpressed_data_wrong_ztrailer_zero.expected | 1 + .../zcompressed_data_wrong_ztrailer_zero.sack | 2 +- 38 files changed, 368 insertions(+), 55 deletions(-) diff --git a/rust/doc/src/system-file.md b/rust/doc/src/system-file.md index 8fdab46ab5..06767966ce 100644 --- a/rust/doc/src/system-file.md +++ b/rust/doc/src/system-file.md @@ -1834,8 +1834,10 @@ case. The format of the data record varies depending on the value of - `int64 zheader_ofs;` - The offset, in bytes, of the beginning of this structure - within the system file. + The offset, in bytes, of the beginning of this structure within + the system file. A reader does not need to use this, so it can + ignore it (PSPP issues a warning if it does not match its own + offset). - `int64 ztrailer_ofs;` diff --git a/rust/pspp/src/format/display/mod.rs b/rust/pspp/src/format/display/mod.rs index 6fcce1791b..b531099e53 100644 --- a/rust/pspp/src/format/display/mod.rs +++ b/rust/pspp/src/format/display/mod.rs @@ -49,7 +49,7 @@ impl DisplayPlain for f64 { } } -pub struct DisplayPlainF64(f64); +pub struct DisplayPlainF64(pub f64); impl Display for DisplayPlainF64 { fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { diff --git a/rust/pspp/src/format/mod.rs b/rust/pspp/src/format/mod.rs index 945cfa6c60..508d0d88f7 100644 --- a/rust/pspp/src/format/mod.rs +++ b/rust/pspp/src/format/mod.rs @@ -18,7 +18,7 @@ use crate::{ mod display; mod parse; -pub use display::{DisplayDatum, DisplayPlain}; +pub use display::{DisplayDatum, DisplayPlain, DisplayPlainF64}; #[derive(Clone, ThisError, Debug, PartialEq, Eq)] pub enum Error { diff --git a/rust/pspp/src/sys/raw.rs b/rust/pspp/src/sys/raw.rs index f81a671ea6..44c40c4570 100644 --- a/rust/pspp/src/sys/raw.rs +++ b/rust/pspp/src/sys/raw.rs @@ -1,7 +1,7 @@ use crate::{ dictionary::{Attributes, Datum, VarWidth}, endian::{Endian, Parse, ToBytes}, - format::DisplayPlain, + format::{DisplayPlain, DisplayPlainF64}, identifier::{Error as IdError, Identifier}, sys::encoding::{default_encoding, get_encoding, Error as EncodingError}, }; @@ -94,9 +94,13 @@ pub enum Error { }, #[error( - "Unexpected end of file at offset {offset:#x}, {case_ofs} bytes into a compressed case." + "Unexpected end of file at offset {offset:#x}, {case_ofs} bytes and {n_chunks} compression chunks into a compressed case." )] - EofInCompressedCase { offset: u64, case_ofs: u64 }, + EofInCompressedCase { + offset: u64, + case_ofs: u64, + n_chunks: usize, + }, #[error("Data ends at offset {offset:#x}, {case_ofs} bytes into a compressed case.")] PartialCompressedCase { offset: u64, case_ofs: u64 }, @@ -107,6 +111,27 @@ pub enum Error { #[error("At {case_ofs} bytes into compressed case starting at offset {offset:#x}, a number was found where a string was expected.")] CompressedStringExpected { offset: u64, case_ofs: u64 }, + #[error("Impossible ztrailer_offset {0:#x}.")] + ImpossibleZTrailerOffset(u64), + + #[error("ZLIB header's zlib_offset is {actual:#x} instead of expected {expected:#x}.")] + UnexpectedZHeaderOffset { actual: u64, expected: u64 }, + + #[error("Invalid ZLIB trailer length {0}.")] + InvalidZTrailerLength(u64), + + #[error( + "ZLIB trailer bias {actual} is not {} as expected from file header bias.", + DisplayPlainF64(*expected) + )] + WrongZlibTrailerBias { actual: i64, expected: f64 }, + + #[error("ZLIB trailer \"zero\" field has nonzero value {0}.")] + WrongZlibTrailerZero(u64), + + #[error("ZLIB trailer specifies unexpected {0}-byte block size.")] + WrongZlibTrailerBlockSize(u32), + #[error("Block count {n_blocks} in ZLIB trailer at offset {offset:#x} differs from expected block count {expected_n_blocks} calculated from trailer length {ztrailer_len}.")] BadZlibTrailerNBlocks { offset: u64, @@ -115,6 +140,30 @@ pub enum Error { ztrailer_len: u64, }, + #[error("ZLIB block descriptor {index} reported uncompressed data offset {actual:#x}, when {expected:#x} was expected.")] + ZlibTrailerBlockWrongUncmpOfs { + index: usize, + actual: u64, + expected: u64, + }, + + #[error("ZLIB block descriptor {index} reported compressed data offset {actual:#x}, when {expected:#x} was expected.")] + ZlibTrailerBlockWrongCmpOfs { + index: usize, + actual: u64, + expected: u64, + }, + + #[error("ZLIB block descriptor {index} reports compressed size {compressed_size} and uncompressed size {uncompressed_size}.")] + ZlibExpansion { + index: usize, + compressed_size: u32, + uncompressed_size: u32, + }, + + #[error("ZLIB trailer is at offset {zheader:#x} but {descriptors:#x} would be expected from block descriptors.")] + ZlibTrailerOffsetInconsistency { descriptors: u64, zheader: u64 }, + #[error("File metadata says it contains {expected} cases, but {actual} cases were read.")] WrongNumberOfCases { expected: u64, actual: u64 }, @@ -280,6 +329,20 @@ pub enum Warning { #[error("Compression bias is {0} instead of the usual values of 0 or 100.")] UnexpectedBias(f64), + #[error("ZLIB block descriptor {index} reported block size {actual:#x}, when {expected:#x} was expected.")] + ZlibTrailerBlockWrongSize { + index: usize, + actual: u32, + expected: u32, + }, + + #[error("ZLIB block descriptor {index} reported block size {actual:#x}, when at most {max_expected:#x} was expected.")] + ZlibTrailerBlockTooBig { + index: usize, + actual: u32, + max_expected: u32, + }, + #[error("Details TBD (raw)")] TBD, } @@ -854,28 +917,31 @@ impl Datum { reader: &mut R, case_vars: &[CaseVar], case_start: u64, + n_chunks: usize, ) -> Result>, Error> { let offset = reader.stream_position()?; - if offset == case_start { - Ok(None) - } else { - Err(Error::EofInCase { - offset, + if n_chunks > 0 { + Err(Error::EofInCompressedCase { case_ofs: offset - case_start, - case_len: case_vars.iter().map(CaseVar::bytes).sum(), + n_chunks, + offset, }) + } else { + Ok(None) } } let case_start = reader.stream_position()?; + let mut n_chunks = 0; let mut values = Vec::with_capacity(case_vars.len()); for var in case_vars { match var { CaseVar::Numeric => { let Some(raw) = Self::read_compressed_chunk(reader, codes, endian, bias)? else { - return eof(reader, case_vars, case_start); + return eof(reader, case_vars, case_start, n_chunks); }; + n_chunks += 1; values.push(Datum::Number(endian.parse(raw))); } CaseVar::String { width, encoding } => { @@ -887,12 +953,13 @@ impl Datum { let Some(raw) = Self::read_compressed_chunk(reader, codes, endian, bias)? else { - return eof(reader, case_vars, case_start); + return eof(reader, case_vars, case_start, n_chunks); }; let n_data = data_bytes.min(8); datum.extend_from_slice(&raw[..n_data]); data_bytes -= n_data; padding_bytes -= 8 - n_data; + n_chunks += 1; } } values.push(Datum::String(RawString(datum))); @@ -963,10 +1030,7 @@ enum ReaderState { Start, Headers, ZlibHeader, - ZlibTrailer { - ztrailer_offset: u64, - ztrailer_len: u64, - }, + ZlibTrailer(ZHeader), End, } @@ -1058,26 +1122,24 @@ where Some(Ok(record)) } ReaderState::ZlibHeader => { - let zheader = - match ZHeader::read(self.0.reader.as_mut().unwrap(), self.0.header.endian) { - Ok(zheader) => zheader, - Err(error) => return Some(Err(error)), - }; - self.0.state = ReaderState::ZlibTrailer { - ztrailer_offset: zheader.ztrailer_offset, - ztrailer_len: zheader.ztrailer_len, + let zheader = match ZHeader::read( + self.0.reader.as_mut().unwrap(), + self.0.header.endian, + &mut self.0.warn, + ) { + Ok(zheader) => zheader, + Err(error) => return Some(Err(error)), }; + self.0.state = ReaderState::ZlibTrailer(zheader.clone()); Some(Ok(Record::ZHeader(zheader))) } - ReaderState::ZlibTrailer { - ztrailer_offset, - ztrailer_len, - } => { + ReaderState::ZlibTrailer(ref zheader) => { match ZTrailer::read( self.0.reader.as_mut().unwrap(), self.0.header.endian, - ztrailer_offset, - ztrailer_len, + self.0.header.bias, + zheader, + &mut self.0.warn, ) { Ok(None) => { self.cases(); @@ -3345,12 +3407,31 @@ pub struct ZHeader { } impl ZHeader { - fn read(r: &mut R, endian: Endian) -> Result { + fn read( + r: &mut R, + endian: Endian, + warn: &mut dyn FnMut(Warning), + ) -> Result { let offset = r.stream_position()?; let zheader_offset: u64 = endian.parse(read_bytes(r)?); let ztrailer_offset: u64 = endian.parse(read_bytes(r)?); let ztrailer_len: u64 = endian.parse(read_bytes(r)?); + if zheader_offset != offset { + return Err(Error::UnexpectedZHeaderOffset { + actual: zheader_offset, + expected: offset, + }); + } + + if ztrailer_offset < offset { + return Err(Error::ImpossibleZTrailerOffset(ztrailer_offset)); + } + + if ztrailer_len < 24 || ztrailer_len % 24 != 0 { + return Err(Error::InvalidZTrailerLength(ztrailer_len)); + } + Ok(ZHeader { offset, zheader_offset, @@ -3411,32 +3492,106 @@ impl ZTrailer { fn read( reader: &mut R, endian: Endian, - ztrailer_ofs: u64, - ztrailer_len: u64, + bias: f64, + zheader: &ZHeader, + warn: &mut dyn FnMut(Warning), ) -> Result, Error> { let start_offset = reader.stream_position()?; - if reader.seek(SeekFrom::Start(ztrailer_ofs)).is_err() { + if reader + .seek(SeekFrom::Start(zheader.ztrailer_offset)) + .is_err() + { return Ok(None); } let int_bias = endian.parse(read_bytes(reader)?); + if int_bias as f64 != -bias { + return Err(Error::WrongZlibTrailerBias { + actual: int_bias, + expected: -bias, + }); + } let zero = endian.parse(read_bytes(reader)?); + if zero != 0 { + return Err(Error::WrongZlibTrailerZero(zero)); + } let block_size = endian.parse(read_bytes(reader)?); + if block_size != 0x3ff000 { + return Err(Error::WrongZlibTrailerBlockSize(block_size)); + } let n_blocks: u32 = endian.parse(read_bytes(reader)?); - let expected_n_blocks = (ztrailer_len - 24) / 24; + let expected_n_blocks = (zheader.ztrailer_len - 24) / 24; if n_blocks as u64 != expected_n_blocks { return Err(Error::BadZlibTrailerNBlocks { - offset: ztrailer_ofs, + offset: zheader.ztrailer_offset, n_blocks, expected_n_blocks, - ztrailer_len, + ztrailer_len: zheader.ztrailer_len, }); } let blocks = (0..n_blocks) .map(|_| ZBlock::read(reader, endian)) .collect::, _>>()?; + + let mut expected_uncmp_ofs = zheader.zheader_offset; + let mut expected_cmp_ofs = zheader.zheader_offset + 24; + for (index, block) in blocks.iter().enumerate() { + if block.uncompressed_ofs != expected_uncmp_ofs { + return Err(Error::ZlibTrailerBlockWrongUncmpOfs { + index, + actual: block.uncompressed_ofs, + expected: expected_cmp_ofs, + }); + } + if block.compressed_ofs != expected_cmp_ofs { + return Err(Error::ZlibTrailerBlockWrongCmpOfs { + index, + actual: block.compressed_ofs, + expected: expected_cmp_ofs, + }); + } + if index < blocks.len() - 1 { + if block.uncompressed_size != block_size { + warn(Warning::ZlibTrailerBlockWrongSize { + index, + actual: block.uncompressed_size, + expected: block_size, + }); + } + } else { + if block.uncompressed_size > block_size { + warn(Warning::ZlibTrailerBlockTooBig { + index, + actual: block.uncompressed_size, + max_expected: block_size, + }); + } + } + // http://www.zlib.net/zlib_tech.html says that the maximum + // expansion from compression, with worst-case parameters, is 13.5% + // plus 11 bytes. This code checks for an expansion of more than + // 14.3% plus 11 bytes. + if block.compressed_size > block.uncompressed_size + block.uncompressed_size / 7 + 11 { + return Err(Error::ZlibExpansion { + index, + compressed_size: block.compressed_size, + uncompressed_size: block.uncompressed_size, + }); + } + + expected_cmp_ofs += block.compressed_size as u64; + expected_uncmp_ofs += block.uncompressed_size as u64; + } + + if expected_cmp_ofs != zheader.ztrailer_offset { + return Err(Error::ZlibTrailerOffsetInconsistency { + descriptors: expected_cmp_ofs, + zheader: zheader.ztrailer_offset, + }); + } + reader.seek(SeekFrom::Start(start_offset))?; Ok(Some(ZTrailer { - offset: ztrailer_ofs, + offset: zheader.ztrailer_offset, int_bias, zero, block_size, diff --git a/rust/pspp/src/sys/testdata/partial_compressed_data_record.expected b/rust/pspp/src/sys/testdata/partial_compressed_data_record.expected index e69de29bb2..bd5f2635de 100644 --- a/rust/pspp/src/sys/testdata/partial_compressed_data_record.expected +++ b/rust/pspp/src/sys/testdata/partial_compressed_data_record.expected @@ -0,0 +1,31 @@ +╭──────────────────────┬────────────────────────╮ +│ Created │ 01-JAN-2011 20:53:52│ +├──────────────────────┼────────────────────────┤ +│Writer Product │PSPP synthetic test file│ +├──────────────────────┼────────────────────────┤ +│ Compression │SAV │ +│ Number of Cases│Unknown │ +╰──────────────────────┴────────────────────────╯ + +╭─────────┬────────────────────────╮ +│Label │PSPP synthetic test file│ +│Variables│ 5│ +╰─────────┴────────────────────────╯ + +╭─────┬────────┬─────┬─────────────────┬─────┬─────┬─────────┬────────────┬────────────┬──────────────╮ +│ │Position│Label│Measurement Level│ Role│Width│Alignment│Print Format│Write Format│Missing Values│ +├─────┼────────┼─────┼─────────────────┼─────┼─────┼─────────┼────────────┼────────────┼──────────────┤ +│num1 │ 1│ │ │Input│ 8│Right │F8.0 │F8.0 │ │ +│num2 │ 2│ │ │Input│ 8│Right │F8.0 │F8.0 │ │ +│str4 │ 3│ │Nominal │Input│ 4│Left │A4 │A4 │ │ +│str8 │ 4│ │Nominal │Input│ 8│Left │A8 │A8 │ │ +│str15│ 5│ │Nominal │Input│ 15│Left │A15 │A15 │ │ +╰─────┴────────┴─────┴─────────────────┴─────┴─────┴─────────┴────────────┴────────────┴──────────────╯ + +Unexpected end of file at offset 0x1ac, 0 bytes and 2 compression chunks into a compressed case. + +╭────┬──────┬────┬────────┬────────────────┬──────────────────────────────╮ +│Case│ num1 │num2│ str4 │ str8 │ str15 │ +├────┼──────┼────┼────────┼────────────────┼──────────────────────────────┤ +│1 │-99.00│ .00│ │abcdefgh │ 0123 │ +╰────┴──────┴────┴────────┴────────────────┴──────────────────────────────╯ diff --git a/rust/pspp/src/sys/testdata/partial_data_record_between_variables.expected b/rust/pspp/src/sys/testdata/partial_data_record_between_variables.expected index e69de29bb2..1b2222eb00 100644 --- a/rust/pspp/src/sys/testdata/partial_data_record_between_variables.expected +++ b/rust/pspp/src/sys/testdata/partial_data_record_between_variables.expected @@ -0,0 +1,27 @@ +╭──────────────────────┬────────────────────────╮ +│ Created │ 01-JAN-2011 20:53:52│ +├──────────────────────┼────────────────────────┤ +│Writer Product │PSPP synthetic test file│ +├──────────────────────┼────────────────────────┤ +│ Compression │None │ +│ Number of Cases│Unknown │ +╰──────────────────────┴────────────────────────╯ + +╭─────────┬─╮ +│Variables│2│ +╰─────────┴─╯ + +╭────┬────────┬─────┬─────────────────┬─────┬─────┬─────────┬────────────┬────────────┬──────────────╮ +│ │Position│Label│Measurement Level│ Role│Width│Alignment│Print Format│Write Format│Missing Values│ +├────┼────────┼─────┼─────────────────┼─────┼─────┼─────────┼────────────┼────────────┼──────────────┤ +│num1│ 1│ │ │Input│ 8│Right │F8.0 │F8.0 │ │ +│num2│ 2│ │ │Input│ 8│Right │F8.0 │F8.0 │ │ +╰────┴────────┴─────┴─────────────────┴─────┴─────┴─────────┴────────────┴────────────┴──────────────╯ + +Unexpected end of file at offset 0x12c, 8 bytes into a 16-byte case. + +╭────┬────┬────╮ +│Case│num1│num2│ +├────┼────┼────┤ +│1 │1.00│2.00│ +╰────┴────┴────╯ diff --git a/rust/pspp/src/sys/testdata/partial_data_record_within_long_string.expected b/rust/pspp/src/sys/testdata/partial_data_record_within_long_string.expected index e69de29bb2..b1c0844998 100644 --- a/rust/pspp/src/sys/testdata/partial_data_record_within_long_string.expected +++ b/rust/pspp/src/sys/testdata/partial_data_record_within_long_string.expected @@ -0,0 +1,26 @@ +╭──────────────────────┬────────────────────────╮ +│ Created │ 01-JAN-2011 20:53:52│ +├──────────────────────┼────────────────────────┤ +│Writer Product │PSPP synthetic test file│ +├──────────────────────┼────────────────────────┤ +│ Compression │None │ +│ Number of Cases│Unknown │ +╰──────────────────────┴────────────────────────╯ + +╭─────────┬─╮ +│Variables│1│ +╰─────────┴─╯ + +╭─────┬────────┬─────┬─────────────────┬─────┬─────┬─────────┬────────────┬────────────┬──────────────╮ +│ │Position│Label│Measurement Level│ Role│Width│Alignment│Print Format│Write Format│Missing Values│ +├─────┼────────┼─────┼─────────────────┼─────┼─────┼─────────┼────────────┼────────────┼──────────────┤ +│str14│ 1│ │Nominal │Input│ 14│Left │A14 │A14 │ │ +╰─────┴────────┴─────┴─────────────────┴─────┴─────┴─────────┴────────────┴────────────┴──────────────╯ + +I/O error (failed to fill whole buffer) + +╭────┬──────────────╮ +│Case│ str14 │ +├────┼──────────────┤ +│1 │one data item │ +╰────┴──────────────╯ diff --git a/rust/pspp/src/sys/testdata/unspecified_number_of_variable_positions.expected b/rust/pspp/src/sys/testdata/unspecified_number_of_variable_positions.expected index bdb0b0da07..12cfebb0c6 100644 --- a/rust/pspp/src/sys/testdata/unspecified_number_of_variable_positions.expected +++ b/rust/pspp/src/sys/testdata/unspecified_number_of_variable_positions.expected @@ -18,3 +18,9 @@ │num1 │ 1│ │ │Input│ 8│Right │F8.0 │F8.0 │ │ │Numeric variable 2's label│ 2│Numeric variable 2's label│ │Input│ 8│Right │F8.0 │F8.0 │ │ ╰──────────────────────────┴────────┴──────────────────────────┴─────────────────┴─────┴─────┴─────────┴────────────┴────────────┴──────────────╯ + +╭────┬────┬──────────────────────────╮ +│Case│num1│Numeric variable 2's label│ +├────┼────┼──────────────────────────┤ +│1 │1.00│ 2.00│ +╰────┴────┴──────────────────────────╯ diff --git a/rust/pspp/src/sys/testdata/variable_labels_and_missing_values.expected b/rust/pspp/src/sys/testdata/variable_labels_and_missing_values.expected index a448646811..2648723946 100644 --- a/rust/pspp/src/sys/testdata/variable_labels_and_missing_values.expected +++ b/rust/pspp/src/sys/testdata/variable_labels_and_missing_values.expected @@ -38,3 +38,9 @@ │str10 │ 20│ │Nominal │Input│ 11│Left │A11 │A11 │"abcdefgh"; "01234567"; "0"│ │25-byte string │ 21│25-byte string │Nominal │Input│ 25│Left │A25 │A25 │ │ ╰────────────────────────────────┴────────┴────────────────────────────────┴─────────────────┴─────┴─────┴─────────┴────────────┴────────────┴───────────────────────────╯ + +╭────┬────┬────────────────────────────────┬────┬──────────────────────────────┬────┬────┬────┬────┬────┬────────┬────┬─────────────────────────┬────┬─────────────────────────────┬────┬────┬───────────┬─────────┬──────────┬───────────┬─────────────────────────╮ +│Case│num1│Numeric variable 2's label (ùúû)│num3│Another numeric variable label│num5│num6│num7│num8│num9│numàèìñò│str1│String variable 2's label│str3│Another string variable label│str5│str6│ str7 │ str8 │ str9 │ str10 │ 25-byte string │ +├────┼────┼────────────────────────────────┼────┼──────────────────────────────┼────┼────┼────┼────┼────┼────────┼────┼─────────────────────────┼────┼─────────────────────────────┼────┼────┼───────────┼─────────┼──────────┼───────────┼─────────────────────────┤ +│1 │1.00│ 2.00│3.00│ 4.00│5.00│6.00│7.00│8.00│9.00│ 10.00│abcd│efgh │ijkl│mnop │qrst│uvwx│yzABCDEFGHI│JKLMNOPQR│STUVWXYZ01│23456789abc│defghijklmnopqstuvwxyzABC│ +╰────┴────┴────────────────────────────────┴────┴──────────────────────────────┴────┴────┴────┴────┴────┴────────┴────┴─────────────────────────┴────┴─────────────────────────────┴────┴────┴───────────┴─────────┴──────────┴───────────┴─────────────────────────╯ diff --git a/rust/pspp/src/sys/testdata/wrong_variable_positions_but_v13.expected b/rust/pspp/src/sys/testdata/wrong_variable_positions_but_v13.expected index 119a3389d8..d701ada126 100644 --- a/rust/pspp/src/sys/testdata/wrong_variable_positions_but_v13.expected +++ b/rust/pspp/src/sys/testdata/wrong_variable_positions_but_v13.expected @@ -19,3 +19,9 @@ │num1 │ 1│ │ │Input│ 8│Right │F8.0 │F8.0 │ │ │Numeric variable 2's label│ 2│Numeric variable 2's label│ │Input│ 8│Right │F8.0 │F8.0 │ │ ╰──────────────────────────┴────────┴──────────────────────────┴─────────────────┴─────┴─────┴─────────┴────────────┴────────────┴──────────────╯ + +╭────┬────┬──────────────────────────╮ +│Case│num1│Numeric variable 2's label│ +├────┼────┼──────────────────────────┤ +│1 │1.00│ 2.00│ +╰────┴────┴──────────────────────────╯ diff --git a/rust/pspp/src/sys/testdata/zcompressed_data.expected b/rust/pspp/src/sys/testdata/zcompressed_data.expected index 396ece9038..41e507a580 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data.expected @@ -21,3 +21,11 @@ │str8 │ 4│ │Nominal │Input│ 8│Left │A8 │A8 │ │ │str15│ 5│ │Nominal │Input│ 15│Left │A15 │A15 │ │ ╰─────┴────────┴─────┴─────────────────┴─────┴─────┴─────────┴────────────┴────────────┴──────────────╯ + +╭────┬──────┬──────┬────────┬────────────────┬──────────────────────────────╮ +│Case│ num1 │ num2 │ str4 │ str8 │ str15 │ +├────┼──────┼──────┼────────┼────────────────┼──────────────────────────────┤ +│1 │-99.00│ .00│ │abcdefgh │ 0123 │ +│2 │ . │151.00│jklm │nopqrstu │vwxyzABC │ +│3 │ 1.00│ 2.00│DEFG │HIJKLMNO │ PQRSTUV │ +╰────┴──────┴──────┴────────┴────────────────┴──────────────────────────────╯ diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_bad_zheader_ofs.expected b/rust/pspp/src/sys/testdata/zcompressed_data_bad_zheader_ofs.expected index e69de29bb2..99b16acc5a 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_bad_zheader_ofs.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_bad_zheader_ofs.expected @@ -0,0 +1 @@ +ZLIB header's zlib_offset is 0x0 instead of expected 0x194. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_bad_zheader_ofs.sack b/rust/pspp/src/sys/testdata/zcompressed_data_bad_zheader_ofs.sack index bbe7020845..7d86604f36 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_bad_zheader_ofs.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_bad_zheader_ofs.sack @@ -27,7 +27,7 @@ i8 0 *3; 999; 0; # ZLIB data header. -i64 0x194; # zheader_ofs +>>i64 0<<; # zheader_ofs i64 0x205; # ztrailer_ofs i64 48; # ztrailer_len diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_bad_ztrailer_ofs.expected b/rust/pspp/src/sys/testdata/zcompressed_data_bad_ztrailer_ofs.expected index e69de29bb2..cced59a32d 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_bad_ztrailer_ofs.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_bad_ztrailer_ofs.expected @@ -0,0 +1 @@ +Impossible ztrailer_offset 0x0. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_bad_ztrailer_ofs.sack b/rust/pspp/src/sys/testdata/zcompressed_data_bad_ztrailer_ofs.sack index bbe7020845..287c656423 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_bad_ztrailer_ofs.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_bad_ztrailer_ofs.sack @@ -28,7 +28,7 @@ i8 0 *3; # ZLIB data header. i64 0x194; # zheader_ofs -i64 0x205; # ztrailer_ofs +>>i64 0<<; # ztrailer_ofs i64 48; # ztrailer_len # ZLIB data block. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_don_t_add_up.expected b/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_don_t_add_up.expected index e69de29bb2..d8b17d3569 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_don_t_add_up.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_don_t_add_up.expected @@ -0,0 +1 @@ +ZLIB trailer is at offset 0x205 but 0x204 would be expected from block descriptors. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_don_t_add_up.sack b/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_don_t_add_up.sack index bbe7020845..06f20d031f 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_don_t_add_up.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_don_t_add_up.sack @@ -60,4 +60,4 @@ i64 0; # ztrailer_zero i64 0x194; # uncompressed_ofs i64 0x1ac; # compressed_ofs 88; # uncompressed_size -89; # compressed_size +>>88<<; # compressed_size diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_dont_add_up.expected b/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_dont_add_up.expected index e69de29bb2..df98439368 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_dont_add_up.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_compressed_sizes_dont_add_up.expected @@ -0,0 +1 @@ +ZLIB block descriptor 1 reported compressed data offset 0x12421, when 0x124f1 was expected. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_compression_expands_data_too_much.expected b/rust/pspp/src/sys/testdata/zcompressed_data_compression_expands_data_too_much.expected index e69de29bb2..29d7c228ab 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_compression_expands_data_too_much.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_compression_expands_data_too_much.expected @@ -0,0 +1 @@ +ZLIB block descriptor 0 reports compressed size 100 and uncompressed size 50. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_compression_expands_data_too_much.sack b/rust/pspp/src/sys/testdata/zcompressed_data_compression_expands_data_too_much.sack index bbe7020845..bc2627e09d 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_compression_expands_data_too_much.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_compression_expands_data_too_much.sack @@ -59,5 +59,5 @@ i64 0; # ztrailer_zero # ZLIB block descriptor: i64 0x194; # uncompressed_ofs i64 0x1ac; # compressed_ofs -88; # uncompressed_size -89; # compressed_size +>>50<<; # uncompressed_size +>>100<<; # compressed_size diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_invalid_ztrailer_len.expected b/rust/pspp/src/sys/testdata/zcompressed_data_invalid_ztrailer_len.expected index e69de29bb2..f98e08b3f7 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_invalid_ztrailer_len.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_invalid_ztrailer_len.expected @@ -0,0 +1 @@ +Invalid ZLIB trailer length 21. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_invalid_ztrailer_len.sack b/rust/pspp/src/sys/testdata/zcompressed_data_invalid_ztrailer_len.sack index bbe7020845..0b03e4a597 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_invalid_ztrailer_len.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_invalid_ztrailer_len.sack @@ -29,7 +29,7 @@ i8 0 *3; # ZLIB data header. i64 0x194; # zheader_ofs i64 0x205; # ztrailer_ofs -i64 48; # ztrailer_len +>>i64 21<<; # ztrailer_len # ZLIB data block. # diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_uncompressed_size_block_size.expected b/rust/pspp/src/sys/testdata/zcompressed_data_uncompressed_size_block_size.expected index e69de29bb2..a1a839cdf2 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_uncompressed_size_block_size.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_uncompressed_size_block_size.expected @@ -0,0 +1,33 @@ +ZLIB block descriptor 0 reported block size 0x400000, when at most 0x3ff000 was expected. + +╭──────────────────────┬────────────────────────╮ +│ Created │ 01-JAN-2011 20:53:52│ +├──────────────────────┼────────────────────────┤ +│Writer Product │PSPP synthetic test file│ +├──────────────────────┼────────────────────────┤ +│ Compression │ZSAV │ +│ Number of Cases│Unknown │ +╰──────────────────────┴────────────────────────╯ + +╭─────────┬────────────────────────╮ +│Label │PSPP synthetic test file│ +│Variables│ 5│ +╰─────────┴────────────────────────╯ + +╭─────┬────────┬─────┬─────────────────┬─────┬─────┬─────────┬────────────┬────────────┬──────────────╮ +│ │Position│Label│Measurement Level│ Role│Width│Alignment│Print Format│Write Format│Missing Values│ +├─────┼────────┼─────┼─────────────────┼─────┼─────┼─────────┼────────────┼────────────┼──────────────┤ +│num1 │ 1│ │ │Input│ 8│Right │F8.0 │F8.0 │ │ +│num2 │ 2│ │ │Input│ 8│Right │F8.0 │F8.0 │ │ +│str4 │ 3│ │Nominal │Input│ 4│Left │A4 │A4 │ │ +│str8 │ 4│ │Nominal │Input│ 8│Left │A8 │A8 │ │ +│str15│ 5│ │Nominal │Input│ 15│Left │A15 │A15 │ │ +╰─────┴────────┴─────┴─────────────────┴─────┴─────┴─────────┴────────────┴────────────┴──────────────╯ + +╭────┬──────┬──────┬────────┬────────────────┬──────────────────────────────╮ +│Case│ num1 │ num2 │ str4 │ str8 │ str15 │ +├────┼──────┼──────┼────────┼────────────────┼──────────────────────────────┤ +│1 │-99.00│ .00│ │abcdefgh │ 0123 │ +│2 │ . │151.00│jklm │nopqrstu │vwxyzABC │ +│3 │ 1.00│ 2.00│DEFG │HIJKLMNO │ PQRSTUV │ +╰────┴──────┴──────┴────────┴────────────────┴──────────────────────────────╯ diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_uncompressed_size_block_size.sack b/rust/pspp/src/sys/testdata/zcompressed_data_uncompressed_size_block_size.sack index bbe7020845..11b2651164 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_uncompressed_size_block_size.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_uncompressed_size_block_size.sack @@ -59,5 +59,5 @@ i64 0; # ztrailer_zero # ZLIB block descriptor: i64 0x194; # uncompressed_ofs i64 0x1ac; # compressed_ofs -88; # uncompressed_size +>>0x400000<<; # uncompressed_size 89; # compressed_size diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_block_size.expected b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_block_size.expected index e69de29bb2..4c3ed56a7c 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_block_size.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_block_size.expected @@ -0,0 +1 @@ +ZLIB trailer specifies unexpected 4096-byte block size. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_block_size.sack b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_block_size.sack index bbe7020845..83ef9b18fb 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_block_size.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_block_size.sack @@ -53,7 +53,7 @@ hex "90 d0 b0 70 00 0f 3f 23 d7"; # ZLIB data trailer fixed header: i64 -100; # ztrailer_bias i64 0; # ztrailer_zero -0x3ff000; # block_size +>>0x1000<<; # block_size 1; # n_blocks # ZLIB block descriptor: diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_compressed_ofs.expected b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_compressed_ofs.expected index e69de29bb2..0f86bbcc44 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_compressed_ofs.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_compressed_ofs.expected @@ -0,0 +1 @@ +ZLIB block descriptor 0 reported compressed data offset 0x191, when 0x1ac was expected. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_compressed_ofs.sack b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_compressed_ofs.sack index bbe7020845..e575aa6b87 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_compressed_ofs.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_compressed_ofs.sack @@ -58,6 +58,6 @@ i64 0; # ztrailer_zero # ZLIB block descriptor: i64 0x194; # uncompressed_ofs -i64 0x1ac; # compressed_ofs +>>i64 0x191<<;# compressed_ofs 88; # uncompressed_size 89; # compressed_size diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_n_blocks.expected b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_n_blocks.expected index e69de29bb2..e2cd618f70 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_n_blocks.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_n_blocks.expected @@ -0,0 +1 @@ +Block count 2 in ZLIB trailer at offset 0x205 differs from expected block count 1 calculated from trailer length 48. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_n_blocks.sack b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_n_blocks.sack index bbe7020845..6f5920707c 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_n_blocks.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_n_blocks.sack @@ -54,7 +54,7 @@ hex "90 d0 b0 70 00 0f 3f 23 d7"; i64 -100; # ztrailer_bias i64 0; # ztrailer_zero 0x3ff000; # block_size -1; # n_blocks +>>2<<; # n_blocks # ZLIB block descriptor: i64 0x194; # uncompressed_ofs diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_uncompressed_ofs.expected b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_uncompressed_ofs.expected index e69de29bb2..0a2af92d87 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_uncompressed_ofs.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_uncompressed_ofs.expected @@ -0,0 +1 @@ +ZLIB block descriptor 0 reported uncompressed data offset 0x177, when 0x1ac was expected. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_uncompressed_ofs.sack b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_uncompressed_ofs.sack index bbe7020845..7d21286d24 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_uncompressed_ofs.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_uncompressed_ofs.sack @@ -57,7 +57,7 @@ i64 0; # ztrailer_zero 1; # n_blocks # ZLIB block descriptor: -i64 0x194; # uncompressed_ofs +i64 >>0x177<<;# uncompressed_ofs i64 0x1ac; # compressed_ofs 88; # uncompressed_size 89; # compressed_size diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_bias.expected b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_bias.expected index e69de29bb2..dbfa4c1cb0 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_bias.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_bias.expected @@ -0,0 +1 @@ +ZLIB trailer bias 0 is not -100 as expected from file header bias. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_bias.sack b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_bias.sack index bbe7020845..efd526379c 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_bias.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_bias.sack @@ -51,7 +51,7 @@ hex "5e de 3e be 7e fe ff fe fe 61 00 81 80 c0 a0 e0"; hex "90 d0 b0 70 00 0f 3f 23 d7"; # ZLIB data trailer fixed header: -i64 -100; # ztrailer_bias +>>i64 0<<; # ztrailer_bias i64 0; # ztrailer_zero 0x3ff000; # block_size 1; # n_blocks diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_len.expected b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_len.expected index e69de29bb2..0b98a9701c 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_len.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_len.expected @@ -0,0 +1 @@ +Block count 1 in ZLIB trailer at offset 0x205 differs from expected block count 2 calculated from trailer length 72. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_len.sack b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_len.sack index bbe7020845..e81a92a015 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_len.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_len.sack @@ -29,7 +29,7 @@ i8 0 *3; # ZLIB data header. i64 0x194; # zheader_ofs i64 0x205; # ztrailer_ofs -i64 48; # ztrailer_len +>>i64 72<<; # ztrailer_len # ZLIB data block. # diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_zero.expected b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_zero.expected index e69de29bb2..08d1d75614 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_zero.expected +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_zero.expected @@ -0,0 +1 @@ +ZLIB trailer "zero" field has nonzero value 100. diff --git a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_zero.sack b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_zero.sack index bbe7020845..fa59e362b4 100644 --- a/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_zero.sack +++ b/rust/pspp/src/sys/testdata/zcompressed_data_wrong_ztrailer_zero.sack @@ -52,7 +52,7 @@ hex "90 d0 b0 70 00 0f 3f 23 d7"; # ZLIB data trailer fixed header: i64 -100; # ztrailer_bias -i64 0; # ztrailer_zero +>>i64 100<<; # ztrailer_zero 0x3ff000; # block_size 1; # n_blocks -- 2.30.2