[[package]]
name = "libz-rs-sys"
-version = "0.5.0"
+version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a"
+checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221"
dependencies = [
"zlib-rs",
]
[[package]]
name = "zlib-rs"
-version = "0.5.0"
+version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8"
+checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a"
[[package]]
name = "zopfli"
eprintln!("warning: {warning}");
}
- let (dictionary, _, cases) = ReadOptions::new()
+ let (dictionary, _, cases) = ReadOptions::new(warn)
.with_encoding(self.encoding)
.with_password(self.password.clone())
- .open_file(&self.input, warn)?
+ .open_file(&self.input)?
.into_parts();
// Take only the first `self.max_cases` cases.
.with_compression(self.sys_options.compression)
.write_file(&dictionary, output)?;
for case in cases {
- output.write_case(&case?)?;
+ output.write_case(case?.0.iter())?;
}
}
}
collections::BTreeMap,
fmt::{Debug, Display},
fs::File,
- io::{Read, Seek},
+ io::{BufRead, BufReader, Read, Seek},
ops::Range,
path::Path,
};
},
};
use anyhow::{anyhow, Error as AnyError};
-use binrw::{io::BufReader, BinRead, BinWrite};
+use binrw::{BinRead, BinWrite};
use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
use encoding_rs::Encoding;
use indexmap::set::MutableValues;
}
/// Options for reading a system file.
-#[derive(Default, Clone, Debug)]
-pub struct ReadOptions {
+#[derive(Clone, Debug)]
+pub struct ReadOptions<F> {
+ /// Function called to report warnings.
+ pub warn: F,
+
/// Character encoding for text in the system file.
///
/// If not set, the character encoding will be determined from reading the
pub password: Option<String>,
}
-impl ReadOptions {
- /// Construct a new `ReadOptions` that initially does not specify an
- /// encoding or password.
- pub fn new() -> Self {
- Self::default()
+impl<F> ReadOptions<F> {
+ /// Construct a new `ReadOptions` that reports warnings by calling `warn`
+ /// and initially does not specify an encoding or password.
+ pub fn new(warn: F) -> Self {
+ Self {
+ warn,
+ encoding: None,
+ password: None,
+ }
}
/// Causes the file to be read using the specified `encoding`, or with a
Self { password, ..self }
}
- /// Opens the file at `path`, reporting warnings using `warn`.
- pub fn open_file<P, F>(self, path: P, warn: F) -> Result<SystemFile, AnyError>
+ /// Opens the file at `path`.
+ pub fn open_file<P>(mut self, path: P) -> Result<SystemFile, AnyError>
where
P: AsRef<Path>,
F: FnMut(AnyError),
{
let file = File::open(path)?;
- if self.password.is_some() {
+ if let Some(password) = self.password.take() {
// Don't create `BufReader`, because [EncryptedReader] will buffer.
- self.open_reader(file, warn)
+ self.open_reader_encrypted(file, password)
} else {
- self.open_reader(BufReader::new(file), warn)
+ Self::open_reader_inner(BufReader::new(file), self.encoding, self.warn)
}
}
- /// Opens the file read from `reader`, reporting warnings using `warn`.
- pub fn open_reader<R, F>(self, reader: R, warn: F) -> Result<SystemFile, AnyError>
+ /// Opens the file read from `reader`.
+ fn open_reader_encrypted<R>(self, reader: R, password: String) -> Result<SystemFile, AnyError>
where
R: Read + Seek + 'static,
F: FnMut(AnyError),
{
- if let Some(password) = &self.password {
- Self::open_reader_inner(
- EncryptedFile::new(reader)?
- .unlock(password.as_bytes())
- .map_err(|_| anyhow!("Incorrect password."))?,
- self.encoding,
- warn,
- )
+ Self::open_reader_inner(
+ EncryptedFile::new(reader)?
+ .unlock(password.as_bytes())
+ .map_err(|_| anyhow!("Incorrect password."))?,
+ self.encoding,
+ self.warn,
+ )
+ }
+
+ /// Opens the file read from `reader`.
+ pub fn open_reader<R>(mut self, reader: R) -> Result<SystemFile, AnyError>
+ where
+ R: BufRead + Seek + 'static,
+ F: FnMut(AnyError),
+ {
+ if let Some(password) = self.password.take() {
+ self.open_reader_encrypted(reader, password)
} else {
- Self::open_reader_inner(reader, self.encoding, warn)
+ Self::open_reader_inner(reader, self.encoding, self.warn)
}
}
- fn open_reader_inner<R, F>(
+ fn open_reader_inner<R>(
reader: R,
encoding: Option<&'static Encoding>,
mut warn: F,
) -> Result<SystemFile, AnyError>
where
- R: Read + Seek + 'static,
+ R: BufRead + Seek + 'static,
F: FnMut(AnyError),
{
let mut reader = Reader::new(reader, |warning| warn(warning.into()))?;
// this program. If not, see <http://www.gnu.org/licenses/>.
//! Character encodings in system files.
+//!
+//! These are useful for reading and writing system files at a low level.
use std::sync::LazyLock;
};
use encoding_rs::Encoding;
-use flate2::read::ZlibDecoder;
+use flate2::bufread::ZlibDecoder;
use smallvec::SmallVec;
use std::{
borrow::Cow,
cell::RefCell,
collections::VecDeque,
fmt::{Debug, Display, Formatter, Result as FmtResult},
- io::{empty, Error as IoError, Read, Seek, SeekFrom},
+ io::{empty, BufRead, Error as IoError, Read, Seek, SeekFrom},
iter::repeat_n,
mem::take,
num::NonZeroU8,
warn: &mut dyn FnMut(Warning),
) -> Result<Option<Record>, Error>
where
- R: Read + Seek,
+ R: BufRead + Seek,
{
let rec_type: u32 = endian.parse(read_bytes(reader)?);
match rec_type {
struct ZlibDecodeMultiple<R>
where
- R: Read + Seek,
+ R: BufRead + Seek,
{
reader: Option<ZlibDecoder<R>>,
+ limit: u64,
}
impl<R> ZlibDecodeMultiple<R>
where
- R: Read + Seek,
+ R: BufRead + Seek,
{
- fn new(reader: R) -> ZlibDecodeMultiple<R> {
+ fn new(reader: R, limit: u64) -> ZlibDecodeMultiple<R> {
ZlibDecodeMultiple {
reader: Some(ZlibDecoder::new(reader)),
+ limit,
}
}
}
impl<R> Read for ZlibDecodeMultiple<R>
where
- R: Read + Seek,
+ R: BufRead + Seek,
{
fn read(&mut self, buf: &mut [u8]) -> Result<usize, IoError> {
loop {
- match self.reader.as_mut().unwrap().read(buf)? {
- 0 => {
- let inner = self.reader.take().unwrap().into_inner();
+ match self.reader.as_mut().unwrap().read(buf) {
+ Err(error) => return Err(error),
+ Ok(0) => {
+ let mut inner = self.reader.take().unwrap().into_inner();
+ let position = inner.stream_position();
self.reader = Some(ZlibDecoder::new(inner));
+ if position? >= self.limit {
+ return Ok(0);
+ }
}
- n => return Ok(n),
+ Ok(n) => return Ok(n),
};
}
}
impl<R> Seek for ZlibDecodeMultiple<R>
where
- R: Read + Seek,
+ R: BufRead + Seek,
{
fn seek(&mut self, pos: SeekFrom) -> Result<u64, IoError> {
self.reader.as_mut().unwrap().get_mut().seek(pos)
/// Reads records from a system file in their raw form.
pub struct Reader<'a, R>
where
- R: Read + Seek + 'static,
+ R: BufRead + Seek + 'static,
{
reader: Option<R>,
warn: Box<dyn FnMut(Warning) + 'a>,
impl<'a, R> Reader<'a, R>
where
- R: Read + Seek + 'static,
+ R: BufRead + Seek + 'static,
{
/// Constructs a new [Reader] from the underlying `reader`. Any warnings
/// encountered while reading the system file will be reported with `warn`.
/// Reads raw records from a system file.
pub struct Records<'a, 'b, R>(&'b mut Reader<'a, R>)
where
- R: Read + Seek + 'static;
+ R: BufRead + Seek + 'static;
impl<'a, 'b, R> Records<'a, 'b, R>
where
- R: Read + Seek + 'static,
+ R: BufRead + Seek + 'static,
{
- fn cases(&mut self) {
+ fn cases(&mut self, ztrailer_offset: Option<u64>) {
self.0.state = ReaderState::End;
self.0.cases = Some(Cases::new(
self.0.reader.take().unwrap(),
take(&mut self.0.var_types),
&self.0.header,
+ ztrailer_offset,
));
}
self.0.state = if let Some(Compression::ZLib) = self.0.header.compression {
ReaderState::ZlibHeader
} else {
- self.cases();
+ self.cases(None);
ReaderState::End
};
}
&mut self.0.warn,
) {
Ok(None) => {
- self.cases();
+ self.cases(Some(zheader.inner.ztrailer_offset));
None
}
Ok(Some(ztrailer)) => {
- self.cases();
+ self.cases(Some(zheader.inner.ztrailer_offset));
Some(Ok(Record::ZTrailer(ztrailer)))
}
Err(error) => Some(Err(error)),
impl<'a, 'b, R> Iterator for Records<'a, 'b, R>
where
- R: Read + Seek + 'static,
+ R: BufRead + Seek + 'static,
{
type Item = Result<Record, Error>;
}
impl Cases {
- fn new<R>(reader: R, var_types: VarTypes, header: &FileHeader<RawString>) -> Self
+ fn new<R>(
+ reader: R,
+ var_types: VarTypes,
+ header: &FileHeader<RawString>,
+ ztrailer_offset: Option<u64>,
+ ) -> Self
where
- R: Read + Seek + 'static,
+ R: BufRead + Seek + 'static,
{
Self {
reader: if header.compression == Some(Compression::ZLib) {
- Box::new(ZlibDecodeMultiple::new(reader))
+ Box::new(ZlibDecodeMultiple::new(reader, ztrailer_offset.unwrap()))
} else {
Box::new(reader)
},
pub inner: RawIntegerInfoRecord,
}
-/// Machine integer info record in [binrw] format.
+/// Machine integer info record in [mod@binrw] format.
#[derive(Clone, Debug, BinRead, BinWrite)]
pub struct RawIntegerInfoRecord {
/// Version number.
pub inner: RawZHeader,
}
+/// A ZLIB header in a system file.
#[derive(Clone, Debug, BinRead, BinWrite)]
pub struct RawZHeader {
/// File offset to the ZLIB data header.
/// Error reading a [ZHeader].
#[derive(ThisError, Debug)]
pub enum ZHeaderError {
+ /// I/O error via [mod@binrw].
#[error("{}", DisplayBinError(&.0, "ZLIB header"))]
BinError(#[from] BinError),
u64,
),
- /// ZLIB header's zlib_offset is {actual:#x} instead of expected
- /// {expected:#x}.
- #[error("ZLIB header's zlib_offset is {actual:#x} instead of expected {expected:#x}.")]
+ /// zlib_offset is {actual:#x} instead of expected {expected:#x}.
+ #[error("zlib_offset is {actual:#x} instead of expected {expected:#x}.")]
UnexpectedZHeaderOffset {
/// Actual `zlib_offset`.
actual: u64,
/// File offset to the start of the record.
pub offset: u64,
+ /// The raw trailer.
pub inner: RawZTrailer,
}
+/// A ZLIB trailer in a system file.
#[binrw]
#[derive(Clone, Debug)]
pub struct RawZTrailer {
}
impl RawZTrailer {
+ /// Returns the length of the trailer when it is written, in bytes.
pub fn len(&self) -> usize {
24 + self.blocks.len() * 24
}
pub enum ZlibTrailerWarning {
/// Wrong block size.
#[error(
- "ZLIB block descriptor {index} reported block size {actual:#x}, when {expected:#x} was expected."
+ "Block descriptor {index} reported block size {actual:#x}, when {expected:#x} was expected."
)]
ZlibTrailerBlockWrongSize {
/// 0-based block descriptor index.
/// Block too big.
#[error(
- "ZLIB block descriptor {index} reported block size {actual:#x}, when at most {max_expected:#x} was expected."
+ "Block descriptor {index} reported block size {actual:#x}, when at most {max_expected:#x} was expected."
)]
ZlibTrailerBlockTooBig {
/// 0-based block descriptor index.
/// Error reading a [ZTrailer].
#[derive(ThisError, Debug)]
pub enum ZTrailerError {
+ /// I/O error via [mod@binrw].
#[error("{}", DisplayBinError(&.0, "ZLIB trailer"))]
BinError(#[from] BinError),
/// ZLIB trailer bias {actual} is not {} as expected from file header bias.
#[
error(
- "ZLIB trailer bias {actual} is not {} as expected from file header bias.",
+ "Bias {actual} is not {} as expected from file header.",
DisplayPlainF64(*expected)
)]
WrongZlibTrailerBias {
expected: f64,
},
- /// ZLIB trailer \"zero\" field has nonzero value {0}.
- #[error("ZLIB trailer \"zero\" field has nonzero value {0}.")]
+ /// ZLIB trailer zero field has nonzero value {0}.
+ #[error("Expected zero field has nonzero value {0}.")]
WrongZlibTrailerZero(
/// Actual value that should have been zero.
u64,
),
/// ZLIB trailer specifies unexpected {0}-byte block size.
- #[error("ZLIB trailer specifies unexpected {0}-byte block size.")]
+ #[error("Unexpected {0:x}-byte block size (expected 0x3ff000).")]
WrongZlibTrailerBlockSize(
/// Block size read from file.
u32,
),
- /// Block count in ZLIB trailer differs from expected block count calculated
- /// from trailer length.
+ /// Block count differs from expected block count calculated from trailer
+ /// length.
#[error(
- "Block count {n_blocks} in ZLIB trailer differs from expected block count {expected_n_blocks} calculated from trailer length {ztrailer_len}."
+ "Block count {n_blocks} differs from expected block count {expected_n_blocks} calculated from trailer length {ztrailer_len}."
)]
BadZlibTrailerNBlocks {
/// Number of blocks.
/// ZLIB block descriptor reported uncompressed data offset different from
/// expected.
#[error(
- "ZLIB block descriptor {index} reported uncompressed data offset {actual:#x}, when {expected:#x} was expected."
+ "Block descriptor {index} reported uncompressed data offset {actual:#x}, when {expected:#x} was expected."
)]
ZlibTrailerBlockWrongUncmpOfs {
/// Block descriptor index.
expected: u64,
},
- /// ZLIB block descriptor {index} reported compressed data offset
+ /// Block descriptor {index} reported compressed data offset
/// {actual:#x}, when {expected:#x} was expected.
#[error(
- "ZLIB block descriptor {index} reported compressed data offset {actual:#x}, when {expected:#x} was expected."
+ "Block descriptor {index} reported compressed data offset {actual:#x}, when {expected:#x} was expected."
)]
ZlibTrailerBlockWrongCmpOfs {
/// Block descriptor index.
expected: u64,
},
- /// ZLIB block descriptor {index} reports compressed size {compressed_size}
+ /// Block descriptor {index} reports compressed size {compressed_size}
/// and uncompressed size {uncompressed_size}.
#[error(
- "ZLIB block descriptor {index} reports compressed size {compressed_size} and uncompressed size {uncompressed_size}."
+ "Block descriptor {index} reports compressed size {compressed_size} and uncompressed size {uncompressed_size}."
)]
ZlibExpansion {
/// Block descriptor index.
use std::{
fs::File,
- io::{Cursor, Read, Seek},
+ io::{BufRead, BufReader, Cursor, Seek},
path::Path,
sync::Arc,
};
#[test]
fn encrypted_file_without_password() {
- let error = ReadOptions::new()
- .open_file("src/crypto/testdata/test-encrypted.sav", |_| {
- panic!();
- })
- .unwrap_err();
+ let error = ReadOptions::new(|_| {
+ panic!();
+ })
+ .open_file("src/crypto/testdata/test-encrypted.sav")
+ .unwrap_err();
assert!(matches!(
error.downcast::<raw::Error>().unwrap().details,
ErrorDetails::Encrypted
.join("src/sys/testdata")
.join(name)
.with_extension("sav");
- let sysfile = File::open(&input_filename).unwrap();
+ let sysfile = BufReader::new(File::open(&input_filename).unwrap());
let expected_filename = input_filename.with_extension("expected");
let expected = String::from_utf8(std::fs::read(&expected_filename).unwrap()).unwrap();
test_sysfile(sysfile, &expected, &expected_filename);
fn test_sysfile<R>(sysfile: R, expected: &str, expected_filename: &Path)
where
- R: Read + Seek + 'static,
+ R: BufRead + Seek + 'static,
{
let mut warnings = Vec::new();
- let output = match ReadOptions::new().open_reader(sysfile, |warning| warnings.push(warning)) {
+ let output = match ReadOptions::new(|warning| warnings.push(warning)).open_reader(sysfile) {
Ok(system_file) => {
let (dictionary, metadata, cases) = system_file.into_parts();
let (group, data) = metadata.to_pivot_rows();
-Error at file offsets 0x194 to 0x1a0: ZLIB header's zlib_offset is 0x0 instead of expected 0x194.
+Error at file offsets 0x194 to 0x1a0: Error reading ZLIB header: zlib_offset is 0x0 instead of expected 0x194.
-Error at file offsets 0x194 to 0x1a0: Impossible ztrailer_offset 0x0.
+Error at file offsets 0x194 to 0x1a0: Error reading ZLIB header: Impossible ztrailer_offset 0x0.
-Error at file offsets 0x1ac to 0x1dc: ZLIB trailer is at offset 0x205 but 0x204 would be expected from block descriptors.
+Error at file offsets 0x1ac to 0x1dc: Error reading ZLIB trailer: ZLIB trailer is at offset 0x205 but 0x204 would be expected from block descriptors.
-Error at file offsets 0x1dc to 0x1f4: ZLIB block descriptor 1 reported compressed data offset 0x12421, when 0x124f1 was expected.
+Error at file offsets 0x1dc to 0x1f4: Error reading ZLIB trailer: Block descriptor 1 reported compressed data offset 0x12421, when 0x124f1 was expected.
-Error at file offsets 0x1c4 to 0x1dc: ZLIB block descriptor 0 reports compressed size 100 and uncompressed size 50.
+Error at file offsets 0x1c4 to 0x1dc: Error reading ZLIB trailer: Block descriptor 0 reports compressed size 100 and uncompressed size 50.
-Error at file offsets 0x194 to 0x1a0: Invalid ZLIB trailer length 21.
+Error at file offsets 0x194 to 0x1a0: Error reading ZLIB header: Invalid ZLIB trailer length 21.
-Warning at file offsets 0x1c4 to 0x1dc: In ZLIB trailer: ZLIB block descriptor 0 reported block size 0x400000, when at most 0x3ff000 was expected.
+Warning at file offsets 0x1c4 to 0x1dc: In ZLIB trailer: Block descriptor 0 reported block size 0x400000, when at most 0x3ff000 was expected.
╭──────────────────────┬────────────────────────╮
│ Created │ 01-JAN-2011 20:53:52│
-Error at file offsets 0x1ac to 0x1c4: ZLIB trailer specifies unexpected 4096-byte block size.
+Error at file offsets 0x1ac to 0x1c4: Error reading ZLIB trailer: Unexpected 1000-byte block size (expected 0x3ff000).
-Error at file offsets 0x1c4 to 0x1dc: ZLIB block descriptor 0 reported compressed data offset 0x191, when 0x1ac was expected.
+Error at file offsets 0x1c4 to 0x1dc: Error reading ZLIB trailer: Block descriptor 0 reported compressed data offset 0x191, when 0x1ac was expected.
-Error at file offsets 0x1ac to 0x1c4: Block count 2 in ZLIB trailer differs from expected block count 1 calculated from trailer length 48.
+Error at file offsets 0x205 to 0x235: Error reading ZLIB trailer: Unexpected end-of-file reading ZLIB trailer
-Error at file offsets 0x1c4 to 0x1dc: ZLIB block descriptor 0 reported uncompressed data offset 0x177, when 0x1ac was expected.
+Error at file offsets 0x1c4 to 0x1dc: Error reading ZLIB trailer: Block descriptor 0 reported uncompressed data offset 0x177, when 0x1ac was expected.
-Error at file offsets 0x1ac to 0x1c4: ZLIB trailer bias 0 is not -100 as expected from file header bias.
+Error at file offsets 0x1ac to 0x1c4: Error reading ZLIB trailer: Bias 0 is not -100 as expected from file header.
-Error at file offsets 0x1ac to 0x1c4: Block count 1 in ZLIB trailer differs from expected block count 2 calculated from trailer length 72.
+Error at file offsets 0x1ac to 0x1c4: Error reading ZLIB trailer: Block count 1 differs from expected block count 2 calculated from trailer length 72.
-Error at file offsets 0x1ac to 0x1c4: ZLIB trailer "zero" field has nonzero value 100.
+Error at file offsets 0x1ac to 0x1c4: Error reading ZLIB trailer: Expected zero field has nonzero value 100.
projects / pspp / commitdiff