From: Ben Pfaff Date: Sun, 26 Apr 2026 20:28:21 +0000 (-0700) Subject: work on better typing for datasources X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e030c9d21d7f12ce53f8b2f407e17654146ea211;p=pspp work on better typing for datasources --- diff --git a/rust/pspp/src/cli/spv.rs b/rust/pspp/src/cli/spv.rs index c48242aa2d..b9df03f172 100644 --- a/rust/pspp/src/cli/spv.rs +++ b/rust/pspp/src/cli/spv.rs @@ -24,7 +24,7 @@ use pspp::{ }, spv::{ SpvArchive, - legacy_bin::{LegacyBin, data_pivot_table}, + legacy_bin::LegacyBin, read::{ legacy_xml::Visualization, structure::{Content, OutlineItem}, @@ -126,7 +126,7 @@ impl LegacyData { if let Some(FileType::LegacyBin) = FileType::from_file(&self.input.input)? { let data = std::fs::read(&self.input.input)?; let data = LegacyBin::from_data(&data, &mut |w| eprintln!("{w}"))?; - println!("{}", data_pivot_table(&data)); + println!("{}", data.as_pivot_table()); } else { let (mut archive, items) = self.input.read_outline()?; for item in items { @@ -143,7 +143,7 @@ impl LegacyData { )) })?; - println!("{}", data_pivot_table(&data)); + println!("{}", data.as_pivot_table()); } } } diff --git a/rust/pspp/src/spv/read.rs b/rust/pspp/src/spv/read.rs index 3a64a7b31c..01186a64a5 100644 --- a/rust/pspp/src/spv/read.rs +++ b/rust/pspp/src/spv/read.rs @@ -23,7 +23,6 @@ use std::{ }; use displaydoc::Display; -use indexmap::IndexMap; use serde::Deserialize; use zip::{ZipArchive, result::ZipError}; @@ -32,7 +31,7 @@ use crate::{ file::ReadSeek, output::{Criteria, Item, page::PageSetup}, spv::{ - legacy_bin::{DataSeries, LegacyBin, LegacyBinWarning}, + legacy_bin::{DataSources, LegacyBin, LegacyBinWarning}, read::{ graph::GraphWarning, legacy_xml::LegacyXmlWarning, @@ -182,7 +181,7 @@ where &mut self, name: &str, warn: &mut dyn FnMut(LegacyBinWarning), - ) -> Result>, Error> { + ) -> Result { LegacyBin::from_data(&self.read_member(name)?, warn).map_err(Error::BinrwError) } } diff --git a/rust/pspp/src/spv/read/graph.rs b/rust/pspp/src/spv/read/graph.rs index 1e22f9309d..dd633ceef7 100644 --- a/rust/pspp/src/spv/read/graph.rs +++ b/rust/pspp/src/spv/read/graph.rs @@ -18,8 +18,8 @@ use std::{ borrow::{Borrow, Cow}, collections::{BTreeMap, HashMap}, - fmt::{Debug, Display}, - io::{Cursor, ErrorKind, Write}, + fmt::Debug, + io::{Cursor, ErrorKind}, iter::{repeat, zip}, marker::PhantomData, mem::take, @@ -37,7 +37,7 @@ use ordered_float::OrderedFloat; use pango::FontDescription; use quick_xml::{ Reader, Writer, - events::{BytesStart, BytesText, Event}, + events::{BytesStart, Event}, }; use rs_stats::distributions::normal_distribution::normal_pdf; use serde::{Deserialize, Deserializer, de::Error as _}; @@ -63,7 +63,7 @@ use crate::{ }, }, spv::{ - legacy_bin::{DataSeries, LegacyBin, LegacyBinWarning}, + legacy_bin::{DataSeries, DataSource, DataSources, LegacyBin, LegacyBinWarning}, read::graph::{ format::Formatter, frames::{Component, Frames, Position}, @@ -193,7 +193,7 @@ impl Visualization { pub fn decode_series( &self, locale: &'static Locale<'static>, - data: IndexMap>, + data: DataSources, warn: &mut dyn FnMut(GraphWarning), ) -> BTreeMap<&str, Series> { let mut variables: Vec<&dyn Variable> = self @@ -484,8 +484,8 @@ impl Visualization { pub fn decode( &self, - mut data: IndexMap>, - mut csv_data: Option>, + mut data: DataSources, + mut csv_data: Option, warn: &mut dyn FnMut(GraphWarning), ) -> Result { for child in &self.children { @@ -500,7 +500,7 @@ impl Visualization { .filter_map(|child| child.delimited_file_source()) { if let Some(csv_data) = csv_data.take() { - data.insert(delimited_source.id.clone(), csv_data); + data.0.insert(delimited_source.id.clone(), csv_data); } else { warn(GraphWarning::DelimitedSourceWithoutCsvData); } @@ -513,7 +513,8 @@ impl Visualization { .iter() .filter_map(|child| child.embedded_source()) { - data.insert(embedded_source.id.clone(), embedded_source.decode()); + data.0 + .insert(embedded_source.id.clone(), embedded_source.decode()); } let locale = self .lang @@ -1239,7 +1240,7 @@ where } impl EmbeddedSource { - fn decode(&self) -> IndexMap { + fn decode(&self) -> DataSource { let mut series = (0..self.names.len()) .map(|i| DataSeries { label: self.labels.get(i).cloned(), @@ -1262,7 +1263,7 @@ impl EmbeddedSource { series.data.push(datum); } } - zip(self.names.iter().cloned(), series).collect() + DataSource(zip(self.names.iter().cloned(), series).collect()) } } @@ -1285,7 +1286,7 @@ trait Variable { fn decode<'a>( &'a self, locale: &'static Locale<'static>, - data: &IndexMap>, + data: &DataSources, series: &mut BTreeMap<&'a str, Series>, warn: &mut dyn FnMut(GraphWarning), ) -> bool; @@ -1339,7 +1340,7 @@ impl Variable for SourceVariable { fn decode<'a>( &'a self, locale: &'static Locale<'static>, - data: &IndexMap>, + data: &DataSources, series: &mut BTreeMap<&'a str, Series>, _warn: &mut dyn FnMut(GraphWarning), ) -> bool { @@ -1354,8 +1355,8 @@ impl Variable for SourceVariable { }; let mut label = self.label.clone(); - let mut data = if let Some(source) = data.get(&self.source) - && let Some(values) = source.get(&self.variable) + let mut data = if let Some(source) = data.0.get(&self.source) + && let Some(values) = source.0.get(&self.variable) { if label.is_none() { label = values.label.clone(); @@ -1418,7 +1419,7 @@ impl Variable for DerivedVariable { fn decode<'a>( &'a self, locale: &'static Locale<'static>, - _data: &IndexMap>, + _data: &DataSources, series: &mut BTreeMap<&'a str, Series>, warn: &mut dyn FnMut(GraphWarning), ) -> bool { @@ -4704,26 +4705,6 @@ impl<'a> Iterator for FacetIterator<'a> { impl<'a> ExactSizeIterator for FacetIterator<'a> {} -pub struct SemicolonEscape<'a>(&'a str); - -impl<'a> Display for SemicolonEscape<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut s = self.0; - while let Some((head, tail)) = s.split_once(['&', '<', ';']) { - f.write_str(head)?; - let escaped = match s.as_bytes()[head.len()] { - b'&' => "&", - b'<' => ">", - b';' => "\\;", - _ => unreachable!(), - }; - f.write_str(escaped)?; - s = tail; - } - f.write_str(s) - } -} - pub struct SemicolonSplit<'a>(Option<&'a str>); impl<'a> SemicolonSplit<'a> { @@ -4816,10 +4797,7 @@ impl crate::output::Item { } impl crate::output::Graph { - pub fn binary_data( - &self, - mut warn: F, - ) -> Result>, GraphError> + pub fn binary_data(&self, mut warn: F) -> Result where F: FnMut(GraphWarning), { @@ -4833,7 +4811,7 @@ impl crate::output::Graph { } } - pub fn csv_data(&self) -> Result>, GraphError> { + pub fn csv_data(&self) -> Result, GraphError> { if let Some(csv) = &self.csv { let mut reader = csv::Reader::from_reader(Cursor::new(&csv)); let headers = reader.headers().unwrap(); // XXX @@ -4848,13 +4826,13 @@ impl crate::output::Graph { variables[column].data.push(value); } } - Ok(Some( + Ok(Some(DataSource( zip( reader.headers().unwrap().iter().map(|name| name.into()), variables, ) .collect(), - )) + ))) } else { Ok(None) } @@ -4880,74 +4858,6 @@ impl crate::output::Graph { where F: FnMut(GraphWarning), { - fn write_embedded_source( - writer: &mut Writer>, - name: &str, - data: &IndexMap, - ) -> std::io::Result<()> { - writer - .create_element("embeddedSource") - .with_attribute(("id", name)) - .write_inner_content(|w| { - w.create_element("names") - .write_text_content(BytesText::new(&data.keys().join(";")))?; - w.create_element("types") - .write_text_content(BytesText::new( - &data - .values() - .map(|series| { - if series.data.iter().all(|datum| datum.is_string()) { - "string" - } else { - "double" - } - }) - .join(";"), - ))?; - if data.values().any(|series| series.label.is_some()) { - w.create_element("labels").write_inner_content(|w| { - let w = w.get_mut(); - for (column, series) in data.values().enumerate() { - if column > 0 { - w.write_all(b";")?; - } - if let Some(label) = &series.label { - write!(w, "{}", SemicolonEscape(label))?; - } - } - Ok(()) - })?; - } - let rows = data - .values() - .map(|series| series.data.len()) - .max() - .unwrap_or_default(); - let empty = Datum::String(String::new()); - for row in 0..rows { - w.create_element("row").write_inner_content(|w| { - let w = w.get_mut(); - for (column, series) in data.values().enumerate() { - if column > 0 { - w.write_all(b";")?; - } - let datum = series.data.get(row).unwrap_or(&empty); - match datum { - Datum::Number(None) => (), - Datum::Number(Some(number)) => write!(w, "{number}")?, - Datum::String(string) => { - write!(w, "{}", SemicolonEscape(string))? - } - } - } - Ok(()) - })?; - } - Ok(()) - })?; - Ok(()) - } - let mut reader = Reader::from_reader(self.xml.as_slice()); let mut writer = Writer::new(Vec::new()); loop { @@ -4973,15 +4883,15 @@ impl crate::output::Graph { let local_name = start.local_name(); if local_name.as_ref() == b"userSource" && let Some(id) = id_attribute(start) - && let Some(data) = self.binary_data(&mut warn)?.get(&*id) + && let Some(data) = self.binary_data(&mut warn)?.0.get(&*id) { - write_embedded_source(&mut writer, &id, &data) + data.write_as_embedded_source(&mut writer, &id) .expect("writing to a vector shouldn't fail"); } else if local_name.as_ref() == b"delimitedFileSource" && let Some(id) = id_attribute(start) && let Some(csv) = self.csv_data()? { - write_embedded_source(&mut writer, &id, &csv) + csv.write_as_embedded_source(&mut writer, &id) .expect("writing to a vector shouldn't fail"); } else { writer diff --git a/rust/pspp/src/spv/read/legacy_bin.rs b/rust/pspp/src/spv/read/legacy_bin.rs index 7ccf3a2789..6b1313d674 100644 --- a/rust/pspp/src/spv/read/legacy_bin.rs +++ b/rust/pspp/src/spv/read/legacy_bin.rs @@ -1,11 +1,16 @@ //! Legacy binary data. use core::f64; -use std::io::{Cursor, Read, Seek, SeekFrom, Write}; +use std::{ + fmt::Display, + io::{Cursor, Read, Seek, SeekFrom, Write}, +}; use binrw::{BinRead, BinResult, BinWrite, Endian, binread, binwrite}; use displaydoc::Display; use encoding_rs::UTF_8; use indexmap::IndexMap; +use itertools::Itertools as _; +use quick_xml::events::BytesText; use crate::{ data::Datum, @@ -19,93 +24,18 @@ use crate::{ }, }; -/// A single column of data with an optional label. +/// A collection of named [DataSource]s for a table or graph. #[derive(Clone, Debug, Default)] -pub struct DataSeries { - /// Label. - /// - /// Data read from a [LegacyBin] never has a label. - pub label: Option, - - /// Data. - pub data: Vec>, -} - -/// A warning decoding a legacy binary member. -#[derive(Clone, Debug, Display, thiserror::Error)] -pub enum LegacyBinWarning { - /// String map refers to unknown source {0:?}. - UnknownSource(String), - /// String map for source {source_name:?} refers to unknown variable {variable:?}. - UnknownVariable { - /// Source name. - source_name: String, - /// Variable name. - variable: String, - }, - /// Datum mapping {datum_idx} for variable {variable:?} in source {source_name:?} has label index {label_idx} but there are only {n_labels} labels. - OutOfRangeLabelIdx { - /// Source name. - source_name: String, - /// Variable name. - variable: String, - /// Index into variable's datum map. - datum_idx: usize, - /// Too-large index into labels. - label_idx: usize, - /// Number of labels. - n_labels: usize, - }, - /// Datum mapping {datum_idx} for variable {variable:?} in source {source_name:?} has value index {value_idx} but the variable has only {n_values} values. - OutOfRangeValueIdx { - /// Source name. - source_name: String, - /// Variable name. - variable: String, - /// Index into variable's datum map. - datum_idx: usize, - /// Too-large index into values. - value_idx: usize, - /// Number of values in `variable`. - n_values: usize, - }, -} - -/// Legacy binary data. -#[binread] -#[br(little)] -#[derive(Debug)] -pub struct LegacyBin { - #[br(magic(0u8), temp)] - version: Version, - #[br(temp)] - n_sources: u16, - #[br(temp)] - _member_size: u32, - #[br(count(n_sources), args { inner: (version,) })] - metadata: Vec, - #[br(parse_with(parse_data), args(metadata.as_slice()))] - data: Vec, - #[br(parse_with(parse_strings))] - strings: Option, -} - -impl LegacyBin { - /// Decodes legacy binary data from `data`. - pub fn from_data( - data: &[u8], - warn: &mut dyn FnMut(LegacyBinWarning), - ) -> Result>, binrw::Error> { - Ok(Self::read(&mut Cursor::new(&data))?.decode(warn)) - } +pub struct DataSources(pub IndexMap); +impl DataSources { /// Encodes `data` into the legacy detail member format. /// /// # Panic /// /// Panics if `data` is empty. - pub fn encode(data: &IndexMap>) -> Vec { - assert!(!data.is_empty()); + pub fn to_legacy_bin(&self) -> Vec { + assert!(!self.0.is_empty()); let mut bin = Cursor::new(Vec::new()); bin.set_position(8); @@ -114,8 +44,8 @@ impl LegacyBin { /// Size of an individual metadata record const METADATA_SIZE: usize = 4 + 4 + 4 + 64 + 4; - let mut data_offset = METADATA_OFFSET + data.len() * METADATA_SIZE; - for (source_name, source) in data { + let mut data_offset = METADATA_OFFSET + self.0.len() * METADATA_SIZE; + for (source_name, source) in &self.0 { #[binwrite] #[br(little)] #[derive(Debug)] @@ -129,23 +59,27 @@ impl LegacyBin { x: (), } let metadata = Metadata { - n_values: source.values().next().map_or(0, |series| series.data.len()) as u32, - n_variables: source.len() as u32, + n_values: source + .0 + .values() + .next() + .map_or(0, |series| series.data.len()) as u32, + n_variables: source.0.len() as u32, data_offset: data_offset as u32, source_name: source_name.as_str(), x: (), }; metadata.write_le(&mut bin).unwrap(); - for series in source.values() { + for series in source.0.values() { data_offset += 288 + 8 * series.data.len(); } } - let mut n_strings = Vec::with_capacity(data.len()); - for source in data.values() { - let mut variable_strings = Vec::with_capacity(source.len()); - for (variable_name, series) in source { + let mut n_strings = Vec::with_capacity(self.0.len()); + for source in self.0.values() { + let mut variable_strings = Vec::with_capacity(source.0.len()); + for (variable_name, series) in &source.0 { write_fixed_utf8_string(&variable_name.as_str(), &mut bin, Endian::Little, (288,)) .unwrap(); for datum in &series.data { @@ -168,7 +102,7 @@ impl LegacyBin { let mut labels = IndexMap::<&str, u32>::new(); if n_maps > 0 { n_maps.write_le(&mut bin).unwrap(); - for ((source_name, source), variable_strings) in data.iter().zip(&n_strings) { + for ((source_name, source), variable_strings) in self.0.iter().zip(&n_strings) { let Some((_n, last_string)) = variable_strings .iter() .zip(1u32..) @@ -180,7 +114,7 @@ impl LegacyBin { (SpvString(source_name), last_string) .write_le(&mut bin) .unwrap(); - for ((variable_name, series), n_strings) in source.iter().zip(variable_strings) { + for ((variable_name, series), n_strings) in source.0.iter().zip(variable_strings) { (SpvString(variable_name), *n_strings as u32) .write_le(&mut bin) .unwrap(); @@ -213,7 +147,7 @@ impl LegacyBin { } let header = Header { version: 0xb0, - n_sources: data.len() as u16, + n_sources: self.0.len() as u16, member_size: bin.position() as u32, }; bin.set_position(0); @@ -224,10 +158,230 @@ impl LegacyBin { /// Decodes legacy binary data into a map from a series name to a map of /// variables, which in turn contains a vector of [Datum]s. - pub fn decode( - &self, + pub fn from_legacy_bin( + bin: &[u8], warn: &mut dyn FnMut(LegacyBinWarning), - ) -> IndexMap> { + ) -> Result { + LegacyBin::from_data(bin, warn) + } + + /// Renders these data sources as a pivot table. + pub fn as_pivot_table(&self) -> PivotTable { + let n_values = self + .0 + .values() + .flat_map(|map| map.0.values()) + .map(|values| values.data.len()) + .max() + .unwrap_or(0); + let index = Dimension::new( + Group::new("Index") + .with_multiple(Leaf::numbers(0..n_values)) + .with_label_shown(), + ); + let variables = Dimension::new(Group::new("Variables").with_multiple(self.0.iter().map( + |(name, contents)| { + Group::new(name.as_str()).with_multiple(contents.0.keys().map(|name| { + name.replace("categories", "\ncategories") + .replace("labels", "\nlabels") + .replace("group", "\ngroup") + .replace("Label", "\nLabel") + })) + }, + ))); + let mut pivot_table = PivotTable::new([(Axis3::Y, index), (Axis3::X, variables)]); + for (variable_index, (variable_name, values)) in + self.0.values().flat_map(|map| map.0.iter()).enumerate() + { + for (value_index, data_value) in values.data.iter().enumerate() { + let value = Value::new_datum(data_value).with_value_label( + (variable_name == "cellFormat") + .then(|| datum_as_format(data_value).to_string()), + ); + pivot_table.insert([value_index, variable_index], value); + } + } + pivot_table + } +} + +/// A collection of named [DataSeries] in [DataSources]. +/// +/// The series in a single `DataSource` should all have the same number of data +/// values; if not, then they cannot be written to a legacy binary data file. +#[derive(Clone, Debug, Default)] +pub struct DataSource(pub IndexMap); + +impl DataSource { + /// Writes this `DataSource` to [quick_xml] writer `writer` as an + /// `embeddedSource` element, using `name` as the element's `id`. + pub fn write_as_embedded_source( + &self, + writer: &mut quick_xml::Writer, + name: &str, + ) -> std::io::Result<()> + where + W: Write, + { + struct SemicolonEscape<'a>(&'a str); + + impl<'a> Display for SemicolonEscape<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut s = self.0; + while let Some((head, tail)) = s.split_once(['&', '<', ';']) { + f.write_str(head)?; + let escaped = match s.as_bytes()[head.len()] { + b'&' => "&", + b'<' => ">", + b';' => "\\;", + _ => unreachable!(), + }; + f.write_str(escaped)?; + s = tail; + } + f.write_str(s) + } + } + + writer + .create_element("embeddedSource") + .with_attribute(("id", name)) + .write_inner_content(|w| { + w.create_element("names") + .write_text_content(BytesText::new(&self.0.keys().join(";")))?; + w.create_element("types") + .write_text_content(BytesText::new( + &self + .0 + .values() + .map(|series| { + if series.data.iter().all(|datum| datum.is_string()) { + "string" + } else { + "double" + } + }) + .join(";"), + ))?; + if self.0.values().any(|series| series.label.is_some()) { + w.create_element("labels").write_inner_content(|w| { + let w = w.get_mut(); + for (column, series) in self.0.values().enumerate() { + if column > 0 { + w.write_all(b";")?; + } + if let Some(label) = &series.label { + write!(w, "{}", SemicolonEscape(label))?; + } + } + Ok(()) + })?; + } + let rows = self + .0 + .values() + .map(|series| series.data.len()) + .max() + .unwrap_or_default(); + let empty = Datum::String(String::new()); + for row in 0..rows { + w.create_element("row").write_inner_content(|w| { + let w = w.get_mut(); + for (column, series) in self.0.values().enumerate() { + if column > 0 { + w.write_all(b";")?; + } + let datum = series.data.get(row).unwrap_or(&empty); + match datum { + Datum::Number(None) => (), + Datum::Number(Some(number)) => write!(w, "{number}")?, + Datum::String(string) => write!(w, "{}", SemicolonEscape(string))?, + } + } + Ok(()) + })?; + } + Ok(()) + })?; + Ok(()) + } +} + +/// A single column of data with an optional label. +#[derive(Clone, Debug, Default)] +pub struct DataSeries { + /// Label. + /// + /// Data read from a [LegacyBin] never has a label. + pub label: Option, + + /// Data. + pub data: Vec>, +} + +/// A warning decoding a legacy binary member. +#[derive(Clone, Debug, Display, thiserror::Error)] +pub enum LegacyBinWarning { + /// String map refers to unknown source {0:?}. + UnknownSource(String), + /// String map for source {source_name:?} refers to unknown variable {variable:?}. + UnknownVariable { + /// Source name. + source_name: String, + /// Variable name. + variable: String, + }, + /// Datum mapping {datum_idx} for variable {variable:?} in source {source_name:?} has label index {label_idx} but there are only {n_labels} labels. + OutOfRangeLabelIdx { + /// Source name. + source_name: String, + /// Variable name. + variable: String, + /// Index into variable's datum map. + datum_idx: usize, + /// Too-large index into labels. + label_idx: usize, + /// Number of labels. + n_labels: usize, + }, + /// Datum mapping {datum_idx} for variable {variable:?} in source {source_name:?} has value index {value_idx} but the variable has only {n_values} values. + OutOfRangeValueIdx { + /// Source name. + source_name: String, + /// Variable name. + variable: String, + /// Index into variable's datum map. + datum_idx: usize, + /// Too-large index into values. + value_idx: usize, + /// Number of values in `variable`. + n_values: usize, + }, +} + +/// Legacy binary data. +#[binread] +#[br(little)] +#[derive(Debug)] +pub struct LegacyBin { + #[br(magic(0u8), temp)] + version: Version, + #[br(temp)] + n_sources: u16, + #[br(temp)] + _member_size: u32, + #[br(count(n_sources), args { inner: (version,) })] + metadata: Vec, + #[br(parse_with(parse_data), args(metadata.as_slice()))] + data: Vec, + #[br(parse_with(parse_strings))] + strings: Option, +} + +impl LegacyBin { + /// Decodes legacy binary data into a map from a series name to a map of + /// variables, which in turn contains a vector of [Datum]s. + pub fn decode(&self, warn: &mut dyn FnMut(LegacyBinWarning)) -> DataSources { let mut sources = IndexMap::new(); for (metadata, data) in self.metadata.iter().zip(&self.data) { let mut variables = IndexMap::new(); @@ -244,7 +398,7 @@ impl LegacyBin { }, ); } - sources.insert(metadata.source_name.clone(), variables); + sources.insert(metadata.source_name.clone(), DataSource(variables)); } if let Some(strings) = &self.strings { for map in &strings.source_maps { @@ -253,7 +407,7 @@ impl LegacyBin { continue; }; for var_map in &map.variable_maps { - let Some(variable) = source.get_mut(&var_map.variable_name) else { + let Some(variable) = source.0.get_mut(&var_map.variable_name) else { warn(LegacyBinWarning::UnknownVariable { source_name: map.source_name.clone(), variable: var_map.variable_name.clone(), @@ -286,7 +440,15 @@ impl LegacyBin { } } } - sources + DataSources(sources) + } + + /// Decodes legacy binary data from `data`. + pub fn from_data( + data: &[u8], + warn: &mut dyn FnMut(LegacyBinWarning), + ) -> Result { + Ok(Self::read(&mut Cursor::new(&data))?.decode(warn)) } } @@ -464,50 +626,11 @@ where Ok(()) } -pub fn data_pivot_table(data: &IndexMap>) -> PivotTable { - let n_values = data - .values() - .flat_map(|map| map.values()) - .map(|values| values.data.len()) - .max() - .unwrap_or(0); - let index = Dimension::new( - Group::new("Index") - .with_multiple(Leaf::numbers(0..n_values)) - .with_label_shown(), - ); - let variables = Dimension::new(Group::new("Variables").with_multiple(data.iter().map( - |(name, contents)| { - Group::new(name.as_str()).with_multiple(contents.keys().map(|name| { - name.replace("categories", "\ncategories") - .replace("labels", "\nlabels") - .replace("group", "\ngroup") - .replace("Label", "\nLabel") - })) - }, - ))); - let mut pivot_table = PivotTable::new([(Axis3::Y, index), (Axis3::X, variables)]); - for (variable_index, (variable_name, values)) in - data.values().flat_map(|map| map.iter()).enumerate() - { - for (value_index, data_value) in values.data.iter().enumerate() { - let value = Value::new_datum(data_value).with_value_label( - (variable_name == "cellFormat").then(|| datum_as_format(data_value).to_string()), - ); - pivot_table.insert([value_index, variable_index], value); - } - } - pivot_table -} - #[cfg(test)] mod tests { use std::{fmt::Write, path::Path}; - use crate::{ - output::pivot::tests::assert_lines_eq, - spv::legacy_bin::{LegacyBin, data_pivot_table}, - }; + use crate::{output::pivot::tests::assert_lines_eq, spv::legacy_bin::LegacyBin}; fn test_legacy_data_file(name: &str) { let input_filename = Path::new("src/spv/testdata") @@ -520,7 +643,7 @@ mod tests { let data = std::fs::read(&input_filename).unwrap(); let data = LegacyBin::from_data(&data, &mut |w| writeln!(&mut actual, "{w}").unwrap()).unwrap(); - write!(&mut actual, "{}", data_pivot_table(&data)).unwrap(); + write!(&mut actual, "{}", data.as_pivot_table()).unwrap(); if expected != actual { if std::env::var("PSPP_REFRESH_EXPECTED").is_ok() { @@ -532,11 +655,11 @@ mod tests { } assert_lines_eq(&expected, expected_filename.display(), &actual, "actual"); - let written = LegacyBin::encode(&data); + let written = data.to_legacy_bin(); let mut reread = String::new(); let reread_data = LegacyBin::from_data(&written, &mut |w| writeln!(&mut reread, "{w}").unwrap()).unwrap(); - write!(&mut reread, "{}", data_pivot_table(&reread_data)).unwrap(); + write!(&mut reread, "{}", reread_data.as_pivot_table()).unwrap(); if expected != reread { eprintln!("wrote and reread legacy data differs from expected:"); assert_lines_eq(&expected, expected_filename.display(), &reread, "actual"); diff --git a/rust/pspp/src/spv/read/legacy_xml.rs b/rust/pspp/src/spv/read/legacy_xml.rs index b5487e714e..31ce908dd5 100644 --- a/rust/pspp/src/spv/read/legacy_xml.rs +++ b/rust/pspp/src/spv/read/legacy_xml.rs @@ -29,7 +29,6 @@ use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use displaydoc::Display; use enum_map::{Enum, EnumMap}; use hashbrown::HashSet; -use indexmap::IndexMap; use ordered_float::OrderedFloat; use serde::Deserialize; @@ -41,7 +40,7 @@ use crate::{ look::{self, Area, AreaStyle, CellStyle, Color, HorzAlign, Look, RowParity, VertAlign}, value::{Value, ValueFormat}, }, - spv::{legacy_bin::DataSeries, read::light::decode_format}, + spv::{legacy_bin::DataSources, read::light::decode_format}, }; /// Interprets this data value as a [Format], first by looking it up in @@ -216,7 +215,7 @@ impl Visualization { } pub fn decode_series( &self, - data: IndexMap>, + data: DataSources, warn: &mut dyn FnMut(LegacyXmlWarning), ) -> BTreeMap<&str, Series> { let mut variables: Vec<&dyn Variable> = self @@ -343,7 +342,7 @@ impl Visualization { pub fn decode( &self, - binary_data: IndexMap>, + binary_data: DataSources, look: Look, warn: &mut dyn FnMut(LegacyXmlWarning), ) -> Result { @@ -465,7 +464,7 @@ impl VisChild { trait Variable { fn decode<'a>( &'a self, - data: &IndexMap>, + data: &DataSources, series: &mut BTreeMap<&'a str, Series>, warn: &mut dyn FnMut(LegacyXmlWarning), ) -> bool; @@ -514,7 +513,7 @@ impl SourceVariable { impl Variable for SourceVariable { fn decode<'a>( &'a self, - data: &IndexMap>, + data: &DataSources, series: &mut BTreeMap<&'a str, Series>, _warn: &mut dyn FnMut(LegacyXmlWarning), ) -> bool { @@ -528,8 +527,8 @@ impl Variable for SourceVariable { None }; - let (categories, mut data) = if let Some(source) = data.get(&self.source) - && let Some(values) = source.get(&self.variable) + let (categories, mut data) = if let Some(source) = data.0.get(&self.source) + && let Some(values) = source.0.get(&self.variable) { let categories = values .data @@ -619,7 +618,7 @@ impl DerivedVariable { impl Variable for DerivedVariable { fn decode<'a>( &'a self, - _data: &IndexMap>, + _data: &DataSources, series: &mut BTreeMap<&'a str, Series>, warn: &mut dyn FnMut(LegacyXmlWarning), ) -> bool {