]> pintos-os.org Git - pspp/commitdiff
work on better typing for datasources
authorBen Pfaff <blp@cs.stanford.edu>
Sun, 26 Apr 2026 20:28:21 +0000 (13:28 -0700)
committerBen Pfaff <blp@cs.stanford.edu>
Sun, 26 Apr 2026 20:28:21 +0000 (13:28 -0700)
rust/pspp/src/cli/spv.rs
rust/pspp/src/spv/read.rs
rust/pspp/src/spv/read/graph.rs
rust/pspp/src/spv/read/legacy_bin.rs
rust/pspp/src/spv/read/legacy_xml.rs

index c48242aa2dd59d5c3788325db4ca7bf5e5d5eb4b..b9df03f172294918eb61b9e76f278826a79b1307 100644 (file)
@@ -24,7 +24,7 @@ use pspp::{
     },
     spv::{
         SpvArchive,
-        legacy_bin::{LegacyBin, data_pivot_table},
+        legacy_bin::LegacyBin,
         read::{
             legacy_xml::Visualization,
             structure::{Content, OutlineItem},
@@ -126,7 +126,7 @@ impl LegacyData {
         if let Some(FileType::LegacyBin) = FileType::from_file(&self.input.input)? {
             let data = std::fs::read(&self.input.input)?;
             let data = LegacyBin::from_data(&data, &mut |w| eprintln!("{w}"))?;
-            println!("{}", data_pivot_table(&data));
+            println!("{}", data.as_pivot_table());
         } else {
             let (mut archive, items) = self.input.read_outline()?;
             for item in items {
@@ -143,7 +143,7 @@ impl LegacyData {
                                 ))
                             })?;
 
-                        println!("{}", data_pivot_table(&data));
+                        println!("{}", data.as_pivot_table());
                     }
                 }
             }
index 3a64a7b31c0699d7004ac6fdbdfbddb9deb5a877..01186a64a5d0be13e83ca22caa15a5afd881af10 100644 (file)
@@ -23,7 +23,6 @@ use std::{
 };
 
 use displaydoc::Display;
-use indexmap::IndexMap;
 use serde::Deserialize;
 use zip::{ZipArchive, result::ZipError};
 
@@ -32,7 +31,7 @@ use crate::{
     file::ReadSeek,
     output::{Criteria, Item, page::PageSetup},
     spv::{
-        legacy_bin::{DataSeries, LegacyBin, LegacyBinWarning},
+        legacy_bin::{DataSources, LegacyBin, LegacyBinWarning},
         read::{
             graph::GraphWarning,
             legacy_xml::LegacyXmlWarning,
@@ -182,7 +181,7 @@ where
         &mut self,
         name: &str,
         warn: &mut dyn FnMut(LegacyBinWarning),
-    ) -> Result<IndexMap<String, IndexMap<String, DataSeries>>, Error> {
+    ) -> Result<DataSources, Error> {
         LegacyBin::from_data(&self.read_member(name)?, warn).map_err(Error::BinrwError)
     }
 }
index 1e22f9309d7149773ce9674be342f706576b40b2..dd633ceef7350162eed73058a2121ba83f271161 100644 (file)
@@ -18,8 +18,8 @@
 use std::{
     borrow::{Borrow, Cow},
     collections::{BTreeMap, HashMap},
-    fmt::{Debug, Display},
-    io::{Cursor, ErrorKind, Write},
+    fmt::Debug,
+    io::{Cursor, ErrorKind},
     iter::{repeat, zip},
     marker::PhantomData,
     mem::take,
@@ -37,7 +37,7 @@ use ordered_float::OrderedFloat;
 use pango::FontDescription;
 use quick_xml::{
     Reader, Writer,
-    events::{BytesStart, BytesText, Event},
+    events::{BytesStart, Event},
 };
 use rs_stats::distributions::normal_distribution::normal_pdf;
 use serde::{Deserialize, Deserializer, de::Error as _};
@@ -63,7 +63,7 @@ use crate::{
         },
     },
     spv::{
-        legacy_bin::{DataSeries, LegacyBin, LegacyBinWarning},
+        legacy_bin::{DataSeries, DataSource, DataSources, LegacyBin, LegacyBinWarning},
         read::graph::{
             format::Formatter,
             frames::{Component, Frames, Position},
@@ -193,7 +193,7 @@ impl Visualization {
     pub fn decode_series(
         &self,
         locale: &'static Locale<'static>,
-        data: IndexMap<String, IndexMap<String, DataSeries>>,
+        data: DataSources,
         warn: &mut dyn FnMut(GraphWarning),
     ) -> BTreeMap<&str, Series> {
         let mut variables: Vec<&dyn Variable> = self
@@ -484,8 +484,8 @@ impl Visualization {
 
     pub fn decode(
         &self,
-        mut data: IndexMap<String, IndexMap<String, DataSeries>>,
-        mut csv_data: Option<IndexMap<String, DataSeries>>,
+        mut data: DataSources,
+        mut csv_data: Option<DataSource>,
         warn: &mut dyn FnMut(GraphWarning),
     ) -> Result<svg::Document, GraphError> {
         for child in &self.children {
@@ -500,7 +500,7 @@ impl Visualization {
             .filter_map(|child| child.delimited_file_source())
         {
             if let Some(csv_data) = csv_data.take() {
-                data.insert(delimited_source.id.clone(), csv_data);
+                data.0.insert(delimited_source.id.clone(), csv_data);
             } else {
                 warn(GraphWarning::DelimitedSourceWithoutCsvData);
             }
@@ -513,7 +513,8 @@ impl Visualization {
             .iter()
             .filter_map(|child| child.embedded_source())
         {
-            data.insert(embedded_source.id.clone(), embedded_source.decode());
+            data.0
+                .insert(embedded_source.id.clone(), embedded_source.decode());
         }
         let locale = self
             .lang
@@ -1239,7 +1240,7 @@ where
 }
 
 impl EmbeddedSource {
-    fn decode(&self) -> IndexMap<String, DataSeries> {
+    fn decode(&self) -> DataSource {
         let mut series = (0..self.names.len())
             .map(|i| DataSeries {
                 label: self.labels.get(i).cloned(),
@@ -1262,7 +1263,7 @@ impl EmbeddedSource {
                 series.data.push(datum);
             }
         }
-        zip(self.names.iter().cloned(), series).collect()
+        DataSource(zip(self.names.iter().cloned(), series).collect())
     }
 }
 
@@ -1285,7 +1286,7 @@ trait Variable {
     fn decode<'a>(
         &'a self,
         locale: &'static Locale<'static>,
-        data: &IndexMap<String, IndexMap<String, DataSeries>>,
+        data: &DataSources,
         series: &mut BTreeMap<&'a str, Series>,
         warn: &mut dyn FnMut(GraphWarning),
     ) -> bool;
@@ -1339,7 +1340,7 @@ impl Variable for SourceVariable {
     fn decode<'a>(
         &'a self,
         locale: &'static Locale<'static>,
-        data: &IndexMap<String, IndexMap<String, DataSeries>>,
+        data: &DataSources,
         series: &mut BTreeMap<&'a str, Series>,
         _warn: &mut dyn FnMut(GraphWarning),
     ) -> bool {
@@ -1354,8 +1355,8 @@ impl Variable for SourceVariable {
         };
 
         let mut label = self.label.clone();
-        let mut data = if let Some(source) = data.get(&self.source)
-            && let Some(values) = source.get(&self.variable)
+        let mut data = if let Some(source) = data.0.get(&self.source)
+            && let Some(values) = source.0.get(&self.variable)
         {
             if label.is_none() {
                 label = values.label.clone();
@@ -1418,7 +1419,7 @@ impl Variable for DerivedVariable {
     fn decode<'a>(
         &'a self,
         locale: &'static Locale<'static>,
-        _data: &IndexMap<String, IndexMap<String, DataSeries>>,
+        _data: &DataSources,
         series: &mut BTreeMap<&'a str, Series>,
         warn: &mut dyn FnMut(GraphWarning),
     ) -> bool {
@@ -4704,26 +4705,6 @@ impl<'a> Iterator for FacetIterator<'a> {
 
 impl<'a> ExactSizeIterator for FacetIterator<'a> {}
 
-pub struct SemicolonEscape<'a>(&'a str);
-
-impl<'a> Display for SemicolonEscape<'a> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let mut s = self.0;
-        while let Some((head, tail)) = s.split_once(['&', '<', ';']) {
-            f.write_str(head)?;
-            let escaped = match s.as_bytes()[head.len()] {
-                b'&' => "&amp;",
-                b'<' => "&gt;",
-                b';' => "\\;",
-                _ => unreachable!(),
-            };
-            f.write_str(escaped)?;
-            s = tail;
-        }
-        f.write_str(s)
-    }
-}
-
 pub struct SemicolonSplit<'a>(Option<&'a str>);
 
 impl<'a> SemicolonSplit<'a> {
@@ -4816,10 +4797,7 @@ impl crate::output::Item {
 }
 
 impl crate::output::Graph {
-    pub fn binary_data<F>(
-        &self,
-        mut warn: F,
-    ) -> Result<IndexMap<String, IndexMap<String, DataSeries>>, GraphError>
+    pub fn binary_data<F>(&self, mut warn: F) -> Result<DataSources, GraphError>
     where
         F: FnMut(GraphWarning),
     {
@@ -4833,7 +4811,7 @@ impl crate::output::Graph {
         }
     }
 
-    pub fn csv_data(&self) -> Result<Option<IndexMap<String, DataSeries>>, GraphError> {
+    pub fn csv_data(&self) -> Result<Option<DataSource>, GraphError> {
         if let Some(csv) = &self.csv {
             let mut reader = csv::Reader::from_reader(Cursor::new(&csv));
             let headers = reader.headers().unwrap(); // XXX
@@ -4848,13 +4826,13 @@ impl crate::output::Graph {
                     variables[column].data.push(value);
                 }
             }
-            Ok(Some(
+            Ok(Some(DataSource(
                 zip(
                     reader.headers().unwrap().iter().map(|name| name.into()),
                     variables,
                 )
                 .collect(),
-            ))
+            )))
         } else {
             Ok(None)
         }
@@ -4880,74 +4858,6 @@ impl crate::output::Graph {
     where
         F: FnMut(GraphWarning),
     {
-        fn write_embedded_source(
-            writer: &mut Writer<Vec<u8>>,
-            name: &str,
-            data: &IndexMap<String, DataSeries>,
-        ) -> std::io::Result<()> {
-            writer
-                .create_element("embeddedSource")
-                .with_attribute(("id", name))
-                .write_inner_content(|w| {
-                    w.create_element("names")
-                        .write_text_content(BytesText::new(&data.keys().join(";")))?;
-                    w.create_element("types")
-                        .write_text_content(BytesText::new(
-                            &data
-                                .values()
-                                .map(|series| {
-                                    if series.data.iter().all(|datum| datum.is_string()) {
-                                        "string"
-                                    } else {
-                                        "double"
-                                    }
-                                })
-                                .join(";"),
-                        ))?;
-                    if data.values().any(|series| series.label.is_some()) {
-                        w.create_element("labels").write_inner_content(|w| {
-                            let w = w.get_mut();
-                            for (column, series) in data.values().enumerate() {
-                                if column > 0 {
-                                    w.write_all(b";")?;
-                                }
-                                if let Some(label) = &series.label {
-                                    write!(w, "{}", SemicolonEscape(label))?;
-                                }
-                            }
-                            Ok(())
-                        })?;
-                    }
-                    let rows = data
-                        .values()
-                        .map(|series| series.data.len())
-                        .max()
-                        .unwrap_or_default();
-                    let empty = Datum::String(String::new());
-                    for row in 0..rows {
-                        w.create_element("row").write_inner_content(|w| {
-                            let w = w.get_mut();
-                            for (column, series) in data.values().enumerate() {
-                                if column > 0 {
-                                    w.write_all(b";")?;
-                                }
-                                let datum = series.data.get(row).unwrap_or(&empty);
-                                match datum {
-                                    Datum::Number(None) => (),
-                                    Datum::Number(Some(number)) => write!(w, "{number}")?,
-                                    Datum::String(string) => {
-                                        write!(w, "{}", SemicolonEscape(string))?
-                                    }
-                                }
-                            }
-                            Ok(())
-                        })?;
-                    }
-                    Ok(())
-                })?;
-            Ok(())
-        }
-
         let mut reader = Reader::from_reader(self.xml.as_slice());
         let mut writer = Writer::new(Vec::new());
         loop {
@@ -4973,15 +4883,15 @@ impl crate::output::Graph {
                     let local_name = start.local_name();
                     if local_name.as_ref() == b"userSource"
                         && let Some(id) = id_attribute(start)
-                        && let Some(data) = self.binary_data(&mut warn)?.get(&*id)
+                        && let Some(data) = self.binary_data(&mut warn)?.0.get(&*id)
                     {
-                        write_embedded_source(&mut writer, &id, &data)
+                        data.write_as_embedded_source(&mut writer, &id)
                             .expect("writing to a vector shouldn't fail");
                     } else if local_name.as_ref() == b"delimitedFileSource"
                         && let Some(id) = id_attribute(start)
                         && let Some(csv) = self.csv_data()?
                     {
-                        write_embedded_source(&mut writer, &id, &csv)
+                        csv.write_as_embedded_source(&mut writer, &id)
                             .expect("writing to a vector shouldn't fail");
                     } else {
                         writer
index 7ccf3a27898ae1310b8c34e07e8d59175232afb7..6b1313d674e9d06096f9ab39a5524bdc174f1cdb 100644 (file)
@@ -1,11 +1,16 @@
 //! Legacy binary data.
 use core::f64;
-use std::io::{Cursor, Read, Seek, SeekFrom, Write};
+use std::{
+    fmt::Display,
+    io::{Cursor, Read, Seek, SeekFrom, Write},
+};
 
 use binrw::{BinRead, BinResult, BinWrite, Endian, binread, binwrite};
 use displaydoc::Display;
 use encoding_rs::UTF_8;
 use indexmap::IndexMap;
+use itertools::Itertools as _;
+use quick_xml::events::BytesText;
 
 use crate::{
     data::Datum,
@@ -19,93 +24,18 @@ use crate::{
     },
 };
 
-/// A single column of data with an optional label.
+/// A collection of named [DataSource]s for a table or graph.
 #[derive(Clone, Debug, Default)]
-pub struct DataSeries {
-    /// Label.
-    ///
-    /// Data read from a [LegacyBin] never has a label.
-    pub label: Option<String>,
-
-    /// Data.
-    pub data: Vec<Datum<String>>,
-}
-
-/// A warning decoding a legacy binary member.
-#[derive(Clone, Debug, Display, thiserror::Error)]
-pub enum LegacyBinWarning {
-    /// String map refers to unknown source {0:?}.
-    UnknownSource(String),
-    /// String map for source {source_name:?} refers to unknown variable {variable:?}.
-    UnknownVariable {
-        /// Source name.
-        source_name: String,
-        /// Variable name.
-        variable: String,
-    },
-    /// Datum mapping {datum_idx} for variable {variable:?} in source {source_name:?} has label index {label_idx} but there are only {n_labels} labels.
-    OutOfRangeLabelIdx {
-        /// Source name.
-        source_name: String,
-        /// Variable name.
-        variable: String,
-        /// Index into variable's datum map.
-        datum_idx: usize,
-        /// Too-large index into labels.
-        label_idx: usize,
-        /// Number of labels.
-        n_labels: usize,
-    },
-    /// Datum mapping {datum_idx} for variable {variable:?} in source {source_name:?} has value index {value_idx} but the variable has only {n_values} values.
-    OutOfRangeValueIdx {
-        /// Source name.
-        source_name: String,
-        /// Variable name.
-        variable: String,
-        /// Index into variable's datum map.
-        datum_idx: usize,
-        /// Too-large index into values.
-        value_idx: usize,
-        /// Number of values in `variable`.
-        n_values: usize,
-    },
-}
-
-/// Legacy binary data.
-#[binread]
-#[br(little)]
-#[derive(Debug)]
-pub struct LegacyBin {
-    #[br(magic(0u8), temp)]
-    version: Version,
-    #[br(temp)]
-    n_sources: u16,
-    #[br(temp)]
-    _member_size: u32,
-    #[br(count(n_sources), args { inner: (version,) })]
-    metadata: Vec<Metadata>,
-    #[br(parse_with(parse_data), args(metadata.as_slice()))]
-    data: Vec<Data>,
-    #[br(parse_with(parse_strings))]
-    strings: Option<Strings>,
-}
-
-impl LegacyBin {
-    /// Decodes legacy binary data from `data`.
-    pub fn from_data(
-        data: &[u8],
-        warn: &mut dyn FnMut(LegacyBinWarning),
-    ) -> Result<IndexMap<String, IndexMap<String, DataSeries>>, binrw::Error> {
-        Ok(Self::read(&mut Cursor::new(&data))?.decode(warn))
-    }
+pub struct DataSources(pub IndexMap<String, DataSource>);
 
+impl DataSources {
     /// Encodes `data` into the legacy detail member format.
     ///
     /// # Panic
     ///
     /// Panics if `data` is empty.
-    pub fn encode(data: &IndexMap<String, IndexMap<String, DataSeries>>) -> Vec<u8> {
-        assert!(!data.is_empty());
+    pub fn to_legacy_bin(&self) -> Vec<u8> {
+        assert!(!self.0.is_empty());
         let mut bin = Cursor::new(Vec::new());
         bin.set_position(8);
 
@@ -114,8 +44,8 @@ impl LegacyBin {
         /// Size of an individual metadata record
         const METADATA_SIZE: usize = 4 + 4 + 4 + 64 + 4;
 
-        let mut data_offset = METADATA_OFFSET + data.len() * METADATA_SIZE;
-        for (source_name, source) in data {
+        let mut data_offset = METADATA_OFFSET + self.0.len() * METADATA_SIZE;
+        for (source_name, source) in &self.0 {
             #[binwrite]
             #[br(little)]
             #[derive(Debug)]
@@ -129,23 +59,27 @@ impl LegacyBin {
                 x: (),
             }
             let metadata = Metadata {
-                n_values: source.values().next().map_or(0, |series| series.data.len()) as u32,
-                n_variables: source.len() as u32,
+                n_values: source
+                    .0
+                    .values()
+                    .next()
+                    .map_or(0, |series| series.data.len()) as u32,
+                n_variables: source.0.len() as u32,
                 data_offset: data_offset as u32,
                 source_name: source_name.as_str(),
                 x: (),
             };
             metadata.write_le(&mut bin).unwrap();
 
-            for series in source.values() {
+            for series in source.0.values() {
                 data_offset += 288 + 8 * series.data.len();
             }
         }
 
-        let mut n_strings = Vec::with_capacity(data.len());
-        for source in data.values() {
-            let mut variable_strings = Vec::with_capacity(source.len());
-            for (variable_name, series) in source {
+        let mut n_strings = Vec::with_capacity(self.0.len());
+        for source in self.0.values() {
+            let mut variable_strings = Vec::with_capacity(source.0.len());
+            for (variable_name, series) in &source.0 {
                 write_fixed_utf8_string(&variable_name.as_str(), &mut bin, Endian::Little, (288,))
                     .unwrap();
                 for datum in &series.data {
@@ -168,7 +102,7 @@ impl LegacyBin {
         let mut labels = IndexMap::<&str, u32>::new();
         if n_maps > 0 {
             n_maps.write_le(&mut bin).unwrap();
-            for ((source_name, source), variable_strings) in data.iter().zip(&n_strings) {
+            for ((source_name, source), variable_strings) in self.0.iter().zip(&n_strings) {
                 let Some((_n, last_string)) = variable_strings
                     .iter()
                     .zip(1u32..)
@@ -180,7 +114,7 @@ impl LegacyBin {
                 (SpvString(source_name), last_string)
                     .write_le(&mut bin)
                     .unwrap();
-                for ((variable_name, series), n_strings) in source.iter().zip(variable_strings) {
+                for ((variable_name, series), n_strings) in source.0.iter().zip(variable_strings) {
                     (SpvString(variable_name), *n_strings as u32)
                         .write_le(&mut bin)
                         .unwrap();
@@ -213,7 +147,7 @@ impl LegacyBin {
         }
         let header = Header {
             version: 0xb0,
-            n_sources: data.len() as u16,
+            n_sources: self.0.len() as u16,
             member_size: bin.position() as u32,
         };
         bin.set_position(0);
@@ -224,10 +158,230 @@ impl LegacyBin {
 
     /// Decodes legacy binary data into a map from a series name to a map of
     /// variables, which in turn contains a vector of [Datum]s.
-    pub fn decode(
-        &self,
+    pub fn from_legacy_bin(
+        bin: &[u8],
         warn: &mut dyn FnMut(LegacyBinWarning),
-    ) -> IndexMap<String, IndexMap<String, DataSeries>> {
+    ) -> Result<DataSources, binrw::Error> {
+        LegacyBin::from_data(bin, warn)
+    }
+
+    /// Renders these data sources as a pivot table.
+    pub fn as_pivot_table(&self) -> PivotTable {
+        let n_values = self
+            .0
+            .values()
+            .flat_map(|map| map.0.values())
+            .map(|values| values.data.len())
+            .max()
+            .unwrap_or(0);
+        let index = Dimension::new(
+            Group::new("Index")
+                .with_multiple(Leaf::numbers(0..n_values))
+                .with_label_shown(),
+        );
+        let variables = Dimension::new(Group::new("Variables").with_multiple(self.0.iter().map(
+            |(name, contents)| {
+                Group::new(name.as_str()).with_multiple(contents.0.keys().map(|name| {
+                    name.replace("categories", "\ncategories")
+                        .replace("labels", "\nlabels")
+                        .replace("group", "\ngroup")
+                        .replace("Label", "\nLabel")
+                }))
+            },
+        )));
+        let mut pivot_table = PivotTable::new([(Axis3::Y, index), (Axis3::X, variables)]);
+        for (variable_index, (variable_name, values)) in
+            self.0.values().flat_map(|map| map.0.iter()).enumerate()
+        {
+            for (value_index, data_value) in values.data.iter().enumerate() {
+                let value = Value::new_datum(data_value).with_value_label(
+                    (variable_name == "cellFormat")
+                        .then(|| datum_as_format(data_value).to_string()),
+                );
+                pivot_table.insert([value_index, variable_index], value);
+            }
+        }
+        pivot_table
+    }
+}
+
+/// A collection of named [DataSeries] in [DataSources].
+///
+/// The series in a single `DataSource` should all have the same number of data
+/// values; if not, then they cannot be written to a legacy binary data file.
+#[derive(Clone, Debug, Default)]
+pub struct DataSource(pub IndexMap<String, DataSeries>);
+
+impl DataSource {
+    /// Writes this `DataSource` to [quick_xml] writer `writer` as an
+    /// `embeddedSource` element, using `name` as the element's `id`.
+    pub fn write_as_embedded_source<W>(
+        &self,
+        writer: &mut quick_xml::Writer<W>,
+        name: &str,
+    ) -> std::io::Result<()>
+    where
+        W: Write,
+    {
+        struct SemicolonEscape<'a>(&'a str);
+
+        impl<'a> Display for SemicolonEscape<'a> {
+            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                let mut s = self.0;
+                while let Some((head, tail)) = s.split_once(['&', '<', ';']) {
+                    f.write_str(head)?;
+                    let escaped = match s.as_bytes()[head.len()] {
+                        b'&' => "&amp;",
+                        b'<' => "&gt;",
+                        b';' => "\\;",
+                        _ => unreachable!(),
+                    };
+                    f.write_str(escaped)?;
+                    s = tail;
+                }
+                f.write_str(s)
+            }
+        }
+
+        writer
+            .create_element("embeddedSource")
+            .with_attribute(("id", name))
+            .write_inner_content(|w| {
+                w.create_element("names")
+                    .write_text_content(BytesText::new(&self.0.keys().join(";")))?;
+                w.create_element("types")
+                    .write_text_content(BytesText::new(
+                        &self
+                            .0
+                            .values()
+                            .map(|series| {
+                                if series.data.iter().all(|datum| datum.is_string()) {
+                                    "string"
+                                } else {
+                                    "double"
+                                }
+                            })
+                            .join(";"),
+                    ))?;
+                if self.0.values().any(|series| series.label.is_some()) {
+                    w.create_element("labels").write_inner_content(|w| {
+                        let w = w.get_mut();
+                        for (column, series) in self.0.values().enumerate() {
+                            if column > 0 {
+                                w.write_all(b";")?;
+                            }
+                            if let Some(label) = &series.label {
+                                write!(w, "{}", SemicolonEscape(label))?;
+                            }
+                        }
+                        Ok(())
+                    })?;
+                }
+                let rows = self
+                    .0
+                    .values()
+                    .map(|series| series.data.len())
+                    .max()
+                    .unwrap_or_default();
+                let empty = Datum::String(String::new());
+                for row in 0..rows {
+                    w.create_element("row").write_inner_content(|w| {
+                        let w = w.get_mut();
+                        for (column, series) in self.0.values().enumerate() {
+                            if column > 0 {
+                                w.write_all(b";")?;
+                            }
+                            let datum = series.data.get(row).unwrap_or(&empty);
+                            match datum {
+                                Datum::Number(None) => (),
+                                Datum::Number(Some(number)) => write!(w, "{number}")?,
+                                Datum::String(string) => write!(w, "{}", SemicolonEscape(string))?,
+                            }
+                        }
+                        Ok(())
+                    })?;
+                }
+                Ok(())
+            })?;
+        Ok(())
+    }
+}
+
+/// A single column of data with an optional label.
+#[derive(Clone, Debug, Default)]
+pub struct DataSeries {
+    /// Label.
+    ///
+    /// Data read from a [LegacyBin] never has a label.
+    pub label: Option<String>,
+
+    /// Data.
+    pub data: Vec<Datum<String>>,
+}
+
+/// A warning decoding a legacy binary member.
+#[derive(Clone, Debug, Display, thiserror::Error)]
+pub enum LegacyBinWarning {
+    /// String map refers to unknown source {0:?}.
+    UnknownSource(String),
+    /// String map for source {source_name:?} refers to unknown variable {variable:?}.
+    UnknownVariable {
+        /// Source name.
+        source_name: String,
+        /// Variable name.
+        variable: String,
+    },
+    /// Datum mapping {datum_idx} for variable {variable:?} in source {source_name:?} has label index {label_idx} but there are only {n_labels} labels.
+    OutOfRangeLabelIdx {
+        /// Source name.
+        source_name: String,
+        /// Variable name.
+        variable: String,
+        /// Index into variable's datum map.
+        datum_idx: usize,
+        /// Too-large index into labels.
+        label_idx: usize,
+        /// Number of labels.
+        n_labels: usize,
+    },
+    /// Datum mapping {datum_idx} for variable {variable:?} in source {source_name:?} has value index {value_idx} but the variable has only {n_values} values.
+    OutOfRangeValueIdx {
+        /// Source name.
+        source_name: String,
+        /// Variable name.
+        variable: String,
+        /// Index into variable's datum map.
+        datum_idx: usize,
+        /// Too-large index into values.
+        value_idx: usize,
+        /// Number of values in `variable`.
+        n_values: usize,
+    },
+}
+
+/// Legacy binary data.
+#[binread]
+#[br(little)]
+#[derive(Debug)]
+pub struct LegacyBin {
+    #[br(magic(0u8), temp)]
+    version: Version,
+    #[br(temp)]
+    n_sources: u16,
+    #[br(temp)]
+    _member_size: u32,
+    #[br(count(n_sources), args { inner: (version,) })]
+    metadata: Vec<Metadata>,
+    #[br(parse_with(parse_data), args(metadata.as_slice()))]
+    data: Vec<Data>,
+    #[br(parse_with(parse_strings))]
+    strings: Option<Strings>,
+}
+
+impl LegacyBin {
+    /// Decodes legacy binary data into a map from a series name to a map of
+    /// variables, which in turn contains a vector of [Datum]s.
+    pub fn decode(&self, warn: &mut dyn FnMut(LegacyBinWarning)) -> DataSources {
         let mut sources = IndexMap::new();
         for (metadata, data) in self.metadata.iter().zip(&self.data) {
             let mut variables = IndexMap::new();
@@ -244,7 +398,7 @@ impl LegacyBin {
                     },
                 );
             }
-            sources.insert(metadata.source_name.clone(), variables);
+            sources.insert(metadata.source_name.clone(), DataSource(variables));
         }
         if let Some(strings) = &self.strings {
             for map in &strings.source_maps {
@@ -253,7 +407,7 @@ impl LegacyBin {
                     continue;
                 };
                 for var_map in &map.variable_maps {
-                    let Some(variable) = source.get_mut(&var_map.variable_name) else {
+                    let Some(variable) = source.0.get_mut(&var_map.variable_name) else {
                         warn(LegacyBinWarning::UnknownVariable {
                             source_name: map.source_name.clone(),
                             variable: var_map.variable_name.clone(),
@@ -286,7 +440,15 @@ impl LegacyBin {
                 }
             }
         }
-        sources
+        DataSources(sources)
+    }
+
+    /// Decodes legacy binary data from `data`.
+    pub fn from_data(
+        data: &[u8],
+        warn: &mut dyn FnMut(LegacyBinWarning),
+    ) -> Result<DataSources, binrw::Error> {
+        Ok(Self::read(&mut Cursor::new(&data))?.decode(warn))
     }
 }
 
@@ -464,50 +626,11 @@ where
     Ok(())
 }
 
-pub fn data_pivot_table(data: &IndexMap<String, IndexMap<String, DataSeries>>) -> PivotTable {
-    let n_values = data
-        .values()
-        .flat_map(|map| map.values())
-        .map(|values| values.data.len())
-        .max()
-        .unwrap_or(0);
-    let index = Dimension::new(
-        Group::new("Index")
-            .with_multiple(Leaf::numbers(0..n_values))
-            .with_label_shown(),
-    );
-    let variables = Dimension::new(Group::new("Variables").with_multiple(data.iter().map(
-        |(name, contents)| {
-            Group::new(name.as_str()).with_multiple(contents.keys().map(|name| {
-                name.replace("categories", "\ncategories")
-                    .replace("labels", "\nlabels")
-                    .replace("group", "\ngroup")
-                    .replace("Label", "\nLabel")
-            }))
-        },
-    )));
-    let mut pivot_table = PivotTable::new([(Axis3::Y, index), (Axis3::X, variables)]);
-    for (variable_index, (variable_name, values)) in
-        data.values().flat_map(|map| map.iter()).enumerate()
-    {
-        for (value_index, data_value) in values.data.iter().enumerate() {
-            let value = Value::new_datum(data_value).with_value_label(
-                (variable_name == "cellFormat").then(|| datum_as_format(data_value).to_string()),
-            );
-            pivot_table.insert([value_index, variable_index], value);
-        }
-    }
-    pivot_table
-}
-
 #[cfg(test)]
 mod tests {
     use std::{fmt::Write, path::Path};
 
-    use crate::{
-        output::pivot::tests::assert_lines_eq,
-        spv::legacy_bin::{LegacyBin, data_pivot_table},
-    };
+    use crate::{output::pivot::tests::assert_lines_eq, spv::legacy_bin::LegacyBin};
 
     fn test_legacy_data_file(name: &str) {
         let input_filename = Path::new("src/spv/testdata")
@@ -520,7 +643,7 @@ mod tests {
         let data = std::fs::read(&input_filename).unwrap();
         let data =
             LegacyBin::from_data(&data, &mut |w| writeln!(&mut actual, "{w}").unwrap()).unwrap();
-        write!(&mut actual, "{}", data_pivot_table(&data)).unwrap();
+        write!(&mut actual, "{}", data.as_pivot_table()).unwrap();
 
         if expected != actual {
             if std::env::var("PSPP_REFRESH_EXPECTED").is_ok() {
@@ -532,11 +655,11 @@ mod tests {
         }
         assert_lines_eq(&expected, expected_filename.display(), &actual, "actual");
 
-        let written = LegacyBin::encode(&data);
+        let written = data.to_legacy_bin();
         let mut reread = String::new();
         let reread_data =
             LegacyBin::from_data(&written, &mut |w| writeln!(&mut reread, "{w}").unwrap()).unwrap();
-        write!(&mut reread, "{}", data_pivot_table(&reread_data)).unwrap();
+        write!(&mut reread, "{}", reread_data.as_pivot_table()).unwrap();
         if expected != reread {
             eprintln!("wrote and reread legacy data differs from expected:");
             assert_lines_eq(&expected, expected_filename.display(), &reread, "actual");
index b5487e714e252eaa4f754fa9934f94b0d79b5d2f..31ce908dd58c5e811f8522e3b2f834ebe4ed6272 100644 (file)
@@ -29,7 +29,6 @@ use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
 use displaydoc::Display;
 use enum_map::{Enum, EnumMap};
 use hashbrown::HashSet;
-use indexmap::IndexMap;
 use ordered_float::OrderedFloat;
 use serde::Deserialize;
 
@@ -41,7 +40,7 @@ use crate::{
         look::{self, Area, AreaStyle, CellStyle, Color, HorzAlign, Look, RowParity, VertAlign},
         value::{Value, ValueFormat},
     },
-    spv::{legacy_bin::DataSeries, read::light::decode_format},
+    spv::{legacy_bin::DataSources, read::light::decode_format},
 };
 
 /// Interprets this data value as a [Format], first by looking it up in
@@ -216,7 +215,7 @@ impl Visualization {
     }
     pub fn decode_series(
         &self,
-        data: IndexMap<String, IndexMap<String, DataSeries>>,
+        data: DataSources,
         warn: &mut dyn FnMut(LegacyXmlWarning),
     ) -> BTreeMap<&str, Series> {
         let mut variables: Vec<&dyn Variable> = self
@@ -343,7 +342,7 @@ impl Visualization {
 
     pub fn decode(
         &self,
-        binary_data: IndexMap<String, IndexMap<String, DataSeries>>,
+        binary_data: DataSources,
         look: Look,
         warn: &mut dyn FnMut(LegacyXmlWarning),
     ) -> Result<PivotTable, super::Error> {
@@ -465,7 +464,7 @@ impl VisChild {
 trait Variable {
     fn decode<'a>(
         &'a self,
-        data: &IndexMap<String, IndexMap<String, DataSeries>>,
+        data: &DataSources,
         series: &mut BTreeMap<&'a str, Series>,
         warn: &mut dyn FnMut(LegacyXmlWarning),
     ) -> bool;
@@ -514,7 +513,7 @@ impl SourceVariable {
 impl Variable for SourceVariable {
     fn decode<'a>(
         &'a self,
-        data: &IndexMap<String, IndexMap<String, DataSeries>>,
+        data: &DataSources,
         series: &mut BTreeMap<&'a str, Series>,
         _warn: &mut dyn FnMut(LegacyXmlWarning),
     ) -> bool {
@@ -528,8 +527,8 @@ impl Variable for SourceVariable {
             None
         };
 
-        let (categories, mut data) = if let Some(source) = data.get(&self.source)
-            && let Some(values) = source.get(&self.variable)
+        let (categories, mut data) = if let Some(source) = data.0.get(&self.source)
+            && let Some(values) = source.0.get(&self.variable)
         {
             let categories = values
                 .data
@@ -619,7 +618,7 @@ impl DerivedVariable {
 impl Variable for DerivedVariable {
     fn decode<'a>(
         &'a self,
-        _data: &IndexMap<String, IndexMap<String, DataSeries>>,
+        _data: &DataSources,
         series: &mut BTreeMap<&'a str, Series>,
         warn: &mut dyn FnMut(LegacyXmlWarning),
     ) -> bool {