use anyhow::{Result, anyhow};
use clap::{Args, Subcommand};
use pspp::{
- file::ReadSeek,
+ file::{FileType, ReadSeek},
output::{
Criteria, Itemlike, SpvMembers,
pivot::{Axis3, Dimension, Group, Leaf, PivotTable, value::Value},
},
spv::{
SpvArchive,
+ legacy_bin::{LegacyBin, data_pivot_table},
read::{
- legacy_xml::{Visualization, datum_as_format},
+ legacy_xml::Visualization,
structure::{Content, OutlineItem},
},
},
impl LegacyData {
fn run(self) -> Result<()> {
- let (mut archive, items) = self.input.read_outline()?;
- for item in items {
- for item in item.iter_preorder() {
- if let Some(spv_info) = item.spv_info()
- && let Some(members) = &spv_info.members
- && let Some(binary) = members.bin_member()
- {
- let data = archive
- .read_legacy_bin(binary, &mut |w| eprintln!("{w}"))
- .map_err(|error| {
- anyhow::Error::from(error).context(format!(
- "While parsing {binary:?} as legacy binary SPV member"
- ))
- })?;
-
- let n_values = data
- .values()
- .flat_map(|map| map.values())
- .map(|values| values.data.len())
- .max()
- .unwrap_or(0);
- let index = Dimension::new(
- Group::new("Index")
- .with_multiple(Leaf::numbers(0..n_values))
- .with_label_shown(),
- );
- let variables = Dimension::new(Group::new("Variables").with_multiple(
- data.iter().map(|(name, contents)| {
- Group::new(name.as_str()).with_multiple(contents.keys().map(|name| {
- name.replace("categories", "\ncategories")
- .replace("labels", "\nlabels")
- .replace("group", "\ngroup")
- .replace("Label", "\nLabel")
- }))
- }),
- ));
- let mut pivot_table =
- PivotTable::new([(Axis3::Y, index), (Axis3::X, variables)]);
- for (variable_index, (variable_name, values)) in
- data.values().flat_map(|map| map.iter()).enumerate()
+ if let Some(FileType::LegacyBin) = FileType::from_file(&self.input.input)? {
+ let data = std::fs::read(&self.input.input)?;
+ let data = LegacyBin::from_data(&data, &mut |w| eprintln!("{w}"))?;
+ println!("{}", data_pivot_table(&data));
+ } else {
+ let (mut archive, items) = self.input.read_outline()?;
+ for item in items {
+ for item in item.iter_preorder() {
+ if let Some(spv_info) = item.spv_info()
+ && let Some(members) = &spv_info.members
+ && let Some(binary) = members.bin_member()
{
- for (value_index, data_value) in values.data.iter().enumerate() {
- let value = Value::new_datum(data_value).with_value_label(
- (variable_name == "cellFormat")
- .then(|| datum_as_format(data_value).to_string()),
- );
- pivot_table.insert([value_index, variable_index], value);
- }
+ let data = archive
+ .read_legacy_bin(binary, &mut |w| eprintln!("{w}"))
+ .map_err(|error| {
+ anyhow::Error::from(error).context(format!(
+ "While parsing {binary:?} as legacy binary SPV member"
+ ))
+ })?;
+
+ println!("{}", data_pivot_table(&data));
}
- println!("{pivot_table}");
}
}
}
/// A standalone graph exported in XML format.
Graph,
+
+ /// A legacy data binary file.
+ ///
+ /// Normally these are seen only inside SPV files.
+ LegacyBin,
}
impl Display for FileType {
return Ok(Some(Self::Viewer { encrypted: false }));
}
+ if buf.starts_with(&[0, 0xaf]) || buf.starts_with(&[0, 0xb0]) {
+ return Ok(Some(Self::LegacyBin));
+ }
+
if !buf.is_empty() && !buf.contains(&0) {
return Ok(Some(Self::Syntax {
confident: buf.starts_with(b"* Encoding:"),
FileType::Look(LookFormat::Binary) => "tlo",
FileType::Look(LookFormat::Xml) => "stt",
FileType::Graph => "xml",
+ FileType::LegacyBin => "bin",
}
}
} => "encrypted SPSS syntax",
FileType::Look(_) => "SPSS TableLook",
FileType::Graph => "SPSS exported chart",
+ FileType::LegacyBin => "SPSS legacy binary data",
}
}
}
mut csv_data: Option<IndexMap<String, DataSeries>>,
warn: &mut dyn FnMut(GraphWarning),
) -> Result<svg::Document, GraphError> {
+ for child in &self.children {
+ if let Some(style_cycle) = child.style_cycle() {
+ dbg!(style_cycle);
+ }
+ }
let _styles = self.decode_styles();
for delimited_source in self
.children
_ => None,
}
}
+ fn style_cycle(&self) -> Option<&StyleCycle> {
+ match self {
+ Self::StyleCycle(style_cycle) => Some(style_cycle),
+ _ => None,
+ }
+ }
}
#[derive(Deserialize, Debug)]
#[derive(Clone, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct StyleCycle {
- #[serde(default)]
+ #[serde(default, rename = "cycle")]
cycles: Vec<Cycle>,
}
#[derive(Clone, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct Cycle {
- #[serde(default)]
+ #[serde(default, rename = "style")]
styles: Vec<Style>,
}
#[serde(rename = "@size")]
size: Option<LocationLength>,
+ #[serde(rename = "@pattern")]
+ pattern: Option<u32>,
+
#[serde(default, rename = "$value")]
children: Vec<Style>,
}
visualization.decode(self.binary_data(&mut warn)?, self.csv_data()?, &mut warn)
}
+ //pub fn to_spv(self) -> Self {}
+
pub fn to_chart_xml<F>(&self, mut warn: F) -> Result<Vec<u8>, GraphError>
where
F: FnMut(GraphWarning),
//! Legacy binary data.
-use std::io::{Cursor, Read, Seek, SeekFrom};
+use core::f64;
+use std::io::{Cursor, Read, Seek, SeekFrom, Write};
-use binrw::{BinRead, BinResult, binread};
+use binrw::{BinRead, BinResult, BinWrite, Endian, binread, binwrite};
use displaydoc::Display;
use encoding_rs::UTF_8;
use indexmap::IndexMap;
use crate::{
data::Datum,
- spv::read::light::{U32String, parse_vec},
+ output::pivot::{Axis3, Dimension, Group, Leaf, PivotTable, value::Value},
+ spv::{
+ read::{
+ legacy_xml::datum_as_format,
+ light::{U32String, parse_vec},
+ },
+ write::SpvString,
+ },
};
/// A single column of data with an optional label.
Ok(Self::read(&mut Cursor::new(&data))?.decode(warn))
}
+ /// Encodes `data` into the legacy detail member format.
+ ///
+ /// # Panic
+ ///
+ /// Panics if `data` is empty.
+ pub fn encode(data: &IndexMap<String, IndexMap<String, DataSeries>>) -> Vec<u8> {
+ assert!(!data.is_empty());
+ let mut bin = Cursor::new(Vec::new());
+ bin.set_position(8);
+
+ /// Starting offset of the first metadata record.
+ const METADATA_OFFSET: usize = 8;
+ /// Size of an individual metadata record
+ const METADATA_SIZE: usize = 4 + 4 + 4 + 64 + 4;
+
+ let mut data_offset = METADATA_OFFSET + data.len() * METADATA_SIZE;
+ for (source_name, source) in data {
+ #[binwrite]
+ #[br(little)]
+ #[derive(Debug)]
+ struct Metadata<'a> {
+ n_values: u32,
+ n_variables: u32,
+ data_offset: u32,
+ #[bw(write_with(write_fixed_utf8_string), args(64))]
+ source_name: &'a str,
+ #[bw(magic(0u32))]
+ x: (),
+ }
+ let metadata = Metadata {
+ n_values: source.values().next().map_or(0, |series| series.data.len()) as u32,
+ n_variables: source.len() as u32,
+ data_offset: data_offset as u32,
+ source_name: source_name.as_str(),
+ x: (),
+ };
+ metadata.write_le(&mut bin).unwrap();
+
+ for series in source.values() {
+ data_offset += 288 + 8 * series.data.len();
+ }
+ }
+
+ let mut n_strings = Vec::with_capacity(data.len());
+ for source in data.values() {
+ let mut variable_strings = Vec::with_capacity(source.len());
+ for (variable_name, series) in source {
+ write_fixed_utf8_string(&variable_name.as_str(), &mut bin, Endian::Little, (288,))
+ .unwrap();
+ for datum in &series.data {
+ datum
+ .as_number()
+ .flatten()
+ .unwrap_or(-f64::MAX)
+ .write_le(&mut bin)
+ .unwrap();
+ }
+ variable_strings.push(series.data.iter().filter(|datum| datum.is_string()).count());
+ }
+ n_strings.push(variable_strings);
+ }
+
+ let n_maps = n_strings
+ .iter()
+ .filter(|variable_strings| variable_strings.iter().any(|n| *n != 0))
+ .count() as u32;
+ let mut labels = IndexMap::<&str, u32>::new();
+ if n_maps > 0 {
+ n_maps.write_le(&mut bin).unwrap();
+ for ((source_name, source), variable_strings) in data.iter().zip(&n_strings) {
+ let Some((_n, last_string)) = variable_strings
+ .iter()
+ .zip(1u32..)
+ .filter(|(n, _index)| **n > 0)
+ .last()
+ else {
+ continue;
+ };
+ (SpvString(source_name), last_string)
+ .write_le(&mut bin)
+ .unwrap();
+ for ((variable_name, series), n_strings) in source.iter().zip(variable_strings) {
+ (SpvString(variable_name), *n_strings as u32)
+ .write_le(&mut bin)
+ .unwrap();
+ for (index, datum) in series.data.iter().enumerate() {
+ if let Some(string) = datum.as_string() {
+ let entry = labels.entry(string.as_str());
+ let label_index = entry.index();
+ *entry.or_default() += 1;
+ (index as u32, label_index as u32)
+ .write_le(&mut bin)
+ .unwrap();
+ }
+ }
+ }
+ }
+
+ (labels.len() as u32).write_le(&mut bin).unwrap();
+ for (label, frequency) in &labels {
+ (*frequency, SpvString(label)).write_le(&mut bin).unwrap();
+ }
+ }
+
+ #[binwrite]
+ #[bw(little)]
+ struct Header {
+ #[bw(magic(0u8))]
+ version: u8,
+ n_sources: u16,
+ member_size: u32,
+ }
+ let header = Header {
+ version: 0xb0,
+ n_sources: data.len() as u16,
+ member_size: bin.position() as u32,
+ };
+ bin.set_position(0);
+ header.write_le(&mut bin).unwrap();
+
+ bin.into_inner()
+ }
+
/// Decodes legacy binary data into a map from a series name to a map of
/// variables, which in turn contains a vector of [Datum]s.
pub fn decode(
}
Ok(UTF_8.decode_without_bom_handling(&buf).0.into_owned())
}
+
+/// Writes a UTF-8 string, padded it on the right with 0-bytes to length `n`.
+fn write_fixed_utf8_string<W>(
+ s: &&str,
+ writer: &mut W,
+ _endian: Endian,
+ (n,): (usize,),
+) -> BinResult<()>
+where
+ W: Write + Seek,
+{
+ writer.write_all(&s.as_bytes()[..n.min(s.len())])?;
+ for _ in s.len()..n {
+ writer.write_all(&[0])?;
+ }
+ Ok(())
+}
+
+pub fn data_pivot_table(data: &IndexMap<String, IndexMap<String, DataSeries>>) -> PivotTable {
+ let n_values = data
+ .values()
+ .flat_map(|map| map.values())
+ .map(|values| values.data.len())
+ .max()
+ .unwrap_or(0);
+ let index = Dimension::new(
+ Group::new("Index")
+ .with_multiple(Leaf::numbers(0..n_values))
+ .with_label_shown(),
+ );
+ let variables = Dimension::new(Group::new("Variables").with_multiple(data.iter().map(
+ |(name, contents)| {
+ Group::new(name.as_str()).with_multiple(contents.keys().map(|name| {
+ name.replace("categories", "\ncategories")
+ .replace("labels", "\nlabels")
+ .replace("group", "\ngroup")
+ .replace("Label", "\nLabel")
+ }))
+ },
+ )));
+ let mut pivot_table = PivotTable::new([(Axis3::Y, index), (Axis3::X, variables)]);
+ for (variable_index, (variable_name, values)) in
+ data.values().flat_map(|map| map.iter()).enumerate()
+ {
+ for (value_index, data_value) in values.data.iter().enumerate() {
+ let value = Value::new_datum(data_value).with_value_label(
+ (variable_name == "cellFormat").then(|| datum_as_format(data_value).to_string()),
+ );
+ pivot_table.insert([value_index, variable_index], value);
+ }
+ }
+ pivot_table
+}
+
+#[cfg(test)]
+mod tests {
+ use std::{fmt::Write, path::Path};
+
+ use crate::{
+ output::pivot::tests::assert_lines_eq,
+ spv::legacy_bin::{LegacyBin, data_pivot_table},
+ };
+
+ fn test_legacy_data_file(name: &str) {
+ let input_filename = Path::new("src/spv/testdata")
+ .join(name)
+ .with_extension("bin");
+ let expected_filename = input_filename.with_extension("expected");
+ let expected = String::from_utf8(std::fs::read(&expected_filename).unwrap()).unwrap();
+
+ let mut actual = String::new();
+ let data = std::fs::read(&input_filename).unwrap();
+ let data =
+ LegacyBin::from_data(&data, &mut |w| writeln!(&mut actual, "{w}").unwrap()).unwrap();
+ write!(&mut actual, "{}", data_pivot_table(&data)).unwrap();
+
+ if expected != actual {
+ if std::env::var("PSPP_REFRESH_EXPECTED").is_ok() {
+ std::fs::write(&expected_filename, actual).unwrap();
+ panic!("{}: refreshed output", expected_filename.display());
+ } else {
+ eprintln!("note: rerun with PSPP_REFRESH_EXPECTED=1 to refresh expected output");
+ }
+ }
+ assert_lines_eq(&expected, expected_filename.display(), &actual, "actual");
+
+ let written = LegacyBin::encode(&data);
+ let mut reread = String::new();
+ let reread_data =
+ LegacyBin::from_data(&written, &mut |w| writeln!(&mut reread, "{w}").unwrap()).unwrap();
+ write!(&mut reread, "{}", data_pivot_table(&reread_data)).unwrap();
+ if expected != reread {
+ eprintln!("wrote and reread legacy data differs from expected:");
+ assert_lines_eq(&expected, expected_filename.display(), &reread, "actual");
+ }
+ }
+
+ #[test]
+ fn test_legacy_data_1() {
+ test_legacy_data_file("legacy-data1");
+ }
+}
--- /dev/null
+╭─────┬────────────────────────────────────────────────────────────────────────────────────────────╮
+│ │ tableData │
+│ ├──────┬──────────┬──────────┬──────────┬──────────┬───────────┬──────────┬──────────────────┤
+│ │ │ │ │ │dimension0│ │ │ │
+│ │ │ │dimension0│dimension0│ group0 │ dimension0│dimension1│ dimension1 │
+│Index│ cell │cellFormat│categories│ group0 │ Label │ labels │categories│ labels │
+├─────┼──────┼──────────┼──────────┼──────────┼──────────┼───────────┼──────────┼──────────────────┤
+│0 │ 1.00│F40.0 │ .00│ 2.00│Valid │>2 years │ .00│Frequency │
+│1 │ .47│F40.1 │ .00│ 2.00│Valid │>2 years │ 1.00│Percent │
+│2 │ .47│F40.1 │ .00│ 2.00│Valid │>2 years │ 2.00│Valid Percent │
+│3 │ .47│F40.1 │ .00│ 2.00│Valid │>2 years │ 3.00│Cumulative Percent│
+│4 │ 17.00│F40.0 │ 1.00│ 2.00│Valid │2-5 years │ .00│Frequency │
+│5 │ 7.94│F40.1 │ 1.00│ 2.00│Valid │2-5 years │ 1.00│Percent │
+│6 │ 7.98│F40.1 │ 1.00│ 2.00│Valid │2-5 years │ 2.00│Valid Percent │
+│7 │ 8.45│F40.1 │ 1.00│ 2.00│Valid │2-5 years │ 3.00│Cumulative Percent│
+│8 │122.00│F40.0 │ 2.00│ 2.00│Valid │6-10 years │ .00│Frequency │
+│9 │ 57.01│F40.1 │ 2.00│ 2.00│Valid │6-10 years │ 1.00│Percent │
+│10 │ 57.28│F40.1 │ 2.00│ 2.00│Valid │6-10 years │ 2.00│Valid Percent │
+│11 │ 65.73│F40.1 │ 2.00│ 2.00│Valid │6-10 years │ 3.00│Cumulative Percent│
+│12 │ 45.00│F40.0 │ 3.00│ 2.00│Valid │11-15 years│ .00│Frequency │
+│13 │ 21.03│F40.1 │ 3.00│ 2.00│Valid │11-15 years│ 1.00│Percent │
+│14 │ 21.13│F40.1 │ 3.00│ 2.00│Valid │11-15 years│ 2.00│Valid Percent │
+│15 │ 86.85│F40.1 │ 3.00│ 2.00│Valid │11-15 years│ 3.00│Cumulative Percent│
+│16 │ 18.00│F40.0 │ 4.00│ 2.00│Valid │16-20 years│ .00│Frequency │
+│17 │ 8.41│F40.1 │ 4.00│ 2.00│Valid │16-20 years│ 1.00│Percent │
+│18 │ 8.45│F40.1 │ 4.00│ 2.00│Valid │16-20 years│ 2.00│Valid Percent │
+│19 │ 95.31│F40.1 │ 4.00│ 2.00│Valid │16-20 years│ 3.00│Cumulative Percent│
+│20 │ 10.00│F40.0 │ 5.00│ 2.00│Valid │>20 years │ .00│Frequency │
+│21 │ 4.67│F40.1 │ 5.00│ 2.00│Valid │>20 years │ 1.00│Percent │
+│22 │ 4.69│F40.1 │ 5.00│ 2.00│Valid │>20 years │ 2.00│Valid Percent │
+│23 │100.00│F40.1 │ 5.00│ 2.00│Valid │>20 years │ 3.00│Cumulative Percent│
+│24 │213.00│F40.0 │ 6.00│ 2.00│Valid │Total │ .00│Frequency │
+│25 │ 99.53│F40.1 │ 6.00│ 2.00│Valid │Total │ 1.00│Percent │
+│26 │100.00│F40.1 │ 6.00│ 2.00│Valid │Total │ 2.00│Valid Percent │
+│27 │ 1.00│F40.0 │ 7.00│ 4.00│Missing │System │ .00│Frequency │
+│28 │ .47│F40.1 │ 7.00│ 4.00│Missing │System │ 1.00│Percent │
+│29 │214.00│F40.0 │ 8.00│ 5.00│ │Total │ .00│Frequency │
+│30 │100.00│F40.1 │ 8.00│ 5.00│ │Total │ 1.00│Percent │
+╰─────┴──────┴──────────┴──────────┴──────────┴──────────┴───────────┴──────────┴──────────────────╯
}
}
-struct SpvString<T>(T);
+pub struct SpvString<T>(pub T);
impl<'a> SpvString<&'a str> {
fn optional(s: &'a Option<String>) -> Self {
Self(s.as_ref().map_or("", |s| s.as_str()))
projects / pspp / commitdiff