1 use float_next_after::NextAfter;
2 use num::{Bounded, Zero};
3 use ordered_float::OrderedFloat;
5 collections::{hash_map::Entry, HashMap},
6 error::Error as StdError,
7 fmt::{Display, Formatter, Result as FmtResult},
11 use crate::endian::{Endian, ToBytes};
13 pub type Result<T, F = Error> = std::result::Result<T, F>;
17 pub file_name: Option<String>,
18 pub line_number: Option<usize>,
19 pub token: Option<String>,
25 file_name: Option<&str>,
26 line_number: Option<usize>,
31 file_name: file_name.map(String::from),
33 token: token.map(String::from),
39 impl StdError for Error {}
41 impl Display for Error {
42 fn fmt(&self, f: &mut Formatter) -> FmtResult {
43 match (self.file_name.as_ref(), self.line_number) {
44 (Some(ref file_name), Some(line_number)) => write!(f, "{file_name}:{line_number}: ")?,
45 (Some(ref file_name), None) => write!(f, "{file_name}: ")?,
46 (None, Some(line_number)) => write!(f, "line {line_number}: ")?,
49 if let Some(ref token) = self.token {
50 write!(f, "at '{token}': ")?;
52 write!(f, "{}", self.message)
56 pub fn sack(input: &str, input_file_name: Option<&str>, endian: Endian) -> Result<Vec<u8>> {
57 let mut symbol_table = HashMap::new();
58 let output = _sack(input, input_file_name, endian, &mut symbol_table)?;
59 let output = if !symbol_table.is_empty() {
60 for (k, v) in symbol_table.iter() {
61 println!("{k} => {v:?}");
63 for (k, v) in symbol_table.iter() {
69 format!("label {k} used but never defined"),
73 _sack(input, input_file_name, endian, &mut symbol_table)?
82 input_file_name: Option<&str>,
84 symbol_table: &mut HashMap<String, Option<u32>>,
85 ) -> Result<Vec<u8>> {
86 let mut lexer = Lexer::new(input, input_file_name, endian)?;
87 let mut output = Vec::new();
88 while parse_data_item(&mut lexer, &mut output, symbol_table)? {}
95 symbol_table: &mut HashMap<String, Option<u32>>,
97 if lexer.token.is_none() {
101 let initial_len = output.len();
102 match lexer.take()? {
103 Token::Integer(integer) => {
104 let Ok(integer): Result<i32, _> = integer.try_into() else {
105 Err(lexer.error(format!(
106 "{integer} is not in the valid range [{},{}]",
111 output.extend_from_slice(&lexer.endian.to_bytes(integer))},
112 Token::Float(float) => output.extend_from_slice(&lexer.endian.to_bytes(float.0)),
114 output.extend_from_slice(&[0xf5, 0x1e, 0x26, 0x02, 0x8a, 0x8c, 0xed, 0xff])
116 Token::I8 => put_integers::<u8, 1>(lexer, "i8", output)?,
117 Token::I16 => put_integers::<u16, 2>(lexer, "i16", output)?,
118 Token::I64 => put_integers::<i64, 8>(lexer, "i64", output)?,
119 Token::String(string) => output.extend_from_slice(string.as_bytes()),
121 let Some((Token::String(ref string), _)) = lexer.token else {
122 Err(lexer.error(format!("string expected after 's{size}'")))?
124 let len = string.len();
126 Err(lexer.error(format!(
127 "{len}-byte string is longer than pad length {size}"
130 output.extend_from_slice(string.as_bytes());
131 output.extend(repeat(b' ').take(size - len));
135 while !matches!(lexer.token, Some((Token::RParen, _))) {
136 parse_data_item(lexer, output, symbol_table)?;
140 Token::Count => put_counted_items::<u32, 4>(lexer, "COUNT", output, symbol_table)?,
141 Token::Count8 => put_counted_items::<u8, 1>(lexer, "COUNT8", output, symbol_table)?,
143 let Some((Token::String(ref string), _)) = lexer.token else {
144 Err(lexer.error(String::from("string expected after 'hex'")))?
146 let mut i = string.chars();
148 let Some(c0) = i.next() else { return Ok(true) };
149 let Some(c1) = i.next() else {
150 Err(lexer.error(String::from("hex string has odd number of characters")))?
152 let (Some(digit0), Some(digit1)) = (c0.to_digit(16), c1.to_digit(16)) else {
153 Err(lexer.error(String::from("invalid digit in hex string")))?
155 let byte = digit0 * 16 + digit1;
156 output.push(byte as u8);
159 Token::Label(name) => {
160 println!("define {name}");
161 let value = output.len() as u32;
162 match symbol_table.entry(name.clone()) {
163 Entry::Vacant(v) => {
164 v.insert(Some(value));
166 Entry::Occupied(mut o) => {
170 Err(lexer.error(format!("{name}: can't redefine label for offset {:#x} with offset {:#x}", *v, value)))?
173 None => drop(o.insert(Some(value))),
180 let mut value = symbol_table
184 println!("{name} has value {value}");
186 let plus = match lexer.token {
187 Some((Token::Plus, _)) => true,
188 Some((Token::Minus, _)) => false,
193 let operand = match lexer.token {
194 Some((Token::At(ref name), _)) => if let Some(value) = symbol_table.get(name) {
197 symbol_table.insert(name.clone(), None);
201 Some((Token::Integer(integer), _)) => integer
203 .map_err(|msg| lexer.error(format!("bad offset literal ({msg})")))?,
204 _ => Err(lexer.error(String::from("expecting @label or integer literal")))?,
209 value.checked_add(operand)
211 value.checked_sub(operand)
213 .ok_or_else(|| lexer.error(String::from("overflow in offset arithmetic")))?;
215 output.extend_from_slice(&lexer.endian.to_bytes(value));
219 if let Some((Token::Asterisk, _)) = lexer.token {
221 let Token::Integer(count) = lexer.take()? else {
222 Err(lexer.error(String::from("positive integer expected after '*'")))?
225 Err(lexer.error(String::from("positive integer expected after '*'")))?
227 let final_len = output.len();
229 output.extend_from_within(initial_len..final_len);
233 Some((Token::Semicolon, _)) => {
236 Some((Token::RParen, _)) => (),
237 _ => Err(lexer.error(String::from("';' expected")))?,
242 fn put_counted_items<T, const N: usize>(
245 output: &mut Vec<u8>,
246 symbol_table: &mut HashMap<String, Option<u32>>,
249 T: Zero + TryFrom<usize>,
250 Endian: ToBytes<T, N>,
252 let old_size = output.len();
253 output.extend_from_slice(&lexer.endian.to_bytes(T::zero()));
254 let start = output.len();
255 if !matches!(lexer.token, Some((Token::LParen, _))) {
256 Err(lexer.error(format!("'(' expected after '{name}'")))?
259 while !matches!(lexer.token, Some((Token::RParen, _))) {
260 parse_data_item(lexer, output, symbol_table)?;
263 let delta = output.len() - start;
264 let Ok(delta): Result<T, _> = delta.try_into() else {
265 Err(lexer.error(format!("{delta} bytes is too much for '{name}'")))?
267 let dest = &mut output[old_size..old_size + N];
268 dest.copy_from_slice(&lexer.endian.to_bytes(delta));
272 fn put_integers<T, const N: usize>(
275 output: &mut Vec<u8>,
278 T: Bounded + Display + TryFrom<i64> + Copy,
279 Endian: ToBytes<T, N>,
281 println!("put_integers {:?}", lexer.token);
283 while let Some(integer) = lexer.take_if(|t| match t {
284 Token::Integer(integer) => Some(*integer),
287 println!("got integer {integer}");
288 let Ok(integer) = integer.try_into() else {
289 Err(lexer.error(format!(
290 "{integer} is not in the valid range [{},{}]",
295 output.extend_from_slice(&lexer.endian.to_bytes(integer));
298 println!("put_integers {:?} {n}", lexer.token);
300 Err(lexer.error(format!("integer expected after '{name}'")))?
305 #[derive(PartialEq, Eq, Clone, Debug)]
308 Float(OrderedFloat<f64>),
330 token: Option<(Token, &'a str)>,
331 input_file_name: Option<&'a str>,
336 fn skip_comments(mut s: &str) -> (&str, usize) {
337 let mut n_newlines = 0;
339 s = s.trim_start_matches([' ', '\t', '\r', '<', '>']);
340 if let Some(remainder) = s.strip_prefix('#') {
341 let Some((_, remainder)) = remainder.split_once('\n') else {
346 } else if let Some(remainder) = s.strip_prefix('\n') {
357 fn new(input: &'a str, input_file_name: Option<&'a str>, endian: Endian) -> Result<Lexer<'a>> {
358 let mut lexer = Lexer {
365 lexer.token = lexer.next()?;
368 fn error(&self, message: String) -> Error {
369 let repr = self.token.as_ref().map(|(_, repr)| *repr);
370 Error::new(self.input_file_name, Some(self.line_number), repr, message)
372 fn take(&mut self) -> Result<Token> {
373 let Some(token) = self.token.take() else {
374 Err(self.error(String::from("unexpected end of input")))?
376 self.token = self.next()?;
379 fn take_if<F, T>(&mut self, condition: F) -> Result<Option<T>>
381 F: FnOnce(&Token) -> Option<T>,
383 let Some(ref token) = self.token else {
386 match condition(&token.0) {
388 self.token = self.next()?;
394 fn get(&mut self) -> Result<Option<&Token>> {
395 if self.token.is_none() {
396 Err(self.error(String::from("unexpected end of input")))?
398 self.token = self.next()?;
400 Some((ref token, _)) => Ok(Some(token)),
406 fn next(&mut self) -> Result<Option<(Token, &'a str)>> {
407 // Get the first character of the token, skipping past white space and
409 let (s, n_newlines) = skip_comments(self.input);
410 self.line_number += n_newlines;
414 let mut iter = s.chars();
415 let Some(c) = iter.next() else {
418 let (token, rest) = match c {
419 c if c.is_ascii_digit() || c == '-' => {
422 !(c.is_ascii_digit() || c.is_alphabetic() || c == '.' || c == '-')
424 .unwrap_or_else(|| s.len());
425 let (number, rest) = s.split_at(len);
426 let token = if number == "-" {
428 } else if let Some(digits) = number.strip_prefix("0x") {
429 Token::Integer(i64::from_str_radix(digits, 16).map_err(|msg| {
430 self.error(format!("bad integer literal '{number}' ({msg})"))
432 } else if !number.contains('.') {
433 Token::Integer(number.parse().map_err(|msg| {
434 self.error(format!("bad integer literal '{number}' ({msg})"))
437 Token::Float(number.parse().map_err(|msg| {
438 self.error(format!("bad float literal '{number}' ({msg})"))
444 let s = iter.as_str();
445 let Some(len) = s.find(['\n', '"']) else {
446 Err(self.error(String::from("end-of-file inside string")))?
448 let (string, rest) = s.split_at(len);
449 let Some(rest) = rest.strip_prefix('"') else {
450 Err(self.error(format!("new-line inside string ({string}...{rest})")))?
452 (Token::String(string.into()), rest)
454 ';' => (Token::Semicolon, iter.as_str()),
455 '*' => (Token::Asterisk, iter.as_str()),
456 '+' => (Token::Plus, iter.as_str()),
457 '(' => (Token::LParen, iter.as_str()),
458 ')' => (Token::RParen, iter.as_str()),
459 c if c.is_alphabetic() || c == '@' || c == '_' => {
468 .unwrap_or_else(|| s.len());
469 let (s, rest) = s.split_at(len);
470 if let Some(rest) = rest.strip_prefix(':') {
471 (Token::Label(s.into()), rest)
472 } else if let Some(name) = s.strip_prefix('@') {
473 (Token::At(name.into()), rest)
474 } else if let Some(count) = s.strip_prefix('s') {
476 Token::S(count.parse().map_err(|msg| {
477 self.error(format!("bad counted string '{s}' ({msg})"))
481 let token = match &s[..] {
485 "SYSMIS" => Token::Float(OrderedFloat(-f64::MAX)),
486 "PCSYSMIS" => Token::PcSysmis,
487 "LOWEST" => Token::Float((-f64::MAX).next_after(0.0).into()),
488 "HIGHEST" => Token::Float(f64::MAX.into()),
489 "ENDIAN" => Token::Integer(if self.endian == Endian::Big { 1 } else { 2 }),
490 "COUNT" => Token::Count,
491 "COUNT8" => Token::Count8,
493 _ => Err(self.error(format!("invalid token '{s}'")))?,
498 _ => Err(self.error(format!("invalid input byte '{c}'")))?,
501 let repr = &start[..start.len() - rest.len()];
502 println!("{token:?} {repr}");
503 Ok(Some((token, repr)))
509 use crate::endian::Endian;
510 use crate::sack::sack;
512 use hexplay::HexView;
515 fn basic_sack() -> Result<()> {
517 "$FL2"; s60 "$(#) SPSS DATA FILE PSPP synthetic test file";
519 28; # Nominal case size
524 "01 Jan 11"; "20:53:52";
525 "PSPP synthetic test file: "; i8 244; i8 245; i8 246; i8 248; s34 "";
528 let output = sack(input, None, Endian::Big)?;
529 HexView::new(&output).print()?;
534 fn pcp_sack() -> Result<()> {
538 @MAIN; @MAIN_END - @MAIN;
539 @VARS; @VARS_END - @VARS;
540 @LABELS; @LABELS_END - @LABELS;
541 @DATA; @DATA_END - @DATA;
547 s62 "PCSPSS PSPP synthetic test product";
549 0; 0; i16 1; # Fixed.
557 s64 "PSPP synthetic test file";
561 0; 0; 0; 0x050800; s8 "$CASENUM"; PCSYSMIS;
562 0; 0; 0; 0x010800; s8 "$DATE"; PCSYSMIS;
563 0; 0; 0; 0x050802; s8 "$WEIGHT"; PCSYSMIS;
565 # Numeric variable, no label or missing values.
566 0; 0; 0; 0x050800; s8 "NUM1"; PCSYSMIS;
568 # Numeric variable, variable label.
569 0; 0; @NUM2_LABEL - @LABELS_OFS; 0x050800; s8 "NUM2"; PCSYSMIS;
571 # Numeric variable with missing value.
572 0; 0; 0; 0x050800; s8 "NUM3"; 1.0;
574 # Numeric variable, variable label and missing value.
575 0; 0; @NUM4_LABEL - @LABELS_OFS; 0x050800; s8 "NUM4"; 2.0;
577 # String variable, no label or missing values.
578 0; 0; 0; 0x010800; s8 "STR1"; PCSYSMIS;
580 # String variable, variable label.
581 0; 0; @STR2_LABEL - @LABELS_OFS; 0x010400; s8 "STR2"; PCSYSMIS;
583 # String variable with missing value.
584 0; 0; 0; 0x010500; s8 "STR3"; s8 "MISS";
586 # String variable, variable label and missing value.
587 0; 0; @STR4_LABEL - @LABELS_OFS; 0x010100; s8 "STR4"; s8 "OTHR";
589 # Long string variable
590 0; 0; 0; 0x010b00; s8 "STR5"; PCSYSMIS;
593 # Long string variable with variable label
594 0; 0; @STR6_LABEL - @LABELS_OFS; 0x010b00; s8 "STR6"; PCSYSMIS;
599 3; i8 0 0 0; LABELS_OFS: i8 0;
600 NUM2_LABEL: COUNT8("Numeric variable 2's label");
601 NUM4_LABEL: COUNT8("Another numeric variable label");
602 STR2_LABEL: COUNT8("STR2's variable label");
603 STR4_LABEL: COUNT8("STR4's variable label");
604 STR6_LABEL: COUNT8("Another string variable's label");
608 0.0; "11/28/14"; 1.0;
609 0.0; 1.0; 2.0; PCSYSMIS; s8 "abcdefgh"; s8 "ijkl"; s8 "mnopq"; s8 "r";
610 s16 "stuvwxyzAB"; s16 "CDEFGHIJKLM";
613 let output = sack(input, None, Endian::Big)?;
614 HexView::new(&output).print()?;