X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=rust%2Fsrc%2Fsack.rs;h=ab35f8661e8c4963b1e07380082fae995d4031b0;hb=6165ed413d9aa818e3246d0a063c646dc4efc7e5;hp=6be81d8aff1c285473660c7ce75745243102e12f;hpb=b8f1dedca099b32b4b4b4dda5f265cc1e8db9587;p=pspp diff --git a/rust/src/sack.rs b/rust/src/sack.rs index 6be81d8aff..ab35f8661e 100644 --- a/rust/src/sack.rs +++ b/rust/src/sack.rs @@ -1,18 +1,327 @@ -use anyhow::{anyhow, Result}; use float_next_after::NextAfter; -use std::{iter::Peekable, str::Chars}; +use num::{Bounded, Zero}; +use ordered_float::OrderedFloat; +use std::{ + collections::{hash_map::Entry, HashMap}, + error::Error as StdError, + fmt::{Display, Formatter, Result as FmtResult}, + iter::repeat, +}; -use crate::endian::Endian; +use crate::endian::{Endian, ToBytes}; -pub fn sack(input: &str, endian: Endian) -> Result> { - let lexer = Lexer::new(input, endian)?; - //let mut output = Vec::new(); - Ok(Vec::new()) +pub type Result = std::result::Result; + +#[derive(Debug)] +pub struct Error { + pub file_name: Option, + pub line_number: Option, + pub token: Option, + pub message: String, +} + +impl Error { + fn new( + file_name: Option<&str>, + line_number: Option, + token: Option<&str>, + message: String, + ) -> Error { + Error { + file_name: file_name.map(String::from), + line_number, + token: token.map(String::from), + message, + } + } +} + +impl StdError for Error {} + +impl Display for Error { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + match (self.file_name.as_ref(), self.line_number) { + (Some(ref file_name), Some(line_number)) => write!(f, "{file_name}:{line_number}: ")?, + (Some(ref file_name), None) => write!(f, "{file_name}: ")?, + (None, Some(line_number)) => write!(f, "line {line_number}: ")?, + (None, None) => (), + } + if let Some(ref token) = self.token { + write!(f, "at '{token}': ")?; + } + write!(f, "{}", self.message) + } +} + +pub fn sack(input: &str, input_file_name: Option<&str>, endian: Endian) -> Result> { + let mut symbol_table = HashMap::new(); + let output = _sack(input, input_file_name, endian, &mut symbol_table)?; + let output = if !symbol_table.is_empty() { + for (k, v) in symbol_table.iter() { + println!("{k} => {v:?}"); + } + for (k, v) in symbol_table.iter() { + if v.is_none() { + Err(Error::new( + input_file_name, + None, + None, + format!("label {k} used but never defined"), + ))? + } + } + _sack(input, input_file_name, endian, &mut symbol_table)? + } else { + output + }; + Ok(output) +} + +fn _sack( + input: &str, + input_file_name: Option<&str>, + endian: Endian, + symbol_table: &mut HashMap>, +) -> Result> { + let mut lexer = Lexer::new(input, input_file_name, endian)?; + let mut output = Vec::new(); + while parse_data_item(&mut lexer, &mut output, symbol_table)? {} + Ok(output) +} + +fn parse_data_item( + lexer: &mut Lexer, + output: &mut Vec, + symbol_table: &mut HashMap>, +) -> Result { + if lexer.token.is_none() { + return Ok(false); + }; + + let initial_len = output.len(); + match lexer.take()? { + Token::Integer(integer) => { + if let Ok(integer) = TryInto::::try_into(integer) { + output.extend_from_slice(&lexer.endian.to_bytes(integer)); + } else if let Ok(integer) = TryInto::::try_into(integer) { + output.extend_from_slice(&lexer.endian.to_bytes(integer)); + } else { + Err(lexer.error(format!( + "{integer} is not in the valid range [{},{}]", + i32::min_value(), + u32::max_value() + )))?; + }; + } + Token::Float(float) => output.extend_from_slice(&lexer.endian.to_bytes(float.0)), + Token::PcSysmis => { + output.extend_from_slice(&[0xf5, 0x1e, 0x26, 0x02, 0x8a, 0x8c, 0xed, 0xff]) + } + Token::I8 => put_integers::(lexer, "i8", output)?, + Token::I16 => put_integers::(lexer, "i16", output)?, + Token::I64 => put_integers::(lexer, "i64", output)?, + Token::String(string) => output.extend_from_slice(string.as_bytes()), + Token::S(size) => { + let Some((Token::String(ref string), _)) = lexer.token else { + Err(lexer.error(format!("string expected after 's{size}'")))? + }; + let len = string.len(); + if len > size { + Err(lexer.error(format!( + "{len}-byte string is longer than pad length {size}" + )))? + } + output.extend_from_slice(string.as_bytes()); + output.extend(repeat(b' ').take(size - len)); + lexer.get()?; + } + Token::LParen => { + while !matches!(lexer.token, Some((Token::RParen, _))) { + parse_data_item(lexer, output, symbol_table)?; + } + lexer.get()?; + } + Token::Count => put_counted_items::(lexer, "COUNT", output, symbol_table)?, + Token::Count8 => put_counted_items::(lexer, "COUNT8", output, symbol_table)?, + Token::Hex => { + let Some((Token::String(ref string), _)) = lexer.token else { + Err(lexer.error(String::from("string expected after 'hex'")))? + }; + let mut string = &string[..]; + loop { + string = string.trim_start(); + if string.is_empty() { + break; + }; + + let mut i = string.chars(); + let Some(c0) = i.next() else { return Ok(true) }; + let Some(c1) = i.next() else { + Err(lexer.error(String::from("hex string has odd number of characters")))? + }; + + let (Some(digit0), Some(digit1)) = (c0.to_digit(16), c1.to_digit(16)) else { + Err(lexer.error(String::from("invalid digit in hex string")))? + }; + let byte = digit0 * 16 + digit1; + output.push(byte as u8); + + string = i.as_str(); + } + lexer.get()?; + } + Token::Label(name) => { + println!("define {name}"); + let value = output.len() as u32; + match symbol_table.entry(name.clone()) { + Entry::Vacant(v) => { + v.insert(Some(value)); + } + Entry::Occupied(mut o) => { + match o.get() { + Some(v) => { + if *v != value { + Err(lexer.error(format!("{name}: can't redefine label for offset {:#x} with offset {:#x}", *v, value)))? + } + } + None => drop(o.insert(Some(value))), + } + } + }; + return Ok(true); + } + Token::At(name) => { + let mut value = *symbol_table.entry(name.clone()).or_insert(None); + loop { + let plus = match lexer.token { + Some((Token::Plus, _)) => true, + Some((Token::Minus, _)) => false, + _ => break, + }; + lexer.get()?; + + let operand = match lexer.token { + Some((Token::At(ref name), _)) => { + *symbol_table.entry(name.clone()).or_insert(None) + } + Some((Token::Integer(integer), _)) => Some( + integer + .try_into() + .map_err(|msg| lexer.error(format!("bad offset literal ({msg})")))?, + ), + _ => Err(lexer.error(String::from("expecting @label or integer literal")))?, + }; + lexer.get()?; + + value = match (value, operand) { + (Some(a), Some(b)) => Some( + if plus { + a.checked_add(b) + } else { + a.checked_sub(b) + } + .ok_or_else(|| { + lexer.error(String::from("overflow in offset arithmetic")) + })?, + ), + _ => None, + }; + } + let value = value.unwrap_or(0); + output.extend_from_slice(&lexer.endian.to_bytes(value)); + } + _ => (), + }; + if let Some((Token::Asterisk, _)) = lexer.token { + lexer.get()?; + let Token::Integer(count) = lexer.take()? else { + Err(lexer.error(String::from("positive integer expected after '*'")))? + }; + if count < 1 { + Err(lexer.error(String::from("positive integer expected after '*'")))? + }; + let final_len = output.len(); + for _ in 1..count { + output.extend_from_within(initial_len..final_len); + } + } + match lexer.token { + Some((Token::Semicolon, _)) => { + lexer.get()?; + } + Some((Token::RParen, _)) => (), + _ => Err(lexer.error(String::from("';' expected")))?, + } + Ok(true) +} + +fn put_counted_items( + lexer: &mut Lexer, + name: &str, + output: &mut Vec, + symbol_table: &mut HashMap>, +) -> Result<()> +where + T: Zero + TryFrom, + Endian: ToBytes, +{ + let old_size = output.len(); + output.extend_from_slice(&lexer.endian.to_bytes(T::zero())); + let start = output.len(); + if !matches!(lexer.token, Some((Token::LParen, _))) { + Err(lexer.error(format!("'(' expected after '{name}'")))? + } + lexer.get()?; + while !matches!(lexer.token, Some((Token::RParen, _))) { + parse_data_item(lexer, output, symbol_table)?; + } + lexer.get()?; + let delta = output.len() - start; + let Ok(delta): Result = delta.try_into() else { + Err(lexer.error(format!("{delta} bytes is too much for '{name}'")))? + }; + let dest = &mut output[old_size..old_size + N]; + dest.copy_from_slice(&lexer.endian.to_bytes(delta)); + Ok(()) +} + +fn put_integers( + lexer: &mut Lexer, + name: &str, + output: &mut Vec, +) -> Result<()> +where + T: Bounded + Display + TryFrom + Copy, + Endian: ToBytes, +{ + println!("put_integers {:?}", lexer.token); + let mut n = 0; + while let Some(integer) = lexer.take_if(|t| match t { + Token::Integer(integer) => Some(*integer), + _ => None, + })? { + println!("got integer {integer}"); + let Ok(integer) = integer.try_into() else { + Err(lexer.error(format!( + "{integer} is not in the valid range [{},{}]", + T::min_value(), + T::max_value() + )))? + }; + output.extend_from_slice(&lexer.endian.to_bytes(integer)); + n += 1; + } + println!("put_integers {:?} {n}", lexer.token); + if n == 0 { + Err(lexer.error(format!("integer expected after '{name}'")))? + } + Ok(()) } +#[derive(PartialEq, Eq, Clone, Debug)] enum Token { Integer(i64), - Float(f64), + Float(OrderedFloat), PcSysmis, String(String), Semicolon, @@ -33,136 +342,292 @@ enum Token { } struct Lexer<'a> { - iter: Peekable>, - token: Option, + input: &'a str, + token: Option<(Token, &'a str)>, + input_file_name: Option<&'a str>, line_number: usize, endian: Endian, } +fn skip_comments(mut s: &str) -> (&str, usize) { + let mut n_newlines = 0; + let s = loop { + s = s.trim_start_matches([' ', '\t', '\r', '<', '>']); + if let Some(remainder) = s.strip_prefix('#') { + let Some((_, remainder)) = remainder.split_once('\n') else { + break ""; + }; + s = remainder; + n_newlines += 1; + } else if let Some(remainder) = s.strip_prefix('\n') { + s = remainder; + n_newlines += 1; + } else { + break s; + } + }; + (s, n_newlines) +} + impl<'a> Lexer<'a> { - fn new(input: &'a str, endian: Endian) -> Result> { + fn new(input: &'a str, input_file_name: Option<&'a str>, endian: Endian) -> Result> { let mut lexer = Lexer { - iter: input.chars().peekable(), + input, token: None, + input_file_name, line_number: 1, endian, }; - lexer.next()?; + lexer.token = lexer.next()?; Ok(lexer) } - fn get(&'a mut self) -> Result> { + fn error(&self, message: String) -> Error { + let repr = self.token.as_ref().map(|(_, repr)| *repr); + Error::new(self.input_file_name, Some(self.line_number), repr, message) + } + fn take(&mut self) -> Result { + let Some(token) = self.token.take() else { + Err(self.error(String::from("unexpected end of input")))? + }; + self.token = self.next()?; + Ok(token.0) + } + fn take_if(&mut self, condition: F) -> Result> + where + F: FnOnce(&Token) -> Option, + { + let Some(ref token) = self.token else { + return Ok(None); + }; + match condition(&token.0) { + Some(value) => { + self.token = self.next()?; + Ok(Some(value)) + } + None => Ok(None), + } + } + fn get(&mut self) -> Result> { if self.token.is_none() { - Err(anyhow!("unexpected end of input")) + Err(self.error(String::from("unexpected end of input")))? } else { self.token = self.next()?; - Ok((&self.token).into()) + match self.token { + Some((ref token, _)) => Ok(Some(token)), + None => Ok(None), + } } } - fn next(&mut self) -> Result> { + fn next(&mut self) -> Result> { // Get the first character of the token, skipping past white space and // comments. - let c = loop { - let Some(c) = self.iter.next() else { - return Ok(None); - }; - let c = if c == '#' { - loop { - match self.iter.next() { - None => return Ok(None), - Some('\n') => break, - _ => (), - } - } - '\n' - } else { - c - }; - if c == '\n' { - self.line_number += 1 - } else if !c.is_whitespace() && c != '<' && c != '>' { - break c; - } - }; + let (s, n_newlines) = skip_comments(self.input); + self.line_number += n_newlines; + self.input = s; - let token = match c { + let start = s; + let mut iter = s.chars(); + let Some(c) = iter.next() else { + return Ok(None); + }; + let (token, rest) = match c { c if c.is_ascii_digit() || c == '-' => { - let mut s = String::from(c); - while let Some(c) = self - .iter - .next_if(|&c| c.is_ascii_digit() || c.is_alphabetic() || c == '.') - { - s.push(c); - } - - if s == "-" { + let len = s + .find(|c: char| { + !(c.is_ascii_digit() || c.is_alphabetic() || c == '.' || c == '-') + }) + .unwrap_or_else(|| s.len()); + let (number, rest) = s.split_at(len); + let token = if number == "-" { Token::Minus - } else if !s.contains('.') { - Token::Integer( - s.parse() - .map_err(|msg| anyhow!("bad integer literal '{s}' ({msg})"))?, - ) + } else if let Some(digits) = number.strip_prefix("0x") { + Token::Integer(i64::from_str_radix(digits, 16).map_err(|msg| { + self.error(format!("bad integer literal '{number}' ({msg})")) + })?) + } else if !number.contains('.') { + Token::Integer(number.parse().map_err(|msg| { + self.error(format!("bad integer literal '{number}' ({msg})")) + })?) } else { - Token::Float( - s.parse() - .map_err(|msg| anyhow!("bad float literal '{s}' ({msg})"))?, - ) - } + Token::Float(number.parse().map_err(|msg| { + self.error(format!("bad float literal '{number}' ({msg})")) + })?) + }; + (token, rest) } '"' => { - let mut s = String::from(c); - loop { - match self.iter.next() { - None => return Err(anyhow!("end-of-file inside string")), - Some('\n') => return Err(anyhow!("new-line inside string")), - Some('"') => break, - Some(c) => s.push(c), - } - } - Token::String(s) + let s = iter.as_str(); + let Some(len) = s.find(['\n', '"']) else { + Err(self.error(String::from("end-of-file inside string")))? + }; + let (string, rest) = s.split_at(len); + let Some(rest) = rest.strip_prefix('"') else { + Err(self.error(format!("new-line inside string ({string}...{rest})")))? + }; + (Token::String(string.into()), rest) } - ';' => Token::Semicolon, - '*' => Token::Asterisk, - '+' => Token::Plus, - '(' => Token::LParen, - ')' => Token::RParen, + ';' => (Token::Semicolon, iter.as_str()), + '*' => (Token::Asterisk, iter.as_str()), + '+' => (Token::Plus, iter.as_str()), + '(' => (Token::LParen, iter.as_str()), + ')' => (Token::RParen, iter.as_str()), c if c.is_alphabetic() || c == '@' || c == '_' => { - let mut s = String::from(c); - while let Some(c) = self - .iter - .next_if(|&c| c.is_ascii_digit() || c.is_alphabetic() || c == '.' || c == '_') - { - s.push(c); - } - if self.iter.next_if_eq(&':').is_some() { - Token::Label(s) - } else if s.starts_with('@') { - Token::At(s) + let len = s + .find(|c: char| { + !(c.is_ascii_digit() + || c.is_alphabetic() + || c == '@' + || c == '.' + || c == '_') + }) + .unwrap_or_else(|| s.len()); + let (s, rest) = s.split_at(len); + if let Some(rest) = rest.strip_prefix(':') { + (Token::Label(s.into()), rest) + } else if let Some(name) = s.strip_prefix('@') { + (Token::At(name.into()), rest) } else if let Some(count) = s.strip_prefix('s') { - Token::S( - count - .parse() - .map_err(|msg| anyhow!("bad counted string '{s}' ({msg})"))?, - ) + let token = + Token::S(count.parse().map_err(|msg| { + self.error(format!("bad counted string '{s}' ({msg})")) + })?); + (token, rest) } else { - match &s[..] { + let token = match &s[..] { "i8" => Token::I8, "i16" => Token::I16, "i64" => Token::I64, - "SYSMIS" => Token::Float(-f64::MAX), + "SYSMIS" => Token::Float(OrderedFloat(-f64::MAX)), "PCSYSMIS" => Token::PcSysmis, - "LOWEST" => Token::Float((-f64::MAX).next_after(0.0)), - "HIGHEST" => Token::Float(f64::MAX), + "LOWEST" => Token::Float((-f64::MAX).next_after(0.0).into()), + "HIGHEST" => Token::Float(f64::MAX.into()), "ENDIAN" => Token::Integer(if self.endian == Endian::Big { 1 } else { 2 }), "COUNT" => Token::Count, "COUNT8" => Token::Count8, "hex" => Token::Hex, - _ => return Err(anyhow!("invalid token '{s}'")), - } + _ => Err(self.error(format!("invalid token '{s}'")))?, + }; + (token, rest) } } - _ => return Err(anyhow!("invalid input byte '{c}'")), + _ => Err(self.error(format!("invalid input byte '{c}'")))?, }; - Ok(Some(token)) + self.input = rest; + let repr = &start[..start.len() - rest.len()]; + println!("{token:?} {repr}"); + Ok(Some((token, repr))) + } +} + +#[cfg(test)] +mod test { + use crate::endian::Endian; + use crate::sack::sack; + use anyhow::Result; + use hexplay::HexView; + + #[test] + fn basic_sack() -> Result<()> { + let input = r#" +"$FL2"; s60 "$(#) SPSS DATA FILE PSPP synthetic test file"; +2; # Layout code +28; # Nominal case size +0; # Not compressed +0; # Not weighted +1; # 1 case. +100.0; # Bias. +"01 Jan 11"; "20:53:52"; +"PSPP synthetic test file: "; i8 244; i8 245; i8 246; i8 248; s34 ""; +i8 0 *3; +"#; + let output = sack(input, None, Endian::Big)?; + HexView::new(&output).print()?; + Ok(()) + } + + #[test] + fn pcp_sack() -> Result<()> { + let input = r#" +# File header. +2; 0; +@MAIN; @MAIN_END - @MAIN; +@VARS; @VARS_END - @VARS; +@LABELS; @LABELS_END - @LABELS; +@DATA; @DATA_END - @DATA; +(0; 0) * 11; +i8 0 * 128; + +MAIN: + i16 1; # Fixed. + s62 "PCSPSS PSPP synthetic test product"; + PCSYSMIS; + 0; 0; i16 1; # Fixed. + i16 0; + i16 15; + 1; + i16 0; # Fixed. + 1; + s8 "11/28/14"; + s8 "15:11:00"; + s64 "PSPP synthetic test file"; +MAIN_END: + +VARS: + 0; 0; 0; 0x050800; s8 "$CASENUM"; PCSYSMIS; + 0; 0; 0; 0x010800; s8 "$DATE"; PCSYSMIS; + 0; 0; 0; 0x050802; s8 "$WEIGHT"; PCSYSMIS; + + # Numeric variable, no label or missing values. + 0; 0; 0; 0x050800; s8 "NUM1"; PCSYSMIS; + + # Numeric variable, variable label. + 0; 0; @NUM2_LABEL - @LABELS_OFS; 0x050800; s8 "NUM2"; PCSYSMIS; + + # Numeric variable with missing value. + 0; 0; 0; 0x050800; s8 "NUM3"; 1.0; + + # Numeric variable, variable label and missing value. + 0; 0; @NUM4_LABEL - @LABELS_OFS; 0x050800; s8 "NUM4"; 2.0; + + # String variable, no label or missing values. + 0; 0; 0; 0x010800; s8 "STR1"; PCSYSMIS; + + # String variable, variable label. + 0; 0; @STR2_LABEL - @LABELS_OFS; 0x010400; s8 "STR2"; PCSYSMIS; + + # String variable with missing value. + 0; 0; 0; 0x010500; s8 "STR3"; s8 "MISS"; + + # String variable, variable label and missing value. + 0; 0; @STR4_LABEL - @LABELS_OFS; 0x010100; s8 "STR4"; s8 "OTHR"; + + # Long string variable + 0; 0; 0; 0x010b00; s8 "STR5"; PCSYSMIS; + 0 * 8; + + # Long string variable with variable label + 0; 0; @STR6_LABEL - @LABELS_OFS; 0x010b00; s8 "STR6"; PCSYSMIS; + 0 * 8; +VARS_END: + +LABELS: + 3; i8 0 0 0; LABELS_OFS: i8 0; + NUM2_LABEL: COUNT8("Numeric variable 2's label"); + NUM4_LABEL: COUNT8("Another numeric variable label"); + STR2_LABEL: COUNT8("STR2's variable label"); + STR4_LABEL: COUNT8("STR4's variable label"); + STR6_LABEL: COUNT8("Another string variable's label"); +LABELS_END: + +DATA: + 0.0; "11/28/14"; 1.0; + 0.0; 1.0; 2.0; PCSYSMIS; s8 "abcdefgh"; s8 "ijkl"; s8 "mnopq"; s8 "r"; + s16 "stuvwxyzAB"; s16 "CDEFGHIJKLM"; +DATA_END: +"#; + let output = sack(input, None, Endian::Big)?; + HexView::new(&output).print()?; + Ok(()) } }