X-Git-Url: https://pintos-os.org/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=rust%2Fsrc%2Fsack.rs;h=ab35f8661e8c4963b1e07380082fae995d4031b0;hb=6165ed413d9aa818e3246d0a063c646dc4efc7e5;hp=41012c467e4eb40a880333cfbeb28a9d2c141b8e;hpb=339b774ab6e5ac5ac64626c0dd30430fa918d925;p=pspp diff --git a/rust/src/sack.rs b/rust/src/sack.rs index 41012c467e..ab35f8661e 100644 --- a/rust/src/sack.rs +++ b/rust/src/sack.rs @@ -1,26 +1,76 @@ -use anyhow::{anyhow, Result}; use float_next_after::NextAfter; use num::{Bounded, Zero}; use ordered_float::OrderedFloat; use std::{ collections::{hash_map::Entry, HashMap}, - fmt::Display, - iter::{repeat, Peekable}, - str::Chars, + error::Error as StdError, + fmt::{Display, Formatter, Result as FmtResult}, + iter::repeat, }; use crate::endian::{Endian, ToBytes}; -pub fn sack(input: &str, endian: Endian) -> Result> { +pub type Result = std::result::Result; + +#[derive(Debug)] +pub struct Error { + pub file_name: Option, + pub line_number: Option, + pub token: Option, + pub message: String, +} + +impl Error { + fn new( + file_name: Option<&str>, + line_number: Option, + token: Option<&str>, + message: String, + ) -> Error { + Error { + file_name: file_name.map(String::from), + line_number, + token: token.map(String::from), + message, + } + } +} + +impl StdError for Error {} + +impl Display for Error { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + match (self.file_name.as_ref(), self.line_number) { + (Some(ref file_name), Some(line_number)) => write!(f, "{file_name}:{line_number}: ")?, + (Some(ref file_name), None) => write!(f, "{file_name}: ")?, + (None, Some(line_number)) => write!(f, "line {line_number}: ")?, + (None, None) => (), + } + if let Some(ref token) = self.token { + write!(f, "at '{token}': ")?; + } + write!(f, "{}", self.message) + } +} + +pub fn sack(input: &str, input_file_name: Option<&str>, endian: Endian) -> Result> { let mut symbol_table = HashMap::new(); - let output = _sack(input, endian, &mut symbol_table)?; + let output = _sack(input, input_file_name, endian, &mut symbol_table)?; let output = if !symbol_table.is_empty() { + for (k, v) in symbol_table.iter() { + println!("{k} => {v:?}"); + } for (k, v) in symbol_table.iter() { if v.is_none() { - return Err(anyhow!("label {k} used but never defined")); + Err(Error::new( + input_file_name, + None, + None, + format!("label {k} used but never defined"), + ))? } } - _sack(input, endian, &mut symbol_table)? + _sack(input, input_file_name, endian, &mut symbol_table)? } else { output }; @@ -29,10 +79,11 @@ pub fn sack(input: &str, endian: Endian) -> Result> { fn _sack( input: &str, + input_file_name: Option<&str>, endian: Endian, symbol_table: &mut HashMap>, ) -> Result> { - let mut lexer = Lexer::new(input, endian)?; + let mut lexer = Lexer::new(input, input_file_name, endian)?; let mut output = Vec::new(); while parse_data_item(&mut lexer, &mut output, symbol_table)? {} Ok(output) @@ -49,7 +100,19 @@ fn parse_data_item( let initial_len = output.len(); match lexer.take()? { - Token::Integer(integer) => output.extend_from_slice(&lexer.endian.to_bytes(integer)), + Token::Integer(integer) => { + if let Ok(integer) = TryInto::::try_into(integer) { + output.extend_from_slice(&lexer.endian.to_bytes(integer)); + } else if let Ok(integer) = TryInto::::try_into(integer) { + output.extend_from_slice(&lexer.endian.to_bytes(integer)); + } else { + Err(lexer.error(format!( + "{integer} is not in the valid range [{},{}]", + i32::min_value(), + u32::max_value() + )))?; + }; + } Token::Float(float) => output.extend_from_slice(&lexer.endian.to_bytes(float.0)), Token::PcSysmis => { output.extend_from_slice(&[0xf5, 0x1e, 0x26, 0x02, 0x8a, 0x8c, 0xed, 0xff]) @@ -59,21 +122,21 @@ fn parse_data_item( Token::I64 => put_integers::(lexer, "i64", output)?, Token::String(string) => output.extend_from_slice(string.as_bytes()), Token::S(size) => { - let Some(Token::String(ref string)) = lexer.token else { - return Err(anyhow!("string expected after 's{size}'")); + let Some((Token::String(ref string), _)) = lexer.token else { + Err(lexer.error(format!("string expected after 's{size}'")))? }; let len = string.len(); if len > size { - return Err(anyhow!( + Err(lexer.error(format!( "{len}-byte string is longer than pad length {size}" - )); + )))? } output.extend_from_slice(string.as_bytes()); output.extend(repeat(b' ').take(size - len)); lexer.get()?; } Token::LParen => { - while lexer.token != Some(Token::RParen) { + while !matches!(lexer.token, Some((Token::RParen, _))) { parse_data_item(lexer, output, symbol_table)?; } lexer.get()?; @@ -81,81 +144,101 @@ fn parse_data_item( Token::Count => put_counted_items::(lexer, "COUNT", output, symbol_table)?, Token::Count8 => put_counted_items::(lexer, "COUNT8", output, symbol_table)?, Token::Hex => { - let Some(Token::String(ref string)) = lexer.token else { - return Err(anyhow!("string expected after 'hex'")); + let Some((Token::String(ref string), _)) = lexer.token else { + Err(lexer.error(String::from("string expected after 'hex'")))? }; - let mut i = string.chars(); + let mut string = &string[..]; loop { + string = string.trim_start(); + if string.is_empty() { + break; + }; + + let mut i = string.chars(); let Some(c0) = i.next() else { return Ok(true) }; let Some(c1) = i.next() else { - return Err(anyhow!("hex string has odd number of characters")); + Err(lexer.error(String::from("hex string has odd number of characters")))? }; + let (Some(digit0), Some(digit1)) = (c0.to_digit(16), c1.to_digit(16)) else { - return Err(anyhow!("invalid digit in hex string")); + Err(lexer.error(String::from("invalid digit in hex string")))? }; let byte = digit0 * 16 + digit1; output.push(byte as u8); + + string = i.as_str(); } + lexer.get()?; } Token::Label(name) => { + println!("define {name}"); let value = output.len() as u32; - match symbol_table.entry(name) { + match symbol_table.entry(name.clone()) { Entry::Vacant(v) => { v.insert(Some(value)); } - Entry::Occupied(o) => { - if let Some(v) = o.get() { - if *v != value { - return Err(anyhow!("syntax error")); + Entry::Occupied(mut o) => { + match o.get() { + Some(v) => { + if *v != value { + Err(lexer.error(format!("{name}: can't redefine label for offset {:#x} with offset {:#x}", *v, value)))? + } } + None => drop(o.insert(Some(value))), } } }; + return Ok(true); } Token::At(name) => { - let mut value = symbol_table.entry(name).or_insert(None).unwrap_or(0); - lexer.get()?; + let mut value = *symbol_table.entry(name.clone()).or_insert(None); loop { let plus = match lexer.token { - Some(Token::Plus) => true, - Some(Token::Minus) => false, + Some((Token::Plus, _)) => true, + Some((Token::Minus, _)) => false, _ => break, }; lexer.get()?; let operand = match lexer.token { - Some(Token::At(ref name)) => if let Some(value) = symbol_table.get(name) { - *value - } else { - symbol_table.insert(name.clone(), None); - None + Some((Token::At(ref name), _)) => { + *symbol_table.entry(name.clone()).or_insert(None) } - .unwrap_or(0), - Some(Token::Integer(integer)) => integer - .try_into() - .map_err(|msg| anyhow!("bad offset literal ({msg})"))?, - _ => return Err(anyhow!("expecting @label or integer literal")), + Some((Token::Integer(integer), _)) => Some( + integer + .try_into() + .map_err(|msg| lexer.error(format!("bad offset literal ({msg})")))?, + ), + _ => Err(lexer.error(String::from("expecting @label or integer literal")))?, }; lexer.get()?; - value = if plus { - value.checked_add(operand) - } else { - value.checked_sub(operand) - } - .ok_or_else(|| anyhow!("overflow in offset arithmetic"))?; + value = match (value, operand) { + (Some(a), Some(b)) => Some( + if plus { + a.checked_add(b) + } else { + a.checked_sub(b) + } + .ok_or_else(|| { + lexer.error(String::from("overflow in offset arithmetic")) + })?, + ), + _ => None, + }; } + let value = value.unwrap_or(0); output.extend_from_slice(&lexer.endian.to_bytes(value)); } _ => (), }; - if lexer.token == Some(Token::Asterisk) { + if let Some((Token::Asterisk, _)) = lexer.token { lexer.get()?; let Token::Integer(count) = lexer.take()? else { - return Err(anyhow!("positive integer expected after '*'")); + Err(lexer.error(String::from("positive integer expected after '*'")))? }; if count < 1 { - return Err(anyhow!("positive integer expected after '*'")); + Err(lexer.error(String::from("positive integer expected after '*'")))? }; let final_len = output.len(); for _ in 1..count { @@ -163,11 +246,11 @@ fn parse_data_item( } } match lexer.token { - Some(Token::Semicolon) => { + Some((Token::Semicolon, _)) => { lexer.get()?; } - Some(Token::RParen) => (), - _ => return Err(anyhow!("';' expected")), + Some((Token::RParen, _)) => (), + _ => Err(lexer.error(String::from("';' expected")))?, } Ok(true) } @@ -184,17 +267,18 @@ where { let old_size = output.len(); output.extend_from_slice(&lexer.endian.to_bytes(T::zero())); - if lexer.token != Some(Token::LParen) { - return Err(anyhow!("'(' expected after '{name}'")); + let start = output.len(); + if !matches!(lexer.token, Some((Token::LParen, _))) { + Err(lexer.error(format!("'(' expected after '{name}'")))? } lexer.get()?; - while lexer.token != Some(Token::RParen) { + while !matches!(lexer.token, Some((Token::RParen, _))) { parse_data_item(lexer, output, symbol_table)?; } lexer.get()?; - let delta = output.len() - old_size; + let delta = output.len() - start; let Ok(delta): Result = delta.try_into() else { - return Err(anyhow!("{delta} bytes is too much for '{name}'")); + Err(lexer.error(format!("{delta} bytes is too much for '{name}'")))? }; let dest = &mut output[old_size..old_size + N]; dest.copy_from_slice(&lexer.endian.to_bytes(delta)); @@ -210,28 +294,31 @@ where T: Bounded + Display + TryFrom + Copy, Endian: ToBytes, { + println!("put_integers {:?}", lexer.token); let mut n = 0; while let Some(integer) = lexer.take_if(|t| match t { Token::Integer(integer) => Some(*integer), _ => None, })? { + println!("got integer {integer}"); let Ok(integer) = integer.try_into() else { - return Err(anyhow!( + Err(lexer.error(format!( "{integer} is not in the valid range [{},{}]", T::min_value(), T::max_value() - )); + )))? }; output.extend_from_slice(&lexer.endian.to_bytes(integer)); n += 1; } + println!("put_integers {:?} {n}", lexer.token); if n == 0 { - return Err(anyhow!("integer expected after '{name}'")); + Err(lexer.error(format!("integer expected after '{name}'")))? } Ok(()) } -#[derive(PartialEq, Eq, Clone)] +#[derive(PartialEq, Eq, Clone, Debug)] enum Token { Integer(i64), Float(OrderedFloat), @@ -255,29 +342,55 @@ enum Token { } struct Lexer<'a> { - iter: Peekable>, - token: Option, + input: &'a str, + token: Option<(Token, &'a str)>, + input_file_name: Option<&'a str>, line_number: usize, endian: Endian, } +fn skip_comments(mut s: &str) -> (&str, usize) { + let mut n_newlines = 0; + let s = loop { + s = s.trim_start_matches([' ', '\t', '\r', '<', '>']); + if let Some(remainder) = s.strip_prefix('#') { + let Some((_, remainder)) = remainder.split_once('\n') else { + break ""; + }; + s = remainder; + n_newlines += 1; + } else if let Some(remainder) = s.strip_prefix('\n') { + s = remainder; + n_newlines += 1; + } else { + break s; + } + }; + (s, n_newlines) +} + impl<'a> Lexer<'a> { - fn new(input: &'a str, endian: Endian) -> Result> { + fn new(input: &'a str, input_file_name: Option<&'a str>, endian: Endian) -> Result> { let mut lexer = Lexer { - iter: input.chars().peekable(), + input, token: None, + input_file_name, line_number: 1, endian, }; - lexer.next()?; + lexer.token = lexer.next()?; Ok(lexer) } + fn error(&self, message: String) -> Error { + let repr = self.token.as_ref().map(|(_, repr)| *repr); + Error::new(self.input_file_name, Some(self.line_number), repr, message) + } fn take(&mut self) -> Result { let Some(token) = self.token.take() else { - return Err(anyhow!("unexpected end of input")); + Err(self.error(String::from("unexpected end of input")))? }; self.token = self.next()?; - Ok(token) + Ok(token.0) } fn take_if(&mut self, condition: F) -> Result> where @@ -286,7 +399,7 @@ impl<'a> Lexer<'a> { let Some(ref token) = self.token else { return Ok(None); }; - match condition(token) { + match condition(&token.0) { Some(value) => { self.token = self.next()?; Ok(Some(value)) @@ -296,100 +409,92 @@ impl<'a> Lexer<'a> { } fn get(&mut self) -> Result> { if self.token.is_none() { - Err(anyhow!("unexpected end of input")) + Err(self.error(String::from("unexpected end of input")))? } else { self.token = self.next()?; - Ok((&self.token).into()) + match self.token { + Some((ref token, _)) => Ok(Some(token)), + None => Ok(None), + } } } - fn next(&mut self) -> Result> { + fn next(&mut self) -> Result> { // Get the first character of the token, skipping past white space and // comments. - let c = loop { - let Some(c) = self.iter.next() else { - return Ok(None); - }; - let c = if c == '#' { - loop { - match self.iter.next() { - None => return Ok(None), - Some('\n') => break, - _ => (), - } - } - '\n' - } else { - c - }; - if c == '\n' { - self.line_number += 1 - } else if !c.is_whitespace() && c != '<' && c != '>' { - break c; - } - }; + let (s, n_newlines) = skip_comments(self.input); + self.line_number += n_newlines; + self.input = s; - let token = match c { + let start = s; + let mut iter = s.chars(); + let Some(c) = iter.next() else { + return Ok(None); + }; + let (token, rest) = match c { c if c.is_ascii_digit() || c == '-' => { - let mut s = String::from(c); - while let Some(c) = self - .iter - .next_if(|&c| c.is_ascii_digit() || c.is_alphabetic() || c == '.') - { - s.push(c); - } - - if s == "-" { + let len = s + .find(|c: char| { + !(c.is_ascii_digit() || c.is_alphabetic() || c == '.' || c == '-') + }) + .unwrap_or_else(|| s.len()); + let (number, rest) = s.split_at(len); + let token = if number == "-" { Token::Minus - } else if !s.contains('.') { - Token::Integer( - s.parse() - .map_err(|msg| anyhow!("bad integer literal '{s}' ({msg})"))?, - ) + } else if let Some(digits) = number.strip_prefix("0x") { + Token::Integer(i64::from_str_radix(digits, 16).map_err(|msg| { + self.error(format!("bad integer literal '{number}' ({msg})")) + })?) + } else if !number.contains('.') { + Token::Integer(number.parse().map_err(|msg| { + self.error(format!("bad integer literal '{number}' ({msg})")) + })?) } else { - Token::Float( - s.parse() - .map_err(|msg| anyhow!("bad float literal '{s}' ({msg})"))?, - ) - } + Token::Float(number.parse().map_err(|msg| { + self.error(format!("bad float literal '{number}' ({msg})")) + })?) + }; + (token, rest) } '"' => { - let mut s = String::from(c); - loop { - match self.iter.next() { - None => return Err(anyhow!("end-of-file inside string")), - Some('\n') => return Err(anyhow!("new-line inside string")), - Some('"') => break, - Some(c) => s.push(c), - } - } - Token::String(s) + let s = iter.as_str(); + let Some(len) = s.find(['\n', '"']) else { + Err(self.error(String::from("end-of-file inside string")))? + }; + let (string, rest) = s.split_at(len); + let Some(rest) = rest.strip_prefix('"') else { + Err(self.error(format!("new-line inside string ({string}...{rest})")))? + }; + (Token::String(string.into()), rest) } - ';' => Token::Semicolon, - '*' => Token::Asterisk, - '+' => Token::Plus, - '(' => Token::LParen, - ')' => Token::RParen, + ';' => (Token::Semicolon, iter.as_str()), + '*' => (Token::Asterisk, iter.as_str()), + '+' => (Token::Plus, iter.as_str()), + '(' => (Token::LParen, iter.as_str()), + ')' => (Token::RParen, iter.as_str()), c if c.is_alphabetic() || c == '@' || c == '_' => { - let mut s = String::from(c); - while let Some(c) = self - .iter - .next_if(|&c| c.is_ascii_digit() || c.is_alphabetic() || c == '.' || c == '_') - { - s.push(c); - } - if self.iter.next_if_eq(&':').is_some() { - Token::Label(s) - } else if s.starts_with('@') { - Token::At(s) + let len = s + .find(|c: char| { + !(c.is_ascii_digit() + || c.is_alphabetic() + || c == '@' + || c == '.' + || c == '_') + }) + .unwrap_or_else(|| s.len()); + let (s, rest) = s.split_at(len); + if let Some(rest) = rest.strip_prefix(':') { + (Token::Label(s.into()), rest) + } else if let Some(name) = s.strip_prefix('@') { + (Token::At(name.into()), rest) } else if let Some(count) = s.strip_prefix('s') { - Token::S( - count - .parse() - .map_err(|msg| anyhow!("bad counted string '{s}' ({msg})"))?, - ) + let token = + Token::S(count.parse().map_err(|msg| { + self.error(format!("bad counted string '{s}' ({msg})")) + })?); + (token, rest) } else { - match &s[..] { + let token = match &s[..] { "i8" => Token::I8, "i16" => Token::I16, "i64" => Token::I64, @@ -401,12 +506,128 @@ impl<'a> Lexer<'a> { "COUNT" => Token::Count, "COUNT8" => Token::Count8, "hex" => Token::Hex, - _ => return Err(anyhow!("invalid token '{s}'")), - } + _ => Err(self.error(format!("invalid token '{s}'")))?, + }; + (token, rest) } } - _ => return Err(anyhow!("invalid input byte '{c}'")), + _ => Err(self.error(format!("invalid input byte '{c}'")))?, }; - Ok(Some(token)) + self.input = rest; + let repr = &start[..start.len() - rest.len()]; + println!("{token:?} {repr}"); + Ok(Some((token, repr))) + } +} + +#[cfg(test)] +mod test { + use crate::endian::Endian; + use crate::sack::sack; + use anyhow::Result; + use hexplay::HexView; + + #[test] + fn basic_sack() -> Result<()> { + let input = r#" +"$FL2"; s60 "$(#) SPSS DATA FILE PSPP synthetic test file"; +2; # Layout code +28; # Nominal case size +0; # Not compressed +0; # Not weighted +1; # 1 case. +100.0; # Bias. +"01 Jan 11"; "20:53:52"; +"PSPP synthetic test file: "; i8 244; i8 245; i8 246; i8 248; s34 ""; +i8 0 *3; +"#; + let output = sack(input, None, Endian::Big)?; + HexView::new(&output).print()?; + Ok(()) + } + + #[test] + fn pcp_sack() -> Result<()> { + let input = r#" +# File header. +2; 0; +@MAIN; @MAIN_END - @MAIN; +@VARS; @VARS_END - @VARS; +@LABELS; @LABELS_END - @LABELS; +@DATA; @DATA_END - @DATA; +(0; 0) * 11; +i8 0 * 128; + +MAIN: + i16 1; # Fixed. + s62 "PCSPSS PSPP synthetic test product"; + PCSYSMIS; + 0; 0; i16 1; # Fixed. + i16 0; + i16 15; + 1; + i16 0; # Fixed. + 1; + s8 "11/28/14"; + s8 "15:11:00"; + s64 "PSPP synthetic test file"; +MAIN_END: + +VARS: + 0; 0; 0; 0x050800; s8 "$CASENUM"; PCSYSMIS; + 0; 0; 0; 0x010800; s8 "$DATE"; PCSYSMIS; + 0; 0; 0; 0x050802; s8 "$WEIGHT"; PCSYSMIS; + + # Numeric variable, no label or missing values. + 0; 0; 0; 0x050800; s8 "NUM1"; PCSYSMIS; + + # Numeric variable, variable label. + 0; 0; @NUM2_LABEL - @LABELS_OFS; 0x050800; s8 "NUM2"; PCSYSMIS; + + # Numeric variable with missing value. + 0; 0; 0; 0x050800; s8 "NUM3"; 1.0; + + # Numeric variable, variable label and missing value. + 0; 0; @NUM4_LABEL - @LABELS_OFS; 0x050800; s8 "NUM4"; 2.0; + + # String variable, no label or missing values. + 0; 0; 0; 0x010800; s8 "STR1"; PCSYSMIS; + + # String variable, variable label. + 0; 0; @STR2_LABEL - @LABELS_OFS; 0x010400; s8 "STR2"; PCSYSMIS; + + # String variable with missing value. + 0; 0; 0; 0x010500; s8 "STR3"; s8 "MISS"; + + # String variable, variable label and missing value. + 0; 0; @STR4_LABEL - @LABELS_OFS; 0x010100; s8 "STR4"; s8 "OTHR"; + + # Long string variable + 0; 0; 0; 0x010b00; s8 "STR5"; PCSYSMIS; + 0 * 8; + + # Long string variable with variable label + 0; 0; @STR6_LABEL - @LABELS_OFS; 0x010b00; s8 "STR6"; PCSYSMIS; + 0 * 8; +VARS_END: + +LABELS: + 3; i8 0 0 0; LABELS_OFS: i8 0; + NUM2_LABEL: COUNT8("Numeric variable 2's label"); + NUM4_LABEL: COUNT8("Another numeric variable label"); + STR2_LABEL: COUNT8("STR2's variable label"); + STR4_LABEL: COUNT8("STR4's variable label"); + STR6_LABEL: COUNT8("Another string variable's label"); +LABELS_END: + +DATA: + 0.0; "11/28/14"; 1.0; + 0.0; 1.0; 2.0; PCSYSMIS; s8 "abcdefgh"; s8 "ijkl"; s8 "mnopq"; s8 "r"; + s16 "stuvwxyzAB"; s16 "CDEFGHIJKLM"; +DATA_END: +"#; + let output = sack(input, None, Endian::Big)?; + HexView::new(&output).print()?; + Ok(()) } }