+++ /dev/null
-#![allow(dead_code)]
-use std::{fmt::Write, ops::RangeFrom, sync::OnceLock};
-
-use flagset::{flags, FlagSet};
-use pspp_derive::FromTokens;
-
-use crate::{
- identifier::Identifier,
- integer::ToInteger,
- lex::{
- command_name::CommandMatcher,
- lexer::{LexToken, TokenSlice},
- token::{Punct, Token},
- },
- message::{Diagnostic, Diagnostics},
-};
-
-flags! {
- enum State: u8 {
- /// No active dataset yet defined.
- Initial,
-
- /// Active dataset has been defined.
- Data,
-
- /// Inside `INPUT PROGRAM`.
- InputProgram,
-
- /// Inside `FILE TYPE`.
- FileType,
-
- /// State nested inside `LOOP` or `DO IF`, inside [State::Data].
- NestedData,
-
- /// State nested inside `LOOP` or `DO IF`, inside [State::InputProgram].
- NestedInputProgram,
- }
-}
-
-struct Command {
- allowed_states: FlagSet<State>,
- enhanced_only: bool,
- testing_only: bool,
- no_abbrev: bool,
- name: &'static str,
- run: Box<dyn Fn(&mut Context) + Send + Sync>, //-> Box<dyn ParsedCommand> + Send + Sync>,
-}
-
-#[derive(Debug)]
-enum ParseError {
- Error(Diagnostics),
- Mismatch(Diagnostics),
-}
-
-#[derive(Debug)]
-struct Parsed<'a, T> {
- value: T,
- rest: TokenSlice<'a>,
- diagnostics: Diagnostics,
-}
-
-impl<'a, T> Parsed<'a, T> {
- pub fn new(value: T, rest: TokenSlice<'a>, warnings: Diagnostics) -> Self {
- Self {
- value,
- rest: rest,
- diagnostics: warnings,
- }
- }
- pub fn ok(value: T, rest: TokenSlice<'a>) -> Self {
- Self {
- value,
- rest: rest,
- diagnostics: Diagnostics::default(),
- }
- }
- pub fn into_tuple(self) -> (T, TokenSlice<'a>, Diagnostics) {
- (self.value, self.rest, self.diagnostics)
- }
- pub fn map<F, R>(self, f: F) -> Parsed<'a, R>
- where
- F: FnOnce(T) -> R,
- {
- Parsed {
- value: f(self.value),
- rest: self.rest,
- diagnostics: self.diagnostics,
- }
- }
- pub fn warn(self, mut warnings: Diagnostics) -> Self {
- Self {
- value: self.value,
- rest: self.rest,
- diagnostics: {
- let mut vec = self.diagnostics.0;
- vec.append(&mut warnings.0);
- Diagnostics(vec)
- },
- }
- }
-}
-
-type ParseResult<'a, T> = Result<Parsed<'a, T>, ParseError>;
-
-trait MismatchToError {
- fn mismatch_to_error(self) -> Self;
-}
-
-impl<'a, T> MismatchToError for ParseResult<'a, T> {
- fn mismatch_to_error(self) -> Self {
- match self {
- Err(ParseError::Mismatch(diagnostic)) => Err(ParseError::Error(diagnostic)),
- rest => rest,
- }
- }
-}
-
-trait FromTokens<'a> {
- fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
- where
- Self: Sized;
-}
-
-impl<'a, T> FromTokens<'a> for Option<T>
-where
- T: FromTokens<'a>,
-{
- fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
- where
- Self: Sized,
- {
- match T::from_tokens(input) {
- Ok(p) => Ok(p.map(Some)),
- Err(ParseError::Mismatch(_)) => Ok(Parsed::ok(None, input)),
- Err(ParseError::Error(error)) => Err(ParseError::Error(error)),
- }
- }
-}
-
-impl<'a, T> FromTokens<'a> for Vec<T>
-where
- T: FromTokens<'a>,
-{
- fn from_tokens(mut input: TokenSlice<'a>) -> ParseResult<'a, Self>
- where
- Self: Sized,
- {
- let mut values_vec = Vec::new();
- let mut warnings_vec = Vec::new();
- loop {
- match T::from_tokens(input) {
- Ok(Parsed {
- value,
- rest,
- diagnostics: mut warnings,
- }) => {
- values_vec.push(value);
- warnings_vec.append(&mut warnings.0);
- input = rest;
- }
- Err(ParseError::Mismatch(_)) => break,
- Err(ParseError::Error(e)) => return Err(ParseError::Error(e)),
- }
- }
- Ok(Parsed {
- value: values_vec,
- rest: input,
- diagnostics: Diagnostics(warnings_vec),
- })
- }
-}
-
-impl<'a> FromTokens<'a> for TokenSlice<'a> {
- fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
- where
- Self: Sized,
- {
- Ok(Parsed::ok(input, input.end()))
- }
-}
-
-#[derive(Debug, FromTokens)]
-#[pspp(add_lifetime)]
-struct Descriptives<'a> {
- subcommands: Vec<Subcommand<DescriptivesSubcommand<'a>>>,
-}
-
-#[derive(Debug)]
-struct Subcommand<T>(pub T);
-
-impl<'a, T> FromTokens<'a> for Subcommand<T>
-where
- T: FromTokens<'a>,
-{
- fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
- where
- Self: Sized,
- {
- let start = input.skip_until(|token| token != &Token::Punct(Punct::Slash));
- if start.is_empty() {
- return Err(ParseError::Error(
- input.error("Syntax error at end of input.").into(),
- ));
- }
- let end = start.skip_to(&Token::Punct(Punct::Slash));
- let subcommand = start.subslice(0..start.len() - end.len());
- let (value, rest, mut warnings) = T::from_tokens(subcommand)?.into_tuple();
- if !rest.is_empty() {
- warnings
- .0
- .push(rest.warning("Syntax error expecting end of subcommand."));
- }
- Ok(Parsed::new(Self(value), end, warnings))
- }
-}
-
-#[derive(Debug, FromTokens)]
-#[pspp(add_lifetime, required_equals)]
-enum DescriptivesSubcommand<'a> {
- #[pspp(default)]
- Variables(Vec<DescriptivesVars<'a>>),
- Missing(Vec<Missing>),
- Save,
- Statistics(Vec<Statistic>),
- Sort(Sort),
- Format(Vec<Format>),
-}
-
-#[derive(Debug, FromTokens)]
-enum Missing {
- Variable,
- Listwise,
- Include,
-}
-
-#[derive(Debug, FromTokens)]
-enum Format {
- Labels,
- NoLabels,
- Index,
- NoIndex,
- Line,
- Serial,
-}
-
-#[derive(Debug, FromTokens)]
-#[pspp(add_lifetime)]
-struct DescriptivesVars<'a> {
- vars: Vars<'a>,
- z_name: Option<InParens<&'a Identifier>>,
-}
-
-#[derive(Debug)]
-struct InParens<T>(pub T);
-
-impl<'a, T> FromTokens<'a> for InParens<T>
-where
- T: FromTokens<'a>,
-{
- fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
- where
- Self: Sized,
- {
- let ((), rest, _) = parse_token(input, &Token::Punct(Punct::LParen))?.into_tuple();
- let (value, rest, warnings) = T::from_tokens(rest)?.into_tuple();
- let ((), rest, _) = parse_token(rest, &Token::Punct(Punct::RParen))?.into_tuple();
- Ok(Parsed {
- value: Self(value),
- rest,
- diagnostics: warnings,
- })
- }
-}
-
-fn parse_token<'a>(input: TokenSlice<'a>, token: &Token) -> ParseResult<'a, ()> {
- if let Some(rest) = input.skip(token) {
- Ok(Parsed::ok((), rest))
- } else {
- Err(ParseError::Mismatch(
- input.error(format!("expecting {token}")).into(),
- ))
- }
-}
-
-fn parse_keyword<'a>(input: TokenSlice<'a>, keyword: &str) -> ParseResult<'a, ()> {
- if let Some(rest) = input.skip_if(|token| token.matches_keyword(keyword)) {
- Ok(Parsed::ok((), rest))
- } else {
- Err(ParseError::Mismatch(
- input.error(format!("expecting {keyword}")).into(),
- ))
- }
-}
-
-#[derive(Debug)]
-enum Vars<'a> {
- Single(&'a Identifier),
- Range(&'a Identifier, &'a Identifier),
- All,
-}
-
-impl<'a> FromTokens<'a> for Vars<'a> {
- fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
- where
- Self: Sized,
- {
- if let Ok(Parsed { rest, .. }) = parse_token(input, &Token::Punct(Punct::All)) {
- Ok(Parsed::ok(Self::All, rest))
- } else {
- let (from, rest, _) = parse_id(input)?.into_tuple();
- if let Ok(Parsed { rest, .. }) = parse_token(rest, &Token::Punct(Punct::To)) {
- if let Ok(p) = parse_id(rest) {
- return Ok(p.map(|to| Self::Range(from, to)));
- }
- }
- Ok(Parsed::ok(Self::Single(from), rest))
- }
- }
-}
-
-fn parse_id<'a>(input: TokenSlice<'a>) -> ParseResult<'a, &'a Identifier> {
- let mut iter = input.iter();
- if let Some(LexToken {
- token: Token::Id(id),
- ..
- }) = iter.next()
- {
- Ok(Parsed::ok(id, iter.remainder()))
- } else {
- Err(ParseError::Mismatch(
- input.error("Syntax error expecting identifier.").into(),
- ))
- }
-}
-
-impl<'a> FromTokens<'a> for &'a Identifier {
- fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
- where
- Self: Sized,
- {
- parse_id(input)
- }
-}
-
-#[derive(Debug, FromTokens)]
-struct Sort {
- key: SortKey,
- direction: Option<Direction>,
-}
-
-#[derive(Debug, FromTokens)]
-enum SortKey {
- Mean,
- SMean,
- Stddev,
- Variance,
- Range,
- Min,
- Max,
- Sum,
- Skewness,
- Kurtosis,
- Name,
-}
-
-#[derive(Debug, FromTokens)]
-enum Direction {
- #[pspp(syntax = "(A)")]
- Ascending,
- #[pspp(syntax = "(D)")]
- Descending,
-}
-
-#[derive(Debug, FromTokens)]
-enum Statistic {
- Default,
- Mean,
- SeMean,
- Stddev,
- Variance,
- Range,
- Sum,
- Min,
- Max,
- Skewness,
- Kurtosis,
- All,
-}
-
-fn collect_subcommands<'a>(src: &'a TokenSlice) -> Vec<TokenSlice<'a>> {
- src.split(|token| token.token == Token::Punct(Punct::Slash))
- .filter(|slice| !slice.is_empty())
- .collect()
-}
-
-fn commands() -> &'static [Command] {
- fn new_commands() -> Vec<Command> {
- vec![
- Command {
- allowed_states: FlagSet::full(),
- enhanced_only: false,
- testing_only: false,
- no_abbrev: false,
- name: "DESCRIPTIVES",
- run: Box::new(|context| {
- let mut input = context.lexer;
- while !input.is_empty() {
- match <Subcommand<DescriptivesSubcommand>>::from_tokens(input) {
- Ok(Parsed {
- value: subcommand,
- rest,
- diagnostics,
- }) => {
- println!("\n{subcommand:?}");
- //println!("rest: {rest:?}");
- println!("warnings: {diagnostics:?}");
- //println!("{:?}", DescriptivesSubcommand::from_tokens(subcommand.0));
- input = rest;
- }
- Err(error) => {
- println!("{error:?}");
- break;
- }
- }
- }
- }),
- },
- Command {
- allowed_states: FlagSet::full(),
- enhanced_only: false,
- testing_only: false,
- no_abbrev: false,
- name: "ECHO",
- run: Box::new(|_context| todo!()),
- },
- ]
- }
-
- static COMMANDS: OnceLock<Vec<Command>> = OnceLock::new();
- COMMANDS.get_or_init(|| new_commands()).as_slice()
-}
-
-fn parse_command_word(lexer: &mut TokenSlice, s: &mut String, n: usize) -> bool {
- let separator = match s.chars().next_back() {
- Some(c) if c != '-' => " ",
- _ => "",
- };
-
- match lexer.get_token(n) {
- Some(Token::Punct(Punct::Dash)) => {
- s.push('-');
- true
- }
- Some(Token::Id(id)) => {
- write!(s, "{separator}{id}").unwrap();
- true
- }
- Some(Token::Number(number)) if number.is_sign_positive() => {
- if let Some(integer) = number.to_exact_usize() {
- write!(s, "{separator}{integer}").unwrap();
- true
- } else {
- false
- }
- }
- _ => false,
- }
-}
-
-fn find_best_match(s: &str) -> (Option<&'static Command>, isize) {
- let mut cm = CommandMatcher::new(s);
- for command in commands() {
- cm.add(command.name, command);
- }
- cm.get_match()
-}
-
-fn parse_command_name(
- lexer: &mut TokenSlice,
- error: &Box<dyn Fn(Diagnostic)>,
-) -> Result<(&'static Command, usize), ()> {
- let mut s = String::new();
- let mut word = 0;
- let mut missing_words = 0;
- let mut command = None;
- while parse_command_word(lexer, &mut s, word) {
- (command, missing_words) = find_best_match(&s);
- if missing_words <= 0 {
- break;
- }
- word += 1;
- }
- if command.is_none() && missing_words > 0 {
- s.push_str(" .");
- (command, missing_words) = find_best_match(&s);
- s.truncate(s.len() - 2);
- }
-
- match command {
- Some(command) => Ok((command, ((word as isize + 1) + missing_words) as usize)),
- None => {
- if word == 0 {
- error(
- lexer
- .subslice(0..1)
- .error("Syntax error expecting command name"),
- )
- } else {
- error(lexer.subslice(0..word + 1).error("Unknown command `{s}`."))
- };
- Err(())
- }
- }
-}
-
-pub enum Success {
- Success,
- Eof,
- Finish,
-}
-
-pub fn end_of_command(context: &Context, range: RangeFrom<usize>) -> Result<Success, ()> {
- match context.lexer.get_token(range.start) {
- None | Some(Token::End) => Ok(Success::Success),
- _ => {
- context.error(
- context
- .lexer
- .subslice(range.start..context.lexer.len())
- .error("Syntax error expecting end of command."),
- );
- Err(())
- }
- }
-}
-
-fn parse_in_state(mut lexer: TokenSlice, error: &Box<dyn Fn(Diagnostic)>, _state: State) {
- match lexer.get_token(0) {
- None | Some(Token::End) => (),
- _ => match parse_command_name(&mut lexer, error) {
- Ok((command, n_tokens)) => {
- let mut context = Context {
- error,
- lexer: lexer.subslice(n_tokens..lexer.len()),
- command_name: Some(command.name),
- };
- (command.run)(&mut context);
- }
- Err(error) => println!("{error:?}"),
- },
- }
-}
-
-pub fn parse_command(lexer: TokenSlice, error: &Box<dyn Fn(Diagnostic)>) {
- parse_in_state(lexer, error, State::Initial)
-}
-
-pub struct Context<'a> {
- error: &'a Box<dyn Fn(Diagnostic)>,
- lexer: TokenSlice<'a>,
- command_name: Option<&'static str>,
-}
-
-impl<'a> Context<'a> {
- pub fn error(&self, diagnostic: Diagnostic) {
- (self.error)(diagnostic);
- }
-}
-
-#[cfg(test)]
-mod tests {
- mod descriptives {
- use std::sync::Arc;
-
- use encoding_rs::UTF_8;
-
- use crate::{
- engine::Engine,
- lex::lexer::{Source, SourceFile},
- };
-
- fn test(syntax: &str) {
- let mut engine = Engine::new();
- engine.run(Source::new_default(&Arc::new(
- SourceFile::for_file_contents(
- syntax.to_string(),
- Some("test.sps".to_string()),
- UTF_8,
- ),
- )));
- }
-
- #[test]
- fn basics() {
- test("descript a to b (c) all/stat=all/format=serial.");
- }
-
- #[test]
- fn include_missing() {
- test("descript all/stat=all/format=serial/missing=include.");
- }
-
- #[test]
- fn include_missing_listwise() {
- test("descript all/stat=all/format=serial/missing=listwise.");
- test("descript all/stat=all/format=serial/missing=listwise include.");
- }
-
- #[test]
- fn mean_only() {
- test("descript all/stat=mean.");
- }
-
- #[test]
- fn z_scores() {
- test("DESCRIPTIVES /VAR=a b /SAVE.");
- }
-
- #[test]
- fn syntax_errors() {
- test("\
-DESCRIPTIVES MISSING=**.
-DESCRIPTIVES FORMAT=**.
-DESCRIPTIVES STATISTICS=**.
-DESCRIPTIVES SORT=**.
-DESCRIPTIVES SORT=NAME (**).
-DESCRIPTIVES SORT=NAME (A **).
-DESCRIPTIVES **.
-DESCRIPTIVES x/ **.
-DESCRIPTIVES MISSING=INCLUDE.
-");
- }
- }
-}
--- /dev/null
+#![allow(dead_code)]
+use std::{fmt::Write, ops::RangeFrom, sync::OnceLock};
+
+use flagset::{flags, FlagSet};
+use pspp_derive::FromTokens;
+
+use crate::{
+ identifier::Identifier,
+ integer::ToInteger,
+ lex::{
+ command_name::CommandMatcher,
+ lexer::{LexToken, TokenSlice},
+ token::{Punct, Token},
+ },
+ message::{Diagnostic, Diagnostics},
+};
+
+flags! {
+ enum State: u8 {
+ /// No active dataset yet defined.
+ Initial,
+
+ /// Active dataset has been defined.
+ Data,
+
+ /// Inside `INPUT PROGRAM`.
+ InputProgram,
+
+ /// Inside `FILE TYPE`.
+ FileType,
+
+ /// State nested inside `LOOP` or `DO IF`, inside [State::Data].
+ NestedData,
+
+ /// State nested inside `LOOP` or `DO IF`, inside [State::InputProgram].
+ NestedInputProgram,
+ }
+}
+
+struct Command {
+ allowed_states: FlagSet<State>,
+ enhanced_only: bool,
+ testing_only: bool,
+ no_abbrev: bool,
+ name: &'static str,
+ run: Box<dyn Fn(&mut Context) + Send + Sync>, //-> Box<dyn ParsedCommand> + Send + Sync>,
+}
+
+#[derive(Debug)]
+enum ParseError {
+ Error(Diagnostics),
+ Mismatch(Diagnostics),
+}
+
+#[derive(Debug)]
+struct Parsed<'a, T> {
+ value: T,
+ rest: TokenSlice<'a>,
+ diagnostics: Diagnostics,
+}
+
+impl<'a, T> Parsed<'a, T> {
+ pub fn new(value: T, rest: TokenSlice<'a>, warnings: Diagnostics) -> Self {
+ Self {
+ value,
+ rest: rest,
+ diagnostics: warnings,
+ }
+ }
+ pub fn ok(value: T, rest: TokenSlice<'a>) -> Self {
+ Self {
+ value,
+ rest: rest,
+ diagnostics: Diagnostics::default(),
+ }
+ }
+ pub fn into_tuple(self) -> (T, TokenSlice<'a>, Diagnostics) {
+ (self.value, self.rest, self.diagnostics)
+ }
+ pub fn map<F, R>(self, f: F) -> Parsed<'a, R>
+ where
+ F: FnOnce(T) -> R,
+ {
+ Parsed {
+ value: f(self.value),
+ rest: self.rest,
+ diagnostics: self.diagnostics,
+ }
+ }
+ pub fn warn(self, mut warnings: Diagnostics) -> Self {
+ Self {
+ value: self.value,
+ rest: self.rest,
+ diagnostics: {
+ let mut vec = self.diagnostics.0;
+ vec.append(&mut warnings.0);
+ Diagnostics(vec)
+ },
+ }
+ }
+}
+
+type ParseResult<'a, T> = Result<Parsed<'a, T>, ParseError>;
+
+trait MismatchToError {
+ fn mismatch_to_error(self) -> Self;
+}
+
+impl<'a, T> MismatchToError for ParseResult<'a, T> {
+ fn mismatch_to_error(self) -> Self {
+ match self {
+ Err(ParseError::Mismatch(diagnostic)) => Err(ParseError::Error(diagnostic)),
+ rest => rest,
+ }
+ }
+}
+
+trait FromTokens<'a> {
+ fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
+ where
+ Self: Sized;
+}
+
+impl<'a, T> FromTokens<'a> for Option<T>
+where
+ T: FromTokens<'a>,
+{
+ fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
+ where
+ Self: Sized,
+ {
+ match T::from_tokens(input) {
+ Ok(p) => Ok(p.map(Some)),
+ Err(ParseError::Mismatch(_)) => Ok(Parsed::ok(None, input)),
+ Err(ParseError::Error(error)) => Err(ParseError::Error(error)),
+ }
+ }
+}
+
+impl<'a, T> FromTokens<'a> for Vec<T>
+where
+ T: FromTokens<'a>,
+{
+ fn from_tokens(mut input: TokenSlice<'a>) -> ParseResult<'a, Self>
+ where
+ Self: Sized,
+ {
+ let mut values_vec = Vec::new();
+ let mut warnings_vec = Vec::new();
+ loop {
+ match T::from_tokens(input) {
+ Ok(Parsed {
+ value,
+ rest,
+ diagnostics: mut warnings,
+ }) => {
+ values_vec.push(value);
+ warnings_vec.append(&mut warnings.0);
+ input = rest;
+ }
+ Err(ParseError::Mismatch(_)) => break,
+ Err(ParseError::Error(e)) => return Err(ParseError::Error(e)),
+ }
+ }
+ Ok(Parsed {
+ value: values_vec,
+ rest: input,
+ diagnostics: Diagnostics(warnings_vec),
+ })
+ }
+}
+
+impl<'a> FromTokens<'a> for TokenSlice<'a> {
+ fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
+ where
+ Self: Sized,
+ {
+ Ok(Parsed::ok(input, input.end()))
+ }
+}
+
+#[derive(Debug, FromTokens)]
+#[pspp(add_lifetime)]
+struct Descriptives<'a> {
+ subcommands: Vec<Subcommand<DescriptivesSubcommand<'a>>>,
+}
+
+#[derive(Debug)]
+struct Subcommand<T>(pub T);
+
+impl<'a, T> FromTokens<'a> for Subcommand<T>
+where
+ T: FromTokens<'a>,
+{
+ fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
+ where
+ Self: Sized,
+ {
+ let start = input.skip_until(|token| token != &Token::Punct(Punct::Slash));
+ if start.is_empty() {
+ return Err(ParseError::Error(
+ input.error("Syntax error at end of input.").into(),
+ ));
+ }
+ let end = start.skip_to(&Token::Punct(Punct::Slash));
+ let subcommand = start.subslice(0..start.len() - end.len());
+ let (value, rest, mut warnings) = T::from_tokens(subcommand)?.into_tuple();
+ if !rest.is_empty() {
+ warnings
+ .0
+ .push(rest.warning("Syntax error expecting end of subcommand."));
+ }
+ Ok(Parsed::new(Self(value), end, warnings))
+ }
+}
+
+#[derive(Debug, FromTokens)]
+#[pspp(add_lifetime, required_equals)]
+enum DescriptivesSubcommand<'a> {
+ #[pspp(default)]
+ Variables(Vec<DescriptivesVars<'a>>),
+ Missing(Vec<Missing>),
+ Save,
+ Statistics(Vec<Statistic>),
+ Sort(Sort),
+ Format(Vec<Format>),
+}
+
+#[derive(Debug, FromTokens)]
+enum Missing {
+ Variable,
+ Listwise,
+ Include,
+}
+
+#[derive(Debug, FromTokens)]
+enum Format {
+ Labels,
+ NoLabels,
+ Index,
+ NoIndex,
+ Line,
+ Serial,
+}
+
+#[derive(Debug, FromTokens)]
+#[pspp(add_lifetime)]
+struct DescriptivesVars<'a> {
+ vars: Vars<'a>,
+ z_name: Option<InParens<&'a Identifier>>,
+}
+
+#[derive(Debug)]
+struct InParens<T>(pub T);
+
+impl<'a, T> FromTokens<'a> for InParens<T>
+where
+ T: FromTokens<'a>,
+{
+ fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
+ where
+ Self: Sized,
+ {
+ let ((), rest, _) = parse_token(input, &Token::Punct(Punct::LParen))?.into_tuple();
+ let (value, rest, warnings) = T::from_tokens(rest)?.into_tuple();
+ let ((), rest, _) = parse_token(rest, &Token::Punct(Punct::RParen))?.into_tuple();
+ Ok(Parsed {
+ value: Self(value),
+ rest,
+ diagnostics: warnings,
+ })
+ }
+}
+
+fn parse_token<'a>(input: TokenSlice<'a>, token: &Token) -> ParseResult<'a, ()> {
+ if let Some(rest) = input.skip(token) {
+ Ok(Parsed::ok((), rest))
+ } else {
+ Err(ParseError::Mismatch(
+ input.error(format!("expecting {token}")).into(),
+ ))
+ }
+}
+
+fn parse_keyword<'a>(input: TokenSlice<'a>, keyword: &str) -> ParseResult<'a, ()> {
+ if let Some(rest) = input.skip_if(|token| token.matches_keyword(keyword)) {
+ Ok(Parsed::ok((), rest))
+ } else {
+ Err(ParseError::Mismatch(
+ input.error(format!("expecting {keyword}")).into(),
+ ))
+ }
+}
+
+#[derive(Debug)]
+enum Vars<'a> {
+ Single(&'a Identifier),
+ Range(&'a Identifier, &'a Identifier),
+ All,
+}
+
+impl<'a> FromTokens<'a> for Vars<'a> {
+ fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
+ where
+ Self: Sized,
+ {
+ if let Ok(Parsed { rest, .. }) = parse_token(input, &Token::Punct(Punct::All)) {
+ Ok(Parsed::ok(Self::All, rest))
+ } else {
+ let (from, rest, _) = parse_id(input)?.into_tuple();
+ if let Ok(Parsed { rest, .. }) = parse_token(rest, &Token::Punct(Punct::To)) {
+ if let Ok(p) = parse_id(rest) {
+ return Ok(p.map(|to| Self::Range(from, to)));
+ }
+ }
+ Ok(Parsed::ok(Self::Single(from), rest))
+ }
+ }
+}
+
+fn parse_id<'a>(input: TokenSlice<'a>) -> ParseResult<'a, &'a Identifier> {
+ let mut iter = input.iter();
+ if let Some(LexToken {
+ token: Token::Id(id),
+ ..
+ }) = iter.next()
+ {
+ Ok(Parsed::ok(id, iter.remainder()))
+ } else {
+ Err(ParseError::Mismatch(
+ input.error("Syntax error expecting identifier.").into(),
+ ))
+ }
+}
+
+impl<'a> FromTokens<'a> for &'a Identifier {
+ fn from_tokens(input: TokenSlice<'a>) -> ParseResult<'a, Self>
+ where
+ Self: Sized,
+ {
+ parse_id(input)
+ }
+}
+
+#[derive(Debug, FromTokens)]
+struct Sort {
+ key: SortKey,
+ direction: Option<Direction>,
+}
+
+#[derive(Debug, FromTokens)]
+enum SortKey {
+ Mean,
+ SMean,
+ Stddev,
+ Variance,
+ Range,
+ Min,
+ Max,
+ Sum,
+ Skewness,
+ Kurtosis,
+ Name,
+}
+
+#[derive(Debug, FromTokens)]
+enum Direction {
+ #[pspp(syntax = "(A)")]
+ Ascending,
+ #[pspp(syntax = "(D)")]
+ Descending,
+}
+
+#[derive(Debug, FromTokens)]
+enum Statistic {
+ Default,
+ Mean,
+ SeMean,
+ Stddev,
+ Variance,
+ Range,
+ Sum,
+ Min,
+ Max,
+ Skewness,
+ Kurtosis,
+ All,
+}
+
+fn collect_subcommands<'a>(src: &'a TokenSlice) -> Vec<TokenSlice<'a>> {
+ src.split(|token| token.token == Token::Punct(Punct::Slash))
+ .filter(|slice| !slice.is_empty())
+ .collect()
+}
+
+fn commands() -> &'static [Command] {
+ fn new_commands() -> Vec<Command> {
+ vec![
+ Command {
+ allowed_states: FlagSet::full(),
+ enhanced_only: false,
+ testing_only: false,
+ no_abbrev: false,
+ name: "DESCRIPTIVES",
+ run: Box::new(|context| {
+ let mut input = context.lexer;
+ while !input.is_empty() {
+ match <Subcommand<DescriptivesSubcommand>>::from_tokens(input) {
+ Ok(Parsed {
+ value: subcommand,
+ rest,
+ diagnostics,
+ }) => {
+ println!("\n{subcommand:?}");
+ //println!("rest: {rest:?}");
+ println!("warnings: {diagnostics:?}");
+ //println!("{:?}", DescriptivesSubcommand::from_tokens(subcommand.0));
+ input = rest;
+ }
+ Err(error) => {
+ println!("{error:?}");
+ break;
+ }
+ }
+ }
+ }),
+ },
+ Command {
+ allowed_states: FlagSet::full(),
+ enhanced_only: false,
+ testing_only: false,
+ no_abbrev: false,
+ name: "ECHO",
+ run: Box::new(|_context| todo!()),
+ },
+ ]
+ }
+
+ static COMMANDS: OnceLock<Vec<Command>> = OnceLock::new();
+ COMMANDS.get_or_init(|| new_commands()).as_slice()
+}
+
+fn parse_command_word(lexer: &mut TokenSlice, s: &mut String, n: usize) -> bool {
+ let separator = match s.chars().next_back() {
+ Some(c) if c != '-' => " ",
+ _ => "",
+ };
+
+ match lexer.get_token(n) {
+ Some(Token::Punct(Punct::Dash)) => {
+ s.push('-');
+ true
+ }
+ Some(Token::Id(id)) => {
+ write!(s, "{separator}{id}").unwrap();
+ true
+ }
+ Some(Token::Number(number)) if number.is_sign_positive() => {
+ if let Some(integer) = number.to_exact_usize() {
+ write!(s, "{separator}{integer}").unwrap();
+ true
+ } else {
+ false
+ }
+ }
+ _ => false,
+ }
+}
+
+fn find_best_match(s: &str) -> (Option<&'static Command>, isize) {
+ let mut cm = CommandMatcher::new(s);
+ for command in commands() {
+ cm.add(command.name, command);
+ }
+ cm.get_match()
+}
+
+fn parse_command_name(
+ lexer: &mut TokenSlice,
+ error: &Box<dyn Fn(Diagnostic)>,
+) -> Result<(&'static Command, usize), ()> {
+ let mut s = String::new();
+ let mut word = 0;
+ let mut missing_words = 0;
+ let mut command = None;
+ while parse_command_word(lexer, &mut s, word) {
+ (command, missing_words) = find_best_match(&s);
+ if missing_words <= 0 {
+ break;
+ }
+ word += 1;
+ }
+ if command.is_none() && missing_words > 0 {
+ s.push_str(" .");
+ (command, missing_words) = find_best_match(&s);
+ s.truncate(s.len() - 2);
+ }
+
+ match command {
+ Some(command) => Ok((command, ((word as isize + 1) + missing_words) as usize)),
+ None => {
+ if word == 0 {
+ error(
+ lexer
+ .subslice(0..1)
+ .error("Syntax error expecting command name"),
+ )
+ } else {
+ error(lexer.subslice(0..word + 1).error("Unknown command `{s}`."))
+ };
+ Err(())
+ }
+ }
+}
+
+pub enum Success {
+ Success,
+ Eof,
+ Finish,
+}
+
+pub fn end_of_command(context: &Context, range: RangeFrom<usize>) -> Result<Success, ()> {
+ match context.lexer.get_token(range.start) {
+ None | Some(Token::End) => Ok(Success::Success),
+ _ => {
+ context.error(
+ context
+ .lexer
+ .subslice(range.start..context.lexer.len())
+ .error("Syntax error expecting end of command."),
+ );
+ Err(())
+ }
+ }
+}
+
+fn parse_in_state(mut lexer: TokenSlice, error: &Box<dyn Fn(Diagnostic)>, _state: State) {
+ match lexer.get_token(0) {
+ None | Some(Token::End) => (),
+ _ => match parse_command_name(&mut lexer, error) {
+ Ok((command, n_tokens)) => {
+ let mut context = Context {
+ error,
+ lexer: lexer.subslice(n_tokens..lexer.len()),
+ command_name: Some(command.name),
+ };
+ (command.run)(&mut context);
+ }
+ Err(error) => println!("{error:?}"),
+ },
+ }
+}
+
+pub fn parse_command(lexer: TokenSlice, error: &Box<dyn Fn(Diagnostic)>) {
+ parse_in_state(lexer, error, State::Initial)
+}
+
+pub struct Context<'a> {
+ error: &'a Box<dyn Fn(Diagnostic)>,
+ lexer: TokenSlice<'a>,
+ command_name: Option<&'static str>,
+}
+
+impl<'a> Context<'a> {
+ pub fn error(&self, diagnostic: Diagnostic) {
+ (self.error)(diagnostic);
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ mod descriptives {
+ use std::sync::Arc;
+
+ use encoding_rs::UTF_8;
+
+ use crate::{
+ engine::Engine,
+ lex::lexer::{Source, SourceFile},
+ };
+
+ fn test(syntax: &str) {
+ let mut engine = Engine::new();
+ engine.run(Source::new_default(&Arc::new(
+ SourceFile::for_file_contents(
+ syntax.to_string(),
+ Some("test.sps".to_string()),
+ UTF_8,
+ ),
+ )));
+ }
+
+ #[test]
+ fn basics() {
+ test("descript a to b (c) all/stat=all/format=serial.");
+ }
+
+ #[test]
+ fn include_missing() {
+ test("descript all/stat=all/format=serial/missing=include.");
+ }
+
+ #[test]
+ fn include_missing_listwise() {
+ test("descript all/stat=all/format=serial/missing=listwise.");
+ test("descript all/stat=all/format=serial/missing=listwise include.");
+ }
+
+ #[test]
+ fn mean_only() {
+ test("descript all/stat=mean.");
+ }
+
+ #[test]
+ fn z_scores() {
+ test("DESCRIPTIVES /VAR=a b /SAVE.");
+ }
+
+ #[test]
+ fn syntax_errors() {
+ test("\
+DESCRIPTIVES MISSING=**.
+DESCRIPTIVES FORMAT=**.
+DESCRIPTIVES STATISTICS=**.
+DESCRIPTIVES SORT=**.
+DESCRIPTIVES SORT=NAME (**).
+DESCRIPTIVES SORT=NAME (A **).
+DESCRIPTIVES **.
+DESCRIPTIVES x/ **.
+DESCRIPTIVES MISSING=INCLUDE.
+");
+ }
+ }
+}
projects / pspp / commitdiff