From 6d2284cd8737e0377bea8d7379f99b6bc9c7cf5a Mon Sep 17 00:00:00 2001
From: Ben Pfaff <blp@cs.stanford.edu>
Date: Sat, 17 Aug 2024 09:41:14 -0700
Subject: [PATCH] work

---
 rust/src/lex/lexer.rs | 50 +++++++++++++++++++++++++++++++------------
 rust/src/macros.rs    | 12 +++++------
 2 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/rust/src/lex/lexer.rs b/rust/src/lex/lexer.rs
index 1eb11e61ca..00611ffb9f 100644
--- a/rust/src/lex/lexer.rs
+++ b/rust/src/lex/lexer.rs
@@ -1,7 +1,6 @@
 use std::{
     borrow::Borrow,
     collections::{HashMap, VecDeque},
-    io::Result,
     num::NonZeroU32,
     ops::RangeInclusive,
     sync::Arc,
@@ -41,16 +40,16 @@ pub enum ErrorHandling {
 
 pub trait LexRead {
     /// Read some input from the source. If successful, returns the input that
-    /// was read.  At end of file, returns `Ok(None)`.
+    /// was read.  At end of file or on error, returns an empty string.
     ///
     /// `prompt` provides a hint to interactive readers as to what kind of
     /// syntax is being read right now.
-    fn read(&mut self, prompt: PromptStyle) -> Result<Option<String>>;
+    fn read(&mut self, prompt: PromptStyle) -> String;
 }
 
 impl LexRead for () {
-    fn read(&mut self, _prompt: PromptStyle) -> Result<Option<String>> {
-        Ok(None)
+    fn read(&mut self, _prompt: PromptStyle) -> String {
+        String::from("")
     }
 }
 
@@ -146,7 +145,18 @@ impl Source {
     }
 
     fn read(&mut self) {
-        todo!()
+        loop {
+            let prompt = self.segmenter.prompt();
+            let s = self.reader.read(prompt);
+            if s.is_empty() {
+                self.eof = true;
+                return;
+            }
+            self.buffer.push_str(&s);
+            if self.buffer[self.seg_pos..].contains('\n') {
+                return;
+            }
+        }
     }
     fn try_get_pp(&mut self) -> bool {
         let (seg_len, seg_type) = loop {
@@ -255,8 +265,9 @@ impl Source {
                 unreachable!();
             }
             let token = &self.pp[ofs];
-            if parser.push(todo!(), &self.buffer[token.pos], &|e| println!("{e:?}"))
-                == ParseStatus::Complete
+            if parser.push(&token.token, &self.buffer[token.pos], &|e| {
+                println!("{e:?}")
+            }) == ParseStatus::Complete
             {
                 break;
             }
@@ -278,6 +289,7 @@ impl Source {
             &mut expansion,
             |e| println!("{e:?}"),
         );
+        let retval = !expansion.is_empty();
 
         const MPRINT: bool = false;
         if MPRINT {
@@ -285,18 +297,27 @@ impl Source {
         }
 
         // Append the macro expansion tokens to the lookahead.
-        let macro_rep = Arc::new(macro_tokens_to_syntax(expansion.as_slice()).collect());
-        for token in expansion {
+        let mut macro_rep = String::new();
+        let mut pos = Vec::with_capacity(expansion.len());
+        for [prefix, token] in macro_tokens_to_syntax(expansion.as_slice()) {
+            macro_rep.push_str(prefix);
+            let len = macro_rep.len();
+            pos.push(len..=len + token.len() - 1);
+        }
+        let macro_rep = Arc::new(macro_rep);
+        for (index, token) in expansion.into_iter().enumerate() {
             let lt = LexToken {
                 token: token.token,
-                pos: todo!(),
+                pos: *c0.pos.start()..=*c1.pos.end(),
                 macro_rep: Some(MacroRepresentation {
                     expansion: Arc::clone(&macro_rep),
-                    pos: todo!(),
+                    pos: pos[index].clone(),
                 }),
             };
+            self.merge.push_back(lt);
         }
-        todo!()
+        self.pp.drain(..call.len());
+        retval
     }
 
     /// Attempts to obtain at least one new token into `self.merge`.
@@ -351,7 +372,8 @@ impl Source {
         unreachable!();
     }
     fn get_parse(&mut self) -> bool {
-        todo!()
+        // XXX deal with accumulate messages
+         self.get_parse__()
     }
 
     fn offset_to_point(&self, offset: usize) -> Point {
diff --git a/rust/src/macros.rs b/rust/src/macros.rs
index f00262aac6..c7a122197d 100644
--- a/rust/src/macros.rs
+++ b/rust/src/macros.rs
@@ -423,12 +423,12 @@ impl From<&Token> for TokenClass {
     }
 }
 
-pub fn macro_tokens_to_syntax(input: &[MacroToken]) -> impl Iterator<Item = &str> {
+pub fn macro_tokens_to_syntax(input: &[MacroToken]) -> impl Iterator<Item = [&str; 2]> {
     input
         .iter()
         .take(1)
-        .map(|token| token.syntax.as_str())
-        .chain(input.windows(2).flat_map(|w| {
+        .map(|token| ["", token.syntax.as_str()])
+        .chain(input.windows(2).map(|w| {
             let c0 = (&w[0].token).into();
             let c1 = (&w[1].token).into();
             [TokenClass::separator(c0, c1), w[1].syntax.as_str()]
@@ -996,7 +996,7 @@ impl<'a> Expander<'a> {
         subexpander.expand(&mut MacroTokens(tokens.as_slice()), &mut output);
         subexpander.stack.pop();
         e.stack = subexpander.stack;
-        Some(macro_tokens_to_syntax(&output).collect())
+        Some(macro_tokens_to_syntax(&output).flatten().collect())
     }
 
     fn expand_head(e: &mut Expander, mut args: Vec<String>) -> Option<String> {
@@ -1162,7 +1162,7 @@ impl<'a> Expander<'a> {
                         input.advance();
                         return Some(
                             macro_tokens_to_syntax(self.args.unwrap()[param_idx].as_ref().unwrap())
-                                .collect(),
+                                .flatten().collect(),
                         );
                     }
                     if let Some(value) = self.vars.borrow().get(id) {
@@ -1184,7 +1184,7 @@ impl<'a> Expander<'a> {
                         }
                         arg.extend(macro_tokens_to_syntax(
                             self.args.unwrap()[i].as_ref().unwrap(),
-                        ));
+                        ).flatten());
                     }
                     input.advance();
                     return Some(arg);
-- 
2.30.2