fix(parser): ignore or push out trivia tokens around arguments and groups

crates/mitex-parser/src/parser.rs

@@ -137,6 +137,8 @@ pub struct Parser<'a, S: BumpTokenStream<'a> = ()> {
     spec: CommandSpec,
     /// Argument matcher builder containing cached regexes
     arg_matchers: ArgMatcherBuilder,
+    /// trivia buffer
+    trivia_buffer: Vec<(Token, &'a str)>,
 
     /// State used by item_list/argument_list parser
     /// The current state
@@ -153,6 +155,7 @@ impl<'a> Parser<'a> {
             spec,
             arg_matchers: ArgMatcherBuilder::default(),
             list_state: Default::default(),
+            trivia_buffer: Vec::new(),
         }
     }
 
@@ -165,6 +168,7 @@ impl<'a> Parser<'a> {
             spec,
             arg_matchers: ArgMatcherBuilder::default(),
             list_state: Default::default(),
+            trivia_buffer: Vec::new(),
         }
     }
 }
@@ -231,7 +235,25 @@ impl<'a, S: BumpTokenStream<'a>> Parser<'a, S> {
     fn eat_if(&mut self, kind: Token) {
         if self.peek() == Some(kind) {
             self.eat();
-            self.trivia();
+        }
+    }
+
+    /// Lexer Interface
+    /// Hold the next trivia token
+    fn hold_trivia(&mut self) {
+        self.trivia_buffer.push(self.lexer.eat().unwrap());
+    }
+
+    /// Lexer Interface
+    fn ignore_holding_trivia(&mut self) {
+        self.trivia_buffer.clear();
+    }
+
+    /// Lexer Interface
+    fn extract_holding_trivia(&mut self) {
+        for (kind, text) in self.trivia_buffer.drain(..) {
+            let kind: SyntaxKind = kind.into();
+            self.builder.token(kind.into(), text);
         }
     }
 
@@ -500,6 +522,8 @@ impl<'a, S: BumpTokenStream<'a>> Parser<'a, S> {
 
         self.builder.finish_node();
 
+        self.extract_holding_trivia();
+
         !is_greedy
     }
 
@@ -527,6 +551,8 @@ impl<'a, S: BumpTokenStream<'a>> Parser<'a, S> {
             }
 
             self.builder.finish_node();
+
+            self.extract_holding_trivia();
         }
 
         self.item_list(ParseScope::Environment);
@@ -620,6 +646,7 @@ impl<'a, S: BumpTokenStream<'a>> Parser<'a, S> {
             f: impl FnOnce(&mut Parser<'a, S>) -> T,
         ) -> T {
             if k_wrap_args!() {
+                this.ignore_holding_trivia();
                 this.builder.start_node(ClauseArgument.into());
                 let res = f(this);
                 this.builder.finish_node();
@@ -638,7 +665,13 @@ impl<'a, S: BumpTokenStream<'a>> Parser<'a, S> {
         while let Some(kind) = self.peek() {
             match kind {
                 // trivials
-                Token::LineBreak | Token::Whitespace | Token::LineComment => self.eat(),
+                Token::LineBreak | Token::Whitespace | Token::LineComment => {
+                    if GREEDY {
+                        self.eat();
+                    } else {
+                        self.hold_trivia();
+                    }
+                }
                 // Argument matches is stopped on these tokens
                 // However, newline is also a command (with name `\`), hence this is different from
                 // mark and (`&`)

crates/mitex-parser/tests/ast.rs

@@ -12,19 +12,22 @@ mod ast {
     mod arg_match;
 
     #[cfg(test)]
-    mod command;
+    mod attachment;
 
     #[cfg(test)]
-    mod environment;
+    mod block_comment;
 
     #[cfg(test)]
-    mod attachment;
+    mod command;
+
+    #[cfg(test)]
+    mod environment;
 
     #[cfg(test)]
     mod left_right;
 
     #[cfg(test)]
-    mod block_comment;
+    mod trivia;
 
     /// Convenient function to launch/debug a test case
     #[test]
@@ -76,11 +79,9 @@ mod ast {
         ||args
         |||cmd
         ||||cmd-name("\\frac")
-        ||||space'(" ")
         ||||args(word'("1"))
-        ||||space'(" ")
         ||||args(word'("2"))
-        ||||space'(" ")
+        |||space'(" ")
         ||underscore'("_")
         ||word'("3")
         "###);
@@ -108,7 +109,7 @@ mod ast {
         ||||lbrace'("{")
         ||||text(word'("d"))
         ||||rbrace'("}")
-        ||||space'(" ")
+        |space'(" ")
         |text(word'("x"))
         "###);
     }

crates/mitex-parser/tests/ast/arg_match.rs

@@ -6,7 +6,6 @@ fn split_char() {
     root
     |cmd
     ||cmd-name("\\frac")
-    ||space'(" ")
     ||args(word'("a"))
     ||args(word'("b"))
     |text(word'("cd"))
@@ -15,15 +14,13 @@ fn split_char() {
     root
     |cmd
     ||cmd-name("\\frac")
-    ||space'(" ")
     ||args(word'("a"))
     ||args(word'("b"))
     "###);
     assert_debug_snapshot!(parse(r#"\frac a"#), @r###"
     root
     |cmd
     ||cmd-name("\\frac")
-    ||space'(" ")
     ||args(word'("a"))
     "###);
 }
@@ -52,7 +49,7 @@ fn eat_regular_brace() {
     ||cmd-name("\\mathrm")
     ||args
     |||cmd(cmd-name("\\lbrace"))
-    ||space'(" ")
+    |space'(" ")
     |text(word'("x"),space'(" "))
     |cmd(cmd-name("\\rbrace"))
     "###);
@@ -140,7 +137,7 @@ fn special_marks() {
     |||||lbrace'("{")
     |||||text(word'("1"))
     |||||rbrace'("}")
-    |||||space'(" ")
+    ||space'(" ")
     ||ampersand'("&")
     ||space'(" ")
     ||curly
@@ -166,9 +163,8 @@ fn special_marks() {
     |||||lbrace'("{")
     |||||text(word'("1"))
     |||||rbrace'("}")
-    |||||space'(" ")
     |||args(newline("\\\\"))
-    |||space'(" ")
+    ||space'(" ")
     ||curly
     |||lbrace'("{")
     |||text(word'("2"))
@@ -220,7 +216,7 @@ fn special_marks_in_env() {
     ||||||lbrace'("{")
     ||||||text(word'("2"))
     ||||||rbrace'("}")
-    ||||||space'(" ")
+    |||space'(" ")
     |||newline("\\\\")
     |||space'(" ")
     |||cmd
@@ -235,9 +231,9 @@ fn special_marks_in_env() {
     ||||||lbrace'("{")
     ||||||text(word'("2"))
     ||||||rbrace'("}")
-    ||||||space'(" ")
-    ||||||br'("\n")
-    ||||||space'("    ")
+    |||space'(" ")
+    |||br'("\n")
+    |||space'("    ")
     "###);
     assert_debug_snapshot!(parse(r#"
     \left. \displaystyle \frac{1}{2} \\ \frac{1}{2} \right.
@@ -264,7 +260,7 @@ fn special_marks_in_env() {
     |||||||lbrace'("{")
     |||||||text(word'("2"))
     |||||||rbrace'("}")
-    |||||||space'(" ")
+    ||||space'(" ")
     ||||newline("\\\\")
     ||||space'(" ")
     ||||cmd
@@ -279,7 +275,7 @@ fn special_marks_in_env() {
     |||||||lbrace'("{")
     |||||||text(word'("2"))
     |||||||rbrace'("}")
-    |||||||space'(" ")
+    ||||space'(" ")
     ||clause-lr(cmd-name("\\right"),word'("."))
     |br'("\n")
     |space'("    ")
@@ -311,7 +307,7 @@ fn special_marks_in_env() {
     |||||||||lbrace'("{")
     |||||||||text(word'("2"))
     |||||||||rbrace'("}")
-    |||||||||space'(" ")
+    ||||||space'(" ")
     ||||||newline("\\\\")
     ||||||space'(" ")
     ||||||cmd
@@ -326,10 +322,12 @@ fn special_marks_in_env() {
     |||||||||lbrace'("{")
     |||||||||text(word'("2"))
     |||||||||rbrace'("}")
-    |||||||||space'(" ")
+    ||||||space'(" ")
     ||||rbracket'("]")
     ||args
-    |||curly(lbrace'("{"),rbrace'("}"),br'("\n"),space'("    "))
+    |||curly(lbrace'("{"),rbrace'("}"))
+    |br'("\n")
+    |space'("    ")
     "###);
     assert_debug_snapshot!(parse(r#"
     \begin{matrix}a \over b \\ c\end{matrix}
@@ -361,7 +359,6 @@ fn sqrt_pattern() {
     root
     |cmd
     ||cmd-name("\\sqrt")
-    ||space'(" ")
     ||args(word'("1"))
     |text(word'("2"))
     "###);

crates/mitex-parser/tests/ast/attachment.rs

@@ -207,7 +207,6 @@ fn test_attachment_may_weird() {
     ||args
     |||cmd
     ||||cmd-name("\\frac")
-    ||||space'(" ")
     ||||args(word'("a"))
     ||||args(word'("b"))
     ||underscore'("_")
@@ -219,7 +218,6 @@ fn test_attachment_may_weird() {
     ||args
     |||cmd
     ||||cmd-name("\\frac")
-    ||||space'(" ")
     ||||args(word'("a"))
     ||underscore'("_")
     ||word'("c")
@@ -230,7 +228,6 @@ fn test_attachment_may_weird() {
     root
     |cmd
     ||cmd-name("\\frac")
-    ||space'(" ")
     ||args
     |||curly
     ||||lbrace'("{")
@@ -240,7 +237,6 @@ fn test_attachment_may_weird() {
     |||||underscore'("_")
     |||||word'("c")
     ||||rbrace'("}")
-    ||||space'(" ")
     ||args(word'("b"))
     "###);
 }

crates/mitex-parser/tests/ast/command.rs

@@ -281,7 +281,7 @@ fn right_greedy() {
     ||||space'(" ")
     ||||text(word'("a"),space'(" "),word'("b"))
     ||rbrace'("}")
-    ||space'(" ")
+    |space'(" ")
     |text(word'("c"))
     "###);
     // Description: doeesn't affect left side

crates/mitex-parser/tests/ast/environment.rs

@@ -69,7 +69,7 @@ c & d
     |||||lbrace'("{")
     |||||text(word'("lc"))
     |||||rbrace'("}")
-    |||||br'("\n")
+    ||br'("\n")
     ||text(word'("a"),space'(" "))
     ||ampersand'("&")
     ||space'(" ")