Skip to content

Commit

Permalink
refactor(converter): convert \text{} to \textmath[] (#147)
Browse files Browse the repository at this point in the history 
* fix: convert \text{} to \textmath[]

* docs: update example.typ

* fix: add more escapes

* fix: make escapes work correctly

* fix: fix test

* feat: add escape for at sign
  • Loading branch information
@OrangeX4
OrangeX4 committed Mar 29, 2024
1 parent 8e5045a commit 92dff01
Show file tree
Hide file tree
Showing 8 changed files with 73 additions and 70 deletions.
10 changes: 9 additions & 1 deletion crates/mitex-lexer/src/token.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,12 +91,20 @@ pub enum Token {
#[token("#")]
Hash,

/// An ascii asterisk
#[token("*")]
Asterisk,

/// An ascii atsign
#[token("@")]
AtSign,

/// An ascii underscore
#[token("_", priority = 2)]
Underscore,

/// A character sequence that doesn't contain any above tokens
#[regex(r#"[^\s\\%\{\},\$\[\]\(\)\~/_'";&^#]+"#, priority = 1)]
#[regex(r#"[^\s\\%\{\},\$\[\]\(\)\~/_\*@'";&^#]+"#, priority = 1)]
Word,

/// Special dollar signs
Expand Down
2 changes: 2 additions & 0 deletions crates/mitex-parser/src/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,8 @@ impl<'a, S: TokenStream<'a>> Parser<'a, S> {
| Token::Whitespace
| Token::LineComment
| Token::Hash
| Token::Asterisk
| Token::AtSign
| Token::Error => {
self.eat();
return false;
Expand Down
4 changes: 4 additions & 0 deletions crates/mitex-parser/src/syntax.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ pub enum SyntaxKind {
TokenEndMath,
TokenAmpersand,
TokenHash,
TokenAsterisk,
TokenAtSign,
TokenUnderscore,
TokenCaret,
TokenApostrophe,
Expand Down Expand Up @@ -126,6 +128,8 @@ impl From<Token> for SyntaxKind {
Token::Dollar => SyntaxKind::TokenDollar,
Token::Ampersand => SyntaxKind::TokenAmpersand,
Token::Hash => SyntaxKind::TokenHash,
Token::Asterisk => SyntaxKind::TokenAsterisk,
Token::AtSign => SyntaxKind::TokenAtSign,
Token::NewLine => SyntaxKind::ItemNewLine,
Token::MacroArg(_) => SyntaxKind::TokenWord,
Token::CommandName(
Expand Down
3 changes: 2 additions & 1 deletion crates/mitex-parser/tests/ast/command.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ fn test_starrd_command() {
assert_debug_snapshot!(parse(r#"\varphi*1"#), @r###"
root
|cmd(cmd-name("\\varphi"))
|text(word'("*1"))
|asterisk'("*")
|text(word'("1"))
"###
);
}
Expand Down
2 changes: 2 additions & 0 deletions crates/mitex-parser/tests/common/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ pub mod ast_snapshot {
SyntaxKind::TokenEndMath => "end-math",
SyntaxKind::TokenAmpersand => "ampersand'",
SyntaxKind::TokenHash => "hash'",
SyntaxKind::TokenAsterisk => "asterisk'",
SyntaxKind::TokenAtSign => "at-sign'",
SyntaxKind::TokenUnderscore => "underscore'",
SyntaxKind::TokenCaret => "caret'",
SyntaxKind::TokenApostrophe => "apostrophe'",
Expand Down
117 changes: 52 additions & 65 deletions crates/mitex/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ use mitex_parser::syntax::FormulaItem;
use mitex_parser::syntax::SyntaxNode;
use mitex_spec_gen::DEFAULT_SPEC;
use rowan::ast::AstNode;
use rowan::SyntaxToken;

// use bitflags::bitflags;
//
Expand Down Expand Up @@ -249,26 +248,32 @@ impl Converter {
}
}
ItemAttachComponent => {
let mut based = false;
let mut first = true;
for child in elem.as_node().unwrap().children_with_tokens() {
if first {
let kind = child.as_token().map(|n| n.kind());
if matches!(kind, Some(TokenUnderscore | TokenCaret)) {
if !based {
f.write_str("zws")?;
if matches!(self.mode, LaTeXMode::Math) {
let mut based = false;
let mut first = true;
for child in elem.as_node().unwrap().children_with_tokens() {
if first {
let kind = child.as_token().map(|n| n.kind());
if matches!(kind, Some(TokenUnderscore | TokenCaret)) {
if !based {
f.write_str("zws")?;
}
write!(f, "{}(", child.as_token().unwrap().text())?;
first = false;
continue;
} else if !matches!(kind, Some(TokenWhiteSpace)) {
based = true;
}
write!(f, "{}(", child.as_token().unwrap().text())?;
first = false;
continue;
} else if !matches!(kind, Some(TokenWhiteSpace)) {
based = true;
}
self.convert(f, child, spec)?;
}
if !first {
f.write_char(')')?;
}
} else {
for child in elem.as_node().unwrap().children_with_tokens() {
self.convert(f, child, spec)?;
}
self.convert(f, child, spec)?;
}
if !first {
f.write_char(')')?;
}
}
TokenApostrophe => {
Expand Down Expand Up @@ -326,6 +331,12 @@ impl Converter {
TokenHash => {
f.write_str("\\#")?;
}
TokenAsterisk => {
f.write_str("\\*")?;
}
TokenAtSign => {
f.write_str("\\@")?;
}
TokenDitto => {
f.write_str("\\\"")?;
}
Expand Down Expand Up @@ -442,49 +453,17 @@ impl Converter {
}

if typst_name.starts_with("text") {
f.write_char('#')?;
f.write_str(typst_name)?;
f.write_str("(\"")?;

fn is_trivia_elem(elem: &LatexSyntaxElem) -> bool {
elem.as_token()
.map(SyntaxToken::kind)
.map_or(false, LatexSyntaxKind::is_trivia)
}

let mut args = args.as_slice();
while args.first().map_or(false, is_trivia_elem) {
args = &args[1..];
}
while args.last().map_or(false, is_trivia_elem) {
args = &args[..args.len() - 1];
}
f.write_char('[')?;

let prev_mode = self.enter_mode(LaTeXMode::Text);
for arg in args {
if let Some(text) = arg.as_token() {
if matches!(text.kind(), TokenLBrace | TokenRBrace) {
continue;
}
f.write_str(text.text())?;
} else {
arg.as_node()
.unwrap()
.descendants_with_tokens()
.for_each(|child| {
if let Some(text) = child.as_token() {
if matches!(text.kind(), TokenLBrace | TokenRBrace) {
return;
}
if matches!(text.kind(), TokenDitto) {
f.write_str("\\\"").unwrap();
return;
}
f.write_str(text.text()).unwrap();
}
});
}
self.convert(f, arg, spec)?;
}
self.exit_mode(prev_mode);

f.write_str("\")")?;
f.write_str("];")?;
return Ok(());
}

Expand Down Expand Up @@ -842,7 +821,7 @@ mod tests {
);
assert_debug_snapshot!(convert_math(r#"$\overbrace{a + b + c}^{\text{This is an overbrace}}$"#), @r###"
Ok(
"mitexoverbrace(a + b + c )^(textmath(\"This is an overbrace\"))",
"mitexoverbrace(a + b + c )^(#textmath[This is an overbrace];)",
)
"###
);
Expand Down Expand Up @@ -921,8 +900,12 @@ mod tests {
)
"###
);
assert_debug_snapshot!(convert_math(r#"$a*b * c$"#).unwrap(), @r###""a \\*b \\* c ""###
);
assert_debug_snapshot!(convert_math(r#"$"$"#).unwrap(), @r###""\\\"""###
);
assert_debug_snapshot!(convert_text(r#"@abc"#).unwrap(), @r###""\\@abc""###
);
}

#[test]
Expand Down Expand Up @@ -1110,19 +1093,23 @@ a & b & c
fn test_convert_ditto() {
assert_debug_snapshot!(convert_math(r#"$"$"#).unwrap(), @r###""\\\"""###);
assert_debug_snapshot!(convert_math(r#"$a"b"c$"#).unwrap(), @r###""a \\\"b \\\"c ""###);
assert_debug_snapshot!(convert_math(r#"$\text{a"b"c}$"#).unwrap(), @r###""textmath(\"a\\\"b\\\"c\")""###);
assert_debug_snapshot!(convert_math(r#"$\text{a " b " c}$"#).unwrap(), @r###""textmath(\"a \\\" b \\\" c\")""###);
assert_debug_snapshot!(convert_math(r#"$\text{a"b"c}$"#).unwrap(), @r###""#textmath[a\\\"b\\\"c];""###);
assert_debug_snapshot!(convert_math(r#"$\text{a " b " c}$"#).unwrap(), @r###""#textmath[a \\\" b \\\" c];""###);
}

#[test]
fn test_convert_text() {
assert_debug_snapshot!(convert_math(r#"$\text{abc}$"#).unwrap(), @r###""textmath(\"abc\")""###);
assert_debug_snapshot!(convert_math(r#"$\text{ a b c }$"#).unwrap(), @r###""textmath(\" a b c \")""###);
assert_debug_snapshot!(convert_math(r#"$\text{abc{}}$"#).unwrap(), @r###""textmath(\"abc\")""###);
assert_debug_snapshot!(convert_math(r#"$\text{ab{}c}$"#).unwrap(), @r###""textmath(\"abc\")""###);
assert_debug_snapshot!(convert_math(r#"$\text{ab c}$"#).unwrap(), @r###""textmath(\"ab c\")""###);
assert_debug_snapshot!(convert_math(r#"$\text{abc}$"#).unwrap(), @r###""#textmath[abc];""###);
assert_debug_snapshot!(convert_math(r#"$\text{ a b c }$"#).unwrap(), @r###""#textmath[ a b c ];""###);
assert_debug_snapshot!(convert_math(r#"$\text{abc{}}$"#).unwrap(), @r###""#textmath[abc];""###);
assert_debug_snapshot!(convert_math(r#"$\text{ab{}c}$"#).unwrap(), @r###""#textmath[abc];""###);
assert_debug_snapshot!(convert_math(r#"$\text{ab c}$"#).unwrap(), @r###""#textmath[ab c];""###);
assert_debug_snapshot!(convert_math(r#"$\text{ab$x$c}$"#).unwrap(), @r###""#textmath[ab#math.equation(block: false, $x $);c];""###);
assert_debug_snapshot!(convert_math(r#"$\text{ab*c}$"#).unwrap(), @r###""#textmath[ab\\*c];""###);
assert_debug_snapshot!(convert_math(r#"$\text{ab_c}$"#).unwrap(), @r###""#textmath[ab\\_c];""###);
assert_debug_snapshot!(convert_math(r#"$\text{ab^c}$"#).unwrap(), @r###""#textmath[ab\\^c];""###);
// note: hack doesn't work in this case
assert_debug_snapshot!(convert_math(r#"$\text{ab\color{red}c}$"#).unwrap(), @r###""textmath(\"ab\\colorredc\")""###);
assert_debug_snapshot!(convert_math(r#"$\text{ab\color{red}c}$"#).unwrap(), @r###""#textmath[abmitexcolor(red,c)];""###);
}

#[test]
Expand Down
2 changes: 1 addition & 1 deletion packages/mitex/examples/example.typ
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#import "@preview/mitex:0.2.2": *
#import "../lib.typ": *

#set page(width: 500pt, height: auto, margin: 1em)

Expand Down
3 changes: 1 addition & 2 deletions packages/mitex/specs/latex/standard.typ
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
} else {
texcolor.text
}
#let text-end-space(it) = if it.len() > 1 and it.ends-with(" ") { " " }

// 1. functions created to make it easier to define a spec
#let operatornamewithlimits(it) = math.op(limits: true, math.upright(it))
Expand All @@ -41,7 +40,7 @@
#let greedy-handle(alias, fn) = define-greedy-cmd(alias, handle: _greedy-handle(fn))
#let limits-handle(alias, wrap) = define-cmd(1, alias: alias, handle: (it) => math.limits(wrap(it)))
#let matrix-handle(delim: none, handle: none) = define-env(none, kind: "is-matrix", alias: none, handle: math.mat.with(delim: delim))
#let text-handle(wrap) = define-cmd(1, handle: it => $wrap(it)$ + text-end-space(it),)
#let text-handle(handle) = define-cmd(1, handle: handle)
#let call-or-ignore(fn) = (..args) => if args.pos().len() > 0 { fn(..args) } else { math.zws }
#let ignore-me = it => {}
#let ignore-sym = define-sym("")
Expand Down

0 comments on commit 92dff01

Please sign in to comment.