From 1bc2d5676e475f28ab53792c7b605367ffd03a08 Mon Sep 17 00:00:00 2001 From: Myriad-Dreamin Date: Thu, 1 Aug 2024 01:45:45 +0800 Subject: [PATCH 1/3] dev: work on arxiv.2301.00029 --- crates/mitex-cli/src/main.rs | 8 ++-- packages/mitex/specs/latex/standard.typ | 52 +++++++++++++++++++------ 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/crates/mitex-cli/src/main.rs b/crates/mitex-cli/src/main.rs index d9129aa..79fde6a 100644 --- a/crates/mitex-cli/src/main.rs +++ b/crates/mitex-cli/src/main.rs @@ -89,9 +89,9 @@ fn compile(input_path: &str, output_path: &str, is_ast: bool) -> Result<(), Erro // Insert preludes // todo: better way? let mut builtin_set = std::collections::HashSet::<&'static str>::new(); - builtin_set.extend(["and", "or", "in", "not"]); + builtin_set.extend(["and", "or", "in", "not", "footnote", "dot"]); let mut alias_set = std::collections::HashSet::>::new(); - for (_, cmd) in spec.items() { + for (key, cmd) in spec.items() { let alias = match cmd { CommandSpecItem::Cmd(CmdShape { alias: Some(alias), .. @@ -99,8 +99,10 @@ fn compile(input_path: &str, output_path: &str, is_ast: bool) -> Result<(), Erro CommandSpecItem::Env(EnvShape { alias: Some(alias), .. }) => alias.as_str(), - _ => continue, + _ => key, }; + // todo: this is hacky + let alias = alias.trim_start_matches('#'); if alias.is_empty() || !alias.chars().all(|c| c.is_ascii_alphanumeric()) { continue; } diff --git a/packages/mitex/specs/latex/standard.typ b/packages/mitex/specs/latex/standard.typ index 72c302d..09c2878 100644 --- a/packages/mitex/specs/latex/standard.typ +++ b/packages/mitex/specs/latex/standard.typ @@ -49,13 +49,16 @@ alias: none, handle: math.mat.with(delim: delim), ) -#let call-or-ignore(fn) = (..args) => if args.pos().len() > 0 { - fn(..args) -} else { - math.zws -} +#let call-or-ignore(fn) = ( + (..args) => if args.pos().len() > 0 { + fn(..args) + } else { + math.zws + } +) #let ignore-me = (..args) => { } #let ignore-sym = define-sym("") +#let ignore-command(n, alias: none) = define-cmd(n, alias: alias, handle: ignore-me) // 2. Standard package definitions, generate specs and scopes, // for parser/convert and typst respectively @@ -80,12 +83,12 @@ enumerate: define-env(none, kind: "is-enumerate"), quote: define-env(none, alias: "quote(block: true)"), abstract: define-env(none, alias: "quote(block: true)"), + document: define-env(none, alias: ""), figure: define-glob-env("{,b}", kind: "is-figure", alias: "figure"), table: define-glob-env("{,b}", kind: "is-figure", alias: "figure"), tabular: define-env(1, kind: "is-table", alias: "table"), // commands for figure centering: ignore-sym, - textwidth: sym, caption: define-cmd(1, alias: "mitexcaption", handle: ignore-me), includegraphics: define-glob-cmd("{,b}t", "#miteximage", handle: ignore-me), // commands for tabular @@ -972,7 +975,7 @@ xlongequal: arrow-handle(math.eq), pmod: define-cmd(1, handle: it => $quad (mod thick it)$), pod: define-cmd(1, handle: it => $quad (it)$), - "set": define-cmd(1, handle: it => $\{it\}$), + "set": define-cmd(1, alias: "mitexset", handle: it => $\{it\}$), Set: define-cmd(1, handle: it => $lr(\{it\})$), bra: define-cmd(1, handle: it => $angle.l it|$), Bra: define-cmd(1, handle: it => $lr(angle.l it|)$), @@ -1092,10 +1095,6 @@ ProvideTextCommandDefault: ignore-sym, providecommand: ignore-sym, "providecommand*": ignore-sym, - newenvironment: ignore-sym, - "newenvironment*": ignore-sym, - renewenvironment: ignore-sym, - "renewenvironment*": ignore-sym, AtEndOfClass: ignore-sym, AtEndOfPackage: ignore-sym, AtBeginDocument: ignore-sym, @@ -1140,6 +1139,37 @@ ProvidesClass: ignore-sym, LoadClass: ignore-sym, LoadClassWithOptions: ignore-sym, + + title: ignore-command(1), + author: ignore-command(1), + date: ignore-command(1), + usepackage: ignore-command(1), + thanks: ignore-command(1), + comment: define-env(none, alias: "mitexcomment", handle: ignore-me), + + // todo: set/length commands + topmargin: ignore-sym, + oddsidemargin: ignore-sym, + textwidth: ignore-sym, + today: ignore-sym, + maketitle: ignore-sym, + textheight: ignore-sym, + footskip: ignore-sym, + interfootnotelinepenalty: ignore-sym, + + makeatother: ignore-sym, + makeatletter: ignore-sym, + "env@matrix": ignore-command(1), + arraystretch: ignore-command(1), + bibliography: ignore-command(1, alias: "mitexbibliography"), + bibliographystyle: ignore-command(1), + arraycolsep: ignore-sym, + "@ifnextchar": ignore-sym, + "new@ifnextchar": ignore-sym, + // \edef\arraystretch{#1}% + "let": ignore-sym, + edef: ignore-sym, + // todo: \newcommand{\newtheorem}[2]{\newenvironment{#1}{}{}} )) // export: include package name, spec and scope From 863eb8767fe7c967361cfa5b5cb798304a9bb016 Mon Sep 17 00:00:00 2001 From: Myriad-Dreamin Date: Sat, 9 May 2026 17:44:08 +0800 Subject: [PATCH 2/3] test: add mitex testset hash regression --- crates/mitex-lexer/src/macro_engine.rs | 53 +++++++- crates/mitex-lexer/tests/expand_macro.rs | 8 ++ crates/mitex/src/converter.rs | 1 + crates/mitex/tests/cvt.rs | 3 + crates/mitex/tests/cvt/testset.rs | 119 ++++++++++++++++++ .../arxiv.2301.00029/YM_sym.typ.sha256 | 1 + packages/mitex/specs/latex/standard.typ | 1 + 7 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 crates/mitex/tests/cvt/testset.rs create mode 100644 fixtures/mitex-testset-hashes/source-tex/arxiv.2301.00029/YM_sym.typ.sha256 diff --git a/crates/mitex-lexer/src/macro_engine.rs b/crates/mitex-lexer/src/macro_engine.rs index 64e5ba9..65dbc13 100644 --- a/crates/mitex-lexer/src/macro_engine.rs +++ b/crates/mitex-lexer/src/macro_engine.rs @@ -192,6 +192,14 @@ pub enum DeclareMacro { /// Synopsis, /// \AtEndDocument{code} AtEndDocument, + /// Command macro for NewTheorem. + /// + /// Synopsis, one of: + /// + /// \newtheorem{name}{caption} + /// \newtheorem{name}[numbered_like]{caption} + /// \newtheorem{name}{caption}[within] + NewTheorem, } fn define_declarative_macros(macros: &mut MacroMap) { @@ -282,6 +290,7 @@ fn define_declarative_macros(macros: &mut MacroMap) { ("AtEndOfPackage", DeclareMacro::AtEndOfPackage), ("AtBeginDocument", DeclareMacro::AtBeginDocument), ("AtEndDocument", DeclareMacro::AtEndDocument), + ("newtheorem", DeclareMacro::NewTheorem), ] .into_iter() { @@ -611,6 +620,21 @@ impl<'a> MacroEngine<'a> { ctx.next_token(); None } + Macro::Declare(NewTheorem) => { + let name = Self::identify_theorem_environment(ctx)?; + self.add_macro( + name, + Macro::Env(Arc::new(EnvMacro { + name: name.to_owned(), + num_args: 0, + opt: None, + begin_def: vec![], + end_def: vec![], + })), + ); + + None + } Macro::Cmd(cmd) => { ctx.next_token(); @@ -730,7 +754,7 @@ impl<'a> MacroEngine<'a> { DeclareCmdOrEnv::NewEnvironment { renew, star: _ } => { is_env = true; - if matches!(ctx.peek()?, Token::Left(BraceKind::Curly)) { + if matches!(ctx.peek_not_trivia()?, Token::Left(BraceKind::Curly)) { ctx.next_token(); end_def = Some(ctx.read_until_balanced(BraceKind::Curly)); } @@ -766,6 +790,33 @@ impl<'a> MacroEngine<'a> { Some((name, action, m)) } + fn identify_theorem_environment(ctx: &mut StreamContext<'a>) -> Option<&'a str> { + ctx.next_not_trivia() + .filter(|nx| *nx == Token::Left(BraceKind::Curly))?; + ctx.next_not_trivia(); + + let name = ctx.peek_word_opt(BraceKind::Curly)?.1; + + if matches!(ctx.peek_not_trivia(), Some(Token::Left(BraceKind::Bracket))) { + ctx.next_token(); + ctx.read_until_balanced(BraceKind::Bracket); + } + + if matches!(ctx.peek_not_trivia()?, Token::Left(BraceKind::Curly)) { + ctx.next_token(); + ctx.read_until_balanced(BraceKind::Curly); + } else { + return None; + } + + if matches!(ctx.peek_not_trivia(), Some(Token::Left(BraceKind::Bracket))) { + ctx.next_token(); + ctx.read_until_balanced(BraceKind::Bracket); + } + + Some(name) + } + // todo: insufficient macro arguments fn read_macro_args( ctx: &mut StreamContext<'a>, diff --git a/crates/mitex-lexer/tests/expand_macro.rs b/crates/mitex-lexer/tests/expand_macro.rs index 0ad1f57..2e1797d 100644 --- a/crates/mitex-lexer/tests/expand_macro.rs +++ b/crates/mitex-lexer/tests/expand_macro.rs @@ -188,6 +188,14 @@ fn subst_macro() { Word("test") Word("end") "###); + assert_snapshot!(tokens( + "\\newenvironment{f}{begin}\n{end}\\begin{f}test\\end{f}" + ), @r###" + Word("begin") + Word("test") + Word("end") + "###); + assert_snapshot!(tokens(r#"\newtheorem{mydef}{Definition}\begin{mydef}test\end{mydef}"#), @r###"Word("test")"###); } #[test] diff --git a/crates/mitex/src/converter.rs b/crates/mitex/src/converter.rs index 7418cb2..5d0c22d 100644 --- a/crates/mitex/src/converter.rs +++ b/crates/mitex/src/converter.rs @@ -274,6 +274,7 @@ impl Converter { "includegraphics" => { self.convert_command_includegraphics(f, &cmd)?; } + "array" => {} _ => { self.convert_normal_command(f, elem, spec)?; } diff --git a/crates/mitex/tests/cvt.rs b/crates/mitex/tests/cvt.rs index 7c492d8..e26a924 100644 --- a/crates/mitex/tests/cvt.rs +++ b/crates/mitex/tests/cvt.rs @@ -60,6 +60,9 @@ mod cvt { #[cfg(test)] mod misc; + + #[cfg(test)] + mod testset; /// Convenient function to launch/debug a test case #[test] fn bug_playground() {} diff --git a/crates/mitex/tests/cvt/testset.rs b/crates/mitex/tests/cvt/testset.rs new file mode 100644 index 0000000..791904b --- /dev/null +++ b/crates/mitex/tests/cvt/testset.rs @@ -0,0 +1,119 @@ +use std::fs; +use std::path::PathBuf; + +use super::prelude::*; + +const CASE: &str = "source-tex/arxiv.2301.00029/YM_sym"; + +#[test] +#[ignore = "requires local/mitex-testset checkout"] +fn test_ym_sym_typ_hash() { + let root = workspace_root(); + let input_path = root.join("local/mitex-testset").join(format!("{CASE}.tex")); + let output_path = root.join("local/mitex-output").join(format!("{CASE}.typ")); + let hash_path = root + .join("fixtures/mitex-testset-hashes") + .join(format!("{CASE}.typ.sha256")); + + assert!( + input_path.exists(), + "missing test input: {}\nrun: git clone --depth 1 https://github.com/mitex-rs/mitex-testset/ local/mitex-testset", + input_path.display() + ); + + let input = fs::read_to_string(&input_path) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", input_path.display())); + let typ = convert_text(&input).unwrap_or_else(|err| panic!("failed to convert {CASE}: {err}")); + let actual_hash = sha256_hex(typ.as_bytes()); + + if let Some(parent) = output_path.parent() { + fs::create_dir_all(parent) + .unwrap_or_else(|err| panic!("failed to create {}: {err}", parent.display())); + } + fs::write(&output_path, &typ) + .unwrap_or_else(|err| panic!("failed to write {}: {err}", output_path.display())); + + let expected_hash = fs::read_to_string(&hash_path) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", hash_path.display())); + assert_eq!(actual_hash, expected_hash.trim()); +} + +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|path| path.parent()) + .expect("crate manifest should be under crates/mitex") + .to_path_buf() +} + +fn sha256_hex(input: &[u8]) -> String { + const H0: [u32; 8] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, + 0x5be0cd19, + ]; + const K: [u32; 64] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, + 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, + 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, + 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, + 0xc67178f2, + ]; + + let mut data = input.to_vec(); + let bit_len = (data.len() as u64) * 8; + data.push(0x80); + while data.len() % 64 != 56 { + data.push(0); + } + data.extend_from_slice(&bit_len.to_be_bytes()); + + let mut h = H0; + for chunk in data.chunks_exact(64) { + let mut w = [0_u32; 64]; + for (word, bytes) in w.iter_mut().zip(chunk.chunks_exact(4)) { + *word = u32::from_be_bytes(bytes.try_into().unwrap()); + } + for i in 16..64 { + let s0 = w[i - 15].rotate_right(7) ^ w[i - 15].rotate_right(18) ^ (w[i - 15] >> 3); + let s1 = w[i - 2].rotate_right(17) ^ w[i - 2].rotate_right(19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + .wrapping_add(s0) + .wrapping_add(w[i - 7]) + .wrapping_add(s1); + } + + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut hh] = h; + for i in 0..64 { + let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); + let ch = (e & f) ^ (!e & g); + let temp1 = hh + .wrapping_add(s1) + .wrapping_add(ch) + .wrapping_add(K[i]) + .wrapping_add(w[i]); + let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); + let maj = (a & b) ^ (a & c) ^ (b & c); + let temp2 = s0.wrapping_add(maj); + + hh = g; + g = f; + f = e; + e = d.wrapping_add(temp1); + d = c; + c = b; + b = a; + a = temp1.wrapping_add(temp2); + } + + for (dst, value) in h.iter_mut().zip([a, b, c, d, e, f, g, hh]) { + *dst = dst.wrapping_add(value); + } + } + + h.iter().map(|word| format!("{word:08x}")).collect() +} diff --git a/fixtures/mitex-testset-hashes/source-tex/arxiv.2301.00029/YM_sym.typ.sha256 b/fixtures/mitex-testset-hashes/source-tex/arxiv.2301.00029/YM_sym.typ.sha256 new file mode 100644 index 0000000..f731493 --- /dev/null +++ b/fixtures/mitex-testset-hashes/source-tex/arxiv.2301.00029/YM_sym.typ.sha256 @@ -0,0 +1 @@ +14b20b42bb2fa11ac8293aa554b8e99be59f2421ec8cd01f97ea4a0ac7fd2a2c diff --git a/packages/mitex/specs/latex/standard.typ b/packages/mitex/specs/latex/standard.typ index 09c2878..8b94acf 100644 --- a/packages/mitex/specs/latex/standard.typ +++ b/packages/mitex/specs/latex/standard.typ @@ -1166,6 +1166,7 @@ arraycolsep: ignore-sym, "@ifnextchar": ignore-sym, "new@ifnextchar": ignore-sym, + "c@MaxMatrixCols": ignore-sym, // \edef\arraystretch{#1}% "let": ignore-sym, edef: ignore-sym, From 6228e0e27ed38b043e25638391634f4d6b9612fe Mon Sep 17 00:00:00 2001 From: Myriad-Dreamin Date: Sat, 9 May 2026 18:06:49 +0800 Subject: [PATCH 3/3] chore: add TODOs for theorem and array handling --- crates/mitex-lexer/src/macro_engine.rs | 2 ++ crates/mitex/src/converter.rs | 2 ++ packages/mitex/specs/latex/standard.typ | 1 + 3 files changed, 5 insertions(+) diff --git a/crates/mitex-lexer/src/macro_engine.rs b/crates/mitex-lexer/src/macro_engine.rs index 65dbc13..5b9ef31 100644 --- a/crates/mitex-lexer/src/macro_engine.rs +++ b/crates/mitex-lexer/src/macro_engine.rs @@ -621,6 +621,8 @@ impl<'a> MacroEngine<'a> { None } Macro::Declare(NewTheorem) => { + // TODO(testset): preserve theorem metadata instead of registering + // theorem environments as empty wrappers. let name = Self::identify_theorem_environment(ctx)?; self.add_macro( name, diff --git a/crates/mitex/src/converter.rs b/crates/mitex/src/converter.rs index 480fa1d..83b0c35 100644 --- a/crates/mitex/src/converter.rs +++ b/crates/mitex/src/converter.rs @@ -274,6 +274,8 @@ impl Converter { "includegraphics" => { self.convert_command_includegraphics(f, &cmd)?; } + // TODO(testset): replace this preamble-only no-op with proper + // command/environment disambiguation for \array. "array" => {} "color" | "textcolor" | "colorbox" => { self.convert_command_color(f, &cmd, spec)?; diff --git a/packages/mitex/specs/latex/standard.typ b/packages/mitex/specs/latex/standard.typ index f0d1f07..859594b 100644 --- a/packages/mitex/specs/latex/standard.typ +++ b/packages/mitex/specs/latex/standard.typ @@ -1209,6 +1209,7 @@ arraycolsep: ignore-sym, "@ifnextchar": ignore-sym, "new@ifnextchar": ignore-sym, + // TODO(testset): model TeX register-style commands from matrix preambles. "c@MaxMatrixCols": ignore-sym, // \edef\arraystretch{#1}% "let": ignore-sym,