diff --git a/crates/mitex-cli/src/main.rs b/crates/mitex-cli/src/main.rs index 1473539..c6a4bb5 100644 --- a/crates/mitex-cli/src/main.rs +++ b/crates/mitex-cli/src/main.rs @@ -89,9 +89,9 @@ fn compile(input_path: &str, output_path: &str, is_ast: bool) -> Result<(), Erro // Insert preludes // todo: better way? let mut builtin_set = std::collections::HashSet::<&'static str>::new(); - builtin_set.extend(["and", "or", "in", "not"]); + builtin_set.extend(["and", "or", "in", "not", "footnote", "dot"]); let mut alias_set = std::collections::HashSet::>::new(); - for (_, cmd) in spec.items() { + for (key, cmd) in spec.items() { let alias = match cmd { CommandSpecItem::Cmd(CmdShape { alias: Some(alias), .. @@ -99,8 +99,10 @@ fn compile(input_path: &str, output_path: &str, is_ast: bool) -> Result<(), Erro CommandSpecItem::Env(EnvShape { alias: Some(alias), .. }) => alias.as_str(), - _ => continue, + _ => key, }; + // todo: this is hacky + let alias = alias.trim_start_matches('#'); if alias.is_empty() || !alias.chars().all(|c| c.is_ascii_alphanumeric()) { continue; } diff --git a/crates/mitex-lexer/src/macro_engine.rs b/crates/mitex-lexer/src/macro_engine.rs index 64e5ba9..5b9ef31 100644 --- a/crates/mitex-lexer/src/macro_engine.rs +++ b/crates/mitex-lexer/src/macro_engine.rs @@ -192,6 +192,14 @@ pub enum DeclareMacro { /// Synopsis, /// \AtEndDocument{code} AtEndDocument, + /// Command macro for NewTheorem. + /// + /// Synopsis, one of: + /// + /// \newtheorem{name}{caption} + /// \newtheorem{name}[numbered_like]{caption} + /// \newtheorem{name}{caption}[within] + NewTheorem, } fn define_declarative_macros(macros: &mut MacroMap) { @@ -282,6 +290,7 @@ fn define_declarative_macros(macros: &mut MacroMap) { ("AtEndOfPackage", DeclareMacro::AtEndOfPackage), ("AtBeginDocument", DeclareMacro::AtBeginDocument), ("AtEndDocument", DeclareMacro::AtEndDocument), + ("newtheorem", DeclareMacro::NewTheorem), ] .into_iter() { @@ -611,6 +620,23 @@ impl<'a> MacroEngine<'a> { ctx.next_token(); None } + Macro::Declare(NewTheorem) => { + // TODO(testset): preserve theorem metadata instead of registering + // theorem environments as empty wrappers. + let name = Self::identify_theorem_environment(ctx)?; + self.add_macro( + name, + Macro::Env(Arc::new(EnvMacro { + name: name.to_owned(), + num_args: 0, + opt: None, + begin_def: vec![], + end_def: vec![], + })), + ); + + None + } Macro::Cmd(cmd) => { ctx.next_token(); @@ -730,7 +756,7 @@ impl<'a> MacroEngine<'a> { DeclareCmdOrEnv::NewEnvironment { renew, star: _ } => { is_env = true; - if matches!(ctx.peek()?, Token::Left(BraceKind::Curly)) { + if matches!(ctx.peek_not_trivia()?, Token::Left(BraceKind::Curly)) { ctx.next_token(); end_def = Some(ctx.read_until_balanced(BraceKind::Curly)); } @@ -766,6 +792,33 @@ impl<'a> MacroEngine<'a> { Some((name, action, m)) } + fn identify_theorem_environment(ctx: &mut StreamContext<'a>) -> Option<&'a str> { + ctx.next_not_trivia() + .filter(|nx| *nx == Token::Left(BraceKind::Curly))?; + ctx.next_not_trivia(); + + let name = ctx.peek_word_opt(BraceKind::Curly)?.1; + + if matches!(ctx.peek_not_trivia(), Some(Token::Left(BraceKind::Bracket))) { + ctx.next_token(); + ctx.read_until_balanced(BraceKind::Bracket); + } + + if matches!(ctx.peek_not_trivia()?, Token::Left(BraceKind::Curly)) { + ctx.next_token(); + ctx.read_until_balanced(BraceKind::Curly); + } else { + return None; + } + + if matches!(ctx.peek_not_trivia(), Some(Token::Left(BraceKind::Bracket))) { + ctx.next_token(); + ctx.read_until_balanced(BraceKind::Bracket); + } + + Some(name) + } + // todo: insufficient macro arguments fn read_macro_args( ctx: &mut StreamContext<'a>, diff --git a/crates/mitex-lexer/tests/expand_macro.rs b/crates/mitex-lexer/tests/expand_macro.rs index 7380547..51862de 100644 --- a/crates/mitex-lexer/tests/expand_macro.rs +++ b/crates/mitex-lexer/tests/expand_macro.rs @@ -189,6 +189,14 @@ fn subst_macro() { Word("test") Word("end") "###); + assert_snapshot!(tokens( + "\\newenvironment{f}{begin}\n{end}\\begin{f}test\\end{f}" + ), @r###" + Word("begin") + Word("test") + Word("end") + "###); + assert_snapshot!(tokens(r#"\newtheorem{mydef}{Definition}\begin{mydef}test\end{mydef}"#), @r###"Word("test")"###); } #[test] diff --git a/crates/mitex/src/converter.rs b/crates/mitex/src/converter.rs index 32cc3c0..83b0c35 100644 --- a/crates/mitex/src/converter.rs +++ b/crates/mitex/src/converter.rs @@ -274,6 +274,9 @@ impl Converter { "includegraphics" => { self.convert_command_includegraphics(f, &cmd)?; } + // TODO(testset): replace this preamble-only no-op with proper + // command/environment disambiguation for \array. + "array" => {} "color" | "textcolor" | "colorbox" => { self.convert_command_color(f, &cmd, spec)?; } diff --git a/crates/mitex/tests/cvt.rs b/crates/mitex/tests/cvt.rs index 7c492d8..e26a924 100644 --- a/crates/mitex/tests/cvt.rs +++ b/crates/mitex/tests/cvt.rs @@ -60,6 +60,9 @@ mod cvt { #[cfg(test)] mod misc; + + #[cfg(test)] + mod testset; /// Convenient function to launch/debug a test case #[test] fn bug_playground() {} diff --git a/crates/mitex/tests/cvt/testset.rs b/crates/mitex/tests/cvt/testset.rs new file mode 100644 index 0000000..791904b --- /dev/null +++ b/crates/mitex/tests/cvt/testset.rs @@ -0,0 +1,119 @@ +use std::fs; +use std::path::PathBuf; + +use super::prelude::*; + +const CASE: &str = "source-tex/arxiv.2301.00029/YM_sym"; + +#[test] +#[ignore = "requires local/mitex-testset checkout"] +fn test_ym_sym_typ_hash() { + let root = workspace_root(); + let input_path = root.join("local/mitex-testset").join(format!("{CASE}.tex")); + let output_path = root.join("local/mitex-output").join(format!("{CASE}.typ")); + let hash_path = root + .join("fixtures/mitex-testset-hashes") + .join(format!("{CASE}.typ.sha256")); + + assert!( + input_path.exists(), + "missing test input: {}\nrun: git clone --depth 1 https://github.com/mitex-rs/mitex-testset/ local/mitex-testset", + input_path.display() + ); + + let input = fs::read_to_string(&input_path) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", input_path.display())); + let typ = convert_text(&input).unwrap_or_else(|err| panic!("failed to convert {CASE}: {err}")); + let actual_hash = sha256_hex(typ.as_bytes()); + + if let Some(parent) = output_path.parent() { + fs::create_dir_all(parent) + .unwrap_or_else(|err| panic!("failed to create {}: {err}", parent.display())); + } + fs::write(&output_path, &typ) + .unwrap_or_else(|err| panic!("failed to write {}: {err}", output_path.display())); + + let expected_hash = fs::read_to_string(&hash_path) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", hash_path.display())); + assert_eq!(actual_hash, expected_hash.trim()); +} + +fn workspace_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|path| path.parent()) + .expect("crate manifest should be under crates/mitex") + .to_path_buf() +} + +fn sha256_hex(input: &[u8]) -> String { + const H0: [u32; 8] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, + 0x5be0cd19, + ]; + const K: [u32; 64] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, + 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, + 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, + 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, + 0xc67178f2, + ]; + + let mut data = input.to_vec(); + let bit_len = (data.len() as u64) * 8; + data.push(0x80); + while data.len() % 64 != 56 { + data.push(0); + } + data.extend_from_slice(&bit_len.to_be_bytes()); + + let mut h = H0; + for chunk in data.chunks_exact(64) { + let mut w = [0_u32; 64]; + for (word, bytes) in w.iter_mut().zip(chunk.chunks_exact(4)) { + *word = u32::from_be_bytes(bytes.try_into().unwrap()); + } + for i in 16..64 { + let s0 = w[i - 15].rotate_right(7) ^ w[i - 15].rotate_right(18) ^ (w[i - 15] >> 3); + let s1 = w[i - 2].rotate_right(17) ^ w[i - 2].rotate_right(19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + .wrapping_add(s0) + .wrapping_add(w[i - 7]) + .wrapping_add(s1); + } + + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut hh] = h; + for i in 0..64 { + let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); + let ch = (e & f) ^ (!e & g); + let temp1 = hh + .wrapping_add(s1) + .wrapping_add(ch) + .wrapping_add(K[i]) + .wrapping_add(w[i]); + let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); + let maj = (a & b) ^ (a & c) ^ (b & c); + let temp2 = s0.wrapping_add(maj); + + hh = g; + g = f; + f = e; + e = d.wrapping_add(temp1); + d = c; + c = b; + b = a; + a = temp1.wrapping_add(temp2); + } + + for (dst, value) in h.iter_mut().zip([a, b, c, d, e, f, g, hh]) { + *dst = dst.wrapping_add(value); + } + } + + h.iter().map(|word| format!("{word:08x}")).collect() +} diff --git a/fixtures/mitex-testset-hashes/source-tex/arxiv.2301.00029/YM_sym.typ.sha256 b/fixtures/mitex-testset-hashes/source-tex/arxiv.2301.00029/YM_sym.typ.sha256 new file mode 100644 index 0000000..f731493 --- /dev/null +++ b/fixtures/mitex-testset-hashes/source-tex/arxiv.2301.00029/YM_sym.typ.sha256 @@ -0,0 +1 @@ +14b20b42bb2fa11ac8293aa554b8e99be59f2421ec8cd01f97ea4a0ac7fd2a2c diff --git a/packages/mitex/specs/latex/standard.typ b/packages/mitex/specs/latex/standard.typ index 5b76bae..859594b 100644 --- a/packages/mitex/specs/latex/standard.typ +++ b/packages/mitex/specs/latex/standard.typ @@ -79,13 +79,16 @@ alias: none, handle: math.mat.with(delim: delim), ) -#let call-or-ignore(fn) = (..args) => if args.pos().len() > 0 { - fn(..args) -} else { - math.zws -} +#let call-or-ignore(fn) = ( + (..args) => if args.pos().len() > 0 { + fn(..args) + } else { + math.zws + } +) #let ignore-me = (..args) => { } #let ignore-sym = define-sym("") +#let ignore-command(n, alias: none) = define-cmd(n, alias: alias, handle: ignore-me) // 2. Standard package definitions, generate specs and scopes, // for parser/convert and typst respectively @@ -110,12 +113,12 @@ enumerate: define-env(none, kind: "is-enumerate"), quote: define-env(none, alias: "quote(block: true)"), abstract: define-env(none, alias: "quote(block: true)"), + document: define-env(none, alias: ""), figure: define-glob-env("{,b}", kind: "is-figure", alias: "figure"), table: define-glob-env("{,b}", kind: "is-figure", alias: "figure"), tabular: define-env(1, kind: "is-table", alias: "table"), // commands for figure centering: ignore-sym, - textwidth: sym, caption: define-cmd(1, alias: "mitexcaption", handle: ignore-me), includegraphics: define-glob-cmd("{,b}t", "#miteximage", handle: ignore-me), // commands for tabular @@ -1014,7 +1017,7 @@ xlongequal: arrow-handle(math.eq), pmod: define-cmd(1, handle: it => $quad (mod thick it)$), pod: define-cmd(1, handle: it => $quad (it)$), - "set": define-cmd(1, handle: it => $\{it\}$), + "set": define-cmd(1, alias: "mitexset", handle: it => $\{it\}$), Set: define-cmd(1, handle: it => $lr(\{it\})$), bra: define-cmd(1, handle: it => $chevron.l it|$), Bra: define-cmd(1, handle: it => $lr(chevron.l it|)$), @@ -1135,10 +1138,6 @@ ProvideTextCommandDefault: ignore-sym, providecommand: ignore-sym, "providecommand*": ignore-sym, - newenvironment: ignore-sym, - "newenvironment*": ignore-sym, - renewenvironment: ignore-sym, - "renewenvironment*": ignore-sym, AtEndOfClass: ignore-sym, AtEndOfPackage: ignore-sym, AtBeginDocument: ignore-sym, @@ -1183,6 +1182,39 @@ ProvidesClass: ignore-sym, LoadClass: ignore-sym, LoadClassWithOptions: ignore-sym, + + title: ignore-command(1), + author: ignore-command(1), + date: ignore-command(1), + usepackage: ignore-command(1), + thanks: ignore-command(1), + comment: define-env(none, alias: "mitexcomment", handle: ignore-me), + + // todo: set/length commands + topmargin: ignore-sym, + oddsidemargin: ignore-sym, + textwidth: ignore-sym, + today: ignore-sym, + maketitle: ignore-sym, + textheight: ignore-sym, + footskip: ignore-sym, + interfootnotelinepenalty: ignore-sym, + + makeatother: ignore-sym, + makeatletter: ignore-sym, + "env@matrix": ignore-command(1), + arraystretch: ignore-command(1), + bibliography: ignore-command(1, alias: "mitexbibliography"), + bibliographystyle: ignore-command(1), + arraycolsep: ignore-sym, + "@ifnextchar": ignore-sym, + "new@ifnextchar": ignore-sym, + // TODO(testset): model TeX register-style commands from matrix preambles. + "c@MaxMatrixCols": ignore-sym, + // \edef\arraystretch{#1}% + "let": ignore-sym, + edef: ignore-sym, + // todo: \newcommand{\newtheorem}[2]{\newenvironment{#1}{}{}} )) // export: include package name, spec and scope