Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions crates/mitex-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,18 +89,20 @@ fn compile(input_path: &str, output_path: &str, is_ast: bool) -> Result<(), Erro
// Insert preludes
// todo: better way?
let mut builtin_set = std::collections::HashSet::<&'static str>::new();
builtin_set.extend(["and", "or", "in", "not"]);
builtin_set.extend(["and", "or", "in", "not", "footnote", "dot"]);
let mut alias_set = std::collections::HashSet::<Box<str>>::new();
for (_, cmd) in spec.items() {
for (key, cmd) in spec.items() {
let alias = match cmd {
CommandSpecItem::Cmd(CmdShape {
alias: Some(alias), ..
}) => alias.as_str(),
CommandSpecItem::Env(EnvShape {
alias: Some(alias), ..
}) => alias.as_str(),
_ => continue,
_ => key,
};
// todo: this is hacky
let alias = alias.trim_start_matches('#');
if alias.is_empty() || !alias.chars().all(|c| c.is_ascii_alphanumeric()) {
continue;
}
Expand Down
55 changes: 54 additions & 1 deletion crates/mitex-lexer/src/macro_engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,14 @@ pub enum DeclareMacro {
/// Synopsis,
/// \AtEndDocument{code}
AtEndDocument,
/// Command macro for NewTheorem.
///
/// Synopsis, one of:
///
/// \newtheorem{name}{caption}
/// \newtheorem{name}[numbered_like]{caption}
/// \newtheorem{name}{caption}[within]
NewTheorem,
}

fn define_declarative_macros(macros: &mut MacroMap) {
Expand Down Expand Up @@ -282,6 +290,7 @@ fn define_declarative_macros(macros: &mut MacroMap) {
("AtEndOfPackage", DeclareMacro::AtEndOfPackage),
("AtBeginDocument", DeclareMacro::AtBeginDocument),
("AtEndDocument", DeclareMacro::AtEndDocument),
("newtheorem", DeclareMacro::NewTheorem),
]
.into_iter()
{
Expand Down Expand Up @@ -611,6 +620,23 @@ impl<'a> MacroEngine<'a> {
ctx.next_token();
None
}
Macro::Declare(NewTheorem) => {
// TODO(testset): preserve theorem metadata instead of registering
// theorem environments as empty wrappers.
let name = Self::identify_theorem_environment(ctx)?;
self.add_macro(
name,
Macro::Env(Arc::new(EnvMacro {
name: name.to_owned(),
num_args: 0,
opt: None,
begin_def: vec![],
end_def: vec![],
})),
);

None
}
Macro::Cmd(cmd) => {
ctx.next_token();

Expand Down Expand Up @@ -730,7 +756,7 @@ impl<'a> MacroEngine<'a> {
DeclareCmdOrEnv::NewEnvironment { renew, star: _ } => {
is_env = true;

if matches!(ctx.peek()?, Token::Left(BraceKind::Curly)) {
if matches!(ctx.peek_not_trivia()?, Token::Left(BraceKind::Curly)) {
ctx.next_token();
end_def = Some(ctx.read_until_balanced(BraceKind::Curly));
}
Expand Down Expand Up @@ -766,6 +792,33 @@ impl<'a> MacroEngine<'a> {
Some((name, action, m))
}

fn identify_theorem_environment(ctx: &mut StreamContext<'a>) -> Option<&'a str> {
ctx.next_not_trivia()
.filter(|nx| *nx == Token::Left(BraceKind::Curly))?;
ctx.next_not_trivia();

let name = ctx.peek_word_opt(BraceKind::Curly)?.1;

if matches!(ctx.peek_not_trivia(), Some(Token::Left(BraceKind::Bracket))) {
ctx.next_token();
ctx.read_until_balanced(BraceKind::Bracket);
}

if matches!(ctx.peek_not_trivia()?, Token::Left(BraceKind::Curly)) {
ctx.next_token();
ctx.read_until_balanced(BraceKind::Curly);
} else {
return None;
}

if matches!(ctx.peek_not_trivia(), Some(Token::Left(BraceKind::Bracket))) {
ctx.next_token();
ctx.read_until_balanced(BraceKind::Bracket);
}

Some(name)
}

// todo: insufficient macro arguments
fn read_macro_args(
ctx: &mut StreamContext<'a>,
Expand Down
8 changes: 8 additions & 0 deletions crates/mitex-lexer/tests/expand_macro.rs
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,14 @@ fn subst_macro() {
Word("test")
Word("end")
"###);
assert_snapshot!(tokens(
"\\newenvironment{f}{begin}\n{end}\\begin{f}test\\end{f}"
), @r###"
Word("begin")
Word("test")
Word("end")
"###);
assert_snapshot!(tokens(r#"\newtheorem{mydef}{Definition}\begin{mydef}test\end{mydef}"#), @r###"Word("test")"###);
}

#[test]
Expand Down
3 changes: 3 additions & 0 deletions crates/mitex/src/converter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,9 @@ impl Converter {
"includegraphics" => {
self.convert_command_includegraphics(f, &cmd)?;
}
// TODO(testset): replace this preamble-only no-op with proper
// command/environment disambiguation for \array.
"array" => {}
"color" | "textcolor" | "colorbox" => {
self.convert_command_color(f, &cmd, spec)?;
}
Expand Down
3 changes: 3 additions & 0 deletions crates/mitex/tests/cvt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ mod cvt {

#[cfg(test)]
mod misc;

#[cfg(test)]
mod testset;
/// Convenient function to launch/debug a test case
#[test]
fn bug_playground() {}
Expand Down
119 changes: 119 additions & 0 deletions crates/mitex/tests/cvt/testset.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
use std::fs;
use std::path::PathBuf;

use super::prelude::*;

const CASE: &str = "source-tex/arxiv.2301.00029/YM_sym";

#[test]
#[ignore = "requires local/mitex-testset checkout"]
fn test_ym_sym_typ_hash() {
let root = workspace_root();
let input_path = root.join("local/mitex-testset").join(format!("{CASE}.tex"));
let output_path = root.join("local/mitex-output").join(format!("{CASE}.typ"));
let hash_path = root
.join("fixtures/mitex-testset-hashes")
.join(format!("{CASE}.typ.sha256"));

assert!(
input_path.exists(),
"missing test input: {}\nrun: git clone --depth 1 https://github.com/mitex-rs/mitex-testset/ local/mitex-testset",
input_path.display()
);

let input = fs::read_to_string(&input_path)
.unwrap_or_else(|err| panic!("failed to read {}: {err}", input_path.display()));
let typ = convert_text(&input).unwrap_or_else(|err| panic!("failed to convert {CASE}: {err}"));
let actual_hash = sha256_hex(typ.as_bytes());

if let Some(parent) = output_path.parent() {
fs::create_dir_all(parent)
.unwrap_or_else(|err| panic!("failed to create {}: {err}", parent.display()));
}
fs::write(&output_path, &typ)
.unwrap_or_else(|err| panic!("failed to write {}: {err}", output_path.display()));

let expected_hash = fs::read_to_string(&hash_path)
.unwrap_or_else(|err| panic!("failed to read {}: {err}", hash_path.display()));
assert_eq!(actual_hash, expected_hash.trim());
}

fn workspace_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.and_then(|path| path.parent())
.expect("crate manifest should be under crates/mitex")
.to_path_buf()
}

fn sha256_hex(input: &[u8]) -> String {
const H0: [u32; 8] = [
0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab,
0x5be0cd19,
];
const K: [u32; 64] = [
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4,
0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe,
0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f,
0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc,
0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116,
0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
0xc67178f2,
];

let mut data = input.to_vec();
let bit_len = (data.len() as u64) * 8;
data.push(0x80);
while data.len() % 64 != 56 {
data.push(0);
}
data.extend_from_slice(&bit_len.to_be_bytes());

let mut h = H0;
for chunk in data.chunks_exact(64) {
let mut w = [0_u32; 64];
for (word, bytes) in w.iter_mut().zip(chunk.chunks_exact(4)) {
*word = u32::from_be_bytes(bytes.try_into().unwrap());
}
for i in 16..64 {
let s0 = w[i - 15].rotate_right(7) ^ w[i - 15].rotate_right(18) ^ (w[i - 15] >> 3);
let s1 = w[i - 2].rotate_right(17) ^ w[i - 2].rotate_right(19) ^ (w[i - 2] >> 10);
w[i] = w[i - 16]
.wrapping_add(s0)
.wrapping_add(w[i - 7])
.wrapping_add(s1);
}

let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut hh] = h;
for i in 0..64 {
let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25);
let ch = (e & f) ^ (!e & g);
let temp1 = hh
.wrapping_add(s1)
.wrapping_add(ch)
.wrapping_add(K[i])
.wrapping_add(w[i]);
let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22);
let maj = (a & b) ^ (a & c) ^ (b & c);
let temp2 = s0.wrapping_add(maj);

hh = g;
g = f;
f = e;
e = d.wrapping_add(temp1);
d = c;
c = b;
b = a;
a = temp1.wrapping_add(temp2);
}

for (dst, value) in h.iter_mut().zip([a, b, c, d, e, f, g, hh]) {
*dst = dst.wrapping_add(value);
}
}

h.iter().map(|word| format!("{word:08x}")).collect()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
14b20b42bb2fa11ac8293aa554b8e99be59f2421ec8cd01f97ea4a0ac7fd2a2c
54 changes: 43 additions & 11 deletions packages/mitex/specs/latex/standard.typ
Original file line number Diff line number Diff line change
Expand Up @@ -79,13 +79,16 @@
alias: none,
handle: math.mat.with(delim: delim),
)
#let call-or-ignore(fn) = (..args) => if args.pos().len() > 0 {
fn(..args)
} else {
math.zws
}
#let call-or-ignore(fn) = (
(..args) => if args.pos().len() > 0 {
fn(..args)
} else {
math.zws
}
)
#let ignore-me = (..args) => { }
#let ignore-sym = define-sym("")
#let ignore-command(n, alias: none) = define-cmd(n, alias: alias, handle: ignore-me)

// 2. Standard package definitions, generate specs and scopes,
// for parser/convert and typst respectively
Expand All @@ -110,12 +113,12 @@
enumerate: define-env(none, kind: "is-enumerate"),
quote: define-env(none, alias: "quote(block: true)"),
abstract: define-env(none, alias: "quote(block: true)"),
document: define-env(none, alias: ""),
figure: define-glob-env("{,b}", kind: "is-figure", alias: "figure"),
table: define-glob-env("{,b}", kind: "is-figure", alias: "figure"),
tabular: define-env(1, kind: "is-table", alias: "table"),
// commands for figure
centering: ignore-sym,
textwidth: sym,
caption: define-cmd(1, alias: "mitexcaption", handle: ignore-me),
includegraphics: define-glob-cmd("{,b}t", "#miteximage", handle: ignore-me),
// commands for tabular
Expand Down Expand Up @@ -1014,7 +1017,7 @@
xlongequal: arrow-handle(math.eq),
pmod: define-cmd(1, handle: it => $quad (mod thick it)$),
pod: define-cmd(1, handle: it => $quad (it)$),
"set": define-cmd(1, handle: it => $\{it\}$),
"set": define-cmd(1, alias: "mitexset", handle: it => $\{it\}$),
Set: define-cmd(1, handle: it => $lr(\{it\})$),
bra: define-cmd(1, handle: it => $chevron.l it|$),
Bra: define-cmd(1, handle: it => $lr(chevron.l it|)$),
Expand Down Expand Up @@ -1135,10 +1138,6 @@
ProvideTextCommandDefault: ignore-sym,
providecommand: ignore-sym,
"providecommand*": ignore-sym,
newenvironment: ignore-sym,
"newenvironment*": ignore-sym,
renewenvironment: ignore-sym,
"renewenvironment*": ignore-sym,
AtEndOfClass: ignore-sym,
AtEndOfPackage: ignore-sym,
AtBeginDocument: ignore-sym,
Expand Down Expand Up @@ -1183,6 +1182,39 @@
ProvidesClass: ignore-sym,
LoadClass: ignore-sym,
LoadClassWithOptions: ignore-sym,

title: ignore-command(1),
author: ignore-command(1),
date: ignore-command(1),
usepackage: ignore-command(1),
thanks: ignore-command(1),
comment: define-env(none, alias: "mitexcomment", handle: ignore-me),

// todo: set/length commands
topmargin: ignore-sym,
oddsidemargin: ignore-sym,
textwidth: ignore-sym,
today: ignore-sym,
maketitle: ignore-sym,
textheight: ignore-sym,
footskip: ignore-sym,
interfootnotelinepenalty: ignore-sym,

makeatother: ignore-sym,
makeatletter: ignore-sym,
"env@matrix": ignore-command(1),
arraystretch: ignore-command(1),
bibliography: ignore-command(1, alias: "mitexbibliography"),
bibliographystyle: ignore-command(1),
arraycolsep: ignore-sym,
"@ifnextchar": ignore-sym,
"new@ifnextchar": ignore-sym,
// TODO(testset): model TeX register-style commands from matrix preambles.
"c@MaxMatrixCols": ignore-sym,
// \edef\arraystretch{#1}%
"let": ignore-sym,
edef: ignore-sym,
// todo: \newcommand{\newtheorem}[2]{\newenvironment{#1}{}{}}
))

// export: include package name, spec and scope
Expand Down
Loading