From e6d1ae9f368fa302723d0b67cdc9a5ef45edc77c Mon Sep 17 00:00:00 2001 From: Quin Gillespie Date: Wed, 30 Sep 2026 20:10:14 -0600 Subject: [PATCH] Add lang-* and braille-* features to choose which rules include-zip embeds Every language and braille code has a feature, all on by default through rules-all, so existing builds embed exactly the same rules.zip. A build that turns off default features and names a subset embeds only those directories. en is always kept because the library falls back to it, a subset with no braille code keeps UEB so set_rules_dir still succeeds, and the staged prefs.yaml BrailleCode default is pointed at a kept code when Nemeth was pruned. build.rs fails if a Rules directory has no feature in lang-all or braille-all, so the lists cannot silently fall behind. --- AGENTS.md | 2 +- Cargo.toml | 37 ++++++++++++ build.rs | 63 +++++++++++++++++++- src/rules_archive.rs | 136 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 231 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7093cf698..d752728e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ but add common mistakes of AI agents here instead. - Common per-language files: - `ClearSpeak_Rules.yaml`, `SimpleSpeak_Rules.yaml` - `SharedRules/`, `unicode.yaml`, `unicode-full.yaml`, `definitions.yaml`, `navigate.yaml` -- `build.rs` can bundle rules into `rules.zip` when `include-zip` is enabled (always the minimized tree). Shared zipper: `src/rules_archive.rs`, invoked as `cargo run --bin package-rules -- Rules [--minimize]`. +- `build.rs` can bundle rules into `rules.zip` when `include-zip` is enabled (always the minimized tree). The `lang-*` / `braille-*` features (all on by default via `rules-all`) choose which `Languages/` and `Braille/` directories it embeds; every rules directory needs a feature listed in `lang-all` / `braille-all`, and `build.rs` fails without one. Shared zipper: `src/rules_archive.rs`, invoked as `cargo run --bin package-rules -- Rules [--minimize]`. - CI packages `Rules.zip` (verbatim) and `Rules-minimized.zip` (comments stripped from `Languages/**/unicode.yaml` and `unicode-full.yaml` only). Each language/braille subdir is stored as `/.zip`; top-level and `Intent/` YAML stay loose. Inner language zips use BZIP2; the outer downloadable archive uses DEFLATE so `unzip` works. Both omit `Languages/zz`; the `test-rules-package` job restores `zz` from git after unzip so unit tests still run against release-like archives. ## Translation Conventions diff --git a/Cargo.toml b/Cargo.toml index e07193031..5bd27afa8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,11 +12,48 @@ exclude = ["src/main.rs", "docs", "BrailleDocs", "PythonScripts"] # should ha [features] +default = ["rules-all"] "include-zip" = [] "enable-logs" = ["android_logger"] "tts" = [ "natural-tts" ] "no-unsafe" = ["sxd-document-no-unsafe/no-unsafe", "sxd-xpath-no-unsafe/no-unsafe"] +# Which Rules/Languages and Rules/Braille directories `include-zip` embeds. Every one is on by default. +# To embed fewer, turn off default features and name the ones you want, for example +# `default-features = false, features = ["include-zip", "lang-en", "braille-ueb"]`. +# `en` is always embedded because the library falls back to it, and `build.rs` fails if a rules +# directory has no feature here or is missing from `lang-all` / `braille-all`. +# MathCAT itself never asks for ASCIIMath or LaTeX, but an application that copies math in those +# notations does so through the braille API, so it needs `braille-asciimath` / `braille-latex`. +# These only change what is embedded: without `include-zip`, the Rules directory is read from disk. +rules-all = ["lang-all", "braille-all"] +lang-all = ["lang-de", "lang-el", "lang-en", "lang-es", "lang-fi", "lang-fr", "lang-hu", "lang-id", "lang-nb", "lang-pl", "lang-pt", "lang-ru", "lang-sv", "lang-vi", "lang-zh"] +braille-all = ["braille-asciimath", "braille-cmu", "braille-french", "braille-latex", "braille-nemeth", "braille-russian", "braille-swedish", "braille-ueb", "braille-vietnam"] +lang-de = [] +lang-el = [] +lang-en = [] +lang-es = [] +lang-fi = [] +lang-fr = [] +lang-hu = [] +lang-id = [] +lang-nb = [] +lang-pl = [] +lang-pt = [] +lang-ru = [] +lang-sv = [] +lang-vi = [] +lang-zh = [] +braille-asciimath = [] +braille-cmu = [] +braille-french = [] +braille-latex = [] +braille-nemeth = [] +braille-russian = [] +braille-swedish = [] +braille-ueb = [] +braille-vietnam = [] + [dependencies] sxd-document-no-unsafe = "0.4.2" diff --git a/build.rs b/build.rs index e5a07cd8d..f2f2862fd 100644 --- a/build.rs +++ b/build.rs @@ -6,21 +6,80 @@ include!("src/rules_archive.rs"); +/// The `Rules/` directories whose `-` feature is on, or `None` when all of them +/// are, so a default build stages exactly what it always has. +/// +/// Panics if a directory is missing from `-all` in Cargo.toml. Otherwise a newly added +/// language or braille code would quietly drop out of every build, since nothing would turn it on. +fn subset_from_features(rules_dir: &Path, kind: &str, prefix: &str, manifest: &str) -> Option> { + let all_feature = format!("{}-all", prefix); + // The list can span lines: `cargo publish` rewrites Cargo.toml with one entry per line. + let all_list = manifest + .split(&format!(" +{} = [", all_feature)) + .nth(1) + .and_then(|rest| rest.split(']').next()) + .unwrap_or_else(|| panic!("Cargo.toml has no `{}` feature", all_feature)); + let mut kept = Vec::new(); + let mut everything = true; + for entry in read_dir(rules_dir.join(kind)).unwrap() { + let path = entry.unwrap().path(); + let name = path.file_name().unwrap().to_string_lossy().to_string(); + if !path.is_dir() || (kind == LANGUAGES_DIR && name == SKIP_LANGUAGE_DIR) { + continue; + } + let feature = format!("{}-{}", prefix, name.to_ascii_lowercase()); + if !all_list.contains(&format!("\"{}\"", feature)) { + panic!( + "Rules/{}/{} has no feature: add `{} = []` to Cargo.toml and list it in `{}`", + kind, name, feature, all_feature + ); + } + let env_var = format!("CARGO_FEATURE_{}", feature.to_ascii_uppercase().replace('-', "_")); + if std::env::var_os(env_var).is_some() { + kept.push(name); + } else { + everything = false; + } + } + kept.sort(); + return if everything { None } else { Some(kept) }; +} + fn main() { println!("cargo::rerun-if-changed=build.rs"); println!("cargo::rerun-if-changed=src/rules_archive.rs"); println!("cargo::rerun-if-changed=Rules"); + println!("cargo::rerun-if-changed=Cargo.toml"); + + let rules_dir = std::env::current_dir().unwrap().join("Rules"); + let manifest = fs::read_to_string("Cargo.toml").unwrap(); + // Checked on every build, not only include-zip ones, so a rules directory added without a + // feature is caught by whoever adds it. + let languages = subset_from_features(&rules_dir, LANGUAGES_DIR, "lang", &manifest); + let braille_codes = subset_from_features(&rules_dir, BRAILLE_DIR, "braille", &manifest); // This doesn't work because the build claims OUT_DIR is not defined(?) // let archive = PathBuf::from(concat!(env!("OUT_DIR"),"/rules.zip")); if std::env::var("CARGO_FEATURE_INCLUDE_ZIP").is_ok() { let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()); - let rules_dir = std::env::current_dir().unwrap().join("Rules"); let staging = out_dir.join("rules_src"); if staging.exists() { let _ = std::fs::remove_dir_all(&staging); } - if let Err(e) = copy_rules_tree(&rules_dir, &staging, true) { + let subset = RulesSubset { languages, braille_codes }; + if let Some(langs) = &subset.languages { + let mut langs = langs.clone(); + if !langs.iter().any(|l| l.eq_ignore_ascii_case("en")) { + langs.push("en (the fallback)".to_string()); + } + println!("cargo::warning=MathCAT rules limited to languages: {}", langs.join(", ")); + } + if let Some(codes) = &subset.braille_codes { + let codes = if codes.is_empty() { "UEB (the fallback)".to_string() } else { codes.join(", ") }; + println!("cargo::warning=MathCAT rules limited to braille codes: {}", codes); + } + if let Err(e) = copy_rules_subset(&rules_dir, &staging, true, &subset) { panic!("build.rs failed to stage minimized Rules: {}", e); } let archive_path = out_dir.join("rules.zip"); diff --git a/src/rules_archive.rs b/src/rules_archive.rs index 938937683..e3f022ca9 100644 --- a/src/rules_archive.rs +++ b/src/rules_archive.rs @@ -20,6 +20,56 @@ use zip::CompressionMethod; const YAML_SUFFIXES: [&str; 2] = ["yaml", "yml"]; const SKIP_LANGUAGE_DIR: &str = "zz"; const ARCHIVE_ROOT: &str = "Rules"; +const LANGUAGES_DIR: &str = "Languages"; +const BRAILLE_DIR: &str = "Braille"; +const PREFS_FILE: &str = "prefs.yaml"; +const BRAILLE_CODE_PREF: &str = "BrailleCode"; +/// `find_file` falls back to this language by name, so it cannot be pruned away. +const FALLBACK_LANGUAGE: &str = "en"; +/// `set_rules_dir` fails unless the default braille code's directory exists, even for a caller that +/// never asks for braille, so a subset that keeps no braille code keeps this one. +const FALLBACK_BRAILLE_CODE: &str = "UEB"; + +/// Which `Languages/` and `Braille/` directories to keep, by directory name. +/// +/// The full Rules tree is about 10 MB, and a caller that only ever asks for one language and one +/// braille code pays for all of it: `include-zip` embeds every directory in the binary. `None` +/// means keep everything, which is what every existing caller gets. `build.rs` fills this in from +/// the `lang-*` and `braille-*` features. +#[derive(Clone, Default)] +pub struct RulesSubset { + pub languages: Option>, + pub braille_codes: Option>, +} + +impl RulesSubset { + /// Keeps everything, the behaviour before this existed. + pub fn all() -> Self { + return RulesSubset::default(); + } + + /// Regional variants such as `en/gb` and `zh/tw` live inside their language's directory, so + /// keeping a language keeps them too. + fn keeps(wanted: &Option>, name: &str) -> bool { + let Some(wanted) = wanted else { + return true; + }; + return wanted.iter().any(|w| w.eq_ignore_ascii_case(name)); + } + + fn keeps_language(&self, name: &str) -> bool { + // `find_file` falls back to "en" by name from inside the library, so a subset without it + // cannot resolve anything. + return name.eq_ignore_ascii_case(FALLBACK_LANGUAGE) || RulesSubset::keeps(&self.languages, name); + } + + fn keeps_braille_code(&self, name: &str) -> bool { + if self.braille_codes.as_ref().is_some_and(|codes| codes.is_empty()) { + return name.eq_ignore_ascii_case(FALLBACK_BRAILLE_CODE); + } + return RulesSubset::keeps(&self.braille_codes, name); + } +} /// Compression methods for nested language zips vs the outer archive. #[derive(Clone, Copy)] @@ -249,17 +299,95 @@ fn minimize_yaml_file(src: &Path, dst: &Path) -> Result<(), String> { /// Copy a Rules tree. When `minify` is true, rewrite every YAML file as comment-free flow YAML. /// `Languages/zz` is omitted. Returns the number of files successfully minimized. pub fn copy_rules_tree(src: &Path, dst: &Path, minify: bool) -> io::Result { + return copy_rules_subset(src, dst, minify, &RulesSubset::all()); +} + +/// As [`copy_rules_tree`], keeping only the languages and braille codes `subset` asks for. +pub fn copy_rules_subset(src: &Path, dst: &Path, minify: bool, subset: &RulesSubset) -> io::Result { fs::create_dir_all(dst)?; - return copy_dir(src, dst, minify, false); + let minimized = copy_dir(src, dst, minify, None, subset)?; + if let Some(codes) = &subset.braille_codes { + set_default_braille_code(&dst.join(PREFS_FILE), codes)?; + } + return Ok(minimized); } -fn copy_dir(src: &Path, dst: &Path, minify: bool, parent_is_languages: bool) -> io::Result { +/// Points the `BrailleCode` default in a staged `prefs.yaml` at the first of `kept` (or at +/// [`FALLBACK_BRAILLE_CODE`] when it is empty) when the default was pruned, and leaves it alone when +/// it was kept. +/// +/// Without this, the default (Nemeth) and the library's own fallback (UEB) are both missing from a +/// subset that asked for neither, and `set_rules_dir` reports that it could not read the braille +/// directory. Keeping Nemeth and UEB instead would cost about 50 KB for rules nothing asks for. +fn set_default_braille_code(prefs: &Path, kept: &[String]) -> io::Result<()> { + let text = fs::read_to_string(prefs)?; + let docs = YamlLoader::load_from_str(&text).map_err(io::Error::other)?; + let current = docs.iter().find_map(|doc| find_value(doc, BRAILLE_CODE_PREF)); + if current.is_some_and(|current| kept.iter().any(|code| code.eq_ignore_ascii_case(current))) { + return Ok(()); + } + let code = kept.first().map_or(FALLBACK_BRAILLE_CODE, |code| code.as_str()); + let mut out = String::new(); + for (i, doc) in docs.iter().enumerate() { + if i > 0 { + out.push_str("\n---\n"); + } + emit_flow(&mut out, &with_replaced_value(doc, BRAILLE_CODE_PREF, code)).map_err(io::Error::other)?; + } + out.push('\n'); + fs::write(prefs, out)?; + return Ok(()); +} + +/// The string value of the first `key` entry in `yaml`, at any depth. +fn find_value<'a>(yaml: &'a Yaml, key: &str) -> Option<&'a str> { + match yaml { + Yaml::Hash(hash) => { + return hash.iter().find_map(|(k, v)| match (k, v) { + (Yaml::String(name), Yaml::String(value)) if name == key => Some(value.as_str()), + _ => find_value(v, key), + }); + } + Yaml::Array(items) => return items.iter().find_map(|item| find_value(item, key)), + _ => return None, + } +} + +/// `yaml` with the value of every `key` entry, at any depth, replaced by `value`. +fn with_replaced_value(yaml: &Yaml, key: &str, value: &str) -> Yaml { + match yaml { + Yaml::Hash(hash) => { + let mut out = yaml_rust::yaml::Hash::new(); + for (k, v) in hash { + let replace = matches!(k, Yaml::String(name) if name == key); + let v = if replace { Yaml::String(value.to_string()) } else { with_replaced_value(v, key, value) }; + out.insert(k.clone(), v); + } + return Yaml::Hash(out); + } + Yaml::Array(items) => { + return Yaml::Array(items.iter().map(|item| with_replaced_value(item, key, value)).collect()); + } + other => return other.clone(), + } +} + +fn copy_dir( + src: &Path, + dst: &Path, + minify: bool, + parent: Option<&str>, + subset: &RulesSubset, +) -> io::Result { let mut minimized = 0usize; for entry in read_dir(src)? { let entry = entry?; let src_path = entry.path(); let name = file_name_str(&src_path)?; - if parent_is_languages && name == SKIP_LANGUAGE_DIR { + if parent == Some(LANGUAGES_DIR) && (name == SKIP_LANGUAGE_DIR || !subset.keeps_language(&name)) { + continue; + } + if parent == Some(BRAILLE_DIR) && src_path.is_dir() && !subset.keeps_braille_code(&name) { continue; } if is_zip_file(&src_path) { @@ -268,7 +396,7 @@ fn copy_dir(src: &Path, dst: &Path, minify: bool, parent_is_languages: bool) -> let dst_path = dst.join(&name); if src_path.is_dir() { fs::create_dir_all(&dst_path)?; - minimized += copy_dir(&src_path, &dst_path, minify, name == "Languages")?; + minimized += copy_dir(&src_path, &dst_path, minify, Some(name.as_str()), subset)?; } else if minify && is_yaml_file(&src_path) { match minimize_yaml_file(&src_path, &dst_path) { Ok(()) => minimized += 1,