Add lang-* and braille-* features to choose which rules include-zip embeds

AGENTS.md

@@ -12,7 +12,7 @@ but add common mistakes of AI agents here instead.
 - Common per-language files:
   - `ClearSpeak_Rules.yaml`, `SimpleSpeak_Rules.yaml`
   - `SharedRules/`, `unicode.yaml`, `unicode-full.yaml`, `definitions.yaml`, `navigate.yaml`
-- `build.rs` can bundle rules into `rules.zip` when `include-zip` is enabled (always the minimized tree). Shared zipper: `src/rules_archive.rs`, invoked as `cargo run --bin package-rules -- Rules <output> [--minimize]`.
+- `build.rs` can bundle rules into `rules.zip` when `include-zip` is enabled (always the minimized tree). The `lang-*` / `braille-*` features (all on by default via `rules-all`) choose which `Languages/` and `Braille/` directories it embeds; every rules directory needs a feature listed in `lang-all` / `braille-all`, and `build.rs` fails without one. Shared zipper: `src/rules_archive.rs`, invoked as `cargo run --bin package-rules -- Rules <output> [--minimize]`.
 - CI packages `Rules.zip` (verbatim) and `Rules-minimized.zip` (comments stripped from `Languages/**/unicode.yaml` and `unicode-full.yaml` only). Each language/braille subdir is stored as `<name>/<name>.zip`; top-level and `Intent/` YAML stay loose. Inner language zips use BZIP2; the outer downloadable archive uses DEFLATE so `unzip` works. Both omit `Languages/zz`; the `test-rules-package` job restores `zz` from git after unzip so unit tests still run against release-like archives.
 
 ## Translation Conventions

Cargo.toml

@@ -12,11 +12,48 @@ exclude = ["src/main.rs", "docs", "BrailleDocs", "PythonScripts"]    # should ha
 
 
 [features]
+default = ["rules-all"]
 "include-zip" = []
 "enable-logs" = ["android_logger"]
 "tts" = [ "natural-tts" ]
 "no-unsafe" = ["sxd-document-no-unsafe/no-unsafe", "sxd-xpath-no-unsafe/no-unsafe"]
 
+# Which Rules/Languages and Rules/Braille directories `include-zip` embeds. Every one is on by default.
+# To embed fewer, turn off default features and name the ones you want, for example
+# `default-features = false, features = ["include-zip", "lang-en", "braille-ueb"]`.
+# `en` is always embedded because the library falls back to it, and `build.rs` fails if a rules
+# directory has no feature here or is missing from `lang-all` / `braille-all`.
+# MathCAT itself never asks for ASCIIMath or LaTeX, but an application that copies math in those
+# notations does so through the braille API, so it needs `braille-asciimath` / `braille-latex`.
+# These only change what is embedded: without `include-zip`, the Rules directory is read from disk.
+rules-all = ["lang-all", "braille-all"]
+lang-all = ["lang-de", "lang-el", "lang-en", "lang-es", "lang-fi", "lang-fr", "lang-hu", "lang-id", "lang-nb", "lang-pl", "lang-pt", "lang-ru", "lang-sv", "lang-vi", "lang-zh"]
+braille-all = ["braille-asciimath", "braille-cmu", "braille-french", "braille-latex", "braille-nemeth", "braille-russian", "braille-swedish", "braille-ueb", "braille-vietnam"]
+lang-de = []
+lang-el = []
+lang-en = []
+lang-es = []
+lang-fi = []
+lang-fr = []
+lang-hu = []
+lang-id = []
+lang-nb = []
+lang-pl = []
+lang-pt = []
+lang-ru = []
+lang-sv = []
+lang-vi = []
+lang-zh = []
+braille-asciimath = []
+braille-cmu = []
+braille-french = []
+braille-latex = []
+braille-nemeth = []
+braille-russian = []
+braille-swedish = []
+braille-ueb = []
+braille-vietnam = []
+
 
 [dependencies]
 sxd-document-no-unsafe = "0.4.2"

build.rs

@@ -6,21 +6,80 @@
 
 include!("src/rules_archive.rs");
 
+/// The `Rules/<kind>` directories whose `<prefix>-<name>` feature is on, or `None` when all of them
+/// are, so a default build stages exactly what it always has.
+///
+/// Panics if a directory is missing from `<prefix>-all` in Cargo.toml. Otherwise a newly added
+/// language or braille code would quietly drop out of every build, since nothing would turn it on.
+fn subset_from_features(rules_dir: &Path, kind: &str, prefix: &str, manifest: &str) -> Option<Vec<String>> {
+    let all_feature = format!("{}-all", prefix);
+    // The list can span lines: `cargo publish` rewrites Cargo.toml with one entry per line.
+    let all_list = manifest
+        .split(&format!("
+{} = [", all_feature))
+        .nth(1)
+        .and_then(|rest| rest.split(']').next())
+        .unwrap_or_else(|| panic!("Cargo.toml has no `{}` feature", all_feature));
+    let mut kept = Vec::new();
+    let mut everything = true;
+    for entry in read_dir(rules_dir.join(kind)).unwrap() {
+        let path = entry.unwrap().path();
+        let name = path.file_name().unwrap().to_string_lossy().to_string();
+        if !path.is_dir() || (kind == LANGUAGES_DIR && name == SKIP_LANGUAGE_DIR) {
+            continue;
+        }
+        let feature = format!("{}-{}", prefix, name.to_ascii_lowercase());
+        if !all_list.contains(&format!("\"{}\"", feature)) {
+            panic!(
+                "Rules/{}/{} has no feature: add `{} = []` to Cargo.toml and list it in `{}`",
+                kind, name, feature, all_feature
+            );
+        }
+        let env_var = format!("CARGO_FEATURE_{}", feature.to_ascii_uppercase().replace('-', "_"));
+        if std::env::var_os(env_var).is_some() {
+            kept.push(name);
+        } else {
+            everything = false;
+        }
+    }
+    kept.sort();
+    return if everything { None } else { Some(kept) };
+}
+
 fn main() {
     println!("cargo::rerun-if-changed=build.rs");
     println!("cargo::rerun-if-changed=src/rules_archive.rs");
     println!("cargo::rerun-if-changed=Rules");
+    println!("cargo::rerun-if-changed=Cargo.toml");
+
+    let rules_dir = std::env::current_dir().unwrap().join("Rules");
+    let manifest = fs::read_to_string("Cargo.toml").unwrap();
+    // Checked on every build, not only include-zip ones, so a rules directory added without a
+    // feature is caught by whoever adds it.
+    let languages = subset_from_features(&rules_dir, LANGUAGES_DIR, "lang", &manifest);
+    let braille_codes = subset_from_features(&rules_dir, BRAILLE_DIR, "braille", &manifest);
 
     // This doesn't work because the build claims OUT_DIR is not defined(?)
     // let archive = PathBuf::from(concat!(env!("OUT_DIR"),"/rules.zip"));
     if std::env::var("CARGO_FEATURE_INCLUDE_ZIP").is_ok() {
         let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
-        let rules_dir = std::env::current_dir().unwrap().join("Rules");
         let staging = out_dir.join("rules_src");
         if staging.exists() {
             let _ = std::fs::remove_dir_all(&staging);
         }
-        if let Err(e) = copy_rules_tree(&rules_dir, &staging, true) {
+        let subset = RulesSubset { languages, braille_codes };
+        if let Some(langs) = &subset.languages {
+            let mut langs = langs.clone();
+            if !langs.iter().any(|l| l.eq_ignore_ascii_case("en")) {
+                langs.push("en (the fallback)".to_string());
+            }
+            println!("cargo::warning=MathCAT rules limited to languages: {}", langs.join(", "));
+        }
+        if let Some(codes) = &subset.braille_codes {
+            let codes = if codes.is_empty() { "UEB (the fallback)".to_string() } else { codes.join(", ") };
+            println!("cargo::warning=MathCAT rules limited to braille codes: {}", codes);
+        }
+        if let Err(e) = copy_rules_subset(&rules_dir, &staging, true, &subset) {
             panic!("build.rs failed to stage minimized Rules: {}", e);
         }
         let archive_path = out_dir.join("rules.zip");

src/rules_archive.rs

@@ -20,6 +20,56 @@ use zip::CompressionMethod;
 const YAML_SUFFIXES: [&str; 2] = ["yaml", "yml"];
 const SKIP_LANGUAGE_DIR: &str = "zz";
 const ARCHIVE_ROOT: &str = "Rules";
+const LANGUAGES_DIR: &str = "Languages";
+const BRAILLE_DIR: &str = "Braille";
+const PREFS_FILE: &str = "prefs.yaml";
+const BRAILLE_CODE_PREF: &str = "BrailleCode";
+/// `find_file` falls back to this language by name, so it cannot be pruned away.
+const FALLBACK_LANGUAGE: &str = "en";
+/// `set_rules_dir` fails unless the default braille code's directory exists, even for a caller that
+/// never asks for braille, so a subset that keeps no braille code keeps this one.
+const FALLBACK_BRAILLE_CODE: &str = "UEB";
+
+/// Which `Languages/` and `Braille/` directories to keep, by directory name.
+///
+/// The full Rules tree is about 10 MB, and a caller that only ever asks for one language and one
+/// braille code pays for all of it: `include-zip` embeds every directory in the binary. `None`
+/// means keep everything, which is what every existing caller gets. `build.rs` fills this in from
+/// the `lang-*` and `braille-*` features.
+#[derive(Clone, Default)]
+pub struct RulesSubset {
+    pub languages: Option<Vec<String>>,
+    pub braille_codes: Option<Vec<String>>,
+}
+
+impl RulesSubset {
+    /// Keeps everything, the behaviour before this existed.
+    pub fn all() -> Self {
+        return RulesSubset::default();
+    }
+
+    /// Regional variants such as `en/gb` and `zh/tw` live inside their language's directory, so
+    /// keeping a language keeps them too.
+    fn keeps(wanted: &Option<Vec<String>>, name: &str) -> bool {
+        let Some(wanted) = wanted else {
+            return true;
+        };
+        return wanted.iter().any(|w| w.eq_ignore_ascii_case(name));
+    }
+
+    fn keeps_language(&self, name: &str) -> bool {
+        // `find_file` falls back to "en" by name from inside the library, so a subset without it
+        // cannot resolve anything.
+        return name.eq_ignore_ascii_case(FALLBACK_LANGUAGE) || RulesSubset::keeps(&self.languages, name);
+    }
+
+    fn keeps_braille_code(&self, name: &str) -> bool {
+        if self.braille_codes.as_ref().is_some_and(|codes| codes.is_empty()) {
+            return name.eq_ignore_ascii_case(FALLBACK_BRAILLE_CODE);
+        }
+        return RulesSubset::keeps(&self.braille_codes, name);
+    }
+}
 
 /// Compression methods for nested language zips vs the outer archive.
 #[derive(Clone, Copy)]
@@ -249,17 +299,95 @@ fn minimize_yaml_file(src: &Path, dst: &Path) -> Result<(), String> {
 /// Copy a Rules tree. When `minify` is true, rewrite every YAML file as comment-free flow YAML.
 /// `Languages/zz` is omitted. Returns the number of files successfully minimized.
 pub fn copy_rules_tree(src: &Path, dst: &Path, minify: bool) -> io::Result<usize> {
+    return copy_rules_subset(src, dst, minify, &RulesSubset::all());
+}
+
+/// As [`copy_rules_tree`], keeping only the languages and braille codes `subset` asks for.
+pub fn copy_rules_subset(src: &Path, dst: &Path, minify: bool, subset: &RulesSubset) -> io::Result<usize> {
     fs::create_dir_all(dst)?;
-    return copy_dir(src, dst, minify, false);
+    let minimized = copy_dir(src, dst, minify, None, subset)?;
+    if let Some(codes) = &subset.braille_codes {
+        set_default_braille_code(&dst.join(PREFS_FILE), codes)?;
+    }
+    return Ok(minimized);
 }
 
-fn copy_dir(src: &Path, dst: &Path, minify: bool, parent_is_languages: bool) -> io::Result<usize> {
+/// Points the `BrailleCode` default in a staged `prefs.yaml` at the first of `kept` (or at
+/// [`FALLBACK_BRAILLE_CODE`] when it is empty) when the default was pruned, and leaves it alone when
+/// it was kept.
+///
+/// Without this, the default (Nemeth) and the library's own fallback (UEB) are both missing from a
+/// subset that asked for neither, and `set_rules_dir` reports that it could not read the braille
+/// directory. Keeping Nemeth and UEB instead would cost about 50 KB for rules nothing asks for.
+fn set_default_braille_code(prefs: &Path, kept: &[String]) -> io::Result<()> {
+    let text = fs::read_to_string(prefs)?;
+    let docs = YamlLoader::load_from_str(&text).map_err(io::Error::other)?;
+    let current = docs.iter().find_map(|doc| find_value(doc, BRAILLE_CODE_PREF));
+    if current.is_some_and(|current| kept.iter().any(|code| code.eq_ignore_ascii_case(current))) {
+        return Ok(());
+    }
+    let code = kept.first().map_or(FALLBACK_BRAILLE_CODE, |code| code.as_str());
+    let mut out = String::new();
+    for (i, doc) in docs.iter().enumerate() {
+        if i > 0 {
+            out.push_str("\n---\n");
+        }
+        emit_flow(&mut out, &with_replaced_value(doc, BRAILLE_CODE_PREF, code)).map_err(io::Error::other)?;
+    }
+    out.push('\n');
+    fs::write(prefs, out)?;
+    return Ok(());
+}
+
+/// The string value of the first `key` entry in `yaml`, at any depth.
+fn find_value<'a>(yaml: &'a Yaml, key: &str) -> Option<&'a str> {
+    match yaml {
+        Yaml::Hash(hash) => {
+            return hash.iter().find_map(|(k, v)| match (k, v) {
+                (Yaml::String(name), Yaml::String(value)) if name == key => Some(value.as_str()),
+                _ => find_value(v, key),
+            });
+        }
+        Yaml::Array(items) => return items.iter().find_map(|item| find_value(item, key)),
+        _ => return None,
+    }
+}
+
+/// `yaml` with the value of every `key` entry, at any depth, replaced by `value`.
+fn with_replaced_value(yaml: &Yaml, key: &str, value: &str) -> Yaml {
+    match yaml {
+        Yaml::Hash(hash) => {
+            let mut out = yaml_rust::yaml::Hash::new();
+            for (k, v) in hash {
+                let replace = matches!(k, Yaml::String(name) if name == key);
+                let v = if replace { Yaml::String(value.to_string()) } else { with_replaced_value(v, key, value) };
+                out.insert(k.clone(), v);
+            }
+            return Yaml::Hash(out);
+        }
+        Yaml::Array(items) => {
+            return Yaml::Array(items.iter().map(|item| with_replaced_value(item, key, value)).collect());
+        }
+        other => return other.clone(),
+    }
+}
+
+fn copy_dir(
+    src: &Path,
+    dst: &Path,
+    minify: bool,
+    parent: Option<&str>,
+    subset: &RulesSubset,
+) -> io::Result<usize> {
     let mut minimized = 0usize;
     for entry in read_dir(src)? {
         let entry = entry?;
         let src_path = entry.path();
         let name = file_name_str(&src_path)?;
-        if parent_is_languages && name == SKIP_LANGUAGE_DIR {
+        if parent == Some(LANGUAGES_DIR) && (name == SKIP_LANGUAGE_DIR || !subset.keeps_language(&name)) {
+            continue;
+        }
+        if parent == Some(BRAILLE_DIR) && src_path.is_dir() && !subset.keeps_braille_code(&name) {
             continue;
         }
         if is_zip_file(&src_path) {
@@ -268,7 +396,7 @@ fn copy_dir(src: &Path, dst: &Path, minify: bool, parent_is_languages: bool) ->
         let dst_path = dst.join(&name);
         if src_path.is_dir() {
             fs::create_dir_all(&dst_path)?;
-            minimized += copy_dir(&src_path, &dst_path, minify, name == "Languages")?;
+            minimized += copy_dir(&src_path, &dst_path, minify, Some(name.as_str()), subset)?;
         } else if minify && is_yaml_file(&src_path) {
             match minimize_yaml_file(&src_path, &dst_path) {
                 Ok(()) => minimized += 1,