docs: render multiple_slides with the others and ship a 1.2 GB stomach dataset

.github/workflows/build_notebooks.yaml

@@ -50,34 +50,11 @@ jobs:
       # `uv lock --upgrade` in a PR. The in-house packages track main.
       - name: Install project
         run: uv sync --dev --upgrade-package lazyslide --upgrade-package wsidata --upgrade-package lazyslide-models
-      - name: Restore model and dataset cache
-        id: artifact-cache
-        uses: actions/cache/restore@v5
-        with:
-          path: .cache/huggingface
-          key: ${{ runner.os }}-hf-v1-${{ hashFiles('scripts/setup_env.py', 'uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-hf-v1-
-
+      # No cache for these downloads: they take about 2.5 min, while a cache of
+      # them (26 GB or more) exceeds GitHub's 10 GB limit and never restores.
       - name: Setup data
         run: uv run scripts/setup_env.py
 
-      # Save before notebook execution so extracted or modified tutorial data
-      # cannot contaminate the reusable download cache.
-      - name: Save model and dataset cache
-        if: steps.artifact-cache.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v5
-        with:
-          path: .cache/huggingface
-          key: ${{ steps.artifact-cache.outputs.cache-primary-key }}
-
-      - name: Verify model and dataset cache offline
-        env:
-          HF_HUB_OFFLINE: 1
-          HF_DATASETS_OFFLINE: 1
-          TRANSFORMERS_OFFLINE: 1
-        run: uv run scripts/setup_env.py
-
       - name: Compute notebook source hash
         id: notebook-source
         shell: bash

.github/workflows/render_heavy.yaml

@@ -10,7 +10,7 @@ on:
       notebook:
         description: Notebook in tutorials/heavy/
         type: string
-        default: multiple_slides.ipynb
+        required: true
       flavor:
         description: HF Jobs hardware
         type: choice

scripts/build_agg_conch_features.py

@@ -0,0 +1,159 @@
+"""Build agg_conch_features.h5ad, the 45-slide table that multiple_slides.ipynb
+and genomics_integration.ipynb load.
+
+It runs the steps of multiple_slides.ipynb on every slide in
+GTEx_artery_dataset.csv.gz: CONCH features for each tile, their slide mean, and
+each term's topk_score (k=100) over the tile-to-term similarities. The result has
+one row per slide, indexed by Tissue Sample Id in the order of the table: the
+slide's mean CONCH feature, its metadata and one score column per term.
+
+Usage: uv run scripts/build_agg_conch_features.py <out.h5ad> [slide ids]
+Without slide ids it runs all 45 slides. It writes one zarr store per slide to
+data/ in the working directory, as the tutorial does.
+
+The published file was built from commit 9da4ef9 (PR #9) on HF Jobs, in 36 min
+on one L4 GPU. Its sha256 is
+26ff197c5ab2016c05a00911730d742e064be8602bc2ab2a027f7e8a8cd2584c.
+"""
+
+import sys
+
+import pandas as pd
+import torch
+from anndata import read_zarr
+from huggingface_hub import hf_hub_download
+from wsidata import agg_wsi, open_wsi
+
+import lazyslide as zs
+
+REPO = "RendeiroLab/LazySlide-data"
+
+# The terms of multiple_slides.ipynb.
+TERMS = [
+    "BMP-2",
+    "Monckeberg sclerosis",
+    "Runx2",
+    "adventitia",
+    "apoptosis",
+    "arterial hardening",
+    "arterial narrowing",
+    "arterial remodeling",
+    "arterial stiffness",
+    "arteriole",
+    "artery",
+    "atherosclerosis",
+    "basement membrane",
+    "blood flow",
+    "bone morphogenetic protein",
+    "calcification",
+    "calcified nodule",
+    "calcium deposition",
+    "calcium phosphate",
+    "chronic kidney disease",
+    "collagen",
+    "compliance",
+    "connective tissue",
+    "elastic fibers",
+    "elasticity",
+    "endothelial dysfunction",
+    "endothelium",
+    "epithelium",
+    "external elastic lamina",
+    "extracellular matrix",
+    "fibroblast",
+    "fibrosis",
+    "fibrous cap",
+    "gap junction",
+    "hemodynamics",
+    "hydroxyapatite",
+    "hyperphosphatemia",
+    "inflammation",
+    "internal elastic lamina",
+    "interstitial space",
+    "intima",
+    "intimal calcification",
+    "intimal thickening",
+    "ischemia",
+    "lamina propria",
+    "lumen",
+    "macrocalcification",
+    "macrophage",
+    "matrix vesicle",
+    "mechanotransduction",
+    "media",
+    "medial calcification",
+    "microcalcification",
+    "mineralization",
+    "myofibroblast",
+    "necrotic core",
+    "osteoblast-like cell",
+    "osteocalcin",
+    "osteogenic",
+    "osteopontin",
+    "oxidative stress",
+    "pericyte",
+    "phosphate transporter",
+    "plaque",
+    "shear stress",
+    "smooth muscle",
+    "tight junction",
+    "tunica",
+    "vasa vasorum",
+    "vascular basement membrane",
+    "vascular compliance",
+    "vascular integrity",
+    "vascular niche",
+    "vascular ossification",
+    "vascular remodeling",
+    "vascular smooth muscle cell",
+    "vascular stiffness",
+    "vascular tone",
+    "vascular wall",
+]
+
+
+def wsi_feature_extraction(slide):
+    # The function of the same name in multiple_slides.ipynb.
+    s = hf_hub_download(REPO, f"gtex_artery_data/{slide}.svs", repo_type="dataset")
+    wsi = open_wsi(s, attach_thumbnail=False, store=f"data/{slide}.zarr")
+    zs.pp.find_tissues(wsi)
+    zs.pp.tile_tissues(wsi, 256, mpp=0.5, background_fraction=0.5)
+
+    # conch feature
+    zs.tl.feature_extraction(wsi, "conch", pbar=False)
+    zs.tl.feature_aggregation(wsi, "conch")
+    embed = zs.tl.text_embedding(TERMS, "conch")
+    zs.tl.text_image_similarity(wsi, embed, "conch")
+    wsi.write()
+
+
+def main(out, slides):
+    table = pd.read_csv(
+        hf_hub_download(REPO, "GTEx_artery_dataset.csv.gz", repo_type="dataset")
+    )
+    if slides:
+        table = table[table["Tissue Sample Id"].isin(slides)]
+    table = table.assign(store=[f"data/{s}.zarr" for s in table["Tissue Sample Id"]])
+
+    slide_scores = {}
+    for slide, store in zip(table["Tissue Sample Id"], table["store"]):
+        wsi_feature_extraction(slide)
+        adata = read_zarr(f"{store}/tables/conch_tiles_text_similarity")
+        scores = zs.metrics.topk_score(adata, k=100)
+        slide_scores[slide] = dict(zip(adata.var.index, scores))
+        print(f"{slide}: {adata.n_obs} tiles", flush=True)
+
+    agg_data = agg_wsi(table, "conch", store_col="store", agg_key="agg_slide")
+    agg_data.obs = (
+        agg_data.obs.join(pd.DataFrame(slide_scores).T, on="Tissue Sample Id")
+        .drop(columns="store")
+        .set_index("Tissue Sample Id")
+    )
+    agg_data.write_h5ad(out)
+
+
+if __name__ == "__main__":
+    # The tutorial renders on CPU. Turn off TF32 convolutions so features
+    # computed on a GPU stay close to the CPU ones.
+    torch.backends.cudnn.allow_tf32 = False
+    main(sys.argv[1], sys.argv[2:])

scripts/build_stomach_compact.py

@@ -0,0 +1,58 @@
+"""Build gtex_stomach_subset_compact.zip for the tile model tutorial.
+
+The tutorial undersamples every class to the smallest one (hemorrhage, 7,700
+tiles), so it uses about 38,500 of the 1.5M tiles in gtex_stomach_subset.zip
+across its train, validation and test splits. This keeps that many training
+tiles, drawn with a fixed seed, and the one test slide the tutorial plots.
+
+Usage: uv run scripts/build_stomach_compact.py <extracted gtex-stomach-subset> <out gtex-stomach-subset>
+Then zip from the output folder's parent, so the zip holds one top folder named gtex-stomach-subset:
+    zip -qr gtex_stomach_subset_compact.zip gtex-stomach-subset
+
+Built with spatialdata 0.7.3 (Zarr v3 train stores) and NumPy's default_rng(0);
+other versions may draw different rows. The published zip has sha256
+f4c0480272b86ab6ed2f1d8d632843b850b424b21f9f526f7fcf73379562cc7a.
+"""
+
+import shutil
+import sys
+from pathlib import Path
+
+import geopandas as gpd
+import numpy as np
+import pandas as pd
+import spatialdata as sd
+
+PER_CLASS = 7_700
+SKIP = {"artifact"}
+TEST_SLIDE = "GTEX-1JJE9-2026"
+
+
+def main(src: Path, out: Path) -> None:
+    stores = sorted((src / "train").glob("*.zarr"))
+    labels = pd.concat(
+        pd.DataFrame({"store": i, "row": np.arange(len(d)), "domain": d.to_numpy()})
+        for i, s in enumerate(stores)
+        for d in [gpd.read_parquet(s / "shapes" / "tiles" / "shapes.parquet")["domain"]]
+    )
+    labels = labels[~labels["domain"].isin(SKIP)]
+    rng = np.random.default_rng(0)
+    keep = pd.concat(
+        g.iloc[rng.choice(len(g), size=min(PER_CLASS, len(g)), replace=False)]
+        for _, g in labels.groupby("domain")
+    )
+    (out / "train").mkdir(parents=True)
+    for i, rows in keep.groupby("store")["row"]:
+        rows = np.sort(rows.to_numpy())
+        sdata = sd.read_zarr(stores[i])
+        sdata.shapes["tiles"] = sdata.shapes["tiles"].iloc[rows]
+        sdata.tables["virchow2_tiles"] = sdata.tables["virchow2_tiles"][rows].copy()
+        sdata.write(out / "train" / stores[i].name)
+    (out / "test").mkdir()
+    for f in (src / "test").glob(f"{TEST_SLIDE}*"):
+        (shutil.copytree if f.is_dir() else shutil.copy2)(f, out / "test" / f.name)
+    print(keep["domain"].value_counts().to_string())
+
+
+if __name__ == "__main__":
+    main(Path(sys.argv[1]), Path(sys.argv[2]))

scripts/hf_render.sh

@@ -7,7 +7,7 @@
 #
 # Required env:
 #   CI_SHA       git commit to render
-#   NOTEBOOK     notebook path in the repo, e.g. tutorials/heavy/multiple_slides.ipynb
+#   NOTEBOOK     notebook path in the repo, e.g. tutorials/heavy/<notebook>.ipynb
 #   OUT_DIR      where the executed notebook goes, e.g. /out/<run id>
 # Optional env:
 #   CI_REPO_URL  default: https://github.com/rendeirolab/lazyslide-tutorials.git

scripts/setup_env.py

@@ -30,7 +30,11 @@
         "TCGA_READ_survival.csv",
         "GTEx_artery_RNA.h5ad",
         "agg_conch_features.h5ad",
-        "gtex_stomach_subset.zip",
+        "gtex_stomach_subset_compact.zip",
+        "GTEx_artery_dataset.csv.gz",
+        "gtex_artery_data/GTEX-PW2O-1926.svs",
+        "gtex_artery_data/GTEX-ZYT6-1526.svs",
+        "gtex_artery_data/GTEX-O5YU-0926.svs",
     ),
     "MahmoodLab/hest": (
         "wsis/NCBI776.tif",