ci: green up modular-viscy-staging (CI matrix, deps, lint)

.github/workflows/lint.yml

@@ -2,9 +2,9 @@ name: Lint
 
 on:
   push:
-    branches: [main]
+    branches: [main, modular-viscy-staging]
   pull_request:
-    branches: [main]
+    branches: [main, modular-viscy-staging]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/test.yml

@@ -2,9 +2,9 @@ name: Test
 
 on:
   push:
-    branches: [main]
+    branches: [main, modular-viscy-staging]
   pull_request:
-    branches: [main]
+    branches: [main, modular-viscy-staging]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -18,7 +18,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.11", "3.12", "3.13"]
+        python-version: ["3.12", "3.13"]
         package: [viscy-transforms, viscy-models]
 
     steps:
@@ -47,7 +47,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.11", "3.12", "3.13"]
+        python-version: ["3.12", "3.13"]
 
     steps:
       - name: Checkout repository
@@ -133,19 +133,24 @@ jobs:
           enable-cache: true
           cache-suffix: ubuntu-latest-3.13
 
-      - name: Install minimal dynacell (base deps + test group)
-        run: uv sync --frozen --group test
+      - name: Install dynacell with eval deps + test group (CPU)
+        # The eval-pipeline tests import dynacell.evaluation.pipeline, a hard
+        # consumer of the eval stack (cubic, cellpose, …). cubic is CPU-capable
+        # (falls back to numpy/scikit-image without CUDA), so the eval extra
+        # installs and runs on a GPU-less runner. eval_gpu (cupy/cucim) is NOT
+        # installed — those are the only CUDA-only deps.
+        run: uv sync --frozen --extra eval --group test
         working-directory: applications/dynacell
 
       - name: Run benchmark-schema + submit-tool + eval-runtime tests
-        # tests/test_runtime.py + tests/test_evaluation_pipeline_parallel.py
-        # cover the dynacell.evaluation.runtime module + FovResult pickle
-        # contract. tests/test_evaluation_pipeline_parallel_cpu.py drives
-        # evaluate_predictions end-to-end (serial vs spawn-process) on a
-        # tiny iohub fixture + prebuilt mask cache — no eval extras (no
-        # cellpose, transformers, cubic) needed since target_name=er +
-        # require_complete_cache=true short-circuit segmenter and feature
-        # extractor loads. tests/test_evaluation_grouped.py drives the
+        # test_benchmark_config_composition + test_submit_benchmark_job cover
+        # config composition + the launcher. test_runtime + the two
+        # test_evaluation_pipeline_parallel* suites cover the runtime module,
+        # the FovResult pickle contract, and evaluate_predictions end-to-end
+        # (serial vs spawn-process) on a tiny iohub fixture + prebuilt mask
+        # cache (target_name=er + require_complete_cache=true short-circuit the
+        # segmenter + feature-extractor model loads — but pipeline import still
+        # needs the eval stack). test_evaluation_grouped drives the
         # multi-condition driver against the same cache-only fixture.
         run: |
           uv run --frozen pytest \

applications/airtable/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
 dynamic = [ "version" ]
 dependencies = [
   "click",
-  "iohub",
+  "iohub>=0.3.6",
   "pandas",
   "pyairtable",
   "pydantic",

applications/dynacell/pyproject.toml

@@ -29,6 +29,7 @@ classifiers = [
 ]
 dynamic = [ "version" ]
 dependencies = [
+  "hydra-core>=1.2",
   "lightning>=2.3",
   "monai",
   "omegaconf",
@@ -46,7 +47,7 @@ optional-dependencies.eval = [
   "cubic==0.7.0a9",
   "dynaclr",
   "hydra-core>=1.2",
-  "iohub",
+  "iohub>=0.3.6",
   "itk",
   "matplotlib",
   "pandas",
@@ -72,7 +73,7 @@ optional-dependencies.eval_gpu = [
   "cupy-cuda13x",
 ]
 optional-dependencies.preprocess = [
-  "iohub",
+  "iohub>=0.3.6",
   "tqdm",
 ]
 optional-dependencies.report = [

applications/dynacell/src/dynacell/evaluation/cross_condition_probe.py

@@ -40,6 +40,20 @@
 _SOURCES = ("pred", "gt")
 _CONDITION_TOKENS = ("mock", "denv", "zikv")
 _DEFAULT_PAIRS = (("mock", "denv"), ("mock", "zikv"))
+_FIELDNAMES = (
+    "feature_type",
+    "pair",
+    "source",
+    "n_cells_c0",
+    "n_cells_c1",
+    "n_fovs",
+    "n_folds",
+    "auroc_mean",
+    "auroc_std",
+    "skipped_reason",
+)
+#: Filename written into each infected condition's eval dir by :func:`run_for_group`.
+GROUP_PROBE_FILENAME = "cross_condition_probe.csv"
 
 
 def _detect_condition(eval_dir: Path) -> str:
@@ -51,14 +65,31 @@ def _detect_condition(eval_dir: Path) -> str:
     raise ValueError(f"cannot infer condition from eval_dir name {name!r}: expected trailing _{{mock,denv,zikv}}")
 
 
-def _load_embeddings(eval_dir: Path, source: str, feature: str) -> tuple[np.ndarray, np.ndarray]:
+def _load_embeddings(
+    eval_dir: Path,
+    source: str,
+    feature: str,
+    cache: dict[Path, tuple[np.ndarray, np.ndarray]] | None = None,
+) -> tuple[np.ndarray, np.ndarray]:
     """Return ``(embeddings, fov_ids)`` from one ``*_single_cell_embeddings.npz``.
 
-    ``np.load`` raises ``FileNotFoundError`` when the NPZ is missing.
+    ``np.load`` raises ``FileNotFoundError`` when the NPZ is missing. When
+    *cache* is given, the result is memoized on the resolved NPZ path so the
+    shared reference (``mock``) side is read from disk once per group instead
+    of once per pair (see :func:`run` / :func:`run_for_group`). The key is
+    resolved so the same file reached via different relative paths/symlinks
+    still hits the cache.
     """
-    npz_path = eval_dir / "embeddings" / f"{source}_{feature}_single_cell_embeddings.npz"
+    # ``.resolve()`` (strict=False) canonicalizes the key without requiring the
+    # file to exist — np.load still raises FileNotFoundError below if it's missing.
+    npz_path = (eval_dir / "embeddings" / f"{source}_{feature}_single_cell_embeddings.npz").resolve()
+    if cache is not None and npz_path in cache:
+        return cache[npz_path]
     with np.load(npz_path) as data:
-        return np.asarray(data["embeddings"]), np.asarray(data["fov"])
+        result = (np.asarray(data["embeddings"]), np.asarray(data["fov"]))
+    if cache is not None:
+        cache[npz_path] = result
+    return result
 
 
 def _probe_pair(
@@ -68,6 +99,7 @@ def _probe_pair(
     source: str,
     n_splits: int,
     rng_seed: int,
+    cache: dict[Path, tuple[np.ndarray, np.ndarray]] | None = None,
 ) -> dict:
     """Run one ``fov_stratified_auroc`` call for the given (pair, feature, source).
 
@@ -92,8 +124,8 @@ def _probe_pair(
         row["skipped_reason"] = "missing eval dir for one side of pair"
         return row
     try:
-        x0, fov0 = _load_embeddings(eval_dirs_by_condition[c0], source, feature)
-        x1, fov1 = _load_embeddings(eval_dirs_by_condition[c1], source, feature)
+        x0, fov0 = _load_embeddings(eval_dirs_by_condition[c0], source, feature, cache)
+        x1, fov1 = _load_embeddings(eval_dirs_by_condition[c1], source, feature, cache)
     except FileNotFoundError as e:
         row["skipped_reason"] = f"missing embeddings file: {e}"
         return row
@@ -135,6 +167,75 @@ def _probe_pair(
     return row
 
 
+def _write_rows(out_path: Path, rows: list[dict]) -> None:
+    """Write probe rows as a CSV with the canonical field order."""
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    with out_path.open("w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=_FIELDNAMES)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def run_for_group(
+    eval_dirs: list[Path],
+    n_splits: int = 5,
+    rng_seed: int = 2020,
+) -> list[Path]:
+    """Probe each infected condition against mock and write a per-condition CSV.
+
+    Unlike :func:`run` (long-form CSV over all pairs at one ``out_path``),
+    this writes one :data:`GROUP_PROBE_FILENAME` into *each infected
+    condition's* eval dir, holding only that condition's ``mock_vs_<cond>``
+    rows (every feature × {pred, gt}). This colocates the probe with the eval
+    dir the reporting layer already resolves per (model, pool, organelle,
+    condition), so the table generator can read it without knowing about
+    sibling conditions.
+
+    Requires a ``mock`` reference dir plus at least one infected dir; returns
+    the list of CSV paths written (empty when the group has no mock or no
+    infected condition, e.g. the in-distribution iPSC eval).
+
+    Parameters
+    ----------
+    eval_dirs : list[Path]
+        Per-condition eval dirs of one (model, pool, organelle) group. The
+        condition is inferred from each dir's trailing ``_{mock,denv,zikv}``;
+        dirs without a recognized token are ignored. Two dirs mapping to the
+        same condition raise ``ValueError`` (an ambiguous group) rather than
+        silently picking one.
+    n_splits, rng_seed : int
+        Forwarded to :func:`fov_stratified_auroc`.
+    """
+    by_condition: dict[str, Path] = {}
+    for d in eval_dirs:
+        try:
+            cond = _detect_condition(d)
+        except ValueError:
+            continue
+        if cond in by_condition:
+            raise ValueError(f"duplicate condition {cond!r}: {by_condition[cond]} and {d}")
+        by_condition[cond] = d
+    if "mock" not in by_condition:
+        return []
+
+    # Shared across pairs so the mock reference embeddings are read once, not
+    # re-read for every infected condition. Local to this call -> released on return.
+    cache: dict[Path, tuple[np.ndarray, np.ndarray]] = {}
+    written: list[Path] = []
+    for ref, cond in _DEFAULT_PAIRS:  # ref == "mock" for every default pair
+        if cond not in by_condition:
+            continue
+        rows = [
+            _probe_pair(by_condition, (ref, cond), feature, source, n_splits, rng_seed, cache)
+            for feature in _FEATURE_TYPES
+            for source in _SOURCES
+        ]
+        out_path = by_condition[cond] / GROUP_PROBE_FILENAME
+        _write_rows(out_path, rows)
+        written.append(out_path)
+    return written
+
+
 def run(
     eval_dirs: list[Path],
     out_path: Path,
@@ -163,29 +264,16 @@ def run(
             raise ValueError(f"duplicate condition {cond!r}: {eval_dirs_by_condition[cond]} and {d}")
         eval_dirs_by_condition[cond] = d
 
+    # Shared across pairs so a condition's embeddings (e.g. the mock reference
+    # reused by every pair) are read once. Local to this call -> released on return.
+    cache: dict[Path, tuple[np.ndarray, np.ndarray]] = {}
     rows = []
     for feature in _FEATURE_TYPES:
         for pair in pairs:
             for source in _SOURCES:
-                rows.append(_probe_pair(eval_dirs_by_condition, pair, feature, source, n_splits, rng_seed))
+                rows.append(_probe_pair(eval_dirs_by_condition, pair, feature, source, n_splits, rng_seed, cache))
 
-    out_path.parent.mkdir(parents=True, exist_ok=True)
-    fieldnames = [
-        "feature_type",
-        "pair",
-        "source",
-        "n_cells_c0",
-        "n_cells_c1",
-        "n_fovs",
-        "n_folds",
-        "auroc_mean",
-        "auroc_std",
-        "skipped_reason",
-    ]
-    with out_path.open("w", newline="") as f:
-        writer = csv.DictWriter(f, fieldnames=fieldnames)
-        writer.writeheader()
-        writer.writerows(rows)
+    _write_rows(out_path, rows)
     return out_path
 
 

applications/dynacell/src/dynacell/evaluation/io.py

@@ -29,18 +29,14 @@ def _require_skimage():
 
 
 def _require_cubic():
+    # cubic only: callers fall back to the numpy/CPU path when no GPU is
+    # present, so do not hard-require the eval_gpu (cupy/cucim) stack here — that
+    # would block CPU-only use. ``ascupy`` raises a clear error if a GPU upload
+    # is attempted without cupy.
     if ascupy is None:
         raise ImportError(
             "cubic is required for GPU array operations. Install via the `eval` extra: `uv sync --extra eval`."
         )
-    try:
-        import cucim  # noqa: F401
-        import cupy  # noqa: F401
-    except ImportError as e:
-        raise ImportError(
-            f"{e.name} is required for GPU-backed I/O. Install cupy-cuda12x "
-            "and cucim-cu12 via the `eval_gpu` extra: `uv sync --extra eval_gpu`."
-        ) from e
 
 
 def _is_zarr_path(path: Path) -> bool:

applications/dynacell/src/dynacell/evaluation/metrics.py

@@ -24,19 +24,17 @@
 
 
 def _require_cubic():
+    # Only cubic itself is required: the metric helpers below gate the GPU
+    # upload on ``torch.cuda.is_available()`` and otherwise run on numpy, where
+    # cubic dispatches to its CPU (numpy / scikit-image) path. cucim / cupy are
+    # needed only when a GPU is actually present and used — and ``ascupy`` raises
+    # a clear "GPU requested but not available" there if they are missing — so
+    # this must NOT hard-require the eval_gpu stack (it would block the CPU path).
     if ascupy is None:
         raise ImportError(
             "cubic is required for resolution and feature metrics. "
             "Install via the `eval` extra: `uv sync --extra eval`."
         )
-    try:
-        import cucim  # noqa: F401
-        import cupy  # noqa: F401
-    except ImportError as e:
-        raise ImportError(
-            f"{e.name} is required for GPU-backed metrics. Install cupy-cuda12x "
-            "and cucim-cu12 via the `eval_gpu` extra: `uv sync --extra eval_gpu`."
-        ) from e
 
 
 @torch.inference_mode()