Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,20 @@
_SOURCES = ("pred", "gt")
_CONDITION_TOKENS = ("mock", "denv", "zikv")
_DEFAULT_PAIRS = (("mock", "denv"), ("mock", "zikv"))
_FIELDNAMES = (
"feature_type",
"pair",
"source",
"n_cells_c0",
"n_cells_c1",
"n_fovs",
"n_folds",
"auroc_mean",
"auroc_std",
"skipped_reason",
)
#: Filename written into each infected condition's eval dir by :func:`run_for_group`.
GROUP_PROBE_FILENAME = "cross_condition_probe.csv"


def _detect_condition(eval_dir: Path) -> str:
Expand Down Expand Up @@ -135,6 +149,68 @@ def _probe_pair(
return row


def _write_rows(out_path: Path, rows: list[dict]) -> None:
"""Write probe rows as a CSV with the canonical field order."""
out_path.parent.mkdir(parents=True, exist_ok=True)
with out_path.open("w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=_FIELDNAMES)
writer.writeheader()
writer.writerows(rows)


def run_for_group(
eval_dirs: list[Path],
n_splits: int = 5,
rng_seed: int = 2020,
) -> list[Path]:
"""Probe each infected condition against mock and write a per-condition CSV.

Unlike :func:`run` (long-form CSV over all pairs at one ``out_path``),
this writes one :data:`GROUP_PROBE_FILENAME` into *each infected
condition's* eval dir, holding only that condition's ``mock_vs_<cond>``
rows (every feature × {pred, gt}). This colocates the probe with the eval
dir the reporting layer already resolves per (model, pool, organelle,
condition), so the table generator can read it without knowing about
sibling conditions.

Requires a ``mock`` reference dir plus at least one infected dir; returns
the list of CSV paths written (empty when the group has no mock or no
infected condition, e.g. the in-distribution iPSC eval).

Parameters
----------
eval_dirs : list[Path]
Per-condition eval dirs of one (model, pool, organelle) group. The
condition is inferred from each dir's trailing ``_{mock,denv,zikv}``;
dirs without a recognized token are ignored.
n_splits, rng_seed : int
Forwarded to :func:`fov_stratified_auroc`.
"""
by_condition: dict[str, Path] = {}
for d in eval_dirs:
try:
cond = _detect_condition(d)
except ValueError:
continue
by_condition[cond] = d
Comment thread
alxndrkalinin marked this conversation as resolved.
if "mock" not in by_condition:
return []

written: list[Path] = []
for ref, cond in _DEFAULT_PAIRS: # ref == "mock" for every default pair
if cond not in by_condition:
continue
rows = [
_probe_pair(by_condition, (ref, cond), feature, source, n_splits, rng_seed)
for feature in _FEATURE_TYPES
for source in _SOURCES
]
out_path = by_condition[cond] / GROUP_PROBE_FILENAME
_write_rows(out_path, rows)
written.append(out_path)
return written


def run(
eval_dirs: list[Path],
out_path: Path,
Expand Down Expand Up @@ -169,23 +245,7 @@ def run(
for source in _SOURCES:
rows.append(_probe_pair(eval_dirs_by_condition, pair, feature, source, n_splits, rng_seed))

out_path.parent.mkdir(parents=True, exist_ok=True)
fieldnames = [
"feature_type",
"pair",
"source",
"n_cells_c0",
"n_cells_c1",
"n_fovs",
"n_folds",
"auroc_mean",
"auroc_std",
"skipped_reason",
]
with out_path.open("w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(rows)
_write_rows(out_path, rows)
return out_path


Expand Down
10 changes: 6 additions & 4 deletions applications/dynacell/tests/test_benchmark_config_composition.py
Original file line number Diff line number Diff line change
Expand Up @@ -327,8 +327,8 @@ def test_a549_predict_leaf_composes(
- dataset_ref.{dataset,target} carry through composition (proves the
gene-keyed predict_set + targets overlays took effect).
- experiment_id reflects the cross-eval pairing (per condition).
- sbatch.constraint inherits "h200" from hardware_h200_single (these
are single-GPU predict leaves).
- sbatch.constraint is unset (None): predict leaves use
hardware_predict_any_gpu (any GPU), not the H200 pin.
"""
monkeypatch.setattr("sys.argv", ["dynacell", "predict"])
leaf = BENCHMARKS / organelle / model / "ipsc_confocal" / f"predict__a549_mantis_{condition}.yml"
Expand All @@ -347,8 +347,10 @@ def test_a549_predict_leaf_composes(
assert bench["dataset_ref"]["target"] == gene_slug
assert bench["experiment_id"] == f"{organelle}__ipsc_confocal__{model}__a549_mantis_{gene_slug}_{condition}"

# Single-GPU predict topology: h200 only, not the 4-GPU alternation.
assert cfg["launcher"]["sbatch"].get("constraint") == "h200"
# Single-GPU predict topology, any GPU: the any-GPU profile composes an
# explicit constraint=null. Subscript (not .get) so a dropped hardware
# profile surfaces as a failure instead of silently passing.
assert cfg["launcher"]["sbatch"]["constraint"] is None


def test_manifest_spacing_propagates(monkeypatch) -> None:
Expand Down