diff --git a/capgen/generator/suite_resolver.py b/capgen/generator/suite_resolver.py index 132c23c0..932714a9 100644 --- a/capgen/generator/suite_resolver.py +++ b/capgen/generator/suite_resolver.py @@ -189,6 +189,70 @@ def _index_symbol_name(base_std_name: str) -> str: _INDEX_PREFIX, base_std_name[:max_base_len], sha8, ) + +def _constituent_index_base(std_name: str) -> Optional[str]: + """Return ``X`` for an ``index_of_`` standard name, else ``None``. + + >>> _constituent_index_base('index_of_water_vapor') + 'water_vapor' + >>> _constituent_index_base('air_pressure') is None + True + """ + if not std_name.startswith(_INDEX_PREFIX): + return None + return std_name[len(_INDEX_PREFIX):] + + +def _is_known_constituent(std_name: str, const_stds: Set[str]) -> bool: + """Is *std_name* known — positively — to name a constituent? + + Possible 'positive evidence': + 1. Constituent standard name exists + 2. Constituent tendency standard name exists (tendency_of_X) + + >>> _is_known_constituent('water_vapor', {'water_vapor'}) + True + >>> _is_known_constituent('air_temperature', {'tendency_of_air_temperature'}) + True + >>> _is_known_constituent('air_temperature', {'water_vapor'}) + False + """ + return (std_name in const_stds + or (_TEND_PREFIX + std_name) in const_stds) + + +def _check_constituent_index_evidence( + suite_name: str, + index_names: List[str], + const_stds: Set[str], +) -> None: + """Assert the :func:`_is_known_constituent` invariant suite-wide. + + Each resolution path enforces it individually; re-checking here guards + against future bugs of this class. + + >>> _check_constituent_index_evidence('s', ['water_vapor'], + ... {'water_vapor'}) + + >>> _check_constituent_index_evidence('s', ['shortwave_band'], + ... {'water_vapor'}) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: Internal error: suite 's' allocated constituent index ... + """ + unevidenced = [n for n in index_names + if not _is_known_constituent(n, const_stds)] + if not unevidenced: + return + raise CCPPError( + "Internal error: suite '{}' allocated constituent index integers " + "for standard name(s) {} that no scheme flags as a constituent " + "(advected/constituent/molar_mass). This indicates a resolver bug, " + "not a metadata error".format( + suite_name, ', '.join(repr(n) for n in unevidenced) + ) + ) + + # Std names directly satisfied by host-constituents-module-owned symbols. _FRAMEWORK_CONST_STDS = frozenset({ _CONST_BASE_ARRAY_STD, @@ -2140,8 +2204,17 @@ def _resolve_constituent_arg( scheme_dims = list(scheme_var.dimensions) is_tendency_name = std_name.startswith(_TEND_PREFIX) - is_index_name = std_name.startswith(_INDEX_PREFIX) - is_framework_name = std_name in _FRAMEWORK_CONST_STDS or is_index_name + + # ``index_of_`` is an ordinary naming convention (band, column and level + # indices are spelled the same way), not a reserved framework namespace, + # so the prefix is a precondition and ``const_stds`` is the evidence. + # See :func:`_is_known_constituent`. + index_base = _constituent_index_base(std_name) + has_index_prefix = index_base is not None + is_constituent_index = (has_index_prefix + and _is_known_constituent(index_base, const_stds)) + is_framework_name = (std_name in _FRAMEWORK_CONST_STDS + or is_constituent_index) # Rule (b): an UNFLAGGED consumer of a name that some scheme declares as a # constituent -- a base constituent (``advected``, read via vars_layer) or a @@ -2182,16 +2255,34 @@ def _resolve_constituent_arg( ): return None + # A constituent index is a framework-owned module integer bound by + # ``%const_index``, so a scheme may only read it. Placed after the + # host/suite gate so a host-declared index stays an ordinary variable. + if is_constituent_index and intent in ('out', 'inout'): + raise CCPPError( + "Scheme arg '{}' (standard_name='{}', scheme='{}', phase='{}') " + "has intent={}, but '{}' indexes a constituent: schemes may " + "only read it".format( + local, std_name, scheme_name, phase, intent, index_base + ) + ) + # A scheme that OUTPUTS ``index_of_`` is producing an ordinary index # variable (e.g. ``rrtmgp_inputs_setup`` computing the diagnostic - # shortwave band index), NOT a constituent index -- constituent indices - # are read-only module integers bound by ``%const_index`` and are never - # written by a scheme. Defer so it becomes a suite var that later-phase + # shortwave band index), NOT a constituent index -- those are rejected + # above. Defer so it becomes a suite var that later-phase # consumers resolve via the gate above. (Index names produced by some # OTHER scheme but consumed here have already been caught by the # suite_vars branch above; this handles the producing arg itself, whose # name is not yet in suite_vars on first occurrence.) - if is_index_name and intent == 'out': + if has_index_prefix and intent == 'out': + return None + + # Unevidenced ``index_of_*`` (``index_of_shortwave_band``, + # ``index_of_timestep``, ...) is an ORDINARY index variable, not tied + # to a constituent. Defer so the host or an earlier scheme provides it, + # or the caller raises the missing-provider error. + if has_index_prefix and not is_constituent_index: return None constituent_module = _constituent_module_name(suite_name) @@ -2242,12 +2333,10 @@ def _common_kwargs(base_expr, subscript, call_expr, ) # ---- Path 1a: index_of_ — module-level integer, no per-instance -- - if is_index_name: + # Reached only when X is a KNOWN constituent (see is_constituent_index). + if is_constituent_index: # Mangle long std_names down to a Fortran-legal 63-char symbol; # identity for short names, so existing fixtures are unaffected. - # ``_INDEX_PREFIX`` is already part of std_name -- strip then - # re-add via the helper for uniform truncation. - index_base = std_name[len(_INDEX_PREFIX):] index_sym = _index_symbol_name(index_base) return ResolvedArg(**_common_kwargs( base_expr=index_sym, subscript='', call_expr=index_sym, @@ -2528,6 +2617,11 @@ def resolve_suite( index_names.update(arg.constituent_index_std_names) constituent_index_names = sorted(index_names) + _check_constituent_index_evidence( + suite.name, constituent_index_names, + scheme_store.constituent_stdnames(), + ) + # Under option A the constituent object is generator-owned (lives in # the ccpp_host_constituents module), so the host is no longer # required to declare ``ccpp_model_constituents_object`` in its diff --git a/capgen/metadata/metadata_table.py b/capgen/metadata/metadata_table.py index 92a727db..ac76532c 100644 --- a/capgen/metadata/metadata_table.py +++ b/capgen/metadata/metadata_table.py @@ -802,6 +802,14 @@ def validate(self, require_intent: bool, context: ParseContext) -> None: "Variable '{}' is marked protected but is intent {}, " "at {}".format(self.local_name, self.intent, context) ) + if self.is_constituent and self.standard_name.startswith('index_of_'): + raise CCPPError( + "Variable '{}' (standard_name='{}') is flagged as a " + "constituent (advected/constituent/molar_mass), but " + "'index_of_' names an index, not a field, at {}".format( + self.local_name, self.standard_name, context + ) + ) # ------------------------------------------------------------------ def __repr__(self) -> str: diff --git a/doc/briefing.md b/doc/briefing.md index 243cc74b..20b05a39 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -1,6 +1,6 @@ # capgen — Briefing for CCPP Framework Developers & Power Users -*Prepared for the 2026-05-14 walk-through; last revised 2026-06-05. +*Prepared for the 2026-05-14 walk-through; last revised 2026-09-01. Companion document to `doc/migration.md` (the detailed migration guide) and `doc/redesign_prompt.md` (the implementation spec).* @@ -8,7 +8,16 @@ guide) and `doc/redesign_prompt.md` (the implementation spec).* ## 1. Why a new generator? -The CCPP Framework runs two code generators today: +*This section describes the situation that motivated the redesign. It +is half history as of 2026-09-01: **`ccpp-prebuild` no longer has a +production consumer** — NEPTUNE, CCPP-SCM and UFS have all moved to +capgen v1 (§10) — but **`ccpp-capgen` v0 still does**, because CAM-SIMA +production still runs on it. Both predecessors are slated for deletion +from the tree in the same operation that merges `feature/capgen-v1` to +`develop`, which happens when/after CAM-SIMA transitions (FU-034). The +reasoning is also the rationale for the design choices in §3–§6.* + +The CCPP Framework ran two code generators: - **`ccpp-prebuild`** — simple, procedural Python; fast; DDT-argument passing; in production use by NOAA UFS Weather Model, Navy NEPTUNE, @@ -196,6 +205,21 @@ underlying legacy spelling from host/scheme metadata and the flag can be retired. See `doc/auto_clone_constituents.md` for the full auto-clone reference. +**Status note (2026-09-01).** Two of the three runways are now +load-bearing. When NEPTUNE, CCPP-SCM and UFS transitioned to capgen v1 +(§10) they did so *with* the shims, not by migrating their metadata +first. So adoption did not bring `--legacy-mode` (FU-010) or +`--gfs-dim-aliases` (FU-011) closer to removal — it gave them +**production** consumers, and retiring either is now a coordinated +host-metadata migration rather than a unilateral framework cleanup. +`--legacy-auto-clone-constituents` (FU-012) is different: its consumer +is CAM-SIMA, which is still on capgen v0, so it is pinned only by the +v1 testing/review branches. That makes it the one shim whose removal +can still be folded into a migration that has not happened yet — and +since CAM-SIMA's transition now gates the `develop` merge (FU-034), it +sits on the critical path. Decide it as part of that transition rather +than inheriting it afterwards. + ### 6.4 Required host `type = control` table Every host MUST declare scalar integers (and one character) with @@ -263,21 +287,26 @@ The headline items for a reader of this brief: - **Constituents overhaul** — three proposals on the table (`doc/constituents_overhaul.md` §8); decision pending a meeting and gating the framework setter additions. `followups.md` FU-020, FU-003. -- **Enforce `protected`** — a scheme can currently write a host variable - the host marked read-only; original capgen errored, capgen v1 does not. - FU-014. - **Codegen-time scheme-registration cross-check** — today's check is at runtime. FU-002. - **Nested-subcycle `ccpp_loop_counter` semantics** — resolves to the outermost counter. FU-001. - **Transient shims** — `--legacy-mode`, `--gfs-dim-aliases`, `--legacy-auto-clone-constituents`, CAM-SIMA's `capgen_compat/`, each - with an explicit removal trigger. FU-010 … FU-013. + with an explicit removal trigger. FU-010 … FU-013. `--legacy-mode` + and `--gfs-dim-aliases` are now pinned by production hosts — see the + status note in §6.3b before planning a removal. +- **Merge `feature/capgen-v1` to `develop`, and delete capgen v0 + + `ccpp-prebuild`** — one operation, gated on CAM-SIMA's transition to + v1. Three host models already build from the feature branch, so + until then it is a release branch in all but name. FU-034. Landed since this section was first written: the validator host-metadata -check (FU-008, 2026-06-01). Closed as *decided against*: suppressing -`ccpp_host_constituents.F90` when unused (FU-009) — see that row for why, -and do not re-propose it. +check (FU-008, 2026-06-01) and `protected` enforcement (FU-014, +2026-07-29 — a scheme can no longer write a host variable the host marked +read-only; both the metadata-level and resolver-level checks are in). +Closed as *decided against*: suppressing `ccpp_host_constituents.F90` +when unused (FU-009) — see that row for why, and do not re-propose it. ### 7.2 Intentionally NOT supported @@ -369,15 +398,34 @@ don't rebuild downstream objects unless something actually moved. ## 10. Where things stand right now -- **Unit tests**: 1516 passing on `feature/capgen` (as of - 2026-06-05). -- **End-to-end tests passing** (12): `advection`, - `advection_auto_clone`, `capgen`, `chunked_data`, +- **NEPTUNE, CCPP-SCM and the UFS Weather Model have transitioned to + capgen v1** (2026-09-01). All three came from `ccpp-prebuild`, and + all three now track the **`feature/capgen-v1` branch** of the NCAR + `ccpp-framework` repository directly. That is the milestone the + redesign was aimed at, and it retires `ccpp-prebuild` as a + production generator: the whole prebuild-style host family now runs + on capgen. **CAM-SIMA is the remaining transition** — see its + bullet below; it is still on capgen v0. Two consequences worth + stating plainly: + - `feature/capgen-v1` now has **three production consumers**, so it + is a long-lived release branch until the merge: no force-push, no + breaking generated-API change without notice. **The merge to + `develop` — and the deletion of both capgen v0 and `ccpp-prebuild` + from the tree — happens when/after CAM-SIMA transitions** + (`followups.md` FU-034). That makes CAM-SIMA's transition the + critical path for the whole v1 rollout. + - The three incoming hosts **still rely on the transient migration + shims** (§6.3b). Adoption did *not* retire them — it pinned + them. See the note in that section and FU-010/FU-011. +- **Unit tests**: 1564 passing on `feature/capgen-v1` (verified + 2026-09-01, `python unit-tests/run_tests.py`). +- **End-to-end tests passing** (13): `advection`, + `advection_auto_clone`, `capgen`, `capgen_ng`, `chunked_data`, `constituents_dim`, `ddthost`, `instances`, `instances_advection`, - `nested_suite`, `opt_arg`, `suite_allocate`, `var_compat`. The two - newest — `constituents_dim` (a variable dimensioned by + `nested_suite`, `opt_arg`, `suite_allocate`, `var_compat`. + `constituents_dim` (a variable dimensioned by `number_of_ccpp_constituents`) and `suite_allocate` (suite-owned - allocatable interstitials sized by a scheme-written dimension) — were + allocatable interstitials sized by a scheme-written dimension) were added while hardening the CAM-SIMA HPC build. - **Code size**: ~17.8k LOC of Python under `capgen/` (includes docstrings, inline comments, and the three transient shim modules) @@ -386,15 +434,18 @@ don't rebuild downstream objects unless something actually moved. - **Three transient migration shims now live** (see §6.3b): `--legacy-mode` (2026-05-13), `--gfs-dim-aliases` (2026-05-21), and `--legacy-auto-clone-constituents` (2026-05-21). Each is - isolated in its own module + grep-tag so removal is a single - cleanup pass once the underlying legacy spelling is gone from - host/scheme metadata. -- **CCPP-SCM**: actively driving development — every build / runtime - failure surfaced this month landed as a fix in capgen (rather - than being patched around in the host). Most of the `phys_ps` group - now builds end-to-end via `--legacy-mode` + `--gfs-dim-aliases`. - On 2026-05-20 the per-arg-attribute validator caught **67 real - metadata/Fortran disagreements** in the SCM physics tree (12 missing + isolated in its own module + grep-tag so the *framework-side* + removal is a single cleanup pass. The gating work is on the host + side: the legacy spelling has to be gone from host/scheme metadata + first, and as of 2026-09-01 it is not (§6.3b status note). +- **CCPP-SCM**: **transitioned to capgen v1** (2026-09-01), on + `feature/capgen-v1`. It drove most of the generator's hardening — + every build / runtime failure it surfaced landed as a fix in capgen + rather than a workaround in the host, which is why it was the + proving ground for the other prebuild hosts. Still runs with + `--legacy-mode` + `--gfs-dim-aliases`. On 2026-05-20 the + per-arg-attribute validator caught **67 real metadata/Fortran + disagreements** in the SCM physics tree (12 missing `kind = kind_phys` + 42 intent mismatches + a mix of optional-flag and bare-`real` cases); all fixed. - **Validator** now checks per-argument `intent`, `type`, `kind`, and @@ -421,14 +472,32 @@ don't rebuild downstream objects unless something actually moved. compilation effectively hang). Signatures stay so existing host callers still link; stubbed bodies return `errflg = 1` with a clear `errmsg`. -- **NEPTUNE**: cleanup and acceptance testing in progress. - Regular/lower-atmosphere physics builds and runs and produces - results within tolerance (deviations similar to compiler changes). - High-altitude physics testing is next. -- **UFS Weather Model**: not yet attempted; SCM is the proving - ground first. An anticipated complication is the "fast physics" - called directly from the FV3 dynamical core as a separate group. -- **CAM-SIMA**: **reconnected (2026-06-03 → 06-05).** capgen now +- **NEPTUNE**: **transitioned to capgen v1** (2026-09-01), on + `feature/capgen-v1`. Regular/lower-atmosphere physics builds and + runs and produces results within tolerance (deviations similar to + compiler changes), and **high-altitude physics works with v1 as + expected** — the acceptance item that was outstanding at 2026-06-05 + is closed. +- **UFS Weather Model**: **transitioned to capgen v1** (2026-09-01), + on `feature/capgen-v1` — the largest of the prebuild hosts and the + one the flat-field argument passing of capgen v0 could never have + served (§1). The anticipated complication was "fast physics" + called directly from the FV3 dynamical core as a separate group; + **that group works with v1 as expected**, so no special handling + was needed. +- **CAM-SIMA**: **still on capgen v0 — the remaining transition, and + the critical path.** As of 2026-09-01 CAM-SIMA production builds + with the original ccpp-capgen; capgen v1 support lives on + **branches maintained for testing and review**, not in the + production configuration. It is therefore *not* a consumer of + `feature/capgen-v1` in the sense the other three now are. Because + the `develop` merge and the deletion of capgen v0 + `ccpp-prebuild` + are gated on this transition (FU-034), its two gating items — the + constituent-ordering re-baseline (FU-018/FU-030) and the + compat-layer removal plan (FU-013) — are now blockers on the entire + v1 rollout, not just on CAM-SIMA. What follows is the state of + that v1 branch work. + **Reconnected (2026-06-03 → 06-05):** capgen drives the real CAM-SIMA build on Derecho via a thin compatibility layer (`cime_config/capgen_compat/`, in the CAM-SIMA tree) that re-implements original ccpp-capgen's Python API surface diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index 8a25cfd9..5d508fdd 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -7,30 +7,40 @@ program managers; it summarises the case for `capgen` in terms of product risk, schedule, and cross-organization impact rather than implementation detail.* -*Last revised: 2026-06-05.* +*Last revised: 2026-09-01.* --- ## TL;DR -The CCPP Framework today ships **two** code generators that solve the +**Status as of 2026-09-01: three of the four host models have +transitioned to `capgen`.** NOAA UFS, Navy NEPTUNE and CCPP-SCM — all +previously on `ccpp-prebuild` — now build with the new generator. +**CAM-SIMA is the one remaining transition**, and it is the gate on +finishing the job: the plan of record is to merge `capgen` into +`develop` and delete *both* older generators when/after CAM-SIMA +moves. The rest of this section is the situation that motivated the +work. + +The CCPP Framework shipped **two** code generators that solved the same problem differently: -- **`ccpp-prebuild`** powers NOAA UFS, Navy NEPTUNE, and CCPP-SCM. +- **`ccpp-prebuild`** powered NOAA UFS, Navy NEPTUNE, and CCPP-SCM. Simple and reliable, but feature-light — does not support features CAM-SIMA needs (constituents, framework-owned variables, - introspection). + introspection). **No longer has a production consumer.** - **`ccpp-capgen`** powers NCAR CAM-SIMA. Feature-rich, but built on technical choices that **do not scale** to UFS or NEPTUNE and that - **do not support multi-instance hosts** at all. + **do not support multi-instance hosts** at all. **Still in + production for CAM-SIMA.** -Neither generator can be the basis for a single shared toolchain. +Neither generator could be the basis for a single shared toolchain. **`capgen`** is a third generator, started in early May 2026, designed to do everything both other generators do, in code small enough for a few people to own, with the architectural choices that -make it work at UFS/NEPTUNE scale and beyond. The redesign is -running on the SCM as proving ground; UFS / NEPTUNE / CAM-SIMA -re-integration is sequenced behind that. +make it work at UFS/NEPTUNE scale and beyond. The SCM was the proving +ground; UFS and NEPTUNE followed and are now transitioned, and +CAM-SIMA re-integration is the remaining step. This document explains, in plain language, **why we did not extend capgen instead**, what risks the redesign retires, and where things @@ -265,27 +275,45 @@ Features that exist only in capgen (some exist in prebuild): --- -## 6. Where things stand right now (2026-06-05) - -- **Unit tests**: 1516 passing. No known failures. -- **End-to-end tests**: 12 passing — `advection`, +## 6. Where things stand right now (2026-09-01) + +- **Three of four host models have transitioned to capgen.** NEPTUNE, + CCPP-SCM and the UFS Weather Model — the entire `ccpp-prebuild` user + base — now build with the new generator and track its development + branch directly. `ccpp-prebuild` has no production consumer left. + **CAM-SIMA is the remaining transition** and is still on the older + `ccpp-capgen`. +- **The endgame is defined and has a single trigger.** When/after + CAM-SIMA transitions, capgen merges into `develop` and *both* older + generators are deleted from the tree — one operation. That makes + CAM-SIMA's transition the critical path for the whole programme, and + promotes its two gating items (the constituent-ordering re-baseline + and the retirement of the CAM-SIMA compatibility layer) to + programme-level blockers rather than CAM-SIMA-local work. Schedule + risk concentrates there; see §8. +- **Unit tests**: 1564 passing. No known failures. +- **End-to-end tests**: 13 passing — `advection`, `advection_auto_clone` (CAM-SIMA advection_test port exercising the - auto-clone shim), `capgen`, `chunked_data`, `constituents_dim`, - `ddthost`, `instances`, `instances_advection` + auto-clone shim), `capgen`, `capgen_ng`, `chunked_data`, + `constituents_dim`, `ddthost`, `instances`, `instances_advection` (multi-instance + constituents), `nested_suite`, `opt_arg`, - `suite_allocate`, `var_compat`. The two newest (`constituents_dim`, - `suite_allocate`) were added while hardening the CAM-SIMA HPC build. + `suite_allocate`, `var_compat`. `constituents_dim` and + `suite_allocate` were added while hardening the CAM-SIMA HPC build. - **Code size**: ~17.8k lines of Python under `capgen/` including inline comments and the three transient shim modules; ~18k lines of unit/doctest under `unit-tests/`. Still procedural; still flat data classes; still well below capgen. -- **CCPP-SCM**: actively driving development. Each build / runtime - issue surfaced this month landed as a fix in capgen rather than - a host-side workaround. All available suites in CCPP-SCM now - build and run end-to-end via `--legacy-mode` + `--gfs-dim-aliases`. +- **CCPP-SCM**: **transitioned.** It drove most of the generator's + hardening — each build / runtime issue it surfaced landed as a fix in + capgen rather than a host-side workaround, which is why it was the + proving ground for the other prebuild hosts. All available suites + build and run end-to-end, via `--legacy-mode` + `--gfs-dim-aliases`. - **Three transient migration shims in place** (see §5). Each is - isolated in its own module with a single grep tag, so removal once - hosts migrate is a single cleanup pass. + isolated in its own module with a single grep tag, so the + framework-side removal is a single cleanup pass. **The hosts + transitioned *with* these shims rather than migrating their metadata + first**, so retiring one is now a coordinated host-side migration — + a scheduling item, not a cleanup. It does not block the merge. - **Auto-clone shim landed 2026-05-21**. Reinstates original capgen's auto-clone path behind `--legacy-auto-clone-constituents`. This is the no-decision-needed bridge for CAM-SIMA — the ~16 schemes that @@ -297,26 +325,34 @@ Features that exist only in capgen (some exist in prebuild): bug; the fix moves the per-suite dynamic-constituents buffer per-instance. No coordination with CAM-SIMA / UFS / NEPTUNE required (host-facing API unchanged). -- **NEPTUNE**: Final cleanup and acceptance testing in progress. - All regression tests (~300) pass with the three mandatory - compilers (Intel LLVM, GCC, LLVM native) for regular physics, - mid-altitude, and high-altitude physics (feature-complete). -- **UFS Weather Model**: not yet attempted; SCM is the proving - ground first. Expecting updates due to the "fast physics" - called directly from the FV3 dynamical core as separate group. -- **CAM-SIMA**: **re-connected (2026-06-03 → 06-05).** capgen now - drives the production CAM-SIMA build on the Derecho supercomputer - through a small compatibility layer that lets CAM-SIMA's existing - build scripts call capgen without being rewritten. Three - configurations build **and run to completion under both the Intel and - GNU compilers**, with bit-comparable results: `kessler`, `rrtmgp`, - and `se_cslam`/CSLAM — the last being the full CAM7 physics suite +- **NEPTUNE**: **transitioned.** All regression tests (~300) pass with + the three mandatory compilers (Intel LLVM, GCC, LLVM native) for + regular physics, mid-altitude, and high-altitude physics + (feature-complete). High-altitude physics — the last acceptance item + outstanding in June — works with capgen as expected. +- **UFS Weather Model**: **transitioned.** The largest of the hosts, + and the one the older capgen could never have served (§3.1). The + anticipated complication was the "fast physics" called directly from + the FV3 dynamical core as a separate group; that group works as + expected and needed no special handling. +- **CAM-SIMA**: **not yet transitioned — still on the older + `ccpp-capgen`, and the critical path (see above).** capgen v1 + support lives on branches maintained for testing and review; the + production configuration has not moved. On those branches, capgen + drives the real CAM-SIMA build on the Derecho supercomputer through a + small compatibility layer that lets CAM-SIMA's existing build scripts + call capgen without being rewritten. Three configurations build + **and run to completion under both the Intel and GNU compilers**, + with bit-comparable results: `kessler`, `rrtmgp`, and + `se_cslam`/CSLAM — the last being the full CAM7 physics suite (deep + shallow convection, stratiform microphysics, RRTMGP radiation, gravity-wave drag) on a cubed-sphere/CSLAM-advection - configuration. This is the first time the redesigned generator has - produced a complete, running CAM-SIMA model. The constituent - overhaul decision (see §7) remains a separate track and was not on - the critical path for this milestone. + configuration. That was the first time the redesigned generator + produced a complete, running CAM-SIMA model. Remaining before + transition: a re-baseline caused by a change in constituent ordering + (a known, understood floating-point difference, not a defect) and + retirement of the compatibility layer. The constituent overhaul + decision (see §7) remains a separate track. --- @@ -356,10 +392,11 @@ proposals are implementable on top of it. | capgen diverges from capgen feature set | LOW | Cross-checked by `doc/redesign_analysis.md`; the feature comparison table in §4 / §5 is exhaustive | | Host metadata break for UFS / NEPTUNE / CAM-SIMA | LOW | Three transient shims (`--legacy-mode`, `--gfs-dim-aliases`, `--legacy-auto-clone-constituents`) together cover the known-incompatible standard-name pair, the GFS radiation/composition vertical-dim spellings, and original capgen's auto-clone registration path. Remaining required changes (e.g., `_finalize` → `_final`) are mechanical and listed in `doc/migration.md` §3 | | Constituent overhaul stalls | LOW | Proposal A unblocks the immediate bug; capgen works with the current framework today; `--legacy-auto-clone-constituents` lets CAM-SIMA's atmospheric_physics build without an overhaul decision; the overhaul is a separate decision track | -| Bus-factor on capgen itself | MEDIUM | Procedural code style + flat data classes + 1426-test safety net; significantly lower than capgen's bus factor | +| Bus-factor on capgen itself | MEDIUM | Procedural code style + flat data classes + 1564-test safety net; significantly lower than capgen's bus factor | +| **CAM-SIMA transition slips, delaying the whole programme** | **MEDIUM — the main schedule risk as of 2026-09-01** | The `develop` merge and the deletion of both older generators are gated on this one transition (§6), so its two remaining items — the constituent-ordering re-baseline and retirement of the compatibility layer — are programme-level blockers. Both are understood and scoped; neither is a defect. Mitigation is to track them as such rather than as CAM-SIMA-local work, and to decide the auto-clone shim's fate as part of the transition | | Two host call-shape conventions (prebuild-style vs capgen-style) coexist forever | LOW | capgen emits one shape; downstream host conversions are tracked in `doc/migration.md` | -| Regression discovered during NEPTUNE / UFS testing | EXPECTED | SCM proving ground catches most; remaining issues become capgen tickets, not host-side patches | -| ccpp-prebuild end-of-life requires a sunset plan | OPEN | Not yet scoped; both generators currently coexist in the framework repo | +| Regression discovered during NEPTUNE / UFS testing | LARGELY RETIRED (2026-09-01) | Both models have transitioned; NEPTUNE passes ~300 regression tests on three compilers including high-altitude physics, and the anticipated UFS FV3 fast-physics complication did not materialise. The SCM proving-ground approach worked as intended — issues became capgen fixes, not host-side patches | +| ccpp-prebuild end-of-life requires a sunset plan | SCOPED (2026-09-01) | Decided: `ccpp-prebuild` **and** the older `ccpp-capgen` are both deleted from the framework repo in the same operation that merges capgen into `develop`, triggered when/after CAM-SIMA transitions. prebuild already has no production consumer. The residual risk is schedule, not scope — see the CAM-SIMA row above | --- diff --git a/doc/followups.md b/doc/followups.md index 572b883b..4c8e03a4 100644 --- a/doc/followups.md +++ b/doc/followups.md @@ -37,7 +37,7 @@ Status values: `open`, `in progress`, `blocked`, `closed`. | FU-015 | Validator: capture `protected` and `allocatable` from Fortran declarations | framework | 2026-07-28 | open | `_ArgAttrs` (`ccpp_validator.py:135`) carries only type/kind/intent/optional/rank; `_parse_decl_line:352-354` explicitly discards `protected`, `parameter` and `allocatable`. `allocatable` is the more consequential of the two — metadata declares it (`metadata_table.py:493`) and it *changes codegen* (subscript emission at call sites), so a mismatch is silently wrong output rather than a missing error. A `protected` check must accept Fortran `parameter` as satisfying it: CAM-SIMA `create_readnl_files.py:422` writes `protected = True` for namelist array dimensions that `:523` declares `integer, public, parameter`. Cost note: `_ArgAttrs` reprs appear in 7 doctests in `ccpp_validator.py`. **Deprioritised 2026-07-28** — CAM-SIMA never invokes `ccpp_validator` (no call site in `cime_config/`), so this is CI/developer value only, and FU-014 catches the same class of error where it is load-bearing. | | FU-016 | Expose `advected` on `ResolvedArg` | framework | 2026-07-26 | open | `capgen_compat/_var_wrapper.py:~320` currently *infers* advectedness from the constituent standard-name shape (`_is_base_constituent_name`) because capgen does not surface the flag. The inference is close but not exact; exposing the real flag would make it exact. | | FU-017 | `cime_config/host_framework_deps.py` may now be redundant | cam-sima | 2026-07-28 | open | It was added 2026-07-27 so CAM-SIMA's host code could compile `ccpp_constituent_prop_mod` in constituent-free builds. Making `ccpp_host_constituents.F90` unconditional (FU-009) put the four framework `.F90` files back into `` unconditionally, which likely covers the same ground. ~90 lines plus 8 tests plus 4 documentation sections. Verify end-to-end before the next Derecho run and remove if genuinely redundant. See `constituents_overhaul.md` §4.17. | -| FU-018 | MPAS 120km cam4 aux test fails on constituent ordering | cam-sima | 2026-07 | open | Known failure, distinct from `fadiab` (which also fails on `develop`). Analysis in `doc/cam4_fwaut_constituent_order.md`. The framework-side fix and the re-baseline decision are FU-030. Post-sign-off cleanup: strip the inert DBG-FP instrumentation (`schemes/utilities/debug_fingerprint.F90` + its call sites) from both `EXT/cam-sima-ng` and `EXT/cam-sima-ng-reference`. | +| FU-018 | MPAS 120km cam4 aux test fails on constituent ordering | cam-sima | 2026-07 | open | Known failure, distinct from `fadiab` (which also fails on `develop`). Analysis in `doc/cam4_fwaut_constituent_order.md`. The framework-side fix and the re-baseline decision are FU-030. Post-sign-off cleanup: strip the inert DBG-FP instrumentation (`schemes/utilities/debug_fingerprint.F90` + its call sites) from both `EXT/cam-sima-ng` and `EXT/cam-sima-ng-reference`. **On the critical path as of 2026-09-01** via FU-030 — see FU-034. | | FU-019 | Delete pushed branch `bugfix/constituents_camsima_july2026` | framework | 2026-07-27 | open | Housekeeping. The branch carried framework commit `501d1c0`, which was wrong and has been reverted; `feature/capgen-v1` is the live branch. | | FU-024 | Confirm FU-014 Check B does not fire in a production CAM-SIMA build | cam-sima | 2026-07-29 | open | The unit tests and fixtures are green, but only a Derecho aux-test run exercises the real registry against the real suites. Risk assessed low — the three registry variables that carry `access="protected"` (`fracis`, `do_lagrangian_vertical_coordinate`, `dycore_calculates_geopotential_using_logarithms`) are all consumed `intent = in` (§5) — but `access="protected"` is not the only source: `allocatable="parameter"` also emits `protected = True` (`generate_registry_data.py:694-695`), as does `create_readnl_files.py:422` for namelist array dimensions. Fold the result back here. | | FU-025 | Revisit capgen's logging-output scheme | framework | 2026-07-15 | open | Per-variable transform logging (`group_cap.py:_log_one_transform`, ~:545) is **temporarily emitted at WARNING** (see the `TEMPORARY level choice` comment at ~:553) purely so it shows in a default run — capgen's default level is WARNING; INFO needs `-v`. Decide its real home (INFO + `-v`, a dedicated `--report-transforms` flag [Dom's likely preference: targeted, no flood], or leave) then drop the WARNING abuse. Same pass: reclassify non-warning WARNINGs — the three shim banners (`legacy_compat.py:~85`, `dim_aliases.py`, `auto_clone_constituents.py`; fire every CAM-SIMA/SCM run) and the per-scheme "no Fortran source found … fallback" (`ccpp_capgen.py:~1162`) are informational. Flipping the default to INFO is not an option — `write_if_changed` logs per file. | @@ -45,10 +45,12 @@ Status values: `open`, `in progress`, `blocked`, `closed`. | FU-027 | Consolidate emitter scoping + host-vs-suite audit (generator hardening) | framework | 2026-06-04 | open | Remaining two of a four-item hardening plan (items 1–2 done: e2e already compiles+links+runs every cap; `suite_allocate` + `constituents_dim` corpus tests landed). **#3** — one shared helper that, given a `ResolvedArg`, returns its USE-requirements and access expression, called by all four emitters (`group_cap`/`static_api`/`suite_cap`/`suite_data`); the register-USE divergence bug could not have existed if both paths shared it. **#4** — proactive walk of the four emitters reconciling host-vs-suite handling (USE / dimensions / allocatable / DDT-module / naming) in one pass. Kills the divergence bug class rather than patching instances. e2e tree off-limits without explicit permission. | | FU-028 | CAM-SIMA schemes: undefined `intent(out)` on an early-return path | cam-sima | 2026-06-08 | open | Original capgen zero/false-initialised interstitial storage, masking schemes that leave an `intent(out)` unset on an early-return branch; capgen-ng deliberately does **not** default-init suite-owned vars, so these read heap garbage at runtime. **Decision (Dom 2026-06-08): fix each scheme in place; do NOT add suite-var default-init to capgen-ng** (that would re-mask the whole class). Expect more to surface one-by-one as suites run under capgen-ng. First instance fixed: `solar_irradiance_data_init` (set `do_spectral_scaling = .false.` before the `fixed_scon` return). Edits live in the `EXT/cam-sima-ng/src/physics/ncar_ccpp` submodule. | | FU-029 | Decide `timestep_init` / `timestep_final` phase-call-count semantics | framework | 2026-06-10 | open | For a scheme that appears multiple times in a suite, original capgen calls its `timestep_init`/`final` **once per appearance**; capgen-ng calls it **once per group** (measured cam4: `qneg_timestep_final` 2 vs 12). Benign for cam4 (the affected phases are idempotent/guarded) but a latent b4b/correctness hazard the moment such a phase is stateful (accumulates, zeroes a buffer). CCPP intent is once-per-timestep; neither matches strictly when a scheme spans groups. Decide the intended semantics and make capgen-ng's behaviour intentional + documented. Reproduce via the standalone-capgen driver, diffing `_timestep_(init|final)` call counts. | -| FU-030 | Deterministic + documented constituent registration order in the generator | framework | 2026-06-11 | open | Root cause of the cam4 FWAUT b4b diff (the framework side of FU-018): capgen-ng registers water species alphabetically ([cloud_ice, cloud_liquid, water_vapor]) vs original's declaration order ([cloud_liquid, cloud_ice, water_vapor]), and trace gases differ too, so `air_composition`'s `thermodynamic_active_species_idx` order → `get_hydrostatic_energy` water-sum FP order → energy fixer → pervasive roundoff. Proven b4b by a flag-guarded reorder hack. **Decision (Dom): RE-BASELINE** — give capgen-ng a deterministic, documented order (qv first; an understandable rule for how constituents land in the array), then CAM-SIMA re-baselines against the original-capgen reference; not match-the-old-order. Levers: `host_constituents.py` / the legacy-auto-clone path (FU-012) / `ccpp_register_constituents` emission; intersects the constituents overhaul (FU-020). Analysis: `doc/cam4_fwaut_constituent_order.md`. | +| FU-030 | Deterministic + documented constituent registration order in the generator | framework | 2026-06-11 | open | Root cause of the cam4 FWAUT b4b diff (the framework side of FU-018): capgen-ng registers water species alphabetically ([cloud_ice, cloud_liquid, water_vapor]) vs original's declaration order ([cloud_liquid, cloud_ice, water_vapor]), and trace gases differ too, so `air_composition`'s `thermodynamic_active_species_idx` order → `get_hydrostatic_energy` water-sum FP order → energy fixer → pervasive roundoff. Proven b4b by a flag-guarded reorder hack. **Decision (Dom): RE-BASELINE** — give capgen-ng a deterministic, documented order (qv first; an understandable rule for how constituents land in the array), then CAM-SIMA re-baselines against the original-capgen reference; not match-the-old-order. Levers: `host_constituents.py` / the legacy-auto-clone path (FU-012) / `ccpp_register_constituents` emission; intersects the constituents overhaul (FU-020). Analysis: `doc/cam4_fwaut_constituent_order.md`. **On the critical path as of 2026-09-01** — CAM-SIMA's transition gates the `develop` merge and the removal of capgen v0 + prebuild (FU-034), and this is one of its two gating items. | | FU-031 | Long-term redesign of the `ccpp_static_api.F90` runtime listings | framework | 2026-05-14 | open | The suite-variable / suite-host-data listings made the introspection module ~33k lines (`-O3` effectively hangs). Immediate pressure is off — `--no-host-introspection` stubs them (→ ~800 lines) — so this is **no longer blocking**, but the long-term redesign stays open for team discussion: move the listings to a runtime read of `datatable.xml` (preferred — no recompile when listings change), or a separate `-O0` file, or static string `data` tables, or lazy-emit only the routines the host calls. Do not redesign unilaterally. | | FU-032 | Generator-owned locals can silently shadow a host import — auto-uniquify | framework | 2026-08-07 | open | `_check_host_control_local_collisions` (`capgen/generator/group_cap.py`) now hard-errors when a host variable's local name collides with a control-variable dummy (issue #774 — the silent wrong-value case, closed by that check + `unit-tests/test_suite_resolver.py::TestHostControlLocalNameCollision`). Two other subroutine-scope locals can shadow a use-associated host import the same way but are **generator-owned**, so the right fix is to rename *them*, not error: transformation temporaries (`_l` / `_p`) and subcycle loop counters (`ccpp_loop_counter*`). Seed the temp uniquifier (`used_local_names_phase`, `suite_resolver.py:2446`) with the group's host-import symbols + control-dummy names so `_local_name_conflict` renames generator locals away from them. Rare in practice (suffixed/reserved names) but closes the class. Deliberately deferred out of the #774 fix (Step 2, 2026-08-07). | | FU-033 | Vertical flip on an allocatable host array is rejected, not supported | framework | 2026-08-12 | open | `_resolve_one_arg` (`capgen/generator/suite_resolver.py`, just after `needs_vert_flip` is computed) hard-errors when a host variable is `allocatable = True` **and** needs a vertical flip (host/scheme `top_at_one` disagree on a var with a vertical dim). An allocatable actual must omit subscripts, so the reverse-stride flip subscript cannot be encoded; silently dropping it would hand the scheme vertically-reversed data. **Decision (2026-08-12, PR #762 review finding from jimmielin): error for now** rather than emit a wrong-but-compiling cap. Test: `unit-tests/test_suite_resolver.py::TestVerticalFlipTransform::test_allocatable_host_plus_flip_raises`. Any future support is limited to the allocatable-host → *non-allocatable* (plain assumed-shape) scheme-dummy sub-case, where a flipped section `host(:, ub:lb:-1)` is legal; an allocatable dummy can never receive a flipped section. Parallels FU-032 (guard now, enhance later). | +| FU-034 | Merge `feature/capgen-v1` to `develop` and delete capgen v0 + `ccpp-prebuild` — gated on CAM-SIMA | framework | 2026-09-01 | open | **Decision (Dom, 2026-09-01): the merge to `develop`, and the removal of both capgen v0 and `ccpp-prebuild` from the tree, happen WHEN/AFTER CAM-SIMA transitions to capgen v1.** One operation, one trigger — the sequencing is settled, not open. Context: NEPTUNE, CCPP-SCM and UFS transitioned on 2026-09-01 and build directly off **`feature/capgen-v1`** (`briefing.md` §10); CAM-SIMA is still on capgen v0 with v1 on testing/review branches, and is the last host holding either predecessor alive. Consequences: (1) **CAM-SIMA's transition is the critical path for the entire v1 rollout**, which promotes its gating items — the constituent-ordering re-baseline (FU-018/FU-030) and the compat-layer removal plan (FU-013) — from CAM-SIMA-local concerns to blockers on the whole merge; (2) until then `feature/capgen-v1` is a long-lived release branch with three production consumers, so treat it as released — no force-push, no breaking generated-API change without notice; (3) PR #762 is the umbrella PR and the natural vehicle; (4) `--legacy-auto-clone-constituents` (FU-012) is CAM-SIMA's shim, so it also sits on this critical path — decide it as part of the transition rather than inheriting it (§3). | +| FU-035 | Constituent index evidence is scheme-metadata-only — a runtime-registered constituent nothing flags needs a host declaration | framework | 2026-09-01 | open | Fallout of the `index_of_*` fix (2026-09-01): `index_of_` is auto-provisioned as a constituent index only when capgen has **positive evidence** that X is a constituent, i.e. some scheme flags `X` or `tendency_of_X` `advected`/`constituent`/`molar_mass` (`_is_known_constituent`, `capgen/generator/suite_resolver.py`). That is the only constituent knowledge available at codegen — register-phase Fortran `%instantiate(std_name=…)` is not parsed (FU-002). **Residual gap:** a constituent registered at runtime whose base name is flagged in *no* scheme metadata, and whose index some scheme consumes, now raises the missing-provider error instead of resolving. Workaround (and what every in-tree host already does): declare the index in host metadata — host declarations win before any constituent path. No such case exists in CAM-SIMA, CCPP-SCM or the e2e corpus today; all nine CAM-SIMA `index_of_*` names are either registry-declared or scheme-produced `intent=out`. Closing FU-002 would remove the gap entirely by making registration itself the evidence. | --- @@ -86,12 +88,30 @@ document; the rest are indexed by section only. Each has an explicit removal trigger. Remove the module, its unit tests, its fixtures, and every marked touchpoint together. +**2026-09-01 — FU-010 and FU-011 are now pinned by production hosts.** +NEPTUNE, CCPP-SCM and UFS transitioned to capgen v1 *with* the shims +rather than by migrating their metadata first (`briefing.md` §6.3b status +note, §10). The removal triggers below are unchanged and still correct, +but FU-010 and FU-011 are now coordinated host-metadata migrations rather +than framework-side cleanups — pinned by the GFS-physics-derived hosts +(CCPP-SCM, UFS; NEPTUNE to confirm). Estimate the host-side work before +scheduling either. + +**FU-012 is the exception.** Its consumer is CAM-SIMA's ~16 auto-clone +schemes, and CAM-SIMA is **still on capgen v0** — its v1 support is on +testing/review branches. So FU-012 is pinned only by branch work, not by +a production build, and it is the one shim whose removal can still be +folded into a migration that has not happened yet. Since that migration +now gates the `develop` merge (FU-034), FU-012 is on the critical path: +decide it as part of CAM-SIMA's transition rather than inheriting it +afterwards. + | ID | Shim | Remove when | Touchpoints | |----|------|-------------|-------------| | FU-010 | `--legacy-mode` | scheme metadata has migrated | `capgen/metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, every `# legacy-compat:` marker | | FU-011 | `--gfs-dim-aliases` (added 2026-05-21) | GFS metadata stops spelling `vertical_layer_dimension` as `adjusted_vertical_layer_dimension_for_radiation` / `vertical_composition_dimension` | `capgen/metadata/dim_aliases.py`, `unit-tests/test_dim_aliases.py`, every `# dim-aliases:` marker | | FU-012 | `--legacy-auto-clone-constituents` (added 2026-05-21) | consumers have moved to explicit `host_constituents(:)` declaration or register-phase scheme registration | `capgen/metadata/auto_clone_constituents.py`, `unit-tests/test_auto_clone_constituents.py`, `unit-tests/sample_files/scheme_auto_clone_consumer.meta`, `unit-tests/sample_suite_files/suite_auto_clone.xml`, every `# auto-clone-constituents:` marker | -| FU-013 | CAM-SIMA `cime_config/capgen_compat/` | phased removal plan A–G in that directory's `README.md` completes | whole directory; brief at `doc/capgen_compat_layer.md` | +| FU-013 | CAM-SIMA `cime_config/capgen_compat/` | phased removal plan A–G in that directory's `README.md` completes — **on the critical path as of 2026-09-01**, one of the two items gating CAM-SIMA's transition and therefore the FU-034 merge | whole directory; brief at `doc/capgen_compat_layer.md`. **The surface grows when upstream moves**: a 2026-09-01 merge from CAM-SIMA `development` added a whole-DDT guard to `write_init_files.py`, needing a new `ddt_library.py` shim (`VarDDT`, never instantiated — capgen flattens DDT components, so nothing is ever one) plus `_VarWrapper.is_ddt()`. The same merge brought an upstream test (`test_ddt_scheme_arg_write_init`) whose host files omit the v1-only `ccpp_control_vars.meta` fixture, and a golden that needed regenerating for the v1 host-symbol names (`cam_*` → `ccpp_*`, §6). Budget for this per upstream sync, and prefer a scheduled retirement over an opportunistic one. | --- @@ -147,6 +167,7 @@ file, per the procedure in the repository's `CLAUDE.md`. | Machine | Last reconciled | By | |---------|-----------------|-----| +| `dutchman` | 2026-09-01 | Host-adoption update: NEPTUNE, CCPP-SCM and UFS have transitioned to capgen v1 on `feature/capgen-v1`; **CAM-SIMA has not — it is still on capgen v0 with v1 on testing/review branches**, so it is the remaining migration, not a fourth consumer. Added FU-034, which records Dom's decision that the `develop` merge **and** the deletion of capgen v0 + `ccpp-prebuild` happen when/after CAM-SIMA transitions — one operation, one trigger. That puts CAM-SIMA's transition on the critical path for the whole rollout, so FU-013, FU-018 and FU-030 gained critical-path annotations. Added the §3 pinning note — FU-010/FU-011 were *not* retired by adoption, they gained production consumers, while FU-012 is pinned only by CAM-SIMA branch work. Closed two long-standing unknowns in `briefing.md` §10: NEPTUNE high-altitude physics and the UFS FV3 fast-physics group both work with v1 as expected. Refreshed `briefing.md` §1/§6.3b/§10 (§10 host bullets were stale on all three models; test counts corrected 1516→1564 verified and e2e 12→13, `capgen_ng` was missing) and dropped the stale §7.1 bullet claiming `protected` is unenforced (FU-014 closed 2026-07-29). Swept this machine's auto-memory: every deferred entry already maps to an existing FU row, nothing new to fold in; wrote one new memory for the adoption fact pointing at FU-034. Later the same day, added FU-035 as fallout of the `index_of_*` positive-evidence fix: constituent auto-provisioning now requires positive evidence that a name IS a constituent, and the residual runtime-registration gap is recorded there (FU-002 would close it). | | `dutchman` | 2026-08-12 | PR #762 review-fix session (jimmielin's Claude-generated findings): added FU-033 (vertical-flip + allocatable-host → hard error). The other two findings — backward-transform temp-name collision (`suite_resolver`) and incomplete `[ccpp-table-properties]` missing `name`/`type` now erroring (`metadata_table`) — are fixed, committed, and PR'd, so they live in git, not restated here. Nothing new in this machine's auto-memory to fold beyond the above. | | `dutchman` | 2026-08-07 | folded the issue #772 / #774 session: added FU-032 (generator-local shadow follow-up). #772 shown to be a non-issue in v1 (cld_shadow e2e reproducer) and #774 detect-and-error landed in `group_cap.py` — both tracked in GitHub, not restated here | | `dutchman` | 2026-08-06 | first sweep of this machine; folded its auto-memory investigation notes into new rows FU-025…FU-031, added Codee `use…only:` detail to FU-005, cross-linked FU-018↔FU-030 | diff --git a/unit-tests/test_metadata_table.py b/unit-tests/test_metadata_table.py index 730feb3e..42f7ec13 100644 --- a/unit-tests/test_metadata_table.py +++ b/unit-tests/test_metadata_table.py @@ -1520,7 +1520,8 @@ def test_molar_mass_in_ddt_raises(self): class TestConstituentAttributes(unittest.TestCase): """Parsing and is_constituent rollup for scheme-only constituent hints.""" - def _scheme_var(self, *, extra_attrs: str = '') -> MetaVar: + def _scheme_var(self, *, extra_attrs: str = '', + std_name: str = 'foo') -> MetaVar: text = """ [ccpp-table-properties] name = my_scheme @@ -1530,13 +1531,13 @@ def _scheme_var(self, *, extra_attrs: str = '') -> MetaVar: name = my_scheme_run type = scheme [ x ] - standard_name = foo + standard_name = {std} units = kg kg-1 dimensions = () type = real intent = inout {extra} - """.format(extra=extra_attrs) + """.format(extra=extra_attrs, std=std_name) tables = _parse_text(text) return tables[0].sections()[0].variables[0] @@ -1566,6 +1567,14 @@ def test_negative_molar_mass_rejected(self): with self.assertRaises((CCPPError, ParseSyntaxError)): self._scheme_var(extra_attrs='molar_mass = -1.0') + def test_index_of_cannot_be_a_constituent(self): + for flag in ('constituent = True', 'advected = .true.', + 'molar_mass = 18.0'): + with self.subTest(flag=flag): + with self.assertRaises(CCPPError): + self._scheme_var(extra_attrs=flag, + std_name='index_of_shortwave_band') + ######################################################################## # File-based tests (use sample_files/) diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 82cc2e57..86cd98e5 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -4153,6 +4153,13 @@ def test_index_names_exclude_provided_dry_var(self): dimensions = () type = integer intent = in +[ q_const ] + standard_name = test_constituent + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. [ idx_const ] standard_name = index_of_test_constituent units = index @@ -4228,8 +4235,8 @@ def test_run_consumer_resolves_from_suite_vars(self): self.assertNotEqual(idx.source, 'constituent') def test_genuine_constituent_index_unchanged(self): - # index_of_test_constituent is produced by no scheme -> still a - # constituent index. + # index_of_test_constituent is produced by no scheme and + # 'test_constituent' is flagged advected -> constituent index. idx = self.consumer['idx_const'] self.assertEqual(idx.source, 'constituent') @@ -5113,20 +5120,81 @@ def test_host_index_of_resolves_to_host_local_name(self): self.assertIsNotNone(arg.host_entry) self.assertEqual(arg.host_entry.local_name, 'ntcw') - def test_unclaimed_index_of_still_routes_to_constituents(self): - """The framework auto-provisioning path is preserved for - ``index_of_`` names the host does NOT declare — required for - capgen-owned constituent flows (cf. the advection e2e test).""" + def test_unclaimed_index_of_routes_to_constituents_with_evidence(self): + """Auto-provisioning is preserved for ``index_of_`` names the host + does NOT declare, when some scheme flags X as a constituent.""" hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) suite_var = self._scheme_var( 'idx_other', 'index_of_some_other_constituent_not_in_host', intent='in', ) - arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'some_scheme', set()) + arg = _resolve_one_arg( + suite_var, 'run', hd, {}, 'some_scheme', set(), + const_stds={'some_other_constituent_not_in_host'}, + ) self.assertEqual(arg.source, 'constituent') self.assertEqual(arg.call_expr, 'index_of_some_other_constituent_not_in_host') + def test_unclaimed_index_of_without_evidence_is_an_error(self): + """Regression: without evidence that X is a constituent, + ``index_of_`` is an ordinary variable that nobody provides and + must raise the missing-provider error rather than be auto-provisioned. + This is the shape a one-sided host/scheme rename produces.""" + hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) + suite_var = self._scheme_var( + 'idx_typo', + 'index_of_deep_convection_process_in_cumulative_change_index', + intent='in', + ) + with self.assertRaises(CCPPError) as ctx: + _resolve_one_arg(suite_var, 'run', hd, {}, 'some_scheme', set(), + const_stds=set()) + msg = str(ctx.exception) + self.assertIn('is not provided by the host metadata', msg) + self.assertIn( + 'index_of_deep_convection_process_in_cumulative_change_index', + msg) + + def test_constituent_index_may_not_be_written_by_a_scheme(self): + """A constituent index is a framework-owned module integer bound via + ``%const_index``; a scheme declaring it intent=out/inout is an error. + Unguarded, intent=out became an ordinary suite var (producer and + consumer agreeing with each other, both missing ``%const_index``) and + intent=inout handed the framework integer to a scheme that may + clobber it.""" + hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) + for intent in ('out', 'inout'): + for const_stds in ({'water_vapor'}, {'tendency_of_water_vapor'}): + with self.subTest(intent=intent, const_stds=const_stds): + suite_var = self._scheme_var( + 'idx_wv', 'index_of_water_vapor', intent=intent, + ) + with self.assertRaises(CCPPError) as ctx: + _resolve_one_arg(suite_var, 'run', hd, {}, + 'some_scheme', set(), + const_stds=const_stds) + msg = str(ctx.exception) + self.assertIn('indexes a constituent', msg) + self.assertIn('intent=' + intent, msg) + + def test_host_declared_index_out_reports_the_host_error(self): + """The constituent-index intent guard sits AFTER the host/suite gate, + so a host-declared index stays an ordinary host variable and reports + the host's own (protected) error, not the constituent-index one.""" + hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) + std = ('index_of_cloud_liquid_water_mixing_ratio' + '_in_tracer_concentration_array') + suite_var = self._scheme_var('ntcw', std, intent='out') + with self.assertRaises(CCPPError) as ctx: + _resolve_one_arg(suite_var, 'run', hd, {}, 'some_scheme', set(), + const_stds={ + 'cloud_liquid_water_mixing_ratio' + '_in_tracer_concentration_array'}) + msg = str(ctx.exception) + self.assertIn('protected', msg) + self.assertNotIn('indexes a constituent', msg) + class TestDimDDTComponentResolution(unittest.TestCase): """When a dimension standard name maps to a DDT-component host