diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000..d3abd08a --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,35 @@ +# Copilot Repository Instructions + +## Documentation synchronization rule + +When working on a pull request: + +1. Determine the set of files modified by the PR. +2. From that set, consider only files under: + + - src/metkit/mars2grib + - test/mars2grib + +3. For each of those files: + - Verify that the related documentation is present + - Verify that the documentation is up to date with the code changes + - If documentation is missing or outdated, propose the required updates + +4. Do not request documentation changes for files outside these paths. + +## Definition of “documentation in sync” + +Documentation must: +- Describe the current public behavior and interfaces +- Reflect any new parameters, options, or outputs +- Remove references to deleted functionality + +## PR review behavior + +During PR reviews: +- Explicitly list the impacted files in the two target directories +- State whether documentation is: + - ✅ in sync + - ❌ missing + - ❌ outdated +- Suggest concrete doc patches when needed, referencing the specific lines or sections that require updates. \ No newline at end of file diff --git a/src/metkit/mars2grib/CoreOperations.h b/src/metkit/mars2grib/CoreOperations.h index ea89c7f3..ab80a03c 100644 --- a/src/metkit/mars2grib/CoreOperations.h +++ b/src/metkit/mars2grib/CoreOperations.h @@ -21,6 +21,20 @@ /// 3. **Value Injection**: Physical realization of the GRIB data section. /// 4. **Diagnostic Capture**: Generating regression data for structural validation. /// +/// +/// --- +/// +/// ## Transitional cache preparation +/// +/// This header also contains a temporary staged-encoding path based on +/// `prepare()` and `finaliseEncoding()`. +/// +/// This code is a preparatory step toward a future implementation where +/// cache lifecycle and reuse are fully internalized inside the core layer. +/// +/// At the current stage, the staged API exists to support benchmarking, +/// comparison, and incremental integration of the future cache design. +/// /// @ingroup mars2grib_core /// #pragma once @@ -86,7 +100,7 @@ struct CoreOperations { return {activeMars, activePar}; } - catch (const std::exception& e) { + catch (...) { // Wrap any exception in a CoreOperations-specific exception to provide context std::throw_with_nested( mars2grib::utils::exceptions::Mars2GribGenericException("Error during metadata normalization", Here())); @@ -118,13 +132,14 @@ struct CoreOperations { return SpecializedEncoder{std::move(layout)}.encode(mars, misc, opt); } - catch (const std::exception& e) { + catch (...) { // Wrap any exception in a CoreOperations-specific exception to provide context std::throw_with_nested( mars2grib::utils::exceptions::Mars2GribGenericException("Error during header encoding", Here())); } } + /// /// @brief Inject numeric field values into a GRIB handle. /// @@ -132,18 +147,19 @@ struct CoreOperations { /// data compression. Utilizes spans for zero-copy data passing. /// /// @tparam Val_t Numeric precision (float or double) + /// @tparam ParDict_t Parameter dictionary type /// @tparam OptDict_t Encoding options dictionary type /// @tparam OutDict_t Output GRIB handle/dictionary type /// - template - static std::unique_ptr encodeValues(backend::Span values, const MiscDict_t& misc, + template + static std::unique_ptr encodeValues(backend::Span values, const ParDict_t& misc, const OptDict_t& opt, std::unique_ptr handle) { try { metkit::mars2grib::backend::encodeValues(values, misc, opt, *handle); return handle; } - catch (const std::exception& e) { + catch (...) { // Wrap any exception in a CoreOperations-specific exception to provide context std::throw_with_nested( mars2grib::utils::exceptions::Mars2GribGenericException("Error during value encoding", Here())); @@ -191,7 +207,7 @@ struct CoreOperations { /// Numeric type of the values to be encoded. /// /// @tparam MarsDict_t MARS dictionary type - /// @tparam MiscDict_t Parameterization dictionary type + /// @tparam ParDict_t Parameterization dictionary type /// @tparam OptDict_t Options dictionary type /// @tparam OutDict_t Output dictionary type /// @@ -217,16 +233,16 @@ struct CoreOperations { /// @throws mars2grib::Exception /// If normalization, header encoding, or value encoding fails. /// - template + template static std::unique_ptr encode(const metkit::codes::Span& values, - const MarsDict_t& inputMars, const MiscDict_t& inputMisc, + const MarsDict_t& inputMars, const ParDict_t& inputMisc, const OptDict_t& options, const eckit::Value& language) { using metkit::mars2grib::utils::exceptions::printExtendedStack; // 1. Prepare Scratches for Normalization MarsDict_t scratchMars; - MiscDict_t scratchMisc; + ParDict_t scratchMisc; try { @@ -246,7 +262,7 @@ struct CoreOperations { // 3. Encode Header (SpecializedEncoder creates the CodesHandle here) auto gribHeader = - encodeHeader(activeMars, activeMisc, options); + encodeHeader(activeMars, activeMisc, options); // 4. Inject Values return encodeValues(values, activeMisc, options, std::move(gribHeader)); @@ -262,6 +278,291 @@ struct CoreOperations { } } + // ------------------------------------------------------------------------- + // Temporary staged-cache preparation path + // ------------------------------------------------------------------------- + // + // The following types and functions implement a transitional staged + // preparation/finalisation workflow. + // + // This code exists as a preparatory step toward a future version where + // cache construction, reuse, and lifecycle are fully internalized in the + // core encoding layer. + // + + /// + /// @brief Immutable staged-encoding cache entry. + /// + /// This object stores the reusable state required to split the encoding + /// pipeline into two phases: + /// + /// 1. **Preparation** + /// - metadata normalization + /// - header layout resolution + /// - encoder specialization + /// - creation of a reusable prepared sample + /// + /// 2. **Finalisation** + /// - reuse of the prepared structural state + /// - injection of dynamic metadata if needed + /// - value encoding into a fresh output object + /// + /// The cache entry is intentionally immutable: + /// - copy is disabled + /// - move is disabled + /// - the prepared sample is owned through an immutable smart pointer + /// + /// This guarantees that once constructed, the cache entry represents a + /// stable reusable encoding context. + /// + /// @note + /// This staged cache path is a preparatory step toward a future + /// implementation where cache management is fully internalized in the + /// core encoding layer. + /// + /// @tparam MarsDict_t MARS dictionary type + /// @tparam ParDict_t Parameter/misc dictionary type + /// @tparam OptDict_t Encoding options dictionary type + /// @tparam OutDict_t Output GRIB handle/dictionary type + /// + template + struct CacheEntry { + + using Encoder = + metkit::mars2grib::frontend::header::SpecializedEncoder; + using Layout = metkit::mars2grib::frontend::GribHeaderLayoutData; + + /// + /// @brief Construct a cache entry from resolved layout and active metadata. + /// + /// This constructor: + /// - internalizes the resolved header layout into a specialized encoder + /// - prepares an immutable reusable sample from the active metadata + /// + /// The input metadata is expected to already represent the active + /// dictionaries selected after any optional normalization step. + /// + /// @param[in] layout + /// Resolved GRIB header layout to internalize. + /// + /// @param[in] inputMars + /// Active MARS metadata dictionary. + /// + /// @param[in] inputMisc + /// Active parameter/misc metadata dictionary. + /// + /// @param[in] options + /// Encoding options controlling specialization behavior. + /// + /// @note + /// This constructor is part of the temporary staged-cache path and + /// prepares the structure required for a future internal cache design. + /// + CacheEntry(Layout&& layout, const MarsDict_t& inputMars, const ParDict_t& inputMisc, const OptDict_t& options) : + encoder_{std::move(layout)}, preparedSample_{encoder_.prepare(inputMars, inputMisc, options)} {}; + + CacheEntry(const CacheEntry&) = delete; + CacheEntry& operator=(const CacheEntry&) = delete; + CacheEntry(CacheEntry&&) = delete; + CacheEntry& operator=(CacheEntry&&) = delete; + ~CacheEntry() = default; + + /// + /// @brief Fully specialized immutable encoder. + /// + /// This encoder owns the resolved layout and the precomputed execution + /// plan used during staged finalisation. + /// + const Encoder encoder_; + + /// + /// @brief Immutable prepared sample used as finalisation template. + /// + /// This object represents the reusable structural sample produced + /// during the preparation phase. It is treated as read-only input + /// during `finaliseEncoding()`, which derives a fresh output handle + /// from it. + /// + const std::unique_ptr preparedSample_; + }; + + /// + /// @brief Prepare a reusable staged-encoding cache. + /// + /// This function performs the preparation phase of the staged encoding + /// pipeline: + /// - optional metadata normalization + /// - header layout resolution + /// - construction of a specialized encoder + /// - creation of an immutable reusable sample + /// + /// The returned cache entry can later be passed to + /// `finaliseEncoding()` to complete the encoding with concrete field + /// values. + /// + /// ----------------------------------------------------------------------------- + /// Normalization and lifetime semantics (CRITICAL) + /// ----------------------------------------------------------------------------- + /// + /// As in `encode()`, normalization returns borrowed references to the + /// active metadata dictionaries: + /// - either the original inputs + /// - or local scratch objects containing normalized copies + /// + /// These references must not escape this function except through + /// immediate materialization into owned cache state. + /// + /// The resulting `CacheEntry` stores only owned immutable state and + /// does not retain references to the scratch buffers. + /// + /// ----------------------------------------------------------------------------- + /// + /// @tparam MarsDict_t MARS dictionary type + /// @tparam ParDict_t Parameter/misc dictionary type + /// @tparam OptDict_t Encoding options dictionary type + /// @tparam OutDict_t Output GRIB handle/dictionary type + /// + /// @param[in] inputMars + /// Input MARS description of the data. + /// + /// @param[in] inputMisc + /// Input miscellaneous description of the data. + /// + /// @param[in] options + /// Encoding options controlling specialization and normalization. + /// + /// @param[in] language + /// MARS language definition. + /// + /// @return + /// A unique pointer owning an immutable staged cache entry. + /// + /// @throws mars2grib::Exception + /// If normalization, layout resolution, or sample preparation fails. + /// + /// @note + /// This staged cache path is a preparatory step toward a future + /// implementation where cache lifecycle and reuse are fully + /// internalized inside the core encoding layer. + /// + template + static std::unique_ptr> prepare( + const MarsDict_t& inputMars, const ParDict_t& inputMisc, const OptDict_t& options, + const eckit::Value& language) { + + // 1. Prepare Scratches for Normalization + MarsDict_t scratchMars; + ParDict_t scratchMisc; + + try { + using metkit::mars2grib::frontend::make_HeaderLayout_or_throw; + + auto [activeMars, activeMisc] = + normalize_if_enabled(inputMars, inputMisc, options, language, scratchMars, scratchMisc); + + auto layout = make_HeaderLayout_or_throw(activeMars, options); + + return std::make_unique>( + std::move(layout), activeMars, activeMisc, options); + } + catch (...) { + // Wrap any exception in a CoreOperations-specific exception to provide context + std::throw_with_nested( + mars2grib::utils::exceptions::Mars2GribGenericException("Error during cache preparation", Here())); + } + } + + /// + /// @brief Complete an encoding from a previously prepared cache entry. + /// + /// This function performs the finalisation phase of the staged encoding + /// pipeline: + /// - optional metadata normalization + /// - reuse of the immutable specialized encoder + /// - reuse of the immutable prepared sample + /// - creation of a fresh header object + /// - injection of the field values into the resulting output handle + /// + /// The provided cache entry is treated as read-only and is not consumed. + /// It may therefore be reused across multiple finalisation calls. + /// + /// ----------------------------------------------------------------------------- + /// Normalization and lifetime semantics (CRITICAL) + /// ----------------------------------------------------------------------------- + /// + /// As in `encode()` and `prepare()`, normalization yields borrowed + /// references to the active metadata dictionaries. These references are + /// used only within this function and are not stored. + /// + /// The cache entry itself is fully owned and immutable, and no references + /// to local scratch objects escape the function. + /// + /// ----------------------------------------------------------------------------- + /// + /// @tparam Val_t Numeric type of the values to be encoded + /// @tparam MarsDict_t MARS dictionary type + /// @tparam ParDict_t Parameter/misc dictionary type + /// @tparam OptDict_t Encoding options dictionary type + /// @tparam OutDict_t Output GRIB handle/dictionary type + /// + /// @param[in] cacheEntry + /// Immutable staged cache entry previously produced by `prepare()`. + /// + /// @param[in] values + /// Contiguous span of values to encode. + /// + /// @param[in] inputMars + /// Input MARS description of the data. + /// + /// @param[in] inputMisc + /// Input miscellaneous description of the data. + /// + /// @param[in] options + /// Encoding options controlling behavior such as validation, + /// logging, or feature toggles. + /// + /// @param[in] language + /// MARS language definition. + /// + /// @return + /// A `std::unique_ptr` owning the fully encoded output object. + /// + /// @throws mars2grib::Exception + /// If normalization, staged header finalisation, or value encoding fails. + /// + /// @note + /// This staged cache path is a preparatory step toward a future + /// implementation where cache lifecycle and reuse are fully + /// internalized inside the core encoding layer. + /// + template + static std::unique_ptr finaliseEncoding( + const CacheEntry& cacheEntry, + const metkit::codes::Span& values, const MarsDict_t& inputMars, const ParDict_t& inputMisc, + const OptDict_t& options, const eckit::Value& language) { + + // 1. Prepare Scratches for Normalization + MarsDict_t scratchMars; + ParDict_t scratchMisc; + + try { + + auto [activeMars, activeMisc] = + normalize_if_enabled(inputMars, inputMisc, options, language, scratchMars, scratchMisc); + + auto gribHeader = + cacheEntry.encoder_.finaliseEncoding(*(cacheEntry.preparedSample_), activeMars, activeMisc, options); + + return encodeValues(values, activeMisc, options, std::move(gribHeader)); + } + catch (...) { + // Wrap any exception in a CoreOperations-specific exception to provide context + std::throw_with_nested( + mars2grib::utils::exceptions::Mars2GribGenericException("Error during encoding finalisation", Here())); + } + } + + /// /// @brief Capture a structural test point for regression analysis. /// @@ -281,7 +582,7 @@ struct CoreOperations { return debug_convert_GribHeaderLayoutData_to_json(layout); } - catch (const std::exception& e) { + catch (...) { // Wrap any exception in a CoreOperations-specific exception to provide context std::throw_with_nested( mars2grib::utils::exceptions::Mars2GribGenericException("Error during header test dump", Here())); @@ -289,4 +590,4 @@ struct CoreOperations { } }; -} // namespace metkit::mars2grib \ No newline at end of file +} // namespace metkit::mars2grib diff --git a/src/metkit/mars2grib/api/Mars2Grib.cc b/src/metkit/mars2grib/api/Mars2Grib.cc index 95274bf6..a8e01623 100644 --- a/src/metkit/mars2grib/api/Mars2Grib.cc +++ b/src/metkit/mars2grib/api/Mars2Grib.cc @@ -20,6 +20,8 @@ /// - builds the internal encoder configuration from the MARS dictionary /// - invokes the specialized backend encoder /// - injects field values into the resulting GRIB handle +/// - exposes a temporary staged-encoding cache interface for benchmarking +/// and transitional comparison purposes /// /// This file intentionally contains **no GRIB semantics** and **no deduction /// logic**. All domain-specific decisions are delegated to lower layers. @@ -36,11 +38,24 @@ /// /// --- /// +/// ## Transitional staged API +/// +/// This implementation also exposes a temporary staged-encoding interface +/// based on `prepare()` and `finaliseEncoding()`. +/// +/// This interface is not intended for general use. It exists only to allow +/// short-term benchmarking and comparison against a legacy cache-based +/// implementation before the cache lifecycle is fully internalized in +/// lower layers. +/// +/// --- +/// /// ## Scope /// /// - This file is part of the **Mars2Grib public API implementation** /// - It is not intended for direct use by end users /// - Its behavior defines the observable semantics of `Mars2Grib::encode` +/// and the temporary staged-encoding API /// /// @ingroup mars2grib_api /// @@ -48,6 +63,9 @@ #include "Mars2Grib.h" +// other libraries +#include "eckit/exception/Exceptions.h" + // dictionary access traits #include "metkit/mars2grib/utils/dictionary_traits/dictaccess_codes_handle.h" #include "metkit/mars2grib/utils/dictionary_traits/dictaccess_eckit_configuration.h" @@ -102,6 +120,292 @@ Options readOptions(const eckit::LocalConfiguration& conf) { } // namespace +/// +/// @brief Concrete specialization of the generic core cache entry. +/// +/// This alias binds the generic `CoreOperations::CacheEntry` template to the +/// concrete public API types: +/// - `eckit::LocalConfiguration` for MARS metadata +/// - `eckit::LocalConfiguration` for auxiliary metadata +/// - `Options` for encoding options +/// - `metkit::codes::CodesHandle` for the GRIB output object +/// +/// It is used internally as the implementation payload of the opaque +/// `Mars2Grib::CacheEntry`. +/// +/// @note +/// This staged cache path is temporary and not intended for public use. +/// +using CoreCacheEntry = CoreOperations::CacheEntry; + +/// +/// @brief Opaque API wrapper around the concrete core cache entry. +/// +/// This type hides the concrete staged-encoding cache representation from +/// API users while preserving unique ownership and immutability. +/// +/// Internally, it stores a fully prepared `CoreCacheEntry` specialized for +/// the public `Mars2Grib` API types. +/// +/// @note +/// This staged cache API is temporary and not intended for public use. +/// It is exposed only for transitional benchmarking and comparison +/// purposes and may be changed or removed without notice. +/// +struct Mars2Grib::CacheEntry { + + /// + /// @brief Construct the opaque wrapper from a concrete core cache. + /// + /// Ownership of the prepared core cache is transferred into the wrapper. + /// + /// @param[in] impl + /// Unique pointer owning the prepared concrete core cache entry. + /// + explicit CacheEntry(std::unique_ptr&& impl) : impl_{std::move(impl)} {} + + CacheEntry(const CacheEntry&) = delete; + CacheEntry& operator=(const CacheEntry&) = delete; + CacheEntry(CacheEntry&&) = delete; + CacheEntry& operator=(CacheEntry&&) = delete; + ~CacheEntry() = default; + + /// + /// @brief Owned immutable implementation cache. + /// + /// This pointer stores the concrete staged-encoding cache used by the + /// generic core layer. + /// + const std::unique_ptr impl_; +}; + +/// +/// @brief Destroy an opaque API cache entry. +/// +/// This custom deleter is required because `Mars2Grib::CacheEntry` is +/// incomplete in the public header and only fully defined in this +/// implementation file. +/// +/// @param[in] p +/// Pointer to the opaque cache entry to destroy. +/// +/// @note +/// This staged cache API is temporary and not intended for public use. +/// +void Mars2Grib::CacheEntryDeleter::operator()(const CacheEntry* p) const { + delete p; +} + +/// +/// @brief Prepare an opaque staged-encoding cache. +/// +/// This function delegates to the generic core layer to: +/// - normalize input metadata if enabled, +/// - resolve the structural header layout, +/// - build the specialized encoder state, +/// - prepare an immutable reusable sample for later finalization. +/// +/// The returned cache is opaque at the public API level and can be +/// passed to `finaliseEncoding()` multiple times. +/// +/// @param[in] mars +/// MARS dictionary describing the field metadata. +/// +/// @param[in] misc +/// Auxiliary metadata dictionary. +/// +/// @return +/// A unique pointer owning an opaque immutable cache entry. +/// +/// @note +/// This staged cache API is temporary and not intended for public use. +/// It is exposed only for transitional benchmarking and comparison +/// purposes and may be changed or removed without notice. +/// +Mars2Grib::CacheEntryPtr Mars2Grib::prepare(const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc) { + auto coreCache = CoreOperations::prepare(mars, misc, opts_, language_); + + return CacheEntryPtr(new CacheEntry(std::move(coreCache))); +} + + +/// +/// @brief Finalize a GRIB encoding from a previously prepared cache. +/// +/// This function delegates to the generic core layer to complete the +/// encoding pipeline using a cache previously produced by `prepare()`. +/// +/// The cache provides the pre-specialized, reusable encoding state, +/// while this function injects the supplied field values and produces +/// a final `CodesHandle`. +/// +/// The cache is not consumed and may be reused in subsequent calls. +/// +/// @param[in] cacheEntry +/// Opaque staged-encoding cache previously produced by `prepare()`. +/// +/// @param[in] values +/// Field values to encode as double. +/// +/// @param[in] mars +/// MARS dictionary describing the field metadata. +/// +/// @param[in] misc +/// Auxiliary metadata dictionary. +/// +/// @return +/// A unique pointer to a GRIB handle containing the encoded message. +/// +/// @note +/// This staged cache API is temporary and not intended for public use. +/// It is exposed only for transitional benchmarking and comparison +/// purposes and may be changed or removed without notice. +/// +std::unique_ptr Mars2Grib::finaliseEncoding(const CacheEntryPtr& cacheEntry, + const std::vector& values, + const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc) { + return CoreOperations::finaliseEncoding( + *(cacheEntry->impl_), Span{values}, mars, misc, opts_, language_); +} + + +/// +/// @brief Finalize a GRIB encoding from a previously prepared cache. +/// +/// This overload accepts field values as `float`. +/// +/// The cache supplies the pre-specialized, reusable encoding state, +/// while this function injects the provided field values and applies +/// any remaining dynamic metadata handling required to produce a final +/// GRIB message. +/// +/// The cache is not consumed by this operation and may be reused +/// for subsequent calls. +/// +/// @param[in] cacheEntry +/// Opaque staged-encoding cache previously produced by `prepare()`. +/// +/// @param[in] values +/// Pointer to the field values as double. +/// +/// @param[in] length +/// Number of values in the buffer. +/// +/// @param[in] mars +/// MARS dictionary describing the field metadata. +/// +/// @param[in] misc +/// Auxiliary metadata dictionary. +/// +/// @return +/// A unique pointer to a GRIB handle containing the encoded message. +/// +/// @note +/// This staged cache API is temporary and not intended for public use. +/// It is exposed only for transitional benchmarking and comparison +/// against a legacy implementation. It may be changed or removed +/// without notice. +/// +std::unique_ptr Mars2Grib::finaliseEncoding(const CacheEntryPtr& cacheEntry, + const double* values, size_t length, + const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc) { + return CoreOperations::finaliseEncoding( + *(cacheEntry->impl_), Span{values, length}, mars, misc, opts_, language_); +} + +/// +/// @brief Finalize a GRIB encoding from a previously prepared cache. +/// +/// This overload accepts field values as `float`. +/// +/// The cache provides the pre-specialized, reusable encoding state, +/// while this function injects the supplied field values and produces +/// a final `CodesHandle`. +/// +/// The cache is not consumed and may be reused in subsequent calls. +/// +/// @param[in] cacheEntry +/// Opaque staged-encoding cache previously produced by `prepare()`. +/// +/// @param[in] values +/// Field values to encode as float. +/// +/// @param[in] mars +/// MARS dictionary describing the field metadata. +/// +/// @param[in] misc +/// Auxiliary metadata dictionary. +/// +/// @return +/// A unique pointer to a GRIB handle containing the encoded message. +/// +/// @note +/// This staged cache API is temporary and not intended for public use. +/// It is exposed only for transitional benchmarking and comparison +/// purposes and may be changed or removed without notice. +/// +std::unique_ptr Mars2Grib::finaliseEncoding(const CacheEntryPtr& cacheEntry, + const std::vector& values, + const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc) { + return CoreOperations::finaliseEncoding(*(cacheEntry->impl_), Span{values}, + mars, misc, opts_, language_); +} + +/// +/// @brief Finalize a GRIB encoding from a previously prepared cache. +/// +/// This overload accepts field values as `float`. +/// +/// The cache supplies the pre-specialized, reusable encoding state, +/// while this function injects the provided field values and applies +/// any remaining dynamic metadata handling required to produce a final +/// GRIB message. +/// +/// The cache is not consumed by this operation and may be reused +/// for subsequent calls. +/// +/// @param[in] cacheEntry +/// Opaque staged-encoding cache previously produced by `prepare()`. +/// +/// @param[in] values +/// Pointer to the field values as float. +/// +/// @param[in] length +/// Number of values in the buffer. +/// +/// @param[in] mars +/// MARS dictionary describing the field metadata. +/// +/// @param[in] misc +/// Auxiliary metadata dictionary. +/// +/// @return +/// A unique pointer to a GRIB handle containing the encoded message. +/// +/// @note +/// This staged cache API is temporary and not intended for public use. +/// It is exposed only for transitional benchmarking and comparison +/// against a legacy implementation. It may be changed or removed +/// without notice. +/// +std::unique_ptr Mars2Grib::finaliseEncoding(const CacheEntryPtr& cacheEntry, + const float* values, size_t length, + const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc) { + return CoreOperations::finaliseEncoding( + *(cacheEntry->impl_), Span{values, length}, mars, misc, opts_, language_); +} + // ----------------------------------------------------------------------------- // Mars2Grib construction diff --git a/src/metkit/mars2grib/api/Mars2Grib.h b/src/metkit/mars2grib/api/Mars2Grib.h index c36dbf5a..db237637 100644 --- a/src/metkit/mars2grib/api/Mars2Grib.h +++ b/src/metkit/mars2grib/api/Mars2Grib.h @@ -91,6 +91,19 @@ namespace metkit::mars2grib { +/// --- +/// +/// ## Transitional staged API +/// +/// A temporary staged-encoding interface is also exposed through +/// `prepare()` and `finaliseEncoding()`. +/// +/// This interface exists only for short-term benchmarking and migration +/// purposes and is not intended for external use. It may be changed or +/// removed without notice. +/// + + /// /// @brief High-level encoder for converting MARS fields to GRIB. /// @@ -314,6 +327,264 @@ class Mars2Grib { std::unique_ptr encode(const float* values, size_t length, const eckit::LocalConfiguration& mars); + /// + /// @brief Opaque cache object for staged GRIB encoding. + /// + /// This type represents a precomputed encoding cache produced by + /// `prepare()` and later consumed by `finaliseEncoding()`. + /// + /// The cache internalizes the expensive metadata-specialization phase + /// and stores immutable state that can be reused across multiple + /// finalization calls. + /// + /// The concrete representation of this type is intentionally hidden + /// from API users. + /// + /// @note + /// This staged cache API is temporary and not intended for public use. + /// It is exposed only for transitional benchmarking and comparison + /// against a legacy implementation. It may be changed or removed + /// without notice. + /// + struct CacheEntry; + + /// + /// @brief Custom deleter for opaque cache entries. + /// + /// This deleter is paired with `CacheEntryPtr` to allow ownership of + /// an incomplete, opaque `CacheEntry` type in the public interface. + /// + /// @note + /// This staged cache API is temporary and not intended for public use. + /// It is exposed only for transitional benchmarking and comparison + /// against a legacy implementation. It may be changed or removed + /// without notice. + /// + struct CacheEntryDeleter { + + /// + /// @brief Destroy an opaque cache entry. + /// + /// @param[in] cache + /// Pointer to the cache entry to destroy. + /// + void operator()(const CacheEntry*) const; + }; + + /// + /// @brief Owning smart pointer to an opaque staged-encoding cache. + /// + /// This alias represents unique ownership of a `CacheEntry` + /// produced by `prepare()`. + /// + /// The pointed object is immutable and may be reused across + /// multiple `finaliseEncoding()` calls. + /// + /// @note + /// This staged cache API is temporary and not intended for public use. + /// It is exposed only for transitional benchmarking and comparison + /// against a legacy implementation. It may be changed or removed + /// without notice. + /// + using CacheEntryPtr = std::unique_ptr; + + /// + /// @brief Prepare a reusable staged-encoding cache. + /// + /// This function performs the metadata-specialization phase of the + /// encoding pipeline and returns an opaque cache object that can be + /// reused in subsequent calls to `finaliseEncoding()`. + /// + /// The cache is derived from: + /// - the MARS dictionary, + /// - the auxiliary metadata dictionary, + /// - the current `Mars2Grib` options, + /// - the active language definition. + /// + /// This operation is intended for workflows where the structural + /// encoding setup is reused multiple times with different field values + /// and/or repeated finalization requests. + /// + /// @param[in] mars + /// MARS dictionary describing the field metadata. + /// + /// @param[in] misc + /// Auxiliary metadata dictionary. + /// + /// @return + /// A unique pointer owning an opaque immutable cache entry. + /// + /// @note + /// This staged cache API is temporary and not intended for public use. + /// It is exposed only for transitional benchmarking and comparison + /// against a legacy implementation. It may be changed or removed + /// without notice. + /// + CacheEntryPtr prepare(const eckit::LocalConfiguration& mars, const eckit::LocalConfiguration& misc); + + /// + /// @brief Finalize a GRIB encoding from a previously prepared cache. + /// + /// This function completes the encoding pipeline using a cache + /// previously produced by `prepare()`. + /// + /// The cache supplies the pre-specialized, reusable encoding state, + /// while this function injects the provided field values and applies + /// any remaining dynamic metadata handling required to produce a final + /// GRIB message. + /// + /// The cache is not consumed by this operation and may be reused + /// for subsequent calls. + /// + /// @param[in] cacheEntry + /// Opaque staged-encoding cache previously produced by `prepare()`. + /// + /// @param[in] values + /// Field values to encode as double. + /// + /// @param[in] mars + /// MARS dictionary describing the field metadata. + /// + /// @param[in] misc + /// Auxiliary metadata dictionary. + /// + /// @return + /// A unique pointer to a GRIB handle containing the encoded message. + /// + /// @note + /// This staged cache API is temporary and not intended for public use. + /// It is exposed only for transitional benchmarking and comparison + /// against a legacy implementation. It may be changed or removed + /// without notice. + /// + std::unique_ptr finaliseEncoding(const CacheEntryPtr& cacheEntry, + const std::vector& values, + const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc); + + /// + /// @brief Finalize a GRIB encoding from a previously prepared cache. + /// + /// This function completes the encoding pipeline using a cache + /// previously produced by `prepare()`. + /// + /// The cache supplies the pre-specialized, reusable encoding state, + /// while this function injects the provided field values and applies + /// any remaining dynamic metadata handling required to produce a final + /// GRIB message. + /// + /// The cache is not consumed by this operation and may be reused + /// for subsequent calls. + /// + /// @param[in] cacheEntry + /// Opaque staged-encoding cache previously produced by `prepare()`. + /// + /// @param[in] values + /// Pointer to the field values as double. + /// + /// @param[in] length + /// Number of values in the buffer. + /// + /// @param[in] mars + /// MARS dictionary describing the field metadata. + /// + /// @param[in] misc + /// Auxiliary metadata dictionary. + /// + /// @return + /// A unique pointer to a GRIB handle containing the encoded message. + /// + /// @note + /// This staged cache API is temporary and not intended for public use. + /// It is exposed only for transitional benchmarking and comparison + /// against a legacy implementation. It may be changed or removed + /// without notice. + /// + std::unique_ptr finaliseEncoding(const CacheEntryPtr& cacheEntry, const double* values, + size_t length, const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc); + + /// + /// @brief Finalize a GRIB encoding from a previously prepared cache. + /// + /// This overload accepts field values as `float`. + /// + /// The cache supplies the pre-specialized, reusable encoding state, + /// while this function injects the provided field values and applies + /// any remaining dynamic metadata handling required to produce a final + /// GRIB message. + /// + /// The cache is not consumed by this operation and may be reused + /// for subsequent calls. + /// + /// @param[in] cacheEntry + /// Opaque staged-encoding cache previously produced by `prepare()`. + /// + /// @param[in] values + /// Field values to encode as float. + /// + /// @param[in] mars + /// MARS dictionary describing the field metadata. + /// + /// @param[in] misc + /// Auxiliary metadata dictionary. + /// + /// @return + /// A unique pointer to a GRIB handle containing the encoded message. + /// + /// @note + /// This staged cache API is temporary and not intended for public use. + /// It is exposed only for transitional benchmarking and comparison + /// against a legacy implementation. It may be changed or removed + /// without notice. + /// + std::unique_ptr finaliseEncoding(const CacheEntryPtr& cacheEntry, + const std::vector& values, + const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc); + + + /// + /// @brief Finalize a GRIB encoding from a previously prepared cache. + /// + /// This overload accepts field values as `float`. + /// + /// The cache supplies the pre-specialized, reusable encoding state, + /// while this function injects the provided field values and applies + /// any remaining dynamic metadata handling required to produce a final + /// GRIB message. + /// + /// The cache is not consumed by this operation and may be reused + /// for subsequent calls. + /// + /// @param[in] cacheEntry + /// Opaque staged-encoding cache previously produced by `prepare()`. + /// + /// @param[in] values + /// Pointer to the field values as float. + /// + /// @param[in] length + /// Number of values in the buffer. + /// + /// @param[in] mars + /// MARS dictionary describing the field metadata. + /// + /// @param[in] misc + /// Auxiliary metadata dictionary. + /// + /// @return + /// A unique pointer to a GRIB handle containing the encoded message. + /// + /// @note + /// This staged cache API is temporary and not intended for public use. + /// It is exposed only for transitional benchmarking and comparison + /// against a legacy implementation. It may be changed or removed + /// without notice. + /// + std::unique_ptr finaliseEncoding(const CacheEntryPtr& cacheEntry, const float* values, + size_t length, const eckit::LocalConfiguration& mars, + const eckit::LocalConfiguration& misc); + private: const eckit::Value language_; diff --git a/src/metkit/mars2grib/frontend/header/SpecializedEncoder.h b/src/metkit/mars2grib/frontend/header/SpecializedEncoder.h index 3fc5de1a..fe87bfa0 100644 --- a/src/metkit/mars2grib/frontend/header/SpecializedEncoder.h +++ b/src/metkit/mars2grib/frontend/header/SpecializedEncoder.h @@ -89,6 +89,24 @@ /// @endcode /// /// ----------------------------------------------------------------------------- +/// Transitional staged-encoding support +/// ----------------------------------------------------------------------------- +/// +/// This class also exposes a temporary staged interface via `prepare()` and +/// `finaliseEncoding()`. +/// +/// These functions are introduced in preparation for a cache mechanism that +/// will soon be implemented in the higher-level encoding pipeline. +/// +/// Their purpose is to split the hot-path header construction into: +/// +/// - an immutable reusable preparation phase +/// - a lightweight finalisation phase operating from a prepared sample +/// +/// This staged interface is intended to support incremental integration and +/// performance comparison while the future cache design is being introduced. +/// +/// ----------------------------------------------------------------------------- /// Template parameters /// ----------------------------------------------------------------------------- /// @@ -108,6 +126,7 @@ #include // Project includes +#include "metkit/mars2grib/backend/compile-time-registry-engine/common.h" #include "metkit/mars2grib/frontend/GribHeaderLayoutData.h" #include "metkit/mars2grib/frontend/header/EncodingPlan.h" #include "metkit/mars2grib/utils/generalUtils.h" @@ -286,7 +305,7 @@ class SpecializedEncoder { try { auto samplePtr = make_from_sample_or_throw("GRIB2"); - // Encoding loop as a dnse set of optimized operations + // Encoding loop as a dense set of optimized operations for (const auto& stage : plan_) { for (const auto& section : stage) { for (const auto& conceptCallback : section) { @@ -308,6 +327,187 @@ class SpecializedEncoder { } } + /// + /// @brief Execute the preparation phase of the staged encoder. + /// + /// This method evaluates only the prefix of the execution plan required + /// to build an immutable reusable sample for later completion. + /// + /// In particular, it executes all stages up to and including the + /// override stage, then returns the resulting output object as a + /// read-only prepared sample. + /// + /// This method exists to support a temporary staged-encoding workflow + /// introduced in preparation for a cache mechanism that will soon be + /// implemented in the surrounding encoding pipeline. + /// + /// Characteristics: + /// + /// - No layout resolution + /// - No plan modification + /// - No dynamic dispatch + /// - Execution limited to the preparation prefix of the plan + /// + /// The algorithm is: + /// + /// 1. Create an initial GRIB sample dictionary + /// 2. Execute all sections for stages from the beginning of the plan + /// through `StageOverride` + /// 3. Clone between stages as required + /// 4. Return the resulting object as an immutable prepared sample + /// + /// @param[in] mars + /// MARS metadata dictionary + /// + /// @param[in] par + /// Parameter metadata dictionary + /// + /// @param[in] opt + /// Encoding options dictionary + /// + /// @return + /// A newly allocated immutable dictionary containing the prepared + /// reusable header state. + /// + /// @throws Mars2GribEncoderException + /// On any failure during preparation. The exception includes: + /// - serialized input dictionaries + /// - serialized header layout + /// - full nested exception chain + /// + /// @note + /// This function is part of a temporary staged interface added in + /// preparation for a cache that will soon be implemented. + /// + std::unique_ptr prepare(const MarsDict_t& mars, const ParDict_t& par, const OptDict_t& opt) const { + + using metkit::mars2grib::backend::compile_time_registry_engine::StageOverride; + using metkit::mars2grib::frontend::debug::debug_convert_GribHeaderLayoutData_to_json; + using metkit::mars2grib::utils::dict_traits::clone_or_throw; + using metkit::mars2grib::utils::dict_traits::dict_to_json; + using metkit::mars2grib::utils::dict_traits::make_from_sample_or_throw; + using metkit::mars2grib::utils::exceptions::Mars2GribEncoderException; + + try { + auto samplePtr = make_from_sample_or_throw("GRIB2"); + + // Initialization of the sample + for (const auto& section : plan_[0]) { + for (const auto& conceptCallback : section) { + if (conceptCallback) { + conceptCallback(mars, par, opt, *samplePtr); + } + } + } + samplePtr = clone_or_throw(*samplePtr); + + // Encoding loop as a dense set of optimized operations + for (std::size_t s = 0; s <= StageOverride; ++s) { + for (const auto& section : plan_[s + 1]) { + for (const auto& conceptCallback : section) { + if (conceptCallback) { + conceptCallback(mars, par, opt, *samplePtr); + } + } + } + samplePtr = clone_or_throw(*samplePtr); + } + + return samplePtr; + } + catch (...) { + std::throw_with_nested(Mars2GribEncoderException( + "Critical failure in SpecializedEncoder execution", dict_to_json(mars), + dict_to_json(par), dict_to_json(opt), + debug_convert_GribHeaderLayoutData_to_json(layout_), Here())); + } + } + + + /// + /// @brief Execute the finalisation phase of the staged encoder. + /// + /// This method completes the encoding from a previously prepared sample. + /// + /// It clones the provided prepared sample, then executes only the runtime + /// stage of the execution plan in order to produce a fresh finalized + /// output object. + /// + /// This method exists to support a temporary staged-encoding workflow + /// introduced in preparation for a cache mechanism that will soon be + /// implemented in the surrounding encoding pipeline. + /// + /// Characteristics: + /// + /// - No layout resolution + /// - No plan modification + /// - No dynamic dispatch + /// - Execution limited to the runtime stage of the plan + /// + /// The algorithm is: + /// + /// 1. Clone the provided prepared sample + /// 2. Execute all sections in `StageRuntime` + /// 3. Return the finalized output object + /// + /// @param[in] sample + /// Immutable prepared sample previously produced by `prepare()` + /// + /// @param[in] mars + /// MARS metadata dictionary + /// + /// @param[in] par + /// Parameter metadata dictionary + /// + /// @param[in] opt + /// Encoding options dictionary + /// + /// @return + /// A newly allocated dictionary containing the finalized GRIB header + /// + /// @throws Mars2GribEncoderException + /// On any failure during finalisation. The exception includes: + /// - serialized input dictionaries + /// - serialized header layout + /// - full nested exception chain + /// + /// @note + /// This function is part of a temporary staged interface added in + /// preparation for a cache that will soon be implemented. + /// + std::unique_ptr finaliseEncoding(const OutDict_t& sample, const MarsDict_t& mars, const ParDict_t& par, + const OptDict_t& opt) const { + + using metkit::mars2grib::backend::compile_time_registry_engine::StageRuntime; + using metkit::mars2grib::frontend::debug::debug_convert_GribHeaderLayoutData_to_json; + using metkit::mars2grib::utils::dict_traits::clone_or_throw; + using metkit::mars2grib::utils::dict_traits::dict_to_json; + using metkit::mars2grib::utils::exceptions::Mars2GribEncoderException; + + try { + auto samplePtr = clone_or_throw(sample); + + // Encoding loop as a dense set of optimized operations + for (const auto& section : plan_[StageRuntime + 1]) { + for (const auto& conceptCallback : section) { + if (conceptCallback) { + conceptCallback(mars, par, opt, *samplePtr); + } + } + } + + // @todo eventually need to return another clone to commit modifications + return samplePtr; + } + catch (...) { + std::throw_with_nested(Mars2GribEncoderException( + "Critical failure in SpecializedEncoder execution", dict_to_json(mars), + dict_to_json(par), dict_to_json(opt), + debug_convert_GribHeaderLayoutData_to_json(layout_), Here())); + } + } + + private: ///