Skip to content

Feature/paramid normalisation mode #188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danovaro merged 3 commits into from
Apr 8, 2026
Merged

Feature/paramid normalisation mode #188

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
525ae34
improved Param normalisation
danovaro Mar 27, 2026
3462113
ParamID normalisation - added backward compatibility
danovaro Apr 8, 2026
a3e9634
addressed PR comments
danovaro Apr 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions src/metkit/mars/ParamID.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ static std::vector<ParamID::WindFamily> windFamilies_;

static std::vector<size_t> dropTables_;

static bool fullTableDropping_;
static NormalisationMode normalisationMode_;

static std::set<std::string> mlParamsSingleLevel_;

Expand Down Expand Up @@ -59,11 +59,11 @@ static void readTable() {
dropTables_.push_back(dropTables[i]);
}

fullTableDropping_ = false;
normalisationMode_ = NormalisationMode::Loose;
if (paramMatching.contains("full-table-dropping")) {
const eckit::Value fullTableDropping = paramMatching["full-table-dropping"];
ASSERT(fullTableDropping.isBool());
fullTableDropping_ = fullTableDropping;
normalisationMode_ = NormalisationMode::FullTableDropping;
}

const eckit::Value mlParamsSingleLevel = paramMatching["ml-params-single-level"];
Expand All @@ -86,9 +86,9 @@ const std::set<std::string>& ParamID::getMlParamsSingleLevel() {
return mlParamsSingleLevel_;
}

bool ParamID::fullTableDropping() {
NormalisationMode ParamID::normalisationMode() {
pthread_once(&once, readTable);
return fullTableDropping_;
return normalisationMode_;
}


Expand Down
40 changes: 33 additions & 7 deletions src/metkit/mars/ParamID.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,14 @@ namespace metkit {

//----------------------------------------------------------------------------------------------------------------------

enum class NormalisationMode {
Strict, // Exact match + wind conversion
Loose, // Strict + partial match (dropping tables defined in param-matching.yaml)
FullTableDropping
Comment on lines +39 to +41
Copy link

Copilot AI Apr 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The inline comments on NormalisationMode are misleading: Strict currently disables the partial-match logic (it is guarded by !strict), so Loose is not “Strict + partial match”. Please adjust these comments to reflect the actual behaviors (e.g., whether GRIB1 value matching and table-expansion are enabled) to avoid confusing API users.

Suggested change
Strict, // Exact match + wind conversion
Loose, // Strict + partial match (dropping tables defined in param-matching.yaml)
FullTableDropping
Strict, // Exact match only (no partial table expansion) + wind conversion
Loose, // GRIB1-style/value-based matching with partial table expansion from param-matching.yaml +
// wind conversion
FullTableDropping // GRIB1-style/value-based matching with full configured table dropping + wind conversion

Copilot uses AI. Check for mistakes.
};

//----------------------------------------------------------------------------------------------------------------------

class ParamID {

public: // types
Expand All @@ -53,11 +61,18 @@ class ParamID {

template <typename REQUEST_T, typename AXIS_T>
static void normalise(const REQUEST_T& r, std::vector<Param>& req, const AXIS_T& axis, bool& windConversion,
bool fullTableDropping = ParamID::fullTableDropping(), bool forceGRIBParamID = false);
NormalisationMode mode = ParamID::normalisationMode());

template <typename REQUEST_T, typename AXIS_T>
[[deprecated]] static void normalise(const REQUEST_T& r, std::vector<Param>& req, const AXIS_T& axis,
bool& windConversion, bool fullTableDropping, bool forceGRIBParamID = false);

static const std::vector<WindFamily>& getWindFamilies();
static const std::vector<size_t>& getDropTables();
static bool fullTableDropping();
static NormalisationMode normalisationMode();
[[deprecated]] static bool fullTableDropping() {
return normalisationMode() == NormalisationMode::FullTableDropping;
}
static const std::set<std::string>& getMlParamsSingleLevel();
};

Expand All @@ -71,8 +86,17 @@ template <typename REQUEST_T, typename AXIS_T>
void ParamID::normalise(const REQUEST_T& request, std::vector<Param>& req, const AXIS_T& axis, bool& windConversion,
bool fullTableDropping, bool forceGRIBParamID) {

normalise(request, req, axis, windConversion,
forceGRIBParamID ? NormalisationMode::Strict
: (fullTableDropping ? NormalisationMode::FullTableDropping : NormalisationMode::Loose));
}

template <typename REQUEST_T, typename AXIS_T>
void ParamID::normalise(const REQUEST_T& request, std::vector<Param>& req, const AXIS_T& axis, bool& windConversion,
NormalisationMode mode) {

static const bool useGRIBParamID = eckit::Resource<bool>("useGRIBParamID", false);
bool strict = useGRIBParamID || forceGRIBParamID;
bool strict = useGRIBParamID || mode == NormalisationMode::Strict;

const std::vector<WindFamily>& windFamilies(getWindFamilies());

Expand All @@ -81,9 +105,11 @@ void ParamID::normalise(const REQUEST_T& request, std::vector<Param>& req, const
std::map<long, Param> inAxisParamID;
std::set<Param> wind;


for (typename AXIS_T::const_iterator j = axis.begin(); j != axis.end(); ++j) {
inAxis.emplace(*j);
inAxisParamID[j->paramId()] = *j;
Param p = *j;
inAxis.emplace(p);
inAxisParamID[p.paramId()] = p;
}

std::vector<Param> newreq;
Expand Down Expand Up @@ -162,8 +188,8 @@ void ParamID::normalise(const REQUEST_T& request, std::vector<Param>& req, const
}
}
}
if (fullTableDropping &&
!ok) { // Backward compatibility - Partial match (drop completely table information)
if (mode == NormalisationMode::FullTableDropping && !ok) {
// Backward compatibility - Partial match (drop completely table information)
for (const auto& ap : inAxisParamID) {
if (ap.first % 1000 == paramid % 1000) {
newreq.push_back(ap.second);
Expand Down
Loading
Loading