From 369ff61922a8854756ee2b31c0b3e959ad1cf790 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:37:12 -0600 Subject: [PATCH 01/12] lazy import sqlite3 package only when it is used, deprecate ProdTask to move towards removal --- metplus/produtil/README_produtil.md | 6 ++++++ metplus/produtil/config.py | 19 +++++++++++++++++-- metplus/produtil/datastore.py | 9 ++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/metplus/produtil/README_produtil.md b/metplus/produtil/README_produtil.md index 9bc21acf48..834225445c 100644 --- a/metplus/produtil/README_produtil.md +++ b/metplus/produtil/README_produtil.md @@ -18,3 +18,9 @@ platforms. + @ref produtil.config --- Parses UNIX conf files and makes the result readily available. This is part of the produtil package and is referenced here as a convenience. +METplus note: + +- The METplus runtime does not require `produtil.datastore`. +- sqlite3 is loaded lazily only when datastore APIs are explicitly used. +- Datastore-backed task helpers in `produtil.config` are deprecated for METplus usage. + diff --git a/metplus/produtil/config.py b/metplus/produtil/config.py index ffffa40621..8aadbe473e 100644 --- a/metplus/produtil/config.py +++ b/metplus/produtil/config.py @@ -9,7 +9,7 @@ ##@var __all__ # decides what symbols are imported by "from produtil.config import *" -__all__=['from_file','confwalker','ProdConfig','ENVIRONMENT','ProdTask'] +__all__=['from_file','confwalker','ProdConfig','ENVIRONMENT'] import collections,re,os,logging,threading import os.path,sys @@ -20,7 +20,10 @@ from configparser import ConfigParser from io import StringIO -from metplus.produtil.datastore import Datastore,Task +# NOTE: +# Datastore is intentionally imported lazily inside getdatastore() so +# importing metplus.produtil.config does not require datastore support. +Task=object from metplus.produtil.numerics import to_datetime, to_datetime_rel, fcst_hr_min from string import Formatter @@ -821,6 +824,13 @@ def getdatastore(self): return d with self: if self._datastore is None: + try: + from metplus.produtil.datastore import Datastore + except ImportError as e: + raise ImportError( + 'Datastore support is unavailable. ' + 'METplus no longer requires produtil.datastore.' + ) from e dsfile=self.getstr('config','datastore') self._datastore=Datastore(dsfile, logger=self.log('datastore')) @@ -1365,6 +1375,11 @@ def __init__(self,dstore,conf,section,taskname=None,workdir=None, the taskvars arguments of produtil.config.ProdConfig member functions. @param kwargs passed to the parent class constructor.""" + raise RuntimeError( + 'ProdTask is deprecated in METplus and depends on ' + 'produtil.datastore. Use METplus wrapper/task logic instead.' + ) + if taskname is None: taskname=section conf.register_task(taskname) diff --git a/metplus/produtil/datastore.py b/metplus/produtil/datastore.py index b0ddb39847..9ee3026a42 100644 --- a/metplus/produtil/datastore.py +++ b/metplus/produtil/datastore.py @@ -6,7 +6,7 @@ Datum, which is the base class of anything that can be stored in the Datastore.""" -import sqlite3, threading, collections, re, contextlib, datetime, logging, os, time +import threading, collections, re, contextlib, datetime, logging, os, time import metplus.produtil.fileop as fileop from metplus.produtil.locking import LockFile from metplus.produtil.log import jlogger @@ -195,6 +195,13 @@ def _connection(self): if tid in self._connections: return self._connections[tid] else: + try: + import sqlite3 + except ImportError as e: + raise ImportError( + 'sqlite3 support is required only when using ' + 'metplus.produtil.datastore.Datastore' + ) from e c=sqlite3.connect(self.filename) self._connections[tid]=c return c From dd4fa77472a081184a4b5e6eb08adf39764c9b23 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:11:20 -0600 Subject: [PATCH 02/12] replace produtil ProdConfig with SimpleConfig --- metplus/util/config_metplus.py | 207 ++++++++++++++++++------------ metplus/util/simple_config.py | 226 +++++++++++++++++++++++++++++++++ 2 files changed, 352 insertions(+), 81 deletions(-) create mode 100644 metplus/util/simple_config.py diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 679b81d308..aa5c00d946 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -19,7 +19,7 @@ from pathlib import Path import uuid -from metplus.produtil.config import ProdConfig +from .simple_config import SimpleConfig from .constants import RUNTIME_CONFS, MISSING_DATA_VALUE from .string_template_substitution import do_string_sub @@ -37,10 +37,9 @@ The launch() function does more than just create the conf file though. It creates several initial files and directories -The METplusConfig class is used in place of a produtil.config.ProdConfig -throughout the METplus system. It can be used as a drop-in replacement -for a produtil.config.ProdConfig, but has additional features needed to -support initial creation of the METplus system. +The METplusConfig class extends a METplus-native configuration base class +to provide additional features needed to support initial creation of the +METplus system. """ '''!@var __all__ @@ -434,7 +433,7 @@ def replace_config_from_section(config, section, required=True): return new_config -class METplusConfig(ProdConfig): +class METplusConfig(SimpleConfig): """! Configuration class to store configuration values read from METplus config files. """ @@ -486,7 +485,7 @@ def __del__(self): handler.close() def log(self, sublog=None): - """! Overrides method in ProdConfig + """! Overrides method in SimpleConfig If the sublog argument is provided, then the logger will be under that subdomain of the "metplus" logging domain. Otherwise, this METplusConfig's logger @@ -547,6 +546,59 @@ def remove_current_vars(self): if self.has_option('config', current_var): self._conf.remove_option('config', current_var) + def _substitute_raw_template(self, sec, in_template, default='', count=0, + keep_double_slash=False, extra_vars=None): + """Apply METplus variable substitution to a raw template string.""" + if count >= 10: + self.logger.error("Could not resolve getraw - check for circular " + "references in METplus configuration variables") + return '' + + match_list = re.findall(r'\{([^}{]*)\}', in_template) + for var_name in match_list: + if extra_vars and var_name in extra_vars: + value = str(extra_vars[var_name]) + elif self.has_option(sec, var_name): + value = self.getraw(sec, var_name, default, count+1, + keep_double_slash=keep_double_slash) + elif self.has_option('config', var_name): + value = self.getraw('config', var_name, default, count+1, + keep_double_slash=keep_double_slash) + elif var_name.startswith('ENV[') and var_name.endswith(']'): + value = os.environ.get(var_name[4:-1]) + else: + value = None + + if value is None: + continue + in_template = in_template.replace(f"{{{var_name}}}", value) + + if not keep_double_slash: + in_template = in_template.replace('//', '/') + + return in_template + + def _get_existing_option_value(self, sec, name): + """Return raw value from section/config fallback or raise NoOptionError.""" + if self.has_option(sec, name): + return super().getraw(sec, name) + if sec != 'config' and self.has_option('config', name): + return super().getraw('config', name) + raise NoOptionError(name, sec) + + def strinterp(self, sec, string, **kwargs): + """Apply METplus substitution rules to an arbitrary input string.""" + if not isinstance(string, str): + raise TypeError('strinterp requires string input') + + if sec in self.OLD_SECTIONS: + sec = 'config' + + return self._substitute_raw_template(sec, + string, + keep_double_slash=True, + extra_vars=kwargs) + # override get methods to perform additional error checking def getraw(self, sec, opt, default='', count=0, sub_vars=True, keep_double_slash=False): """ parse parameter and replace any existing parameters @@ -563,11 +615,6 @@ def getraw(self, sec, opt, default='', count=0, sub_vars=True, keep_double_slash @returns Raw string or empty string if function calls itself too many times """ - if count >= 10: - self.logger.error("Could not resolve getraw - check for circular " - "references in METplus configuration variables") - return '' - # if requested section is in the list of sections that are no longer # used, look in the [config] section for the variable if sec in self.OLD_SECTIONS: @@ -583,32 +630,11 @@ def getraw(self, sec, opt, default='', count=0, sub_vars=True, keep_double_slash if not sub_vars: return in_template - # get inner-most tags that could potentially be other variables - match_list = re.findall(r'\{([^}{]*)\}', in_template) - for var_name in match_list: - # check if each tag is an existing METplus config variable - if self.has_option(sec, var_name): - value = self.getraw(sec, var_name, default, count+1, keep_double_slash=keep_double_slash) - elif self.has_option('config', var_name): - value = self.getraw('config', var_name, default, count+1, keep_double_slash=keep_double_slash) - elif var_name.startswith('ENV'): - # if environment variable, ENV[nameofvar], get nameofvar - value = os.environ.get(var_name[4:-1]) - else: - value = None - - if value is None: - continue - in_template = in_template.replace(f"{{{var_name}}}", value) - - # Replace double slash with single slash because MET config files fail - # when they encounter double slash. This is a GitHub issue MET #1277 - # This fix will prevent using URLs with https:// so the MET issue must - # be resolved before we can remove the replace call - if not keep_double_slash: - in_template = in_template.replace('//', '/') - - return in_template + return self._substitute_raw_template(sec, + in_template, + default=default, + count=count, + keep_double_slash=keep_double_slash) def check_default(self, sec, name, default): """! helper function for get methods, report error and raise @@ -646,8 +672,10 @@ def getexe(self, exe_name, default=None, morevars=None, taskvars=None): """! Wraps produtil exe with checks to see if option is set and if exe actually exists. Returns None if not found instead of exiting """ + del default, morevars, taskvars + try: - exe_path = super().getstr('config', exe_name) + exe_path = self.getstr('config', exe_name) except NoOptionError as e: if self.logger: self.logger.error(e) @@ -692,8 +720,15 @@ def getdir(self, name, default=None, must_exist=False, keep_double_slash=False): return dir_path def getdir_nocheck(self, dir_name, default=None): - return super().getstr('config', dir_name, - default=default).replace('//', '/') + try: + raw_value = self._get_existing_option_value('config', dir_name) + return self._substitute_raw_template('config', + raw_value, + keep_double_slash=False) + except NoOptionError: + if default is None: + raise + return default.replace('//', '/') def getstr_nocheck(self, sec, name, default=None): # if requested section is in the list of sections that are @@ -701,7 +736,15 @@ def getstr_nocheck(self, sec, name, default=None): if sec in self.OLD_SECTIONS: sec = 'config' - return super().getstr(sec, name, default=default).replace('//', '/') + try: + raw_value = self._get_existing_option_value(sec, name) + return self._substitute_raw_template(sec, + raw_value, + keep_double_slash=False) + except NoOptionError: + if default is None: + raise + return default.replace('//', '/') def getstr(self, sec, name, default=None, badtypeok=False, morevars=None, taskvars=None): @@ -718,10 +761,13 @@ def getstr(self, sec, name, default=None, badtypeok=False, morevars=None, if sec in self.OLD_SECTIONS: sec = 'config' + # Keep optional compatibility args but process interpolation entirely + # in METplusConfig. + del badtypeok, morevars, taskvars + try: - return super().getstr(sec, name, default=None, - badtypeok=badtypeok, morevars=morevars, - taskvars=taskvars).replace('//', '/') + raw_value = self._get_existing_option_value(sec, name) + return self._substitute_raw_template(sec, raw_value) except NoOptionError: # if config variable is not set self.check_default(sec, name, default) @@ -739,36 +785,39 @@ def getbool(self, sec, name, default=None, badtypeok=False, morevars=None, @returns None if value is not a boolean (or yes/no), value if set, default if not set """ + del badtypeok, morevars, taskvars + if sec in self.OLD_SECTIONS: sec = 'config' try: - return super().getbool(sec, name, default=None, - badtypeok=badtypeok, morevars=morevars, - taskvars=taskvars) + value_string = self.getstr(sec, name) except NoOptionError: # config item was not set self.check_default(sec, name, default) return default - except ValueError: - # check if it was an empty string and return default or False if so - value_string = super().getstr(sec, name) - if not value_string: - if default: - return default - return False - - # check if value is y/Y/n/N and return True/False if so - value_string = remove_quotes(value_string) - if value_string.lower() == 'y': - return True - if value_string.lower() == 'n': - return False - - # if value is not correct type, log error and return None - self.logger.error(f"[{sec}] {name} must be an boolean.") - return None + # check if it was an empty string and return default or False if so + if not value_string: + if default: + return default + return False + + # check if value is y/Y/n/N and return True/False if so + value_string = remove_quotes(value_string) + if value_string.lower() == 'y': + return True + if value_string.lower() == 'n': + return False + + if value_string.lower() in ('true', '.true.', 'yes', 'on', '1', 't'): + return True + if value_string.lower() in ('false', '.false.', 'no', 'off', '0', 'f'): + return False + + # if value is not correct type, log error and return None + self.logger.error(f"[{sec}] {name} must be an boolean.") + return None def getint(self, sec, name, default=None, badtypeok=False, morevars=None, taskvars=None): @@ -776,15 +825,13 @@ def getint(self, sec, name, default=None, badtypeok=False, morevars=None, and no default value is specified @returns Value if set, default of missing value if not set, None if value is an incorrect type""" + del badtypeok, morevars, taskvars + if sec in self.OLD_SECTIONS: sec = 'config' try: - # call ProdConfig function with no default set so - # we can log and set the default - return super().getint(sec, name, default=None, - badtypeok=badtypeok, morevars=morevars, - taskvars=taskvars) + return int(self.getstr(sec, name)) # if config variable is not set except NoOptionError: @@ -797,7 +844,7 @@ def getint(self, sec, name, default=None, badtypeok=False, morevars=None, # if invalid value except ValueError: # check if it was an empty string and return MISSING_DATA_VALUE - value = super().getstr(sec, name) + value = self.getstr(sec, name) if value == '': return MISSING_DATA_VALUE @@ -819,15 +866,13 @@ def getfloat(self, sec, name, default=None, badtypeok=False, morevars=None, and no default value is specified @returns Value if set, default of missing value if not set, None if value is an incorrect type""" + del badtypeok, morevars, taskvars + if sec in self.OLD_SECTIONS: sec = 'config' try: - # call ProdConfig function with no default set so - # we can log and set the default - return super().getfloat(sec, name, default=None, - badtypeok=badtypeok, morevars=morevars, - taskvars=taskvars) + return float(self.getstr(sec, name)) # if config variable is not set except NoOptionError: @@ -840,7 +885,7 @@ def getfloat(self, sec, name, default=None, badtypeok=False, morevars=None, # if invalid value except ValueError: # check if it was an empty string and return MISSING_DATA_VALUE - if super().getstr(sec, name) == '': + if self.getstr(sec, name) == '': return MISSING_DATA_VALUE # if value is not correct type, log error and return None @@ -850,15 +895,15 @@ def getfloat(self, sec, name, default=None, badtypeok=False, morevars=None, def getseconds(self, sec, name, default=None, badtypeok=False, morevars=None, taskvars=None): """!Converts time values ending in H, M, or S to seconds""" + del badtypeok, morevars, taskvars + if sec in self.OLD_SECTIONS: sec = 'config' try: # convert value to seconds # Valid options match format 3600, 3600S, 60M, or 1H - value = super().getstr(sec, name, default=None, - badtypeok=badtypeok, morevars=morevars, - taskvars=taskvars) + value = self.getstr(sec, name) regex_and_multiplier = {r'(-*)(\d+)S': 1, r'(-*)(\d+)M': 60, r'(-*)(\d+)H': 3600, diff --git a/metplus/util/simple_config.py b/metplus/util/simple_config.py new file mode 100644 index 0000000000..58e56383fa --- /dev/null +++ b/metplus/util/simple_config.py @@ -0,0 +1,226 @@ +"""Lightweight configuration utilities used by METplus. + +This module provides a small thread-safe wrapper around ConfigParser. +Higher-level METplus-specific interpolation behavior lives in +metplus.util.config_metplus.METplusConfig. +""" + +import logging +import re +import threading +from configparser import ConfigParser, NoOptionError + + +_NOT_FOUND = object() +_MAX_INTERP_DEPTH = 10 + + +class SimpleConfig(object): + """Minimal ConfigParser wrapper with METplus-compatible interpolation.""" + + def __init__(self, conf=None): + self._lock = threading.RLock() + self._logger = logging.getLogger("metplus.config") + self._conf = ConfigParser(strict=False, inline_comment_prefixes=(";",), interpolation=None) if conf is None else conf + self._conf.optionxform = str + + if not self._conf.has_section("config"): + self._conf.add_section("config") + + def __enter__(self): + self._lock.acquire() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._lock.release() + + def log(self, sublog=None): + if sublog is None: + return self._logger + with self: + return logging.getLogger(f"metplus.{sublog}") + + def read(self, source): + with self: + self._conf.read(source) + return self + + def write(self, fileobject): + with self: + self._conf.write(fileobject) + + def add_section(self, sec): + with self: + if not self._conf.has_section(sec): + self._conf.add_section(sec) + return self + + def has_section(self, sec): + with self: + return self._conf.has_section(sec) + + def has_option(self, sec, opt): + with self: + return self._conf.has_option(sec, opt) + + def keys(self, sec): + with self: + return list(self._conf.options(sec)) + + def sections(self): + with self: + return self._conf.sections() + + def set(self, section, key, value): + with self: + section = str(section) + if not self._conf.has_section(section): + self._conf.add_section(section) + self._conf.set(section, str(key), str(value)) + + def getraw(self, sec, opt, default=None): + try: + with self: + return self._conf.get(sec, opt, raw=True) + except NoOptionError: + if default is not None: + return default + raise + + def _resolve_option(self, sec, opt): + if self._conf.has_option(sec, opt): + return self._conf.get(sec, opt, raw=True) + + if self._conf.has_option("config", opt): + return self._conf.get("config", opt, raw=True) + + return _NOT_FOUND + + def _resolve_tag_value(self, sec, tag_name, kwargs, depth): + if tag_name in kwargs: + return kwargs[tag_name] + + + target_sec = sec + target_opt = tag_name + split_index = tag_name.find("/") + if split_index >= 0: + if split_index > 0: + target_sec = tag_name[:split_index] + target_opt = tag_name[split_index + 1 :] + + if not target_opt: + return None + + resolved = self._resolve_option(target_sec, target_opt) + if resolved is _NOT_FOUND: + return None + + return self._interpolate(target_sec, resolved, kwargs=kwargs, depth=depth + 1) + + def _interpolate(self, sec, value, kwargs=None, depth=0): + if kwargs is None: + kwargs = {} + + if depth >= _MAX_INTERP_DEPTH or "{" not in value: + return value + + interpolated = value + for match in re.findall(r"\{([^{}]+)\}", value): + replacement = self._resolve_tag_value(sec, match, kwargs, depth) + if replacement is None: + continue + interpolated = interpolated.replace(f"{{{match}}}", str(replacement)) + + # Resolve newly expanded nested tags until stable or depth limit. + if interpolated != value and "{" in interpolated and depth < _MAX_INTERP_DEPTH: + return self._interpolate(sec, interpolated, kwargs=kwargs, depth=depth + 1) + + return interpolated + + def strinterp(self, sec, string, **kwargs): + if not isinstance(string, str): + raise TypeError("strinterp requires a string input") + with self: + return self._interpolate(sec, string, kwargs=kwargs) + + def get(self, sec, opt, default=None, morevars=None, taskvars=None): + return self.getstr(sec, opt, default=default, morevars=morevars, taskvars=taskvars) + + def getstr(self, sec, opt, default=None, badtypeok=False, morevars=None, taskvars=None): + del badtypeok, taskvars # kept for interface compatibility + with self: + raw_value = self._resolve_option(sec, opt) + if raw_value is _NOT_FOUND: + if default is not None: + return str(default) + raise NoOptionError(opt, sec) + + kwargs = {} if morevars is None else dict(morevars) + return self._interpolate(sec, str(raw_value), kwargs=kwargs) + + def getint(self, sec, opt, default=None, badtypeok=False, morevars=None, taskvars=None): + try: + return int(self.getstr(sec, opt, default=None, morevars=morevars, taskvars=taskvars)) + except NoOptionError: + if default is not None: + return default + raise + except (TypeError, ValueError): + if badtypeok and default is not None: + return default + raise + + def getfloat(self, sec, opt, default=None, badtypeok=False, morevars=None, taskvars=None): + try: + return float(self.getstr(sec, opt, default=None, morevars=morevars, taskvars=taskvars)) + except NoOptionError: + if default is not None: + return default + raise + except (TypeError, ValueError): + if badtypeok and default is not None: + return default + raise + + def getbool(self, sec, opt, default=None, badtypeok=False, morevars=None, taskvars=None): + try: + value = self.getstr(sec, opt, default=None, morevars=morevars, taskvars=taskvars) + except NoOptionError: + if default is not None: + return bool(default) + raise + + if re.match(r"(?i)\A(?:T|\.true\.|true|yes|on|1)\Z", value): + return True + if re.match(r"(?i)\A(?:F|\.false\.|false|no|off|0)\Z", value): + return False + + try: + return int(value) != 0 + except ValueError: + if badtypeok and default is not None: + return bool(default) + raise ValueError(f"{sec}.{opt}: invalid value for conf file boolean: {value!r}") + + + def items(self, sec, morevars=None, taskvars=None): + with self: + result = [] + for opt in self._conf.options(sec): + result.append((opt, self.getstr(sec, opt, morevars=morevars, taskvars=taskvars))) + return result + + def __getitem__(self, arg): + with self: + if isinstance(arg, str): + return dict(self.items(arg)) + if isinstance(arg, (list, tuple)): + if len(arg) == 1: + return dict(self.items(arg[0])) + if len(arg) == 2: + return self.get(arg[0], arg[1]) + if len(arg) == 3: + return self.get(arg[0], arg[1], default=arg[2]) + return NotImplemented + From 7e5f145fc832439d01c2b23e8747774ef64bc730 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:37:52 -0600 Subject: [PATCH 03/12] remove produtil config --- metplus/produtil/config.py | 1690 ------------------------------------ 1 file changed, 1690 deletions(-) delete mode 100644 metplus/produtil/config.py diff --git a/metplus/produtil/config.py b/metplus/produtil/config.py deleted file mode 100644 index 8aadbe473e..0000000000 --- a/metplus/produtil/config.py +++ /dev/null @@ -1,1690 +0,0 @@ -"""!Parses UNIX conf files and makes the result readily available - -The produtil.config module reads configuration information for a -production system from one or more *.conf files, via the Python -ConfigParser module. This module also automatically fills in certain -information, such as fields calculated from the tcvitals or date. The -result is accessible via the ProdConfig class, which provides many -ways of automatically accessing configuration options.""" - -##@var __all__ -# decides what symbols are imported by "from produtil.config import *" -__all__=['from_file','confwalker','ProdConfig','ENVIRONMENT'] - -import collections,re,os,logging,threading -import os.path,sys -import datetime -import metplus.produtil.fileop as fileop - -import configparser -from configparser import ConfigParser -from io import StringIO - -# NOTE: -# Datastore is intentionally imported lazily inside getdatastore() so -# importing metplus.produtil.config does not require datastore support. -Task=object - -from metplus.produtil.numerics import to_datetime, to_datetime_rel, fcst_hr_min -from string import Formatter -from configparser import NoOptionError,NoSectionError - -UNSPECIFIED=object() - -class DuplicateTaskName(Exception): - """!Raised when more than one task is registered with the same - name in an ProdConfig object.""" - -######################################################################## - -class Environment(object): - """!returns environment variables, allowing substitutions - - This class is used to read (but not write) environment variables - and provide default values if an environment variable is unset or - blank. It is only meant to be used in string formats, by passing - ENV=ENVIRONMENT. There is a global constant in this module, - ENVIRONMENT, which is an instance of this class. You should never - need to instantiate another one.""" - def __contains__(self,s): - """!Determines if __getitem__ will return something (True) or - raise KeyError (False). Same as "s in os.environ" unless s - contains "|-", in which case, the result is True.""" - return s.find('|-')>=0 or s in os.environ - def __getitem__(self,s): - """!Same as os.environ[s] unless s contains "|-". - ENVIRONMENT["VARNAME|-substitute"] - will return os.environ[VARNAME] if VARNAME is defined and - non-empty in os.environ. Otherwise, it will return - "substitute".""" - if not s: return '' - i=s.find('|-') - if i<0: return os.environ[s] - var=s[0:i] - sub=s[(i+2):] - val=os.environ.get(var,'') - if val!='': return val - return sub - -## @var ENVIRONMENT -# an Environment object. You should never need to instantiate another one. -ENVIRONMENT=Environment() - -class ConfFormatter(Formatter): - """!Internal class that implements ProdConfig.strinterp() - - This class is part of the implementation of ProdConfig: it is - used to interpolate strings using a syntax similar to - string.format(), but it allows recursion in the config sections, - and it also is able to use the [config] and [dir] sections as - defaults for variables not found in the current section.""" - def __init__(self,quoted_literals=False): - """!Constructor for ConfFormatter""" - super(ConfFormatter,self).__init__() - if quoted_literals: - self.format=self.slow_format - self.vformat=self.slow_vformat - self.parse=qparse - - @property - def quoted_literals(self): - return self.parse==qparse - - def slow_format(self,format_string,*args,**kwargs): - return self.vformat(format_string,args,kwargs) - def slow_vformat(self,format_string,args,kwargs): - out=StringIO() - for literal_text, field_name, format_spec, conversion in \ - self.parse(format_string): - if literal_text: - out.write(literal_text) - if not field_name: - continue - (obj, used_key) = self.get_field(field_name,args,kwargs) - if obj is None and used_key: - obj=self.get_value(used_key,args,kwargs) - value=obj - if conversion=='s': - value=str(value) - elif conversion=='r': - value=repr(value) - elif conversion: - raise ValueError('Unknown conversion %s'%(repr(conversion),)) - if format_spec: - value=value.__format__(format_spec) - out.write(value) - ret=out.getvalue() - out.close() - assert(ret is not None) - assert(isinstance(ret,str)) - return ret - - def get_value(self,key,args,kwargs): - """!Return the value of variable, or a substitution. - - Never call this function. It is called automatically by - str.format. It provides the value of an variable, - or a string substitution. - @param key the string key being analyzed by str.format() - @param args the indexed arguments to str.format() - @param kwargs the keyword arguments to str.format()""" - kwargs['__depth']+=1 - if kwargs['__depth']>=configparser.MAX_INTERPOLATION_DEPTH: - raise configparser.InterpolationDepthError(kwargs['__key'], - kwargs['__section'],key) - try: - if isinstance(key,int): - return args[key] - v = _get_v(key, kwargs) - if isinstance(v,str) and (v.find('{')>=0 or v.find('%')>=0): - vnew=self.vformat(v,args,kwargs) - assert(vnew is not None) - return vnew - return v - finally: - kwargs['__depth']-=1 - - -def _get_v(key, kwargs): - if key in kwargs: - return kwargs[key] - if ('__taskvars' in kwargs and kwargs['__taskvars'] - and key in kwargs['__taskvars']): - return kwargs['__taskvars'][key] - - isec = key.find('/') - if isec >= 0: - section = key[0:isec] - nkey = key[(isec + 1):] - if not section: - section = kwargs.get('__section', None) - if nkey: - key = nkey - else: - section = kwargs.get('__section', None) - conf = kwargs.get('__conf', None) - v = NOTFOUND - if section is None or conf is None: - return v - if conf.has_option(section, key): - return conf.get(section, key, raw=True) - if conf.has_option(section, '@inc'): - for osec in conf.get(section, '@inc').split(','): - if conf.has_option(osec, key): - v = conf.get(osec, key, raw=True) - if v is not NOTFOUND: - return v - - if conf.has_option('config', key): - return conf.get('config', key, raw=True) - if conf.has_option('dir', key): - return conf.get('dir', key, raw=True) - raise KeyError(key) - - -def qparse(format_string): - """!Replacement for Formatter.parse which can be added to Formatter objects - to turn {'...'} and {"..."} blocks into literal strings (the ... part). - Apply this by doing f=Formatter() ; f.parse=qparse. """ - if not format_string: return [] - if not isinstance(format_string, str): - raise TypeError('iterparse expects a str, not a %s %s'%( - type(format_string).__name__,repr(format_string))) - result=list() - literal_text='' - for m in re.finditer(r'''(?xs) ( - \{ \' (?P (?: \' (?! \} ) | [^'] )* ) \' \} - | \{ \" (?P (?: \" (?! \} ) | [^"] )* ) \" \} - | (?P - \{ - (?P - [^\}:!\['"\{] [^\}:!\[]* - (?: \. [a-zA-Z_]\w+ - | \[ [^\]]+ \] )* - ) - (?: ! (?P[rs]) )? - (?: : - (?P - (?: [^\{\}]+ - | \{[^\}]*\} )* - ) - )? - \} ) - | (?P \{\{ ) - | (?P \}\} ) - | (?P [^\{\}]+ ) - | (?P . ) ) ''',format_string): - if m.group('qescape'): - literal_text+=m.group('qescape') - elif m.group('dqescape'): - literal_text+=m.group('dqescape') - elif m.group('left_set'): - literal_text+='{' - elif m.group('right_set'): - literal_text+='}' - elif m.group('literal_text'): - literal_text+=m.group('literal_text') - elif m.group('replacement_field'): - result.append( ( literal_text, - m.group('field_name'), - m.group('format_spec'), - m.group('conversion') ) ) - literal_text='' - elif not m.group('error'): - continue - if m.group('error')=='{': - raise ValueError("Single '{' encountered in format string") - elif m.group('error')=='}': - raise ValueError("Single '}' encountered in format string") - else: - raise ValueError("Unexpected %s in format string"%( - repr(m.group('error')),)) - if literal_text: - result.append( ( literal_text, None, None, None ) ) - return result - -######################################################################## - -##@var FCST_KEYS -# the list of forecast time keys recognized by ConfTimeFormatter -FCST_KEYS={ 'fYMDHM':'%Y%m%d%H%M', 'fYMDH':'%Y%m%d%H', 'fYMD':'%Y%m%d', - 'fyear':'%Y', 'fYYYY':'%Y', 'fYY':'%y', 'fCC':'%C', 'fcen':'%C', - 'fmonth':'%m', 'fMM':'%m', 'fday':'%d', 'fDD':'%d', 'fhour':'%H', - 'fcyc':'%H', 'fHH':'%H', 'fminute':'%M', 'fmin':'%M' } -"""A list of keys recognized by ConfTimeFormatter if the key is -requested during string interpolation, and the key is not in the -relevant section. This list of keys represents the forecast time. It -is a dict mapping from the key name to the format sent to -datetime.datetime.strftime to generate the string value.""" - -##@var ANL_KEYS -# the list of analysis time keys recognized by ConfTimeFormatter -ANL_KEYS={ 'aYMDHM':'%Y%m%d%H%M', 'aYMDH':'%Y%m%d%H', 'aYMD':'%Y%m%d', - 'ayear':'%Y', 'aYYYY':'%Y', 'aYY':'%y', 'aCC':'%C', 'acen':'%C', - 'amonth':'%m', 'aMM':'%m', 'aday':'%d', 'aDD':'%d', 'ahour':'%H', - 'acyc':'%H', 'aHH':'%H', 'aminute':'%M', 'amin':'%M' } -"""A list of keys recognized by ConfTimeFormatter if the key is -requested during string interpolation, and the key is not in the -relevant section. This list of keys represents the analysis time. It -is a dict mapping from the key name to the format sent to -datetime.datetime.strftime to generate the string value.""" - -##@var M6_KEYS -# the list of analysis time ( -6h ) keys recognized by ConfTimeFormatter -ANL_M6_KEYS={ 'am6YMDHM':'%Y%m%d%H%M', 'am6YMDH':'%Y%m%d%H', 'am6YMD':'%Y%m%d', - 'am6year':'%Y', 'am6YYYY':'%Y', 'am6YY':'%y', 'am6CC':'%C', 'am6cen':'%C', - 'am6month':'%m', 'am6MM':'%m', 'am6day':'%d', 'am6DD':'%d', 'am6hour':'%H', - 'am6cyc':'%H', 'am6HH':'%H', 'am6minute':'%M', 'am6min':'%M' } -"""A list of keys recognized by ConfTimeFormatter if the key is -requested during string interpolation, and the key is not in the -relevant section. This list of keys represents the analysis time. It -is a dict mapping from the key name to the format sent to -datetime.datetime.strftime to generate the string value.""" - -##@var P6_KEYS -# the list of analysis time ( +6h ) keys recognized by ConfTimeFormatter -ANL_P6_KEYS={ 'ap6YMDHM':'%Y%m%d%H%M', 'ap6YMDH':'%Y%m%d%H', 'ap6YMD':'%Y%m%d', - 'ap6year':'%Y', 'ap6YYYY':'%Y', 'ap6YY':'%y', 'ap6CC':'%C', 'ap6cen':'%C', - 'ap6month':'%m', 'ap6MM':'%m', 'ap6day':'%d', 'ap6DD':'%d', 'ap6hour':'%H', - 'ap6cyc':'%H', 'ap6HH':'%H', 'ap6minute':'%M', 'ap6min':'%M' } -"""A list of keys recognized by ConfTimeFormatter if the key is -requested during string interpolation, and the key is not in the -relevant section. This list of keys represents the analysis time. It -is a dict mapping from the key name to the format sent to -datetime.datetime.strftime to generate the string value.""" - -##@var TIME_DIFF_KEYS -# the list of "forecast time minus analysis time" keys recognized by -# ConfTimeFormatter -TIME_DIFF_KEYS=set(['fahr','famin','fahrmin']) -"""A list of keys recognized by ConfTimeFormatter if the key is -requested during string interpolation, and the key is not in the -relevant section. This list of keys represents the time difference -between the forecast and analysis time. Unlike FCST_KEYS and -ANL_KEYS, this is not a mapping: it is a set.""" - -##@var NOTFOUND -# a special constant that represents a key not being found -NOTFOUND=object() - -class ConfTimeFormatter(ConfFormatter): - """!internal function that implements time formatting - - Like its superclass, ConfFormatter, this class is part of the - implementation of ProdConfig, and is used to interpolate strings - in a way similar to string.format(). It works the same way as - ConfFormatter, but accepts additional keys generated based on the - forecast and analysis times: - - fYMDHM - 201409171200 = forecast time September 17, 2014 at 12:00 UTC - fYMDH - 2014091712 - fYMD - 20140917 - fyear - 2014 - fYYYY - 2014 - fYY - 14 (year % 100) - fCC - 20 (century) - fcen - 20 - fmonth - 09 - fMM - 09 - fday - 17 - fDD - 17 - fhour - 12 - fcyc - 12 - fHH - 12 - fminute - 00 - fmin - 00 - - Replace the initial "f" with "a" for analysis times. In addition, - the following are available for the time difference between - forecast and analysis time. Suppose the forecast is twenty-three - hours and nineteen minutes (23:19) after the analysis time: - - fahr - 23 - famin - 1399 ( = 23*60+19) - fahrmin - 19 """ - def __init__(self,quoted_literals=False): - """!constructor for ConfTimeFormatter""" - super(ConfTimeFormatter,self).__init__( - quoted_literals=bool(quoted_literals)) - def get_value(self,key,args,kwargs): - """!return the value of a variable, or a substitution - - Never call this function. It is called automatically by - str.format. It provides the value of an variable, - or a string substitution. - @param key the string key being analyzed by str.format() - @param args the indexed arguments to str.format() - @param kwargs the keyword arguments to str.format()""" - v=NOTFOUND - kwargs['__depth']+=1 - if kwargs['__depth']>=configparser.MAX_INTERPOLATION_DEPTH: - raise configparser.InterpolationDepthError( - kwargs['__key'],kwargs['__section'],v) - try: - if isinstance(key,int): - return args[key] - if key in kwargs: - v=kwargs[key] - elif '__taskvars' in kwargs \ - and kwargs['__taskvars'] \ - and key in kwargs['__taskvars']: - v=kwargs['__taskvars'][key] - elif '__ftime' in kwargs and key in FCST_KEYS: - v=kwargs['__ftime'].strftime(FCST_KEYS[key]) - elif '__atime' in kwargs and key in ANL_KEYS: - v=kwargs['__atime'].strftime(ANL_KEYS[key]) - elif '__atime' in kwargs and key in ANL_M6_KEYS: - am6=kwargs['__atime']-datetime.timedelta(0,3600*6) - v=am6.strftime(ANL_M6_KEYS[key]) - elif '__atime' in kwargs and key in ANL_P6_KEYS: - ap6=kwargs['__atime']+datetime.timedelta(0,3600*6) - v=ap6.strftime(ANL_P6_KEYS[key]) - elif '__ftime' in kwargs and '__atime' in kwargs and \ - key in TIME_DIFF_KEYS: - (ihours,iminutes)=fcst_hr_min( - kwargs['__ftime'],kwargs['__atime']) - if key=='fahr': - v=int(ihours) - elif key=='famin': - v=int(ihours*60+iminutes) - elif key=='fahrmin': - v=int(iminutes) - else: - v=int(ihours*60+iminutes) - else: - isec=key.find('/') - if isec>=0: - section=key[0:isec] - nkey=key[(isec+1):] - if not section: - section=kwargs.get('__section',None) - if nkey: - key=nkey - else: - section=kwargs.get('__section',None) - conf=kwargs.get('__conf',None) - if section and conf: - if conf.has_option(section,key): - v=conf.get(section,key) - elif conf.has_option(section,'@inc'): - for osec in conf.get(section,'@inc').split(','): - if conf.has_option(osec,key): - v=conf.get(osec,key) - if v is NOTFOUND: - if conf.has_option('config',key): - v=conf.get('config',key) - elif conf.has_option('dir',key): - v=conf.get('dir',key) - if v is NOTFOUND: - raise KeyError('Cannot find key %s in section %s' - %(repr(key),repr(section))) - - if isinstance(v,str) and ( v.find('{')!=-1 or - v.find('%')!=-1 ): - try: - vnew=self.vformat(v,args,kwargs) - assert(vnew is not None) - return vnew - except KeyError as e: - # Seriously, does the exception's class name - # really need to be this long? - raise ConfigParser.InterpolationMissingOptionError( - kwargs['__key'],kwargs['__section'],v,str(e)) - return v - finally: - kwargs['__depth']-=1 - -######################################################################## - -def confwalker(conf,start,selector,acceptor,recursevar): - """!walks through a ConfigParser-like object performing some action - - Recurses through a ConfigParser-like object "conf" starting at - section "start", performing a specified action. The special - variable whose name is in recursevar specifies a list of - additional sections to recurse into. No section will be processed - more than once, and sections are processed in breadth-first order. - For each variable seen in each section (including recursevar), - this will call selector(sectionname, varname) to see if the - variable should be processed. If selector returns True, then - acceptor(section, varname, value) will be called. - - @param conf the ConfigParser-like object - @param start the starting section - @param selector a function selector(section,option) that decides - if an option needs processing (True) or not (False) - @param acceptor a function acceptor(section,option,value) - run on all options for which the selector returns True - @param recursevar an option in each section that lists more - sections the confwalker should touch. If the selector returns - True for the recursevar, then the recursevar will be sent to - the acceptor. However, it will be scanned for sections to - recurse into even if the selector rejects it.""" - touched=set() - requested=[str(start)] - while len(requested)>0: - sec=requested.pop(0) - if sec in touched: - continue - touched.add(sec) - for (key,val) in conf.items(sec): - if selector(sec,key): - acceptor(sec,key,val) - if key==recursevar: - for sec2 in reversed(val.split(',')): - trim=sec2.strip() - if len(trim)>0 and trim not in touched: - requested.append(trim) - -######################################################################## - -def from_file(filename,quoted_literals=False): - """!Reads the specified conf file into an ProdConfig object. - - Creates a new ProdConfig object and instructs it to read the specified file. - @param filename the path to the file that is to be read - @return a new ProdConfig object""" - if not isinstance(filename,str): - raise TypeError('First input to produtil.config.from_file must be a string.') - conf=ProdConfig(quoted_literals=bool(quoted_literals)) - conf.read(filename) - return conf - -def from_string(confstr,quoted_literals=False): - """!Reads the given string as if it was a conf file into an ProdConfig object - - Creates a new ProdConfig object and reads the string data into it - as if it was a config file - @param confstr the config data - @return a new ProdConfig object""" - if not isinstance(confstr,str): - raise TypeError('First input to produtil.config.from_string must be a string.') - conf=ProdConfig(quoted_literals=bool(quoted_literals)) - conf.readstr(confstr) - return conf - -class ProdConfig(object): - """!a class that contains configuration information - - This class keeps track of configuration information for all tasks - in a running model. It can be used in a read-only manner as - if it was a ConfigParser object. All ProdTask objects require an - ProdConfig object to keep track of registered task names via the - register_task_name method, the current forecast cycle (cycle - property) and the Datastore object (datastore property). - - This class should never be instantiated directly. Instead, you - should use the produtil.config.from_string or produtil.config.from_file to - read configuration information from an in-memory string or a file.""" - - def __init__(self,conf=None,quoted_literals=False,strict=False, inline_comment_prefixes=(';',)): - """!ProdConfig constructor - - Creates a new ProdConfig object. - @param conf the underlying configparser.ConfigParser object - that stores the actual config data. This was a SafeConfigParser - in Python 2 but in Python 3 the SafeConfigParser is now ConfigParser. - @param quoted_literals if True, then {'...'} and {"..."} will - be interpreted as quoting the contained ... text. Otherwise, - those blocks will be considered errors. - @param strict set default to False so it will not raise - DuplicateOptionError or DuplicateSectionError, This param was - added when ported to Python 3.6, to maintain the previous - python 2 behavior. - @param inline_comment_prefixes, defaults set to ;. This param was - added when ported to Python 3.6, to maintain the previous - python 2 behavior. - - Note: In Python 2, conf was ConfigParser.SafeConfigParser. In - Python 3.2, the old ConfigParser class was removed in favor of - SafeConfigParser which has in turn been renamed to ConfigParser. - Support for inline comments is now turned off by default and - section or option duplicates are not allowed in a single - configuration source.""" - self._logger=logging.getLogger('prodconfig') - self._lock=threading.RLock() - self._formatter=ConfFormatter(bool(quoted_literals)) - self._time_formatter=ConfTimeFormatter(bool(quoted_literals)) - self._datastore=None - self._tasknames=set() - # Added strict=False and inline_comment_prefixes for Python 3, - # so everything works as it did before in Python 2. - #self._conf=ConfigParser(strict=False, inline_comment_prefixes=(';',)) if (conf is None) else conf - self._conf=ConfigParser(strict=strict, inline_comment_prefixes=inline_comment_prefixes) if (conf is None) else conf - self._conf.optionxform=str - - self._conf.add_section('config') - self._conf.add_section('dir') - self._fallback_callbacks=list() - - @property - def quoted_literals(self): - return self._time_formatter.quoted_literals and \ - self._formatter.quoted_literals - - def fallback(self,name,details): - """!Asks whether the specified fallback is allowed. May perform - other tasks, such as alerting the operator. - - Calls the list of functions sent to add_fallback_callback. - Each one receives the result of the last, and the final result - at the end is returned. Note that ALL of the callbacks are - called, even if one returns False; this is not a short-circuit - operation. This is done to allow all reporting methods report - to their operator and decide whether the fallback is allowed. - - Each function called is f(allow,name,details) where: - - - allow = True or False, whether the callbacks called thus far - have allowed the fallback. - - - name = The short name of the fallback. - - - details = A long, human-readable description. May be - several lines long. - - @param name the name of the emergency situation - - @warning This function may take seconds or minutes to return. - It could perform cpu- or time-intensive operations such as - emailing an operator. - - """ - allow=self.getbool('config','allow_fallbacks',False) - for fc in self._fallback_callbacks: - allow=bool(fc(allow,name,details)) - return allow - - def add_fallback_callback(self,function): - """!Appends a function to the list of fallback callback functions - called by fallback() - - Appends the given function to the list that fallback() - searches while determining if a workflow emergency fallback - option is allowed. - - @param function a function f(allow,name,details) - @see fallbacks()""" - self._fallback_callbacks.append(function) - - def readstr(self,source): - - """!read config data and add it to this object - - Given a string with conf data in it, parses the data. - @param source the data to parse - @return self""" - fp=StringIO(str(source)) - self._conf.readfp(fp) - fp.close() - return self - - def from_args(self,args=None,allow_files=True,allow_options=True, - rel_path=None,verbose=False): - """!Given a list of arguments, usually from sys.argv[1:], reads - configuration files or sets option values. - - Reads list of strings of these formats: - - - /path/to/file.conf --- A configuration file to read. - - - section.option=value --- A configuration option to set in a - specified section. - - Will read files in the order listed, and then will override - options in the order listed. Note that specified options - override those read from files. Also, later files override - earlier files. - - @param args Typically argv[1:] or some other list of - arguments. - - @param allow_files If True, filenames are allowed in args. - Otherwise, they are ignored. - - @param allow_options If True, specified options - (section.name=value) are allowed. Otherwise they are detected - and ignored. - - @param rel_path Any filenames that are relative will be - relative to this path. If None or unspecified, the current - working directory as of the entry to this function is used. - - @returns self - """ - allow_files=bool(allow_files) - allow_options=bool(allow_options) - verbose=bool(verbose) - logger=self.log() - if allow_files: - if rel_path is None: - rel_path=os.getcwd() - else: - rel_path=str(rel_path) - elif not allow_options: - # Nothing to do! - return self - infiles=list() - moreopt=collections.defaultdict(dict) - for arg in args: - if not isinstance(arg,str): - raise TypeError( - 'In produtil.ProdConfig.from_args(), the args argument must ' - 'be an iterable of strings. It contained an invalid %s %s ' - 'instead.'%(type(arg).__name__,repr(arg))) - if verbose: logger.info(arg) - m=re.match(r'''(?x) - (?P
[a-zA-Z]\w*) - \.(?P