diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 56f6cda8b4..d5d4690b38 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -6,6 +6,7 @@ import sys import os +import shutil # add METplus directory to sys path so the test suite can be found METPLUS_TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), @@ -54,7 +55,7 @@ def handle_automation_env(host_name, reqs): # if not using docker (automation), # return no setup commands and python embedding argument to command if host_name != 'docker': - if 'py_embed' in reqs and conda_env != METPLUS_BASE_ENV: + if _met_python_exe_should_be_set(reqs, conda_env): return '', 'user_env_vars.MET_PYTHON_EXE=python3' return '', '' @@ -70,51 +71,24 @@ def handle_automation_env(host_name, reqs): _add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT') ] - # add conda bin to beginning of PATH - python_dir = os.path.join('/usr', 'local', 'conda', 'envs', - conda_env_w_ext, 'bin') - python_path = os.path.join(python_dir, 'python3') - setup_env.append(_add_to_bashrc(f'export PATH={python_dir}:$PATH')) - + # get path to conda bin unless metplus_base (default) is used + if conda_env != METPLUS_BASE_ENV: + python_dir = os.path.join('/usr', 'local', 'conda', 'envs', + conda_env_w_ext, 'bin') + python_path = os.path.join(python_dir, 'python3') + # add python path to PATH in bashrc so run_metplus.py is called with that version of Python + setup_env.append(_add_to_bashrc(f'export PATH={python_dir}:$PATH')) + else: + python_path = shutil.which('python3') + if python_path is None: + raise RuntimeError('python3 not found in PATH; cannot determine base-image Python executable') # if py_embed listed in requirements and using a Python # environment that differs from the MET env, set MET_PYTHON_EXE - if 'py_embed' in reqs and conda_env != METPLUS_BASE_ENV: + py_embed_arg = '' + if _met_python_exe_should_be_set(reqs, conda_env): py_embed_arg = f'user_env_vars.MET_PYTHON_EXE={python_path} ' - else: - py_embed_arg = '' - # get METplus version to determine which version of - # METplotpy/METcalcpy/METdataio to use - # If stable release, get main branch, otherwise get develop - metplus_version = get_metplus_version() - - # if any metplotpy/metcalcpy keywords are in requirements list, - # add command to obtain and install METplotpy and METcalcpy - components = [] - if any([item for item in PLOTCALC_KEYWORDS if item in str(reqs).lower()]): - components.extend(('METplotpy', 'METcalcpy')) - - # if metdataio is in requirements list, add command to obtain METdataio - if 'metdataio' in str(reqs).lower(): - components.append('METdataio') - - if components: - setup_env.append(f'cd {METPLUS_DOCKER_LOC}/..') - for component in components: - # get branch if defined, otherwise determine from METplus version - version = os.environ.get(f'INPUT_{component.upper()}_BRANCH') - if not version: - version = get_component_version(input_component='METplus', - input_version=metplus_version, - output_component=component, - output_format='main_v{X}.{Y}', - get_dev=False) - setup_env.extend(( - 'git --version', - f'git clone --single-branch --branch {version} https://github.com/dtcenter/{component}', - f'{python_path} -m pip install --no-deps {METPLUS_DOCKER_LOC}/../{component}', - )) - setup_env.append('cd -') + _handle_metplus_analysis_components(reqs, python_path, setup_env) # if metplus is in requirements list, # add top of METplus repo to PYTHONPATH so metplus can be imported @@ -135,6 +109,64 @@ def handle_automation_env(host_name, reqs): return ';'.join(setup_env), py_embed_arg +def _met_python_exe_should_be_set(reqs, conda_env): + return 'py_embed' in reqs and conda_env != METPLUS_BASE_ENV + +def _handle_metplus_analysis_components(reqs, python_path, setup_env): + """ + Handles the addition of METplus analysis components (METplotpy, METcalcpy, METdataio) + to the setup environment based on specific requirements. This function checks whether + the provided requirements contain specific keywords or component names and, based on + the presence of these, constructs commands to clone and install the appropriate versions + of these components. + + @param reqs List of software requirements that may include keywords indicating the need + for METplotpy, METcalcpy, or METdataio. + + @param python_path Path to the Python executable that should be used for installing + the required components. + + @param setup_env List representing the setup environment configuration. The function + appends the appropriate commands to this list for setting up the required components. + + @returns None + """ + # if any metplotpy/metcalcpy keywords are in requirements list, + # add command to obtain and install METplotpy and METcalcpy + components = [] + if any([item for item in PLOTCALC_KEYWORDS if item in str(reqs).lower()]): + components.extend(('METplotpy', 'METcalcpy')) + + # if metdataio is in requirements list, add command to obtain METdataio + if 'metdataio' in str(reqs).lower(): + components.append('METdataio') + + # do not add any commands if no METplus Analysis components are needed + if not components: + return + + # get METplus version to determine which version of + # METplotpy/METcalcpy/METdataio to use + # If stable release, get main branch, otherwise get develop + metplus_version = get_metplus_version() + + # add commands to install METplus Analysis components in directory above METplus repo + setup_env.append(f'cd {METPLUS_DOCKER_LOC}/..') + for component in components: + # get branch if defined, otherwise determine from METplus version + version = os.environ.get(f'INPUT_{component.upper()}_BRANCH') + if not version: + version = get_component_version(input_component='METplus', + input_version=metplus_version, + output_component=component, + output_format='main_v{X}.{Y}', + get_dev=False) + setup_env.extend(( + 'git --version', + f'git clone --single-branch --branch {version} https://github.com/dtcenter/{component}', + f'{python_path} -m pip install --no-deps {METPLUS_DOCKER_LOC}/../{component}', + )) + setup_env.append('cd -') def _add_to_bashrc(command): return f"echo '{command}' >> /root/.bashrc" diff --git a/internal/tests/pytests/run_metplus/test_run_metplus.py b/internal/tests/pytests/run_metplus/test_run_metplus.py index 493af9e37c..592bf02b48 100644 --- a/internal/tests/pytests/run_metplus/test_run_metplus.py +++ b/internal/tests/pytests/run_metplus/test_run_metplus.py @@ -35,13 +35,20 @@ def test_run_metplus_exists(): @pytest.mark.parametrize( 'command, expected_return_code', [ - ([RUN_METPLUS], 2), - ([RUN_METPLUS, EXAMPLE_CONF, 'config.OUTPUT_BASE=/path/to'], 2), + # no arguments + ([RUN_METPLUS], 1), + # no arguments after deprecated config args are removed + ([RUN_METPLUS, "--config", "-config", "-c"], 1), + # help flag requests usage statement and returns success + ([RUN_METPLUS, "-h"], 0), + ([RUN_METPLUS, "-help"], 0), + ([RUN_METPLUS, "--help"], 0), + ([RUN_METPLUS, EXAMPLE_CONF, 'config.OUTPUT_BASE=/path/to'], 3), ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, OUTPUT_BASE_OVERRIDE], 0), ([RUN_METPLUS, '-c', EXAMPLE_CONF, MINIMUM_CONF, OUTPUT_BASE_OVERRIDE], 0), ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, LIST_CONFIG_OVERRIDE_1], 0), ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, LIST_CONFIG_OVERRIDE_2], 0), - ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, '--fake-arg'], 1), + ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, '--fake-arg'], 2), ] ) @pytest.mark.run_metplus diff --git a/internal/tests/pytests/util/config_metplus/test_config_metplus.py b/internal/tests/pytests/util/config_metplus/test_config_metplus.py index 911f4c704d..7592dcc18b 100644 --- a/internal/tests/pytests/util/config_metplus/test_config_metplus.py +++ b/internal/tests/pytests/util/config_metplus/test_config_metplus.py @@ -905,3 +905,23 @@ def test_get_raw_keep_double_slash(metplus_config): config.set('config', 'URMA_ANLY_URL', TEST_URL) config.set('config', 'DATA_INGEST_1_INPUT_TEMPLATE', '{URMA_ANLY_URL}') assert config.getraw('config', 'DATA_INGEST_1_INPUT_TEMPLATE', keep_double_slash=True) == TEST_URL + +@pytest.mark.util +def test_nocheck_getters(metplus_config): + config = metplus_config + getstr_check_val = config.getstr('config', 'GETSTR_CHECK_VAL', 'getstr_check_default') + getstr_nocheck_val = config.getstr_nocheck('config', 'GETSTR_NO_CHECK_VAL', 'getstr_nocheck_default') + getdir_check_val = config.getdir('GETDIR_CHECK_VAL', 'getdir_check_default') + getdir_nocheck_val = config.getdir_nocheck('GETDIR_NO_CHECK_VAL', 'getdir_nocheck_default') + + # ensure default value is returned and set for config variable that did not exist + # new default value should not be used - config variable should be set in the previous get call + assert getstr_check_val == config.getstr('config', 'GETSTR_CHECK_VAL', 'getstr_check_default2') + assert getdir_check_val == config.getdir('GETDIR_CHECK_VAL', 'getdir_check_default2') + + # ensure default value is returned but not set for config variable that did not exist + # new default value should be used and therefore not match the previous get call + assert getstr_nocheck_val == 'getstr_nocheck_default' + assert getstr_nocheck_val != config.get('config', 'GETSTR_NO_CHECK_VAL', 'getstr_nocheck_default2') + assert getdir_nocheck_val == 'getdir_nocheck_default' + assert getdir_nocheck_val != config.get('config', 'GETDIR_NO_CHECK_VAL', 'getdir_nocheck_default2') diff --git a/metplus/produtil/config.py b/metplus/produtil/config.py deleted file mode 100644 index ffffa40621..0000000000 --- a/metplus/produtil/config.py +++ /dev/null @@ -1,1675 +0,0 @@ -"""!Parses UNIX conf files and makes the result readily available - -The produtil.config module reads configuration information for a -production system from one or more *.conf files, via the Python -ConfigParser module. This module also automatically fills in certain -information, such as fields calculated from the tcvitals or date. The -result is accessible via the ProdConfig class, which provides many -ways of automatically accessing configuration options.""" - -##@var __all__ -# decides what symbols are imported by "from produtil.config import *" -__all__=['from_file','confwalker','ProdConfig','ENVIRONMENT','ProdTask'] - -import collections,re,os,logging,threading -import os.path,sys -import datetime -import metplus.produtil.fileop as fileop - -import configparser -from configparser import ConfigParser -from io import StringIO - -from metplus.produtil.datastore import Datastore,Task - -from metplus.produtil.numerics import to_datetime, to_datetime_rel, fcst_hr_min -from string import Formatter -from configparser import NoOptionError,NoSectionError - -UNSPECIFIED=object() - -class DuplicateTaskName(Exception): - """!Raised when more than one task is registered with the same - name in an ProdConfig object.""" - -######################################################################## - -class Environment(object): - """!returns environment variables, allowing substitutions - - This class is used to read (but not write) environment variables - and provide default values if an environment variable is unset or - blank. It is only meant to be used in string formats, by passing - ENV=ENVIRONMENT. There is a global constant in this module, - ENVIRONMENT, which is an instance of this class. You should never - need to instantiate another one.""" - def __contains__(self,s): - """!Determines if __getitem__ will return something (True) or - raise KeyError (False). Same as "s in os.environ" unless s - contains "|-", in which case, the result is True.""" - return s.find('|-')>=0 or s in os.environ - def __getitem__(self,s): - """!Same as os.environ[s] unless s contains "|-". - ENVIRONMENT["VARNAME|-substitute"] - will return os.environ[VARNAME] if VARNAME is defined and - non-empty in os.environ. Otherwise, it will return - "substitute".""" - if not s: return '' - i=s.find('|-') - if i<0: return os.environ[s] - var=s[0:i] - sub=s[(i+2):] - val=os.environ.get(var,'') - if val!='': return val - return sub - -## @var ENVIRONMENT -# an Environment object. You should never need to instantiate another one. -ENVIRONMENT=Environment() - -class ConfFormatter(Formatter): - """!Internal class that implements ProdConfig.strinterp() - - This class is part of the implementation of ProdConfig: it is - used to interpolate strings using a syntax similar to - string.format(), but it allows recursion in the config sections, - and it also is able to use the [config] and [dir] sections as - defaults for variables not found in the current section.""" - def __init__(self,quoted_literals=False): - """!Constructor for ConfFormatter""" - super(ConfFormatter,self).__init__() - if quoted_literals: - self.format=self.slow_format - self.vformat=self.slow_vformat - self.parse=qparse - - @property - def quoted_literals(self): - return self.parse==qparse - - def slow_format(self,format_string,*args,**kwargs): - return self.vformat(format_string,args,kwargs) - def slow_vformat(self,format_string,args,kwargs): - out=StringIO() - for literal_text, field_name, format_spec, conversion in \ - self.parse(format_string): - if literal_text: - out.write(literal_text) - if not field_name: - continue - (obj, used_key) = self.get_field(field_name,args,kwargs) - if obj is None and used_key: - obj=self.get_value(used_key,args,kwargs) - value=obj - if conversion=='s': - value=str(value) - elif conversion=='r': - value=repr(value) - elif conversion: - raise ValueError('Unknown conversion %s'%(repr(conversion),)) - if format_spec: - value=value.__format__(format_spec) - out.write(value) - ret=out.getvalue() - out.close() - assert(ret is not None) - assert(isinstance(ret,str)) - return ret - - def get_value(self,key,args,kwargs): - """!Return the value of variable, or a substitution. - - Never call this function. It is called automatically by - str.format. It provides the value of an variable, - or a string substitution. - @param key the string key being analyzed by str.format() - @param args the indexed arguments to str.format() - @param kwargs the keyword arguments to str.format()""" - kwargs['__depth']+=1 - if kwargs['__depth']>=configparser.MAX_INTERPOLATION_DEPTH: - raise configparser.InterpolationDepthError(kwargs['__key'], - kwargs['__section'],key) - try: - if isinstance(key,int): - return args[key] - v = _get_v(key, kwargs) - if isinstance(v,str) and (v.find('{')>=0 or v.find('%')>=0): - vnew=self.vformat(v,args,kwargs) - assert(vnew is not None) - return vnew - return v - finally: - kwargs['__depth']-=1 - - -def _get_v(key, kwargs): - if key in kwargs: - return kwargs[key] - if ('__taskvars' in kwargs and kwargs['__taskvars'] - and key in kwargs['__taskvars']): - return kwargs['__taskvars'][key] - - isec = key.find('/') - if isec >= 0: - section = key[0:isec] - nkey = key[(isec + 1):] - if not section: - section = kwargs.get('__section', None) - if nkey: - key = nkey - else: - section = kwargs.get('__section', None) - conf = kwargs.get('__conf', None) - v = NOTFOUND - if section is None or conf is None: - return v - if conf.has_option(section, key): - return conf.get(section, key, raw=True) - if conf.has_option(section, '@inc'): - for osec in conf.get(section, '@inc').split(','): - if conf.has_option(osec, key): - v = conf.get(osec, key, raw=True) - if v is not NOTFOUND: - return v - - if conf.has_option('config', key): - return conf.get('config', key, raw=True) - if conf.has_option('dir', key): - return conf.get('dir', key, raw=True) - raise KeyError(key) - - -def qparse(format_string): - """!Replacement for Formatter.parse which can be added to Formatter objects - to turn {'...'} and {"..."} blocks into literal strings (the ... part). - Apply this by doing f=Formatter() ; f.parse=qparse. """ - if not format_string: return [] - if not isinstance(format_string, str): - raise TypeError('iterparse expects a str, not a %s %s'%( - type(format_string).__name__,repr(format_string))) - result=list() - literal_text='' - for m in re.finditer(r'''(?xs) ( - \{ \' (?P (?: \' (?! \} ) | [^'] )* ) \' \} - | \{ \" (?P (?: \" (?! \} ) | [^"] )* ) \" \} - | (?P - \{ - (?P - [^\}:!\['"\{] [^\}:!\[]* - (?: \. [a-zA-Z_]\w+ - | \[ [^\]]+ \] )* - ) - (?: ! (?P[rs]) )? - (?: : - (?P - (?: [^\{\}]+ - | \{[^\}]*\} )* - ) - )? - \} ) - | (?P \{\{ ) - | (?P \}\} ) - | (?P [^\{\}]+ ) - | (?P . ) ) ''',format_string): - if m.group('qescape'): - literal_text+=m.group('qescape') - elif m.group('dqescape'): - literal_text+=m.group('dqescape') - elif m.group('left_set'): - literal_text+='{' - elif m.group('right_set'): - literal_text+='}' - elif m.group('literal_text'): - literal_text+=m.group('literal_text') - elif m.group('replacement_field'): - result.append( ( literal_text, - m.group('field_name'), - m.group('format_spec'), - m.group('conversion') ) ) - literal_text='' - elif not m.group('error'): - continue - if m.group('error')=='{': - raise ValueError("Single '{' encountered in format string") - elif m.group('error')=='}': - raise ValueError("Single '}' encountered in format string") - else: - raise ValueError("Unexpected %s in format string"%( - repr(m.group('error')),)) - if literal_text: - result.append( ( literal_text, None, None, None ) ) - return result - -######################################################################## - -##@var FCST_KEYS -# the list of forecast time keys recognized by ConfTimeFormatter -FCST_KEYS={ 'fYMDHM':'%Y%m%d%H%M', 'fYMDH':'%Y%m%d%H', 'fYMD':'%Y%m%d', - 'fyear':'%Y', 'fYYYY':'%Y', 'fYY':'%y', 'fCC':'%C', 'fcen':'%C', - 'fmonth':'%m', 'fMM':'%m', 'fday':'%d', 'fDD':'%d', 'fhour':'%H', - 'fcyc':'%H', 'fHH':'%H', 'fminute':'%M', 'fmin':'%M' } -"""A list of keys recognized by ConfTimeFormatter if the key is -requested during string interpolation, and the key is not in the -relevant section. This list of keys represents the forecast time. It -is a dict mapping from the key name to the format sent to -datetime.datetime.strftime to generate the string value.""" - -##@var ANL_KEYS -# the list of analysis time keys recognized by ConfTimeFormatter -ANL_KEYS={ 'aYMDHM':'%Y%m%d%H%M', 'aYMDH':'%Y%m%d%H', 'aYMD':'%Y%m%d', - 'ayear':'%Y', 'aYYYY':'%Y', 'aYY':'%y', 'aCC':'%C', 'acen':'%C', - 'amonth':'%m', 'aMM':'%m', 'aday':'%d', 'aDD':'%d', 'ahour':'%H', - 'acyc':'%H', 'aHH':'%H', 'aminute':'%M', 'amin':'%M' } -"""A list of keys recognized by ConfTimeFormatter if the key is -requested during string interpolation, and the key is not in the -relevant section. This list of keys represents the analysis time. It -is a dict mapping from the key name to the format sent to -datetime.datetime.strftime to generate the string value.""" - -##@var M6_KEYS -# the list of analysis time ( -6h ) keys recognized by ConfTimeFormatter -ANL_M6_KEYS={ 'am6YMDHM':'%Y%m%d%H%M', 'am6YMDH':'%Y%m%d%H', 'am6YMD':'%Y%m%d', - 'am6year':'%Y', 'am6YYYY':'%Y', 'am6YY':'%y', 'am6CC':'%C', 'am6cen':'%C', - 'am6month':'%m', 'am6MM':'%m', 'am6day':'%d', 'am6DD':'%d', 'am6hour':'%H', - 'am6cyc':'%H', 'am6HH':'%H', 'am6minute':'%M', 'am6min':'%M' } -"""A list of keys recognized by ConfTimeFormatter if the key is -requested during string interpolation, and the key is not in the -relevant section. This list of keys represents the analysis time. It -is a dict mapping from the key name to the format sent to -datetime.datetime.strftime to generate the string value.""" - -##@var P6_KEYS -# the list of analysis time ( +6h ) keys recognized by ConfTimeFormatter -ANL_P6_KEYS={ 'ap6YMDHM':'%Y%m%d%H%M', 'ap6YMDH':'%Y%m%d%H', 'ap6YMD':'%Y%m%d', - 'ap6year':'%Y', 'ap6YYYY':'%Y', 'ap6YY':'%y', 'ap6CC':'%C', 'ap6cen':'%C', - 'ap6month':'%m', 'ap6MM':'%m', 'ap6day':'%d', 'ap6DD':'%d', 'ap6hour':'%H', - 'ap6cyc':'%H', 'ap6HH':'%H', 'ap6minute':'%M', 'ap6min':'%M' } -"""A list of keys recognized by ConfTimeFormatter if the key is -requested during string interpolation, and the key is not in the -relevant section. This list of keys represents the analysis time. It -is a dict mapping from the key name to the format sent to -datetime.datetime.strftime to generate the string value.""" - -##@var TIME_DIFF_KEYS -# the list of "forecast time minus analysis time" keys recognized by -# ConfTimeFormatter -TIME_DIFF_KEYS=set(['fahr','famin','fahrmin']) -"""A list of keys recognized by ConfTimeFormatter if the key is -requested during string interpolation, and the key is not in the -relevant section. This list of keys represents the time difference -between the forecast and analysis time. Unlike FCST_KEYS and -ANL_KEYS, this is not a mapping: it is a set.""" - -##@var NOTFOUND -# a special constant that represents a key not being found -NOTFOUND=object() - -class ConfTimeFormatter(ConfFormatter): - """!internal function that implements time formatting - - Like its superclass, ConfFormatter, this class is part of the - implementation of ProdConfig, and is used to interpolate strings - in a way similar to string.format(). It works the same way as - ConfFormatter, but accepts additional keys generated based on the - forecast and analysis times: - - fYMDHM - 201409171200 = forecast time September 17, 2014 at 12:00 UTC - fYMDH - 2014091712 - fYMD - 20140917 - fyear - 2014 - fYYYY - 2014 - fYY - 14 (year % 100) - fCC - 20 (century) - fcen - 20 - fmonth - 09 - fMM - 09 - fday - 17 - fDD - 17 - fhour - 12 - fcyc - 12 - fHH - 12 - fminute - 00 - fmin - 00 - - Replace the initial "f" with "a" for analysis times. In addition, - the following are available for the time difference between - forecast and analysis time. Suppose the forecast is twenty-three - hours and nineteen minutes (23:19) after the analysis time: - - fahr - 23 - famin - 1399 ( = 23*60+19) - fahrmin - 19 """ - def __init__(self,quoted_literals=False): - """!constructor for ConfTimeFormatter""" - super(ConfTimeFormatter,self).__init__( - quoted_literals=bool(quoted_literals)) - def get_value(self,key,args,kwargs): - """!return the value of a variable, or a substitution - - Never call this function. It is called automatically by - str.format. It provides the value of an variable, - or a string substitution. - @param key the string key being analyzed by str.format() - @param args the indexed arguments to str.format() - @param kwargs the keyword arguments to str.format()""" - v=NOTFOUND - kwargs['__depth']+=1 - if kwargs['__depth']>=configparser.MAX_INTERPOLATION_DEPTH: - raise configparser.InterpolationDepthError( - kwargs['__key'],kwargs['__section'],v) - try: - if isinstance(key,int): - return args[key] - if key in kwargs: - v=kwargs[key] - elif '__taskvars' in kwargs \ - and kwargs['__taskvars'] \ - and key in kwargs['__taskvars']: - v=kwargs['__taskvars'][key] - elif '__ftime' in kwargs and key in FCST_KEYS: - v=kwargs['__ftime'].strftime(FCST_KEYS[key]) - elif '__atime' in kwargs and key in ANL_KEYS: - v=kwargs['__atime'].strftime(ANL_KEYS[key]) - elif '__atime' in kwargs and key in ANL_M6_KEYS: - am6=kwargs['__atime']-datetime.timedelta(0,3600*6) - v=am6.strftime(ANL_M6_KEYS[key]) - elif '__atime' in kwargs and key in ANL_P6_KEYS: - ap6=kwargs['__atime']+datetime.timedelta(0,3600*6) - v=ap6.strftime(ANL_P6_KEYS[key]) - elif '__ftime' in kwargs and '__atime' in kwargs and \ - key in TIME_DIFF_KEYS: - (ihours,iminutes)=fcst_hr_min( - kwargs['__ftime'],kwargs['__atime']) - if key=='fahr': - v=int(ihours) - elif key=='famin': - v=int(ihours*60+iminutes) - elif key=='fahrmin': - v=int(iminutes) - else: - v=int(ihours*60+iminutes) - else: - isec=key.find('/') - if isec>=0: - section=key[0:isec] - nkey=key[(isec+1):] - if not section: - section=kwargs.get('__section',None) - if nkey: - key=nkey - else: - section=kwargs.get('__section',None) - conf=kwargs.get('__conf',None) - if section and conf: - if conf.has_option(section,key): - v=conf.get(section,key) - elif conf.has_option(section,'@inc'): - for osec in conf.get(section,'@inc').split(','): - if conf.has_option(osec,key): - v=conf.get(osec,key) - if v is NOTFOUND: - if conf.has_option('config',key): - v=conf.get('config',key) - elif conf.has_option('dir',key): - v=conf.get('dir',key) - if v is NOTFOUND: - raise KeyError('Cannot find key %s in section %s' - %(repr(key),repr(section))) - - if isinstance(v,str) and ( v.find('{')!=-1 or - v.find('%')!=-1 ): - try: - vnew=self.vformat(v,args,kwargs) - assert(vnew is not None) - return vnew - except KeyError as e: - # Seriously, does the exception's class name - # really need to be this long? - raise ConfigParser.InterpolationMissingOptionError( - kwargs['__key'],kwargs['__section'],v,str(e)) - return v - finally: - kwargs['__depth']-=1 - -######################################################################## - -def confwalker(conf,start,selector,acceptor,recursevar): - """!walks through a ConfigParser-like object performing some action - - Recurses through a ConfigParser-like object "conf" starting at - section "start", performing a specified action. The special - variable whose name is in recursevar specifies a list of - additional sections to recurse into. No section will be processed - more than once, and sections are processed in breadth-first order. - For each variable seen in each section (including recursevar), - this will call selector(sectionname, varname) to see if the - variable should be processed. If selector returns True, then - acceptor(section, varname, value) will be called. - - @param conf the ConfigParser-like object - @param start the starting section - @param selector a function selector(section,option) that decides - if an option needs processing (True) or not (False) - @param acceptor a function acceptor(section,option,value) - run on all options for which the selector returns True - @param recursevar an option in each section that lists more - sections the confwalker should touch. If the selector returns - True for the recursevar, then the recursevar will be sent to - the acceptor. However, it will be scanned for sections to - recurse into even if the selector rejects it.""" - touched=set() - requested=[str(start)] - while len(requested)>0: - sec=requested.pop(0) - if sec in touched: - continue - touched.add(sec) - for (key,val) in conf.items(sec): - if selector(sec,key): - acceptor(sec,key,val) - if key==recursevar: - for sec2 in reversed(val.split(',')): - trim=sec2.strip() - if len(trim)>0 and trim not in touched: - requested.append(trim) - -######################################################################## - -def from_file(filename,quoted_literals=False): - """!Reads the specified conf file into an ProdConfig object. - - Creates a new ProdConfig object and instructs it to read the specified file. - @param filename the path to the file that is to be read - @return a new ProdConfig object""" - if not isinstance(filename,str): - raise TypeError('First input to produtil.config.from_file must be a string.') - conf=ProdConfig(quoted_literals=bool(quoted_literals)) - conf.read(filename) - return conf - -def from_string(confstr,quoted_literals=False): - """!Reads the given string as if it was a conf file into an ProdConfig object - - Creates a new ProdConfig object and reads the string data into it - as if it was a config file - @param confstr the config data - @return a new ProdConfig object""" - if not isinstance(confstr,str): - raise TypeError('First input to produtil.config.from_string must be a string.') - conf=ProdConfig(quoted_literals=bool(quoted_literals)) - conf.readstr(confstr) - return conf - -class ProdConfig(object): - """!a class that contains configuration information - - This class keeps track of configuration information for all tasks - in a running model. It can be used in a read-only manner as - if it was a ConfigParser object. All ProdTask objects require an - ProdConfig object to keep track of registered task names via the - register_task_name method, the current forecast cycle (cycle - property) and the Datastore object (datastore property). - - This class should never be instantiated directly. Instead, you - should use the produtil.config.from_string or produtil.config.from_file to - read configuration information from an in-memory string or a file.""" - - def __init__(self,conf=None,quoted_literals=False,strict=False, inline_comment_prefixes=(';',)): - """!ProdConfig constructor - - Creates a new ProdConfig object. - @param conf the underlying configparser.ConfigParser object - that stores the actual config data. This was a SafeConfigParser - in Python 2 but in Python 3 the SafeConfigParser is now ConfigParser. - @param quoted_literals if True, then {'...'} and {"..."} will - be interpreted as quoting the contained ... text. Otherwise, - those blocks will be considered errors. - @param strict set default to False so it will not raise - DuplicateOptionError or DuplicateSectionError, This param was - added when ported to Python 3.6, to maintain the previous - python 2 behavior. - @param inline_comment_prefixes, defaults set to ;. This param was - added when ported to Python 3.6, to maintain the previous - python 2 behavior. - - Note: In Python 2, conf was ConfigParser.SafeConfigParser. In - Python 3.2, the old ConfigParser class was removed in favor of - SafeConfigParser which has in turn been renamed to ConfigParser. - Support for inline comments is now turned off by default and - section or option duplicates are not allowed in a single - configuration source.""" - self._logger=logging.getLogger('prodconfig') - self._lock=threading.RLock() - self._formatter=ConfFormatter(bool(quoted_literals)) - self._time_formatter=ConfTimeFormatter(bool(quoted_literals)) - self._datastore=None - self._tasknames=set() - # Added strict=False and inline_comment_prefixes for Python 3, - # so everything works as it did before in Python 2. - #self._conf=ConfigParser(strict=False, inline_comment_prefixes=(';',)) if (conf is None) else conf - self._conf=ConfigParser(strict=strict, inline_comment_prefixes=inline_comment_prefixes) if (conf is None) else conf - self._conf.optionxform=str - - self._conf.add_section('config') - self._conf.add_section('dir') - self._fallback_callbacks=list() - - @property - def quoted_literals(self): - return self._time_formatter.quoted_literals and \ - self._formatter.quoted_literals - - def fallback(self,name,details): - """!Asks whether the specified fallback is allowed. May perform - other tasks, such as alerting the operator. - - Calls the list of functions sent to add_fallback_callback. - Each one receives the result of the last, and the final result - at the end is returned. Note that ALL of the callbacks are - called, even if one returns False; this is not a short-circuit - operation. This is done to allow all reporting methods report - to their operator and decide whether the fallback is allowed. - - Each function called is f(allow,name,details) where: - - - allow = True or False, whether the callbacks called thus far - have allowed the fallback. - - - name = The short name of the fallback. - - - details = A long, human-readable description. May be - several lines long. - - @param name the name of the emergency situation - - @warning This function may take seconds or minutes to return. - It could perform cpu- or time-intensive operations such as - emailing an operator. - - """ - allow=self.getbool('config','allow_fallbacks',False) - for fc in self._fallback_callbacks: - allow=bool(fc(allow,name,details)) - return allow - - def add_fallback_callback(self,function): - """!Appends a function to the list of fallback callback functions - called by fallback() - - Appends the given function to the list that fallback() - searches while determining if a workflow emergency fallback - option is allowed. - - @param function a function f(allow,name,details) - @see fallbacks()""" - self._fallback_callbacks.append(function) - - def readstr(self,source): - - """!read config data and add it to this object - - Given a string with conf data in it, parses the data. - @param source the data to parse - @return self""" - fp=StringIO(str(source)) - self._conf.readfp(fp) - fp.close() - return self - - def from_args(self,args=None,allow_files=True,allow_options=True, - rel_path=None,verbose=False): - """!Given a list of arguments, usually from sys.argv[1:], reads - configuration files or sets option values. - - Reads list of strings of these formats: - - - /path/to/file.conf --- A configuration file to read. - - - section.option=value --- A configuration option to set in a - specified section. - - Will read files in the order listed, and then will override - options in the order listed. Note that specified options - override those read from files. Also, later files override - earlier files. - - @param args Typically argv[1:] or some other list of - arguments. - - @param allow_files If True, filenames are allowed in args. - Otherwise, they are ignored. - - @param allow_options If True, specified options - (section.name=value) are allowed. Otherwise they are detected - and ignored. - - @param rel_path Any filenames that are relative will be - relative to this path. If None or unspecified, the current - working directory as of the entry to this function is used. - - @returns self - """ - allow_files=bool(allow_files) - allow_options=bool(allow_options) - verbose=bool(verbose) - logger=self.log() - if allow_files: - if rel_path is None: - rel_path=os.getcwd() - else: - rel_path=str(rel_path) - elif not allow_options: - # Nothing to do! - return self - infiles=list() - moreopt=collections.defaultdict(dict) - for arg in args: - if not isinstance(arg,str): - raise TypeError( - 'In produtil.ProdConfig.from_args(), the args argument must ' - 'be an iterable of strings. It contained an invalid %s %s ' - 'instead.'%(type(arg).__name__,repr(arg))) - if verbose: logger.info(arg) - m=re.match(r'''(?x) - (?P
[a-zA-Z]\w*) - \.(?P