Feature #2858 produtil - remove sqlite3 dependency

.github/jobs/get_use_case_commands.py

@@ -6,6 +6,7 @@
 
 import sys
 import os
+import shutil
 
 # add METplus directory to sys path so the test suite can be found
 METPLUS_TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
@@ -54,7 +55,7 @@ def handle_automation_env(host_name, reqs):
     # if not using docker (automation),
     # return no setup commands and python embedding argument to command
     if host_name != 'docker':
-        if 'py_embed' in reqs and conda_env != METPLUS_BASE_ENV:
+        if _met_python_exe_should_be_set(reqs, conda_env):
             return '', 'user_env_vars.MET_PYTHON_EXE=python3'
         return '', ''
 
@@ -70,51 +71,24 @@ def handle_automation_env(host_name, reqs):
         _add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT')
     ]
 
-    # add conda bin to beginning of PATH
-    python_dir = os.path.join('/usr', 'local', 'conda', 'envs',
-                              conda_env_w_ext, 'bin')
-    python_path = os.path.join(python_dir, 'python3')
-    setup_env.append(_add_to_bashrc(f'export PATH={python_dir}:$PATH'))
-
+    # get path to conda bin unless metplus_base (default) is used
+    if conda_env != METPLUS_BASE_ENV:
+        python_dir = os.path.join('/usr', 'local', 'conda', 'envs',
+                                  conda_env_w_ext, 'bin')
+        python_path = os.path.join(python_dir, 'python3')
+        # add python path to PATH in bashrc so run_metplus.py is called with that version of Python
+        setup_env.append(_add_to_bashrc(f'export PATH={python_dir}:$PATH'))
+    else:
+        python_path = shutil.which('python3')
+        if python_path is None:
+            raise RuntimeError('python3 not found in PATH; cannot determine base-image Python executable')
     # if py_embed listed in requirements and using a Python
     # environment that differs from the MET env, set MET_PYTHON_EXE
-    if 'py_embed' in reqs and conda_env != METPLUS_BASE_ENV:
+    py_embed_arg = ''
+    if _met_python_exe_should_be_set(reqs, conda_env):
         py_embed_arg = f'user_env_vars.MET_PYTHON_EXE={python_path} '
-    else:
-        py_embed_arg = ''
 
-    # get METplus version to determine which version of
-    # METplotpy/METcalcpy/METdataio to use
-    # If stable release, get main branch, otherwise get develop
-    metplus_version = get_metplus_version()
-
-    # if any metplotpy/metcalcpy keywords are in requirements list,
-    # add command to obtain and install METplotpy and METcalcpy
-    components = []
-    if any([item for item in PLOTCALC_KEYWORDS if item in str(reqs).lower()]):
-        components.extend(('METplotpy', 'METcalcpy'))
-
-    # if metdataio is in requirements list, add command to obtain METdataio
-    if 'metdataio' in str(reqs).lower():
-        components.append('METdataio')
-
-    if components:
-        setup_env.append(f'cd {METPLUS_DOCKER_LOC}/..')
-        for component in components:
-            # get branch if defined, otherwise determine from METplus version
-            version = os.environ.get(f'INPUT_{component.upper()}_BRANCH')
-            if not version:
-                version = get_component_version(input_component='METplus',
-                                                input_version=metplus_version,
-                                                output_component=component,
-                                                output_format='main_v{X}.{Y}',
-                                                get_dev=False)
-            setup_env.extend((
-                'git --version',
-                f'git clone --single-branch --branch {version} https://github.com/dtcenter/{component}',
-                f'{python_path} -m pip install --no-deps {METPLUS_DOCKER_LOC}/../{component}',
-            ))
-        setup_env.append('cd -')
+    _handle_metplus_analysis_components(reqs, python_path, setup_env)
 
     # if metplus is in requirements list,
     # add top of METplus repo to PYTHONPATH so metplus can be imported
@@ -135,6 +109,64 @@ def handle_automation_env(host_name, reqs):
 
     return ';'.join(setup_env), py_embed_arg
 
+def _met_python_exe_should_be_set(reqs, conda_env):
+    return 'py_embed' in reqs and conda_env != METPLUS_BASE_ENV
+
+def _handle_metplus_analysis_components(reqs, python_path, setup_env):
+    """
+    Handles the addition of METplus analysis components (METplotpy, METcalcpy, METdataio)
+    to the setup environment based on specific requirements. This function checks whether
+    the provided requirements contain specific keywords or component names and, based on
+    the presence of these, constructs commands to clone and install the appropriate versions
+    of these components.
+
+    @param reqs List of software requirements that may include keywords indicating the need
+        for METplotpy, METcalcpy, or METdataio.
+
+    @param python_path Path to the Python executable that should be used for installing
+        the required components.
+
+    @param setup_env List representing the setup environment configuration. The function
+        appends the appropriate commands to this list for setting up the required components.
+
+    @returns None
+    """
+    # if any metplotpy/metcalcpy keywords are in requirements list,
+    # add command to obtain and install METplotpy and METcalcpy
+    components = []
+    if any([item for item in PLOTCALC_KEYWORDS if item in str(reqs).lower()]):
+        components.extend(('METplotpy', 'METcalcpy'))
+
+    # if metdataio is in requirements list, add command to obtain METdataio
+    if 'metdataio' in str(reqs).lower():
+        components.append('METdataio')
+
+    # do not add any commands if no METplus Analysis components are needed
+    if not components:
+        return
+
+    # get METplus version to determine which version of
+    # METplotpy/METcalcpy/METdataio to use
+    # If stable release, get main branch, otherwise get develop
+    metplus_version = get_metplus_version()
+
+    # add commands to install METplus Analysis components in directory above METplus repo
+    setup_env.append(f'cd {METPLUS_DOCKER_LOC}/..')
+    for component in components:
+        # get branch if defined, otherwise determine from METplus version
+        version = os.environ.get(f'INPUT_{component.upper()}_BRANCH')
+        if not version:
+            version = get_component_version(input_component='METplus',
+                                            input_version=metplus_version,
+                                            output_component=component,
+                                            output_format='main_v{X}.{Y}',
+                                            get_dev=False)
+        setup_env.extend((
+            'git --version',
+            f'git clone --single-branch --branch {version} https://github.com/dtcenter/{component}',
+            f'{python_path} -m pip install --no-deps {METPLUS_DOCKER_LOC}/../{component}',
+        ))
+    setup_env.append('cd -')
 
 def _add_to_bashrc(command):
     return f"echo '{command}' >> /root/.bashrc"

internal/tests/pytests/run_metplus/test_run_metplus.py

@@ -35,13 +35,20 @@ def test_run_metplus_exists():
 
 @pytest.mark.parametrize(
     'command, expected_return_code', [
-        ([RUN_METPLUS], 2),
-        ([RUN_METPLUS, EXAMPLE_CONF, 'config.OUTPUT_BASE=/path/to'], 2),
+        # no arguments
+        ([RUN_METPLUS], 1),
+        # no arguments after deprecated config args are removed
+        ([RUN_METPLUS, "--config", "-config", "-c"], 1),
+        # help flag requests usage statement and returns success
+        ([RUN_METPLUS, "-h"], 0),
+        ([RUN_METPLUS, "-help"], 0),
+        ([RUN_METPLUS, "--help"], 0),
+        ([RUN_METPLUS, EXAMPLE_CONF, 'config.OUTPUT_BASE=/path/to'], 3),
         ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, OUTPUT_BASE_OVERRIDE], 0),
         ([RUN_METPLUS, '-c', EXAMPLE_CONF, MINIMUM_CONF, OUTPUT_BASE_OVERRIDE], 0),
         ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, LIST_CONFIG_OVERRIDE_1], 0),
         ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, LIST_CONFIG_OVERRIDE_2], 0),
-        ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, '--fake-arg'], 1),
+        ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, '--fake-arg'], 2),
     ]
 )
 @pytest.mark.run_metplus

internal/tests/pytests/util/config_metplus/test_config_metplus.py

@@ -905,3 +905,23 @@ def test_get_raw_keep_double_slash(metplus_config):
     config.set('config', 'URMA_ANLY_URL', TEST_URL)
     config.set('config', 'DATA_INGEST_1_INPUT_TEMPLATE', '{URMA_ANLY_URL}')
     assert config.getraw('config', 'DATA_INGEST_1_INPUT_TEMPLATE', keep_double_slash=True) == TEST_URL
+
+@pytest.mark.util
+def test_nocheck_getters(metplus_config):
+    config = metplus_config
+    getstr_check_val = config.getstr('config', 'GETSTR_CHECK_VAL', 'getstr_check_default')
+    getstr_nocheck_val = config.getstr_nocheck('config', 'GETSTR_NO_CHECK_VAL', 'getstr_nocheck_default')
+    getdir_check_val = config.getdir('GETDIR_CHECK_VAL', 'getdir_check_default')
+    getdir_nocheck_val = config.getdir_nocheck('GETDIR_NO_CHECK_VAL', 'getdir_nocheck_default')
+
+    # ensure default value is returned and set for config variable that did not exist
+    # new default value should not be used - config variable should be set in the previous get call
+    assert getstr_check_val == config.getstr('config', 'GETSTR_CHECK_VAL', 'getstr_check_default2')
+    assert getdir_check_val == config.getdir('GETDIR_CHECK_VAL', 'getdir_check_default2')
+
+    # ensure default value is returned but not set for config variable that did not exist
+    # new default value should be used and therefore not match the previous get call
+    assert getstr_nocheck_val == 'getstr_nocheck_default'
+    assert getstr_nocheck_val != config.get('config', 'GETSTR_NO_CHECK_VAL', 'getstr_nocheck_default2')
+    assert getdir_nocheck_val == 'getdir_nocheck_default'
+    assert getdir_nocheck_val != config.get('config', 'GETDIR_NO_CHECK_VAL', 'getdir_nocheck_default2')