diff --git a/eessi/testsuite/common_config.py b/eessi/testsuite/common_config.py index 44fbe0dd..71022668 100644 --- a/eessi/testsuite/common_config.py +++ b/eessi/testsuite/common_config.py @@ -42,8 +42,11 @@ def set_common_required_config(site_configuration: dict, set_memory: bool = True :param site_configuration: site configuration dictionary :param set_memory: whether to set memory resources """ - environments = [{'name': 'default'}] - environs = ['default'] + environments = [ + {'name': 'EESSI-2023.06', 'modules': ['EESSI/2023.06']}, + {'name': 'EESSI-2025.06', 'modules': ['EESSI/2025.06']}, + ] + environs = ['EESSI-2023.06', 'EESSI-2025.06'] use_nodes_option = True if set_memory: resources_memory = [{ diff --git a/eessi/testsuite/eessi_mixin.py b/eessi/testsuite/eessi_mixin.py index 314ad45e..6436dc9d 100644 --- a/eessi/testsuite/eessi_mixin.py +++ b/eessi/testsuite/eessi_mixin.py @@ -9,6 +9,8 @@ from reframe.core.pipeline import RegressionMixin as RegressionTestPlugin from reframe.utility.sanity import make_performance_function import reframe.utility.sanity as sn +from reframe.core.runtime import valid_sysenv_comb +from reframe import VERSION as reframe_version from eessi.testsuite import check_process_binding, hooks from eessi.testsuite.constants import COMPUTE_UNITS, DEVICE_TYPES, SCALES, TAGS @@ -36,7 +38,7 @@ class EESSI_Mixin(RegressionTestPlugin): That definition needs to be done 'on time', i.e. early enough in the execution of the ReFrame pipeline. Here, we list which class attributes must be defined by the child class, and by (the end of) what phase: - - Init phase: device_type, scale, module_name, bench_name + - Init phase: device_type, scale, module_info, bench_name - Setup phase: compute_unit, required_mem_per_node The child class may also overwrite the following attributes: @@ -80,8 +82,8 @@ class EESSI_Mixin(RegressionTestPlugin): # Note that the error for an empty parameter is a bit unclear for ReFrame 4.6.2, but that will hopefully improve # see https://github.com/reframe-hpc/reframe/issues/3254 - # If that improves: uncomment the following to force the user to set module_name - # module_name = parameter() + # If that improves: uncomment the following to force the user to set module_info + # module_info = parameter() def __init_subclass__(cls, **kwargs): " set default values for built-in ReFrame attributes " @@ -137,7 +139,7 @@ def mark_all_files_readonly(self): def EESSI_mixin_validate_init(self): """Check that all variables that have to be set for subsequent hooks in the init phase have been set""" # List which variables we will need/use in the run_after('init') hooks - var_list = ['device_type', 'scale', 'module_name', 'measure_memory_usage'] + var_list = ['device_type', 'scale', 'module_info', 'measure_memory_usage'] for var in var_list: if not hasattr(self, var): msg = "The variable '%s' should be defined in any test class that inherits" % var @@ -162,8 +164,6 @@ def EESSI_mixin_run_after_init(self): # Filter on which scales are supported by the partitions defined in the ReFrame configuration hooks.filter_supported_scales(self) - hooks.set_modules(self) - if self.require_buildenv_module: hooks.add_buildenv_module(self) @@ -174,7 +174,41 @@ def EESSI_mixin_run_after_init(self): err_msg = f"Invalid thread_binding value '{thread_binding}'. Valid values: 'true', 'compact', or 'false'." raise EESSIError(err_msg) - hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) + # Unpack module_info + sys, env, mod = self.module_info + self.valid_prog_environs = [env] + self.module_name = [mod] + + # Set modules + hooks.set_modules(self) + + # Checks reframe version, and if newer or equal to 4.10, use the new functionality + # that allows combining sys:part notation with +feat. + syspart_feat_supported = False + try: + import semver + if semver.VersionInfo.parse(reframe_version) >= semver.VersionInfo.parse("4.10.0"): + syspart_feat_supported = True + except ImportError: + pass + + # If we use reframe 4.10.0 or later, we can just set the valid system and the hook will + # append any relevant features. Otherwise, as a fallback, we set the features first + # (by calling the hook), then check if the sys:part combination from the find_modules triplet + # is in the list of valid combinations for the given features + if syspart_feat_supported: + self.valid_systems = [sys] + hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) + else: + # Filter by defice type. E.g. add features based on whether CUDA appears in the module name + hooks.filter_valid_systems_by_device_type(self, required_device_type=self.device_type) + + # Check if partitions returned by find_modules satisfy the current features/extras in valid_systems + valid_partitions = [part.fullname for part in valid_sysenv_comb(self.valid_systems, env)] + if sys in valid_partitions: + self.valid_systems = [sys] + else: + self.valid_systems = [] # Set scales as tags hooks.set_tag_scale(self) diff --git a/eessi/testsuite/tests/apps/MetalWalls.py b/eessi/testsuite/tests/apps/MetalWalls.py index a483e4a3..fc4c8cb5 100644 --- a/eessi/testsuite/tests/apps/MetalWalls.py +++ b/eessi/testsuite/tests/apps/MetalWalls.py @@ -51,7 +51,7 @@ class EESSI_MetalWalls_MW(MetalWallsCheck): # input files are downloaded readonly_files = [''] - module_name = parameter(find_modules('MetalWalls')) + module_info = parameter(find_modules('MetalWalls')) # For now, MetalWalls is being build for CPU targets only # compute_device = parameter([DEVICE_TYPES.CPU, DEVICE_TYPES.GPU]) compute_device = parameter([DEVICE_TYPES.CPU]) diff --git a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py index 32ae0994..7a80a565 100644 --- a/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py +++ b/eessi/testsuite/tests/apps/PyTorch/PyTorch_torchvision.py @@ -15,7 +15,7 @@ class EESSI_PyTorch_torchvision(rfm.RunOnlyRegressionTest, EESSI_Mixin): nn_model = parameter(['vgg16', 'resnet50', 'resnet152', 'densenet121', 'mobilenet_v3_large']) parallel_strategy = parameter([None, 'ddp']) # Both torchvision and PyTorch-bundle modules have everything needed to run this test - module_name = parameter(chain(find_modules('torchvision'), find_modules('PyTorch-bundle'))) + module_info = parameter(chain(find_modules('torchvision'), find_modules('PyTorch-bundle'))) executable = 'python' time_limit = '30m' readonly_files = ['get_free_socket.py', 'pytorch_synthetic_benchmark.py'] diff --git a/eessi/testsuite/tests/apps/QuantumESPRESSO.py b/eessi/testsuite/tests/apps/QuantumESPRESSO.py index eb51abef..52c3c710 100644 --- a/eessi/testsuite/tests/apps/QuantumESPRESSO.py +++ b/eessi/testsuite/tests/apps/QuantumESPRESSO.py @@ -40,7 +40,7 @@ @rfm.simple_test class EESSI_QuantumESPRESSO_PW(QEspressoPWCheck, EESSI_Mixin): time_limit = '30m' - module_name = parameter(find_modules('QuantumESPRESSO')) + module_info = parameter(find_modules('QuantumESPRESSO')) # For now, QE is built for CPU targets only device_type = parameter([DEVICE_TYPES.CPU]) readonly_files = [''] diff --git a/eessi/testsuite/tests/apps/cp2k/cp2k.py b/eessi/testsuite/tests/apps/cp2k/cp2k.py index 53de55d5..78e8605d 100644 --- a/eessi/testsuite/tests/apps/cp2k/cp2k.py +++ b/eessi/testsuite/tests/apps/cp2k/cp2k.py @@ -17,7 +17,7 @@ class EESSI_CP2K(rfm.RunOnlyRegressionTest, EESSI_Mixin): ('QS/H2O-512', -8808.1439, 1e-4), ], fmt=lambda x: x[0], loggable=True) - module_name = parameter(find_modules('CP2K')) + module_info = parameter(find_modules('CP2K')) scale = parameter(SCALES.keys()) executable = 'cp2k.popt' diff --git a/eessi/testsuite/tests/apps/espresso/espresso.py b/eessi/testsuite/tests/apps/espresso/espresso.py index d1e9a56c..d231b9ae 100644 --- a/eessi/testsuite/tests/apps/espresso/espresso.py +++ b/eessi/testsuite/tests/apps/espresso/espresso.py @@ -36,7 +36,7 @@ def filter_scales(): class EESSI_ESPRESSO_base(rfm.RunOnlyRegressionTest): - module_name = parameter(find_modules('^ESPResSo$')) + module_info = parameter(find_modules('^ESPResSo$')) device_type = DEVICE_TYPES.CPU compute_unit = COMPUTE_UNITS.CPU time_limit = '300m' diff --git a/eessi/testsuite/tests/apps/gromacs.py b/eessi/testsuite/tests/apps/gromacs.py index ca64265b..2a0630f8 100644 --- a/eessi/testsuite/tests/apps/gromacs.py +++ b/eessi/testsuite/tests/apps/gromacs.py @@ -49,7 +49,7 @@ def set_device_type(self): class EESSI_GROMACS(EESSI_GROMACS_base, EESSI_Mixin): scale = parameter(SCALES.keys()) time_limit = '30m' - module_name = parameter(find_modules('GROMACS')) + module_info = parameter(find_modules('GROMACS')) # input files are downloaded readonly_files = [''] # executable_opts in addition to those set by the hpctestlib diff --git a/eessi/testsuite/tests/apps/lammps/lammps.py b/eessi/testsuite/tests/apps/lammps/lammps.py index ba6e3e32..4c13449e 100644 --- a/eessi/testsuite/tests/apps/lammps/lammps.py +++ b/eessi/testsuite/tests/apps/lammps/lammps.py @@ -53,7 +53,7 @@ class EESSI_LAMMPS_base(rfm.RunOnlyRegressionTest): device_type = parameter([DEVICE_TYPES.CPU, DEVICE_TYPES.GPU]) # Parameterize over all modules that start with LAMMPS - module_name = parameter(find_modules('LAMMPS')) + module_info = parameter(find_modules('LAMMPS')) all_readonly_files = True is_ci_test = True diff --git a/eessi/testsuite/tests/apps/lbmpy-pssrt/lbmpy-pssrt.py b/eessi/testsuite/tests/apps/lbmpy-pssrt/lbmpy-pssrt.py index 1125721b..5b1d0570 100644 --- a/eessi/testsuite/tests/apps/lbmpy-pssrt/lbmpy-pssrt.py +++ b/eessi/testsuite/tests/apps/lbmpy-pssrt/lbmpy-pssrt.py @@ -48,7 +48,7 @@ class EESSI_lbmpy_pssrt(rfm.RunOnlyRegressionTest, EESSI_Mixin): launcher = 'local' # no MPI module is loaded in this test - module_name = parameter(find_modules('lbmpy-pssrt')) + module_info = parameter(find_modules('lbmpy-pssrt')) readonly_files = ['mixing_layer_2D.py'] diff --git a/eessi/testsuite/tests/apps/lpc3d/lpc3d.py b/eessi/testsuite/tests/apps/lpc3d/lpc3d.py index ed972a41..b3250450 100644 --- a/eessi/testsuite/tests/apps/lpc3d/lpc3d.py +++ b/eessi/testsuite/tests/apps/lpc3d/lpc3d.py @@ -44,7 +44,7 @@ class EESSI_LPC3D(rfm.RunOnlyRegressionTest, EESSI_Mixin): launcher = 'local' # no MPI module is loaded in this test - module_name = parameter(find_modules('LPC3D')) + module_info = parameter(find_modules('LPC3D')) readonly_files = ['lattice_gas.inpt', 'pore_dens_freq_2neg.txt', 'psd.txt'] diff --git a/eessi/testsuite/tests/apps/numpy/numpy.py b/eessi/testsuite/tests/apps/numpy/numpy.py index 8532fa80..82b0229d 100644 --- a/eessi/testsuite/tests/apps/numpy/numpy.py +++ b/eessi/testsuite/tests/apps/numpy/numpy.py @@ -24,7 +24,7 @@ class EESSI_NumPy(rfm.RunOnlyRegressionTest, EESSI_Mixin): executable = './np_ops.py' time_limit = '30m' readonly_files = ['np_ops.py'] - module_name = parameter(find_modules('SciPy-bundle')) + module_info = parameter(find_modules('SciPy-bundle')) device_type = DEVICE_TYPES.CPU compute_unit = COMPUTE_UNITS.NODE scale = parameter([ diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index ad646893..ebf94790 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -87,7 +87,7 @@ class EESSI_OPENFOAM_LID_DRIVEN_CAVITY_64M(rfm.RunOnlyRegressionTest, EESSI_Mixi time_limit = '120m' readonly_files = [''] device_type = parameter([DEVICE_TYPES.CPU]) - module_name = parameter(find_modules('OpenFOAM/v', name_only=False)) + module_info = parameter(find_modules('OpenFOAM/v', name_only=False)) valid_systems = ['*'] scale = parameter(filter_scales_64M()) diff --git a/eessi/testsuite/tests/apps/osu.py b/eessi/testsuite/tests/apps/osu.py index 9d7418f0..d6b44af3 100644 --- a/eessi/testsuite/tests/apps/osu.py +++ b/eessi/testsuite/tests/apps/osu.py @@ -57,7 +57,7 @@ def filter_scales_coll(): class EESSI_OSU_Base(osu_benchmark): """ base class for OSU tests """ time_limit = '30m' - module_name = parameter(find_modules('OSU-Micro-Benchmarks')) + module_info = parameter(find_modules('OSU-Micro-Benchmarks')) used_cpus_per_task = 1 # reset num_tasks_per_node from the hpctestlib: we handle it ourselves diff --git a/eessi/testsuite/tests/apps/tensorflow/tensorflow.py b/eessi/testsuite/tests/apps/tensorflow/tensorflow.py index a3b68bbe..09070f0d 100644 --- a/eessi/testsuite/tests/apps/tensorflow/tensorflow.py +++ b/eessi/testsuite/tests/apps/tensorflow/tensorflow.py @@ -19,7 +19,7 @@ class EESSI_TensorFlow(rfm.RunOnlyRegressionTest, EESSI_Mixin): # Parameterize over all modules that start with TensorFlow - module_name = parameter(find_modules('TensorFlow')) + module_info = parameter(find_modules('TensorFlow')) # Make CPU and GPU versions of this test device_type = parameter([DEVICE_TYPES.CPU, DEVICE_TYPES.GPU]) diff --git a/eessi/testsuite/tests/apps/walberla/walberla.py b/eessi/testsuite/tests/apps/walberla/walberla.py index 8112d8c2..9fbdcc85 100644 --- a/eessi/testsuite/tests/apps/walberla/walberla.py +++ b/eessi/testsuite/tests/apps/walberla/walberla.py @@ -44,7 +44,7 @@ class EESSI_WALBERLA_BACKWARD_FACING_STEP(rfm.RunOnlyRegressionTest, EESSI_Mixin time_limit = '30m' readonly_files = [''] device_type = parameter([DEVICE_TYPES.CPU]) - module_name = parameter(find_modules('waLBerla')) + module_info = parameter(find_modules('waLBerla')) valid_systems = ['*'] scale = parameter(filter_scales()) diff --git a/eessi/testsuite/utils.py b/eessi/testsuite/utils.py index ef32b7c0..e203d3ff 100644 --- a/eessi/testsuite/utils.py +++ b/eessi/testsuite/utils.py @@ -6,13 +6,14 @@ import os import re import sys -from typing import Iterator, List +from typing import Iterator, List, Tuple import reframe as rfm from reframe.core.exceptions import ReframeFatalError from reframe.core.logging import getlogger import reframe.core.runtime as rt from reframe.frontend.printer import PrettyPrinter +from reframe.utility import find_modules as rf_find_modules from eessi.testsuite.constants import DEVICE_TYPES @@ -145,66 +146,13 @@ def get_avail_modules() -> List[str]: return _available_modules -def find_modules(regex: str, name_only=True) -> Iterator[str]: +def find_modules(substr, environ_mapping=None) -> Iterator[Tuple[str, str, str]]: """ - Return all modules matching the regular expression regex. Note that since we use re.search, - a module matches if the regex matches the module name at any place. I.e. the match does - not have to be at the start of the smodule name - - Arguments: - - regex: a regular expression - - name_only: regular expressions will only be matched on the module name, not the version (default: True). - - Note: the name_only feature assumes anything after the last forward '/' is the version, - and strips that before doing a match. - - Example - - Suppose we have the following modules on a system: - - gompic/2022a - gompi/2022a - CGAL/4.14.3-gompi-2022a - - The following calls would return the following respective modules - - find_modules('gompi') => [gompic/2022a, gompi/2022a] - find_modules('gompi$') => [gompi/2022a] - find_modules('gompi', name_only = False) => [gompic/2022a, gompi/2022a, CGAL/4.14.3-gompi-2022a] - find_modules('^gompi', name_only = False) => [gompic/2022a, gompi/2022a] - find_modules('^gompi/', name_only = False) => [gompi/2022a] - find_modules('-gompi-2022a', name_only = False) => [CGAL/4.14.3-gompi-2022a] - + Wraps reframe.utility.find_modules in order to provide caching, so that we don't have to do repeated + module avail calls. """ - - if not isinstance(regex, str): - raise TypeError("'substr' argument must be a string") - - seen = set() - dupes = [] - for mod in get_avail_modules(): - # The thing we yield should always be the original module name (orig_mod), including version - orig_mod = mod - if name_only: - # Remove trailing slashes from the regex (in case the callee forgot) - regex = regex.rstrip('/') - # Remove part after the last forward slash, as we assume this is the version - mod = re.sub('/[^/]*$', '', mod) - # Match the actual regular expression - log(f"Matching module {mod} with regex {regex}") - if re.search(regex, mod): - log("Match!") - if orig_mod in seen: - dupes.append(orig_mod) - else: - seen.add(orig_mod) - yield orig_mod - - if dupes: - err_msg = "EESSI test-suite cannot handle duplicate modules. " - err_msg += "Please make sure that only one is available on your system. " - err_msg += f"The following modules have a duplicate on your system: {dupes}" - raise ValueError(err_msg) + # TODO: implement caching to make this function more efficient + return rf_find_modules(substr, environ_mapping) def get_tc_hierarchy(tcdict):