From 7e09c0dc35e3cb865bb188343b59b48744ab37e7 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Wed, 29 Nov 2023 11:04:06 +0100 Subject: [PATCH 01/10] Skip module sanity checking when module resolution is off --- reframe/core/modules.py | 58 ++++++++++++++++++++++++++++++----------- reframe/core/systems.py | 7 ++--- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 5fc41cac00..056e04320e 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -106,22 +106,22 @@ class ModulesSystem: module_map = fields.TypedField(types.Dict[str, types.List[str]]) @classmethod - def create(cls, modules_kind=None): + def create(cls, modules_kind=None, module_resolution=True): getlogger().debug(f'Initializing modules system {modules_kind!r}') if modules_kind is None or modules_kind == 'nomod': return ModulesSystem(NoModImpl()) elif modules_kind == 'tmod31': - return ModulesSystem(TMod31Impl()) + return ModulesSystem(TMod31Impl(module_resolution)) elif modules_kind == 'tmod': - return ModulesSystem(TModImpl()) + return ModulesSystem(TModImpl(module_resolution)) elif modules_kind == 'tmod32': - return ModulesSystem(TModImpl()) + return ModulesSystem(TModImpl(module_resolution)) elif modules_kind == 'tmod4': - return ModulesSystem(TMod4Impl()) + return ModulesSystem(TMod4Impl(module_resolution)) elif modules_kind == 'lmod': - return ModulesSystem(LModImpl()) + return ModulesSystem(LModImpl(module_resolution)) elif modules_kind == 'spack': - return ModulesSystem(SpackImpl()) + return ModulesSystem(SpackImpl(module_resolution)) else: raise ConfigError('unknown module system: %s' % modules_kind) @@ -585,7 +585,12 @@ class TModImpl(ModulesSystemImpl): MIN_VERSION = (3, 2) - def __init__(self): + def __init__(self, module_resolution=True): + if not module_resolution: + # The module system may not be available locally + self._version = None + return + # Try to figure out if we are indeed using the TCL version try: completed = osext.run_command('modulecmd -V') @@ -717,7 +722,14 @@ class TMod31Impl(TModImpl): MIN_VERSION = (3, 1) - def __init__(self): + def __init__(self, module_resolution=True): + self._version = None + self._command = None + + if not module_resolution: + # The module system may not be available locally + return + # Try to figure out if we are indeed using the TCL version try: modulecmd = os.getenv('MODULESHOME') @@ -792,7 +804,13 @@ class TMod4Impl(TModImpl): MIN_VERSION = (4, 1) - def __init__(self): + def __init__(self, module_resolution=True): + self._version = None + self._extra_module_paths = [] + if not module_resolution: + # The module system may not be available locally + return + try: completed = osext.run_command(self.modulecmd('-V'), check=True) except OSError as e: @@ -915,7 +933,13 @@ def searchpath_remove(self, *dirs): class LModImpl(TMod4Impl): '''Module system for Lmod (Tcl/Lua).''' - def __init__(self): + def __init__(self, module_resolution=True): + self._extra_module_paths = [] + self._version = None + if not module_resolution: + # The module system may not be available locally + return + # Try to figure out if we are indeed using LMOD self._lmod_cmd = os.getenv('LMOD_CMD') if self._lmod_cmd is None: @@ -945,8 +969,6 @@ def __init__(self): raise ConfigError('Python is not supported by ' 'this Lmod installation') - self._extra_module_paths = [] - def name(self): return 'lmod' @@ -1094,7 +1116,14 @@ class SpackImpl(ModulesSystemImpl): ''' - def __init__(self): + def __init__(self, module_resolution=True): + self._name_format = '{name}/{version}-{hash}' + self._version = None + + if not module_resolution: + # The module system may not be available locally + return + # Try to figure out if we are indeed using the TCL version try: completed = osext.run_command('spack -V') @@ -1103,7 +1132,6 @@ def __init__(self): 'could not find a sane Spack installation') from e self._version = completed.stdout.strip() - self._name_format = '{name}/{version}-{hash}' def name(self): return 'spack' diff --git a/reframe/core/systems.py b/reframe/core/systems.py index 53ed8ad40b..8fef375a4c 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -462,12 +462,12 @@ class System(jsonext.JSONSerializable): def __init__(self, name, descr, hostnames, modules_system, preload_env, prefix, outputdir, - resourcesdir, stagedir, partitions): + resourcesdir, stagedir, partitions, module_resolution): getlogger().debug(f'Initializing system {name!r}') self._name = name self._descr = descr self._hostnames = hostnames - self._modules_system = ModulesSystem.create(modules_system) + self._modules_system = ModulesSystem.create(modules_system, module_resolution) self._preload_env = preload_env self._prefix = prefix self._outputdir = outputdir @@ -590,7 +590,8 @@ def create(cls, site_config): outputdir=site_config.get('systems/0/outputdir'), resourcesdir=site_config.get('systems/0/resourcesdir'), stagedir=site_config.get('systems/0/stagedir'), - partitions=partitions + partitions=partitions, + module_resolution=site_config.get('general/resolve_module_conflicts') ) @property From 8fcb7d8ab7bf6134b1e34d8a03ea938ab9ab2e50 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Wed, 29 Nov 2023 11:10:51 +0100 Subject: [PATCH 02/10] Fix long lines --- reframe/core/systems.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/reframe/core/systems.py b/reframe/core/systems.py index 8fef375a4c..d3377f186f 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -467,7 +467,8 @@ def __init__(self, name, descr, hostnames, modules_system, self._name = name self._descr = descr self._hostnames = hostnames - self._modules_system = ModulesSystem.create(modules_system, module_resolution) + self._modules_system = ModulesSystem.create(modules_system, + module_resolution) self._preload_env = preload_env self._prefix = prefix self._outputdir = outputdir @@ -591,7 +592,9 @@ def create(cls, site_config): resourcesdir=site_config.get('systems/0/resourcesdir'), stagedir=site_config.get('systems/0/stagedir'), partitions=partitions, - module_resolution=site_config.get('general/resolve_module_conflicts') + module_resolution=site_config.get( + 'general/resolve_module_conflicts' + ) ) @property From 5641948abca1815e54b08d017e639d9085210c5c Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Wed, 29 Nov 2023 11:16:56 +0100 Subject: [PATCH 03/10] Fix formatting --- reframe/core/modules.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 056e04320e..37bc923d69 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -586,9 +586,9 @@ class TModImpl(ModulesSystemImpl): MIN_VERSION = (3, 2) def __init__(self, module_resolution=True): + self._version = None if not module_resolution: # The module system may not be available locally - self._version = None return # Try to figure out if we are indeed using the TCL version @@ -725,7 +725,6 @@ class TMod31Impl(TModImpl): def __init__(self, module_resolution=True): self._version = None self._command = None - if not module_resolution: # The module system may not be available locally return @@ -1119,7 +1118,6 @@ class SpackImpl(ModulesSystemImpl): def __init__(self, module_resolution=True): self._name_format = '{name}/{version}-{hash}' self._version = None - if not module_resolution: # The module system may not be available locally return From 28863cff23c279d96f0386b7700c9a2a4d889e45 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 30 Nov 2023 10:47:03 +0100 Subject: [PATCH 04/10] Small refactoring and add docs --- docs/config_reference.rst | 5 +++++ docs/manpage.rst | 5 +++++ reframe/core/systems.py | 14 +++++++------- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 7f5774b185..21cc81c106 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1749,6 +1749,11 @@ General Configuration .. versionadded:: 3.6.0 + .. versionchanged:: 4.5.0 + + When the module conflict resolution is off, ReFrame will not check if the module system is available and sane where it is running. + It will only emit the module commands in the build and run scripts. + .. py:attribute:: general.save_log_files diff --git a/docs/manpage.rst b/docs/manpage.rst index 7be39a9016..1537cec9a2 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1632,6 +1632,11 @@ Whenever an environment variable is associated with a configuration option, its .. versionadded:: 3.6.0 + .. versionchanged:: 4.5.0 + + When the module conflict resolution is off, ReFrame will not check if the module system is available and sane where it is running. + It will only emit the module commands in the build and run scripts. + .. table:: :align: left diff --git a/reframe/core/systems.py b/reframe/core/systems.py index d3377f186f..3fad77be77 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -461,14 +461,14 @@ class System(jsonext.JSONSerializable): ''' def __init__(self, name, descr, hostnames, modules_system, - preload_env, prefix, outputdir, - resourcesdir, stagedir, partitions, module_resolution): + modules_system_validate, preload_env, prefix, outputdir, + resourcesdir, stagedir, partitions): getlogger().debug(f'Initializing system {name!r}') self._name = name self._descr = descr self._hostnames = hostnames self._modules_system = ModulesSystem.create(modules_system, - module_resolution) + modules_system_validate) self._preload_env = preload_env self._prefix = prefix self._outputdir = outputdir @@ -582,6 +582,9 @@ def create(cls, site_config): descr=site_config.get('systems/0/descr'), hostnames=site_config.get('systems/0/hostnames'), modules_system=site_config.get('systems/0/modules_system'), + modules_system_validate=site_config.get( + 'general/resolve_module_conflicts' + ), preload_env=Environment( name=f'__rfm_env_{sysname}', modules=site_config.get('systems/0/modules'), @@ -591,10 +594,7 @@ def create(cls, site_config): outputdir=site_config.get('systems/0/outputdir'), resourcesdir=site_config.get('systems/0/resourcesdir'), stagedir=site_config.get('systems/0/stagedir'), - partitions=partitions, - module_resolution=site_config.get( - 'general/resolve_module_conflicts' - ) + partitions=partitions ) @property From b62838cc83227d2f3e32f44e1056d93272cce683 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 30 Nov 2023 10:49:04 +0100 Subject: [PATCH 05/10] Small refactoring and add docs --- reframe/core/modules.py | 54 +++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 37bc923d69..853474bea2 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -106,25 +106,27 @@ class ModulesSystem: module_map = fields.TypedField(types.Dict[str, types.List[str]]) @classmethod - def create(cls, modules_kind=None, module_resolution=True): + def create(cls, modules_kind=None, validate=True): getlogger().debug(f'Initializing modules system {modules_kind!r}') - if modules_kind is None or modules_kind == 'nomod': - return ModulesSystem(NoModImpl()) - elif modules_kind == 'tmod31': - return ModulesSystem(TMod31Impl(module_resolution)) - elif modules_kind == 'tmod': - return ModulesSystem(TModImpl(module_resolution)) - elif modules_kind == 'tmod32': - return ModulesSystem(TModImpl(module_resolution)) - elif modules_kind == 'tmod4': - return ModulesSystem(TMod4Impl(module_resolution)) - elif modules_kind == 'lmod': - return ModulesSystem(LModImpl(module_resolution)) - elif modules_kind == 'spack': - return ModulesSystem(SpackImpl(module_resolution)) - else: + modules_impl = { + None: NoModImpl, + 'nomod': NoModImpl, + 'tmod31': TMod31Impl, + 'tmod': TModImpl, + 'tmod32': TModImpl, + 'tmod4': TMod4Impl, + 'lmod': LModImpl, + 'spack': SpackImpl + } + + try: + impl_cls = modules_impl[modules_kind] + except KeyError: raise ConfigError('unknown module system: %s' % modules_kind) + impl_cls.validate = True + return impl_cls() + def __init__(self, backend): self._backend = backend self.module_map = {} @@ -585,9 +587,9 @@ class TModImpl(ModulesSystemImpl): MIN_VERSION = (3, 2) - def __init__(self, module_resolution=True): + def __init__(self): self._version = None - if not module_resolution: + if not self.validate: # The module system may not be available locally return @@ -722,10 +724,10 @@ class TMod31Impl(TModImpl): MIN_VERSION = (3, 1) - def __init__(self, module_resolution=True): + def __init__(self): self._version = None self._command = None - if not module_resolution: + if not self.validate: # The module system may not be available locally return @@ -803,10 +805,10 @@ class TMod4Impl(TModImpl): MIN_VERSION = (4, 1) - def __init__(self, module_resolution=True): + def __init__(self): self._version = None self._extra_module_paths = [] - if not module_resolution: + if not self.validate: # The module system may not be available locally return @@ -932,10 +934,10 @@ def searchpath_remove(self, *dirs): class LModImpl(TMod4Impl): '''Module system for Lmod (Tcl/Lua).''' - def __init__(self, module_resolution=True): + def __init__(self): self._extra_module_paths = [] self._version = None - if not module_resolution: + if not self.validate: # The module system may not be available locally return @@ -1115,10 +1117,10 @@ class SpackImpl(ModulesSystemImpl): ''' - def __init__(self, module_resolution=True): + def __init__(self): self._name_format = '{name}/{version}-{hash}' self._version = None - if not module_resolution: + if not self.validate: # The module system may not be available locally return From cc1ad30b4d1adf524fabf36f2eb9a8edb47d6f20 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 30 Nov 2023 22:42:58 +0100 Subject: [PATCH 06/10] Fix unit tests --- reframe/core/modules.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 853474bea2..279baf1c7c 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -118,14 +118,13 @@ def create(cls, modules_kind=None, validate=True): 'lmod': LModImpl, 'spack': SpackImpl } - try: impl_cls = modules_impl[modules_kind] except KeyError: - raise ConfigError('unknown module system: %s' % modules_kind) + raise ConfigError(f'unknown module system: {modules_kind}') - impl_cls.validate = True - return impl_cls() + impl_cls.validate = validate + return ModulesSystem(impl_cls()) def __init__(self, backend): self._backend = backend @@ -177,7 +176,7 @@ def resolve_module(self, name): @property def backend(self): - return(self._backend) + return (self._backend) def available_modules(self, substr=None): '''Return a list of available modules that contain ``substr`` in their @@ -445,6 +444,7 @@ class ModulesSystemImpl(abc.ABC): :meta private: ''' + validate = True def execute(self, cmd, *args): '''Execute an arbitrary module command using the modules backend. @@ -590,7 +590,6 @@ class TModImpl(ModulesSystemImpl): def __init__(self): self._version = None if not self.validate: - # The module system may not be available locally return # Try to figure out if we are indeed using the TCL version @@ -728,7 +727,6 @@ def __init__(self): self._version = None self._command = None if not self.validate: - # The module system may not be available locally return # Try to figure out if we are indeed using the TCL version @@ -809,7 +807,6 @@ def __init__(self): self._version = None self._extra_module_paths = [] if not self.validate: - # The module system may not be available locally return try: @@ -938,7 +935,6 @@ def __init__(self): self._extra_module_paths = [] self._version = None if not self.validate: - # The module system may not be available locally return # Try to figure out if we are indeed using LMOD @@ -1121,7 +1117,6 @@ def __init__(self): self._name_format = '{name}/{version}-{hash}' self._version = None if not self.validate: - # The module system may not be available locally return # Try to figure out if we are indeed using the TCL version From 283301751b126fb99fee53761bab34cd1445068a Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Tue, 5 Dec 2023 10:37:22 +0100 Subject: [PATCH 07/10] Move docs --- docs/config_reference.rst | 10 +++++----- docs/manpage.rst | 5 ----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 21cc81c106..cf9c6b2a7f 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -149,6 +149,11 @@ System Configuration .. versionadded:: 3.4 The ``spack`` backend is added. + .. versionchanged:: 4.5.0 + + When the module conflict resolution is off, ReFrame will not check if the module system is available and sane where it is running. + It will only emit the module commands in the build and run scripts. + .. py:attribute:: systems.modules :required: No @@ -1749,11 +1754,6 @@ General Configuration .. versionadded:: 3.6.0 - .. versionchanged:: 4.5.0 - - When the module conflict resolution is off, ReFrame will not check if the module system is available and sane where it is running. - It will only emit the module commands in the build and run scripts. - .. py:attribute:: general.save_log_files diff --git a/docs/manpage.rst b/docs/manpage.rst index 1537cec9a2..7be39a9016 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -1632,11 +1632,6 @@ Whenever an environment variable is associated with a configuration option, its .. versionadded:: 3.6.0 - .. versionchanged:: 4.5.0 - - When the module conflict resolution is off, ReFrame will not check if the module system is available and sane where it is running. - It will only emit the module commands in the build and run scripts. - .. table:: :align: left From 0bc7615d2e85ac1b0820f8c9b6cdad597915626d Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Tue, 5 Dec 2023 13:28:14 +0100 Subject: [PATCH 08/10] postpone module validation instead of cancelling --- reframe/core/modules.py | 53 +++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index 279baf1c7c..adc4c0f30a 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -589,9 +589,13 @@ class TModImpl(ModulesSystemImpl): def __init__(self): self._version = None - if not self.validate: - return + if self.validate: + self._validate_installation() + self._has_validated = True + else: + self._has_validated = False + def _validate_installation(self): # Try to figure out if we are indeed using the TCL version try: completed = osext.run_command('modulecmd -V') @@ -641,6 +645,9 @@ def modulecmd(self, *args): return ' '.join(['modulecmd', 'python', *args]) def _execute(self, cmd, *args): + if not self._has_validated: + self._validate_installation() + modulecmd = self.modulecmd(cmd, *args) completed = osext.run_command(modulecmd) if re.search(r'\bERROR\b', completed.stderr) is not None: @@ -726,9 +733,14 @@ class TMod31Impl(TModImpl): def __init__(self): self._version = None self._command = None - if not self.validate: - return + if self.validate: + self._validate_installation() + self._has_validated = True + else: + self._has_validated = False + + def _validate_installation(self): # Try to figure out if we are indeed using the TCL version try: modulecmd = os.getenv('MODULESHOME') @@ -778,6 +790,9 @@ def modulecmd(self, *args): return ' '.join([self._command, *args]) def _execute(self, cmd, *args): + if not self._has_validated: + self._validate_installation() + modulecmd = self.modulecmd(cmd, *args) completed = osext.run_command(modulecmd) if re.search(r'\bERROR\b', completed.stderr) is not None: @@ -806,9 +821,13 @@ class TMod4Impl(TModImpl): def __init__(self): self._version = None self._extra_module_paths = [] - if not self.validate: - return + if self.validate: + self._validate_installation() + self._has_validated = True + else: + self._has_validated = False + def _validate_installation(self): try: completed = osext.run_command(self.modulecmd('-V'), check=True) except OSError as e: @@ -845,6 +864,9 @@ def modulecmd(self, *args): return ' '.join(['modulecmd', 'python', *args]) def _execute(self, cmd, *args): + if not self._has_validated: + self._validate_installation() + modulecmd = self.modulecmd(cmd, *args) completed = osext.run_command(modulecmd, check=False) namespace = {} @@ -934,9 +956,13 @@ class LModImpl(TMod4Impl): def __init__(self): self._extra_module_paths = [] self._version = None - if not self.validate: - return + if self.validate: + self._validate_installation() + self._has_validated = True + else: + self._has_validated = False + def _validate_installation(self): # Try to figure out if we are indeed using LMOD self._lmod_cmd = os.getenv('LMOD_CMD') if self._lmod_cmd is None: @@ -1116,9 +1142,13 @@ class SpackImpl(ModulesSystemImpl): def __init__(self): self._name_format = '{name}/{version}-{hash}' self._version = None - if not self.validate: - return + if self.validate: + self._validate_installation() + self._has_validated = True + else: + self._has_validated = False + def _validate_installation(self): # Try to figure out if we are indeed using the TCL version try: completed = osext.run_command('spack -V') @@ -1138,6 +1168,9 @@ def modulecmd(self, *args): return ' '.join(['spack', *args]) def _execute(self, cmd, *args): + if not self._has_validated: + self._validate_installation() + modulecmd = self.modulecmd(cmd, *args) completed = osext.run_command(modulecmd, check=True) return completed.stdout From c02c6b1decb53ae939d9390825174066be694da0 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 8 Dec 2023 00:19:38 +0100 Subject: [PATCH 09/10] Improve naming --- reframe/core/modules.py | 116 +++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 54 deletions(-) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index adc4c0f30a..53ec62f5b4 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -589,13 +589,11 @@ class TModImpl(ModulesSystemImpl): def __init__(self): self._version = None + self._validated = False if self.validate: - self._validate_installation() - self._has_validated = True - else: - self._has_validated = False + self._do_validate() - def _validate_installation(self): + def _do_validate(self): # Try to figure out if we are indeed using the TCL version try: completed = osext.run_command('modulecmd -V') @@ -620,8 +618,9 @@ def _validate_installation(self): if (ver_major, ver_minor) < self.MIN_VERSION: raise ConfigError( - 'unsupported TMod version: %s (required >= %s)' % - (version, self.MIN_VERSION)) + f'unsupported TMod version: ' + f'{version} (required >= {self.MIN_VERSION})' + ) self._version = version try: @@ -629,11 +628,15 @@ def _validate_installation(self): completed = osext.run_command(self.modulecmd()) except OSError as e: raise ConfigError( - 'could not get the Python bindings for TMod: ' % e) from e + f'could not get the Python bindings for TMod: {e}' + ) from e if re.search(r'Unknown shell type', completed.stderr): raise ConfigError( - 'Python is not supported by this TMod installation') + 'Python is not supported by this TMod installation' + ) + + self._validated = True def name(self): return 'tmod' @@ -645,8 +648,8 @@ def modulecmd(self, *args): return ' '.join(['modulecmd', 'python', *args]) def _execute(self, cmd, *args): - if not self._has_validated: - self._validate_installation() + if not self._validated: + self._do_validate() modulecmd = self.modulecmd(cmd, *args) completed = osext.run_command(modulecmd) @@ -733,14 +736,11 @@ class TMod31Impl(TModImpl): def __init__(self): self._version = None self._command = None - + self._validated = False if self.validate: - self._validate_installation() - self._has_validated = True - else: - self._has_validated = False + self._do_validate() - def _validate_installation(self): + def _do_validate(self): # Try to figure out if we are indeed using the TCL version try: modulecmd = os.getenv('MODULESHOME') @@ -748,7 +748,8 @@ def _validate_installation(self): completed = osext.run_command(modulecmd) except OSError as e: raise ConfigError( - 'could not find a sane TMod31 installation: %s' % e) from e + f'could not find a sane TMod31 installation: {e}' + ) from e version_match = re.search(r'Release Tcl (\S+)', completed.stderr, re.MULTILINE) @@ -766,22 +767,26 @@ def _validate_installation(self): if (ver_major, ver_minor) < self.MIN_VERSION: raise ConfigError( - 'unsupported TMod version: %s (required >= %s)' % - (version, self.MIN_VERSION)) + f'unsupported TMod version: {version} ' + f'(required >= {self.MIN_VERSION})' + ) self._version = version - self._command = '%s python' % modulecmd - + self._command = f'{modulecmd} python' try: # Try the Python bindings now completed = osext.run_command(self._command) except OSError as e: raise ConfigError( - 'could not get the Python bindings for TMod31: ' % e) from e + f'could not get the Python bindings for TMod31: {e}' + ) if re.search(r'Unknown shell type', completed.stderr): raise ConfigError( - 'Python is not supported by this TMod installation') + 'Python is not supported by this TMod installation' + ) + + self._validated = True def name(self): return 'tmod31' @@ -790,8 +795,8 @@ def modulecmd(self, *args): return ' '.join([self._command, *args]) def _execute(self, cmd, *args): - if not self._has_validated: - self._validate_installation() + if not self._validated: + self._do_validate() modulecmd = self.modulecmd(cmd, *args) completed = osext.run_command(modulecmd) @@ -821,21 +826,21 @@ class TMod4Impl(TModImpl): def __init__(self): self._version = None self._extra_module_paths = [] + self._validated = False if self.validate: - self._validate_installation() - self._has_validated = True - else: - self._has_validated = False + self._do_validate() - def _validate_installation(self): + def _do_validate(self): try: completed = osext.run_command(self.modulecmd('-V'), check=True) except OSError as e: raise ConfigError( - 'could not find a sane TMod4 installation') from e + 'could not find a sane TMod4 installation' + ) from e except SpawnedProcessError as e: raise ConfigError( - 'could not get the Python bindings for TMod4') from e + 'could not get the Python bindings for TMod4' + ) from e version_match = re.match(r'^Modules Release (\S+)\s+', completed.stderr) @@ -847,15 +852,18 @@ def _validate_installation(self): ver_major, ver_minor = [int(v) for v in version.split('.')[:2]] except ValueError: raise ConfigError( - 'could not parse TMod4 version string: ' + version) from None + f'could not parse TMod4 version string: {version}' + ) from None if (ver_major, ver_minor) < self.MIN_VERSION: raise ConfigError( - 'unsupported TMod4 version: %s (required >= %s)' % - (version, self.MIN_VERSION)) + f'unsupported TMod4 version: {version} ' + f'(required >= {self.MIN_VERSION})' + ) self._version = version self._extra_module_paths = [] + self._validated = True def name(self): return 'tmod4' @@ -864,8 +872,8 @@ def modulecmd(self, *args): return ' '.join(['modulecmd', 'python', *args]) def _execute(self, cmd, *args): - if not self._has_validated: - self._validate_installation() + if not self._validated: + self._do_validate() modulecmd = self.modulecmd(cmd, *args) completed = osext.run_command(modulecmd, check=False) @@ -956,13 +964,11 @@ class LModImpl(TMod4Impl): def __init__(self): self._extra_module_paths = [] self._version = None + self._validated = False if self.validate: - self._validate_installation() - self._has_validated = True - else: - self._has_validated = False + self._do_validate() - def _validate_installation(self): + def _do_validate(self): # Try to figure out if we are indeed using LMOD self._lmod_cmd = os.getenv('LMOD_CMD') if self._lmod_cmd is None: @@ -972,8 +978,7 @@ def _validate_installation(self): try: completed = osext.run_command(f'{self._lmod_cmd} --version') except OSError as e: - raise ConfigError( - 'could not find a sane Lmod installation: %s' % e) + raise ConfigError(f'could not find a sane Lmod installation: {e}') version_match = re.search(r'.*Version\s*(\S+)', completed.stderr, re.MULTILINE) @@ -986,12 +991,15 @@ def _validate_installation(self): completed = osext.run_command(self.modulecmd()) except OSError as e: raise ConfigError( - 'could not get the Python bindings for Lmod: ' % e) + f'could not get the Python bindings for Lmod: {e}' + ) if re.search(r'Unknown shell type', completed.stderr): raise ConfigError('Python is not supported by ' 'this Lmod installation') + self._validated = True + def name(self): return 'lmod' @@ -1142,21 +1150,21 @@ class SpackImpl(ModulesSystemImpl): def __init__(self): self._name_format = '{name}/{version}-{hash}' self._version = None + self._validated = False if self.validate: - self._validate_installation() - self._has_validated = True - else: - self._has_validated = False + self._do_validate() - def _validate_installation(self): + def _do_validate(self): # Try to figure out if we are indeed using the TCL version try: completed = osext.run_command('spack -V') except OSError as e: raise ConfigError( - 'could not find a sane Spack installation') from e + 'could not find a sane Spack installation' + ) from e self._version = completed.stdout.strip() + self._validated = True def name(self): return 'spack' @@ -1168,8 +1176,8 @@ def modulecmd(self, *args): return ' '.join(['spack', *args]) def _execute(self, cmd, *args): - if not self._has_validated: - self._validate_installation() + if not self._validated: + self._do_validate() modulecmd = self.modulecmd(cmd, *args) completed = osext.run_command(modulecmd, check=True) From 28b41a34b72c40371db374c4d643c97b44d1b928 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 8 Dec 2023 00:32:39 +0100 Subject: [PATCH 10/10] Expand and fine tune the docs --- docs/config_reference.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index cf9c6b2a7f..3b97160393 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -146,13 +146,16 @@ System Configuration - ``spack``: `Spack `__'s built-in mechanism for managing modules. - ``nomod``: This is to denote that no modules system is used by this system. - .. versionadded:: 3.4 + Normally, upon loading the configuration of the system ReFrame checks that a sane installation exists for the modules system requested and will issue an error if it fails to find one. + The modules system sanity check is skipped when the :attr:`~config.general.resolve_module_conflicts` is set to :obj:`False`. + This is useful in cases where the current system does not have a modules system but the remote partitions have one and you would like ReFrame to generate the module commands. + + .. versionadded:: 3.4 The ``spack`` backend is added. - .. versionchanged:: 4.5.0 + .. versionchanged:: 4.5.0 + The modules system sanity check is skipped when the :attr:`config.general.resolve_module_conflicts` is not set. - When the module conflict resolution is off, ReFrame will not check if the module system is available and sane where it is running. - It will only emit the module commands in the build and run scripts. .. py:attribute:: systems.modules