Merge pull request #3548 from vkarak/feat/custom-config-imports

[feat] Facilitate local imports from configuration files

Author: vkarak

docs/config_reference.rst

@@ -2390,6 +2390,8 @@ Dynamic configuration
 
 One advantage of ReFrame's configuration is that it is programmable, especially if you are using the Python files.
 Since the configuration is loaded as a Python module, you can generate parts of the configuration dynamically.
+You can also import seamlessly other modules that reside inside the configuration directory.
+This is particularly useful when you define custom schedulers or :ref:`parallel launcher <custom-launchers>` backends.
 
 The YAML configuration on the other hand is more static, although not fully.
 Code generation can still be used with the YAML configuration as it is treated as a Jinja2 template, where ReFrame provides the following bindings:

reframe/core/config.py

@@ -343,7 +343,7 @@ def subconfig_system(self):
 
     def load_config_python(self, filename):
         try:
-            mod = util.import_module_from_file(filename)
+            mod = util.import_module_from_file(filename, load_parents=True)
         except ImportError as e:
             # import_module_from_file() may raise an ImportError if the
             # configuration file is under ReFrame's top-level directory
@@ -695,6 +695,7 @@ def load_config(*filenames):
             # The builtin configuration is always loaded at the beginning
             continue
 
+        f = os.path.abspath(f)
         getlogger().debug(f'Loading configuration file: {f!r}')
         _, ext = os.path.splitext(f)
         if ext == '.py':

reframe/frontend/loader.py

@@ -192,33 +192,9 @@ def load_from_file(self, filename, force=False):
             return []
 
         try:
-            dirname = os.path.dirname(filename)
-
-            # Load all parent modules of test file
-            parents = []
-            while os.path.exists(os.path.join(dirname, '__init__.py')):
-                parents.append(os.path.join(dirname))
-                dirname = os.path.split(dirname)[0]
-
-            parent_module = None
-            for pdir in reversed(parents):
-                with osext.change_dir(pdir):
-                    with util.temp_sys_path(pdir):
-                        package_path = os.path.join(pdir, '__init__.py')
-                        parent_module = util.import_module_from_file(
-                            package_path, parent=parent_module
-                        ).__name__
-
-            # Now load the actual test file
-            if not parents:
-                pdir = dirname
-
-            with osext.change_dir(pdir):
-                with util.temp_sys_path(pdir):
-                    return self.load_from_module(
-                        util.import_module_from_file(filename, force,
-                                                     parent_module)
-                    )
+            return self.load_from_module(util.import_module_from_file(
+                filename, force=force, load_parents=True
+            ))
         except Exception:
             exc_info = sys.exc_info()
             if not is_severe(*exc_info):

reframe/utility/__init__.py

@@ -71,18 +71,8 @@ def _do_import_module_from_file(filename, module_name=None):
     return module
 
 
-def import_module_from_file(filename, force=False, parent=None):
-    '''Import module from file.
-
-    If the file location refers to a directory, the contained ``__init__.py``
-    will be loaded. If the filename resolves to a location that is within the
-    current working directory, a module name will be derived from the supplied
-    file name and Python's :func:`importlib.import_module` will be invoked to
-    actually load the module. If the file location refers to a path outside
-    the current working directory, then the module will be loaded directly
-    from the file, but it will be assigned a mangled name in
-    :obj:`sys.modules`, to avoid clashes with other modules loaded using the
-    standard import mechanism.
+def _import_module_from_file(filename, force, parent):
+    '''Low-level non-recusrive import from file
 
     :arg filename: The path to the filename of a Python module.
     :arg force: Force reload of module in case it is already loaded.
@@ -91,9 +81,6 @@ def import_module_from_file(filename, force=False, parent=None):
         ``parent`` so that Python would be able to resolve relative imports in
         the module file.
     :returns: The loaded Python module.
-
-    .. versionchanged:: 4.6
-       The ``parent`` argument is added.
     '''
 
     # Expand and sanitize filename
@@ -130,6 +117,73 @@ def import_module_from_file(filename, force=False, parent=None):
     return importlib.import_module(module_name)
 
 
+def import_module_from_file(filename, *, force=False, load_parents=False):
+    '''Import module from file.
+
+    If the file location refers to a directory, the contained ``__init__.py``
+    will be loaded. If the filename resolves to a location that is within the
+    current working directory, a module name will be derived from the supplied
+    file name and Python's :func:`importlib.import_module` will be invoked to
+    actually load the module. If the file location refers to a path outside
+    the current working directory, then the module will be loaded directly
+    from the file, but it will be assigned a mangled name in
+    :obj:`sys.modules`, to avoid clashes with other modules loaded using the
+    standard import mechanism.
+
+    If ``load_parents`` is set, any modules along the path will also be
+    loaded. A path will considered as a parent module and loaded if it
+    contains an ``__init__.py`` file.
+
+    :arg filename: The path to the filename of a Python module.
+    :arg force: Force reload of module in case it is already loaded. This does
+        not apply to the parent modules that have been loaded with
+        ``load_parents=True``.
+    :arg load_parents: Load parent modules along the path.
+    :returns: The loaded Python module.
+
+    .. versionchanged:: 4.6
+       The ``parent`` argument is added.
+
+    .. versionchanged:: 4.9
+        The ``parent`` argumet is replaced by ``load_parents``. Also, all
+        arguments except ``filename`` are now keyword-only arguments.
+
+        If the old interface is desired, you should use directly the
+        lower-level :func:`_import_module_from_file` function.
+    '''
+
+    import reframe.utility.osext as osext
+
+
+    if not load_parents:
+        return _import_module_from_file(filename, force, None)
+
+    dirname = os.path.dirname(filename)
+
+    # Load all parent modules of test file
+    parents = []
+    while os.path.exists(os.path.join(dirname, '__init__.py')):
+        parents.append(os.path.join(dirname))
+        dirname = os.path.split(dirname)[0]
+
+    parent_module = None
+    for pdir in reversed(parents):
+        with osext.change_dir(pdir):
+            with temp_sys_path(pdir):
+                package_path = os.path.join(pdir, '__init__.py')
+                parent_module = _import_module_from_file(
+                    package_path, force, parent=parent_module
+                ).__name__
+
+    # Now load the actual test file
+    if not parents:
+        pdir = dirname
+
+    with osext.change_dir(pdir):
+        with temp_sys_path(pdir):
+            return _import_module_from_file(filename, force, parent_module)
+
+
 def import_module(module_name, force=False):
     '''Import a module.
 
@@ -162,7 +216,7 @@ def import_module(module_name, force=False):
     else:
         path += '.py'
 
-    return import_module_from_file(path, force)
+    return _import_module_from_file(path, force, None)
 
 
 def import_from_module(module_name, symbol):

unittests/test_config.py

@@ -109,7 +109,7 @@ def test_load_config_json(tmp_path):
     json_file = tmp_path / 'settings.json'
     json_file.write_text(json.dumps(settings.site_configuration, indent=4))
     site_config = config.load_config(json_file)
-    assert site_config.sources == ['<builtin>', json_file]
+    assert site_config.sources == ['<builtin>', f'{json_file}']
 
 
 def test_load_config_json_invalid_syntax(tmp_path):