Manually merge PR244 to add support for new include_sections option in MiGserver.conf which will load SECTIONNAME into parsed configuration for any sectionname.conf files from the corresponding folder. Useful e.g. to keep rather involved cloud sections out of the main MiGserver.conf

git-svn-id: svn+ssh://svn.code.sf.net/p/migrid/code/trunk@6237 b75ad72c-e7d7-11dd-a971-7dbc132099af

Author: jonasbardino

mig/install/MiGserver-cloud_misty-template.conf

@@ -0,0 +1,47 @@
+[CLOUD_MISTY]
+# General cloud provider settings and limits
+service_name = MISTY
+service_title = MISTY-Cloud
+service_desc = A misty cloud service
+service_provider_flavor = openstack
+service_hosts = https://misty:5000/v3
+service_max_user_instances = 1
+service_max_user_instances_map = john.doe@nosuchplace.org=3
+# Semi-colon separated list of img=user login pairs when img and user differs
+service_user_map = rocky9=rocky;ubuntu-noble=ubuntu
+# Cloud instance defaults
+# The general structure is a default option and an optional user override map
+# service_flavor_id = INSERT CLOUD FLAVOR ID
+# Semi-colon separated list of user=flavor pairs to override for some users
+# service_flavor_id_map =
+# service_network_id = INSERT CLOUD NETWORK ID
+# Semi-colon separated list of user=net pairs to override for some users
+# service_network_id_map =
+# service_key_id = INSERT DEFAULT KEY ID
+# Semi-colon separated list of user=keyid pairs to override for some users
+# service_key_id_map =
+# service_sec_group_id = INSERT CLOUD SEC GROUP ID
+# Semi-colon separated list of user=secgrp pairs to override for some users
+# service_sec_group_id_map
+# service_floating_network_id = INSERT CLOUD FLOATING NETWORK ID
+# Semi-colon separated list of user=floatnet pairs to override for some users
+# service_floating_network_id_map =
+# service_availability_zone = INSERT CLOUD AVAILABILITY ZONE
+# Semi-colon separated list of user=availzone pairs to override for some users
+# service_availability_zone_map =
+# Optional jump host so that instances are shielded fom direct ssh access
+service_jumphost_address = jumpy01
+# Semi-colon separated list of user=jumpaddr pairs to override for some users
+# service_jumphost_address_map =
+service_jumphost_user = misty
+# Path to the ssh key used for managing user public keys on cloud jumphost
+service_jumphost_key = ~/.ssh/misty-jumphost-key
+# Semi-colon separated list of user=jumpuser pairs to override for some users
+# service_jumphost_user_map =
+# Helper to automatically add user pub keys on jumphost
+# The script and coding values are used like this under the hood:
+# ssh %(jumphost_user)s@%(jumphost_address)s %(jumphost_manage_keys_script)s add \
+#     %(jumphost_manage_keys_coding)s %(encoded_client_id)s %(encoded_pub_keys)s
+# where coding is applied to client_id and pub_keys to yield encoded_X versions
+service_jumphost_manage_keys_script = manage_misty_keys.py
+service_jumphost_manage_keys_coding = base16

mig/install/MiGserver-template.conf

@@ -400,6 +400,9 @@ scriptlanguages = sh python java
 jobtypes = batch interactive all
 lrmstypes = Native Native-execution-leader Batch Batch-execution-leader
 
+# Include any additional section confs files from a per-section conf folder
+include_sections = %(mig_server_home)s/MiGserver.d
+
 # Jupyter integration sections
 ### E.g. ###
 # [JUPYTER_DAG]

mig/server/MiGserver.d/.placeholder

@@ -65,14 +65,49 @@
         default_css_filename, keyword_any, keyword_auto, keyword_all, \
         keyword_file, keyword_env, cert_valid_days, oid_valid_days, \
         generic_valid_days, DEFAULT_USER_ID_FORMAT, valid_user_id_formats, \
-        valid_filter_methods, default_twofactor_auth_apps
+        valid_filter_methods, default_twofactor_auth_apps, \
+        mig_conf_section_dirname
     from mig.shared.logger import Logger, SYSLOG_GDP
     from mig.shared.htmlgen import menu_items, vgrid_items
     from mig.shared.fileio import read_file, load_json, write_file
 except ImportError as ioe:
     print("could not import migrid modules")
 
 
+def include_section_contents(logger, config, section, load_path, verbose=False):
+    """Include additional section contents from load_path in config."""
+    if not os.path.exists(load_path):
+        msg = "no such %r section config in %s" % (section, load_path)
+        if verbose:
+            print(msg)
+        logger.error(msg)
+        return False
+
+    logger.debug("include %r section from %s" % (section, load_path))
+    section_config = ConfigParser()
+    section_config.read([load_path])
+    if section not in section_config.sections():
+        msg = "missing required %r section in config %s" % (section, load_path)
+        if verbose:
+            print(msg)
+        logger.error(msg)
+        return False
+    other_sections = [i for i in section_config.sections() if not i == section]
+    if other_sections:
+        msg = "only %r section will be read from %s" % (section, load_path)
+        if verbose:
+            print(msg)
+        logger.warning(msg)
+    if section not in config.sections():
+        logger.debug("add %r section to main config" % section)
+        config.add_section(section)
+    for (key, val) in section_config.items(section):
+        logger.debug("add config key %r in %r section" % (key, section))
+        config.set(section, key, val)
+    logger.debug("done including %r section to main config" % section)
+    return True
+
+
 def expand_external_sources(logger, val):
     """Expand a string containing ENV::NAME, FILE::PATH or FILE::PATH$$CACHE
     references to fill in the content of the corresponding environment, file or
@@ -923,6 +958,45 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
         else:
             self.user_db_home = os.path.join(self.state_path, 'user_db_home')
 
+        # Allow section confs included from file
+        if config.has_option('GLOBAL', 'include_sections'):
+            self.include_sections = config.get('GLOBAL', 'include_sections')
+        else:
+            self.include_sections = os.path.join(self.mig_server_home,
+                                                 mig_conf_section_dirname)
+
+        # NOTE: for simplicity we do NOT allow overrides in GLOBAL section
+        no_override_sections = ['GLOBAL']
+        self.include_sections = os.path.normpath(self.include_sections)
+        if os.path.isdir(self.include_sections):
+            msg = "read extra config sections from %s" % self.include_sections
+            if verbose:
+                print(msg)
+            logger.info(msg)
+            for section_filename in os.listdir(self.include_sections):
+                # skip dotfiles and non-confs
+                if section_filename.startswith('.'):
+                    continue
+                if not section_filename.endswith('.conf'):
+                    msg = "%r is not on required sectionname.conf form" % \
+                        section_filename
+                    if verbose:
+                        print(msg)
+                    logger.warning(msg)
+                    continue
+                section_path = os.path.join(self.include_sections,
+                                            section_filename)
+                section = section_filename.replace('.conf', '').upper()
+                if section in no_override_sections:
+                    msg = "skip unsupported %r section override in %r" % \
+                        (section, section_filename)
+                    if verbose:
+                        print(msg)
+                    logger.warning(msg)
+                    continue
+                include_section_contents(logger, config, section, section_path,
+                                         verbose)
+
         if config.has_option('GLOBAL', 'admin_list'):
             # Parse semi-colon separated list of admins with optional spaces
             admins = config.get('GLOBAL', 'admin_list')
@@ -1669,15 +1743,12 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
         self.jupyter_services = []
         # Load generated jupyter sections
         for section in config.sections():
-            if 'JUPYTER_' in section:
+            if section.startswith('JUPYTER_'):
                 # Allow service_desc to be a file that should be read
                 if config.has_option(section, 'service_desc'):
-                    service_desc = config.get(section, 'service_desc')
-                    if os.path.exists(service_desc) \
-                            and os.path.isfile(service_desc):
-                        content = read_file(service_desc, logger)
-                        if content:
-                            config.set(section, 'service_desc', content)
+                    content = expand_external_sources(
+                        logger, config.get(section, 'service_desc'))
+                    config.set(section, 'service_desc', content)
 
                 self.jupyter_services.append({option: config.get(section,
                                                                  option)
@@ -1697,15 +1768,12 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
                              'service_jumphost_key']
         # Load generated cloud sections
         for section in config.sections():
-            if 'CLOUD_' in section:
+            if section.startswith('CLOUD_'):
                 # Allow service_desc to be a file that should be read
                 if config.has_option(section, 'service_desc'):
-                    service_desc = config.get(section, 'service_desc')
-                    if os.path.exists(service_desc) \
-                            and os.path.isfile(service_desc):
-                        content = read_file(service_desc, logger)
-                        if content:
-                            config.set(section, 'service_desc', content)
+                    content = expand_external_sources(
+                        logger, config.get(section, 'service_desc'))
+                    config.set(section, 'service_desc', content)
 
                 service = {option: config.get(section, option) for option in
                            config.options(section)}

mig/shared/configuration.py

@@ -220,6 +220,7 @@
 
 exe_leader_name = "execution-leader"
 
+mig_conf_section_dirname = "MiGserver.d"
 htaccess_filename = '.htaccess'
 welcome_filename = 'welcome.txt'
 default_mrsl_filename = '.default.mrsl'

mig/shared/defaults.py

@@ -2279,6 +2279,7 @@ def _generate_confs_writefiles(options, user_dict, insert_list=[], cleanup_list=
         ("trac-MiG-template.ini", "trac.ini"),
         ("logrotate-MiG-template", "logrotate-migrid"),
         ("MiGserver-template.conf", "MiGserver.conf"),
+        ("MiGserver-cloud_misty-template.conf", "MiGserver-cloud_misty.conf"),
         ("static-skin-template.css", "static-skin.css"),
         ("index-template.html", "index.html"),
         ("openssh-MiG-sftp-subsys-template.conf",
@@ -2404,6 +2405,15 @@ def _generate_confs_instructions(options, user_dict):
 and if Trac is enabled, the generated trac.ini to %(mig_code)s/server/:
 cp %(destination)s/trac.ini %(mig_code)s/server/
 
+If standalone configuration sections are useful to your site you can
+additionally create a suitable per-section include folder and put appropriately
+named section conf files there as shown here for the generated CLOUD_MISTY
+section in the generated MiGserver-cloud_misty.conf example:
+mkdir -p %(mig_code)s/MiGserver.d
+cp %(destination_dir)s/MiGserver-cloud_misty.conf \
+    %(mig_code)s/MiGserver.d/cloud_misty.conf
+such section conf files must accordingly match the lower-cased section name.
+
 On a Debian/Ubuntu MiG developer server the dedicated apache init script is
 added with:
 sudo cp %(destination)s/apache-%(user)s /etc/init.d/apache-%(user)s