Merge the changes introduces in PR #254

git-svn-id: svn+ssh://svn.code.sf.net/p/migrid/code/trunk@6244 b75ad72c-e7d7-11dd-a971-7dbc132099af

Author: rasmunk

README

@@ -413,6 +413,8 @@ settings:
 --ftps_pasv_ports=FTPS_PASV_PORTS
 --davs_address=DAVS_ADDRESS
 --jupyter_services=JUPYTER_SERVICES
+--jupyter_services_enable_proxy_https=JUPYTER_SERVICES_ENABLE_PROXY_HTTPS
+--jupyter_services_proxy_config=JUPYTER_SERVICES_PROXY_CONFIG
 --jupyter_services_desc=JUPYTER_SERVICES_DESC
 --cloud_fqdn=CLOUD_FQDN
 --cloud_services=CLOUD_SERVICES
@@ -798,6 +800,7 @@ integration for data analysis:
                    --enable_vhost_certs=True --enable_verify_certs=True \
                    --enable_notify=True --enable_jupyter=True \
                    --jupyter_services='DAG.https://dag002.science DAG.https://dag003.science DAG.https://dag004.science DAG.https://dag005.science DAG.https://dag006.science DAG.https://dag007.science DAG.https://dag008.science DAG.https://dag009.science DAG.https://dag010.science DAG.https://dag203.science DAG.https://dag204.science MODI.https://dag100.science' \
+                   --jupyter_services_proxy_config="{'DAG': {'SSLProxyCACertificateFile': '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'}, 'MODI': {'SSLProxyCACertificateFile': '/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem'}}"
                    --jupyter_services_desc="{'DAG': '/home/mig/state/wwwpublic/dag_desc.html', 'MODI': '/home/mig/state/wwwpublic/modi_desc.html'}" \
                    --enable_cloud=True --enable_migadmin=True \
                    --enable_peers=True --peers_mandatory=True \

mig/install/generateconfs.py

@@ -99,6 +99,8 @@ def main(argv, _generate_confs=generate_confs, _print=print):
         'ftps_pasv_ports',
         'davs_address',
         'jupyter_services',
+        'jupyter_services_enable_proxy_https',
+        'jupyter_services_proxy_config',
         'jupyter_services_desc',
         'cloud_fqdn',
         'cloud_services',

mig/shared/functionality/reqjupyterservice.py

@@ -54,11 +54,11 @@
 import random
 try:
     import requests
-except ImportError as err:
+except ImportError:
     requests = None
 
 from mig.shared import returnvalues
-from mig.shared.base import client_id_dir, extract_field
+from mig.shared.base import client_id_dir, extract_field, force_native_str
 from mig.shared.conf import get_configuration_object
 from mig.shared.defaults import session_id_bytes
 from mig.shared.fileio import make_symlink, pickle, unpickle, write_file, \
@@ -68,6 +68,7 @@
 from mig.shared.init import initialize_main_variables
 from mig.shared.pwcrypto import generate_random_ascii
 from mig.shared.ssh import generate_ssh_rsa_key_pair, tighten_key_perms
+from mig.shared.url import urljoin
 from mig.shared.workflows import create_workflow_session_id, \
     get_workflow_session_id
 
@@ -281,6 +282,23 @@ def jupyter_host(configuration, output_objects, user, url):
     return (output_objects, returnvalues.OK)
 
 
+def jupyterhub_session_post_request(session, url, params=None, **kwargs):
+    """
+    Sends a post request to a URL
+    :param session: the session object that can be used to conduct the post request
+    :param url: the designated URL that the post request is sent to
+    :param params: parameters to pass to the post request
+    :return: the response object from the post request
+    """
+    if not params:
+        params = {}
+
+    if "_xsrf" in session.cookies:
+        params["_xsrf"] = session.cookies['_xsrf']
+
+    return session.post(url, params=params, **kwargs)
+
+
 def reset(configuration):
     """Helper function to clean up all jupyter directories and mounts
     :param configuration: the MiG Configuration object
@@ -445,10 +463,13 @@ def main(client_id, user_arguments_dict):
     # Make sure ssh daemon does not complain
     tighten_key_perms(configuration, client_id)
 
-    url_base = '/' + service['service_name']
-    url_home = url_base + '/home'
-    url_auth = host + url_base + '/hub/login'
-    url_data = host + url_base + '/hub/set-user-data'
+    url_service = urljoin('/', service['service_name'])
+    url_home = urljoin(url_service + "/", 'home')
+
+    url_base = urljoin(host, service['service_name'])
+    url_hub = urljoin(url_base + "/", 'hub')
+    url_auth = urljoin(url_hub + "/", 'login')
+    url_data = urljoin(url_hub + "/", 'set-user-data')
 
     # Does the client home dir contain an active mount key
     # If so just keep on using it.
@@ -520,15 +541,12 @@ def main(client_id, user_arguments_dict):
 
         with requests.session() as session:
             # Refresh cookies
-            session.get(url_auth)
-            auth_params = {}
-            if "_xsrf" in session.cookies:
-                auth_params = {"_xsrf": session.cookies['_xsrf']}
+            session.get(url_hub)
             # Authenticate and submit data
-            response = session.post(url_auth, headers=auth_header, params=auth_params)
+            response = jupyterhub_session_post_request(session, url_auth, headers=auth_header)
             if response.status_code == 200:
                 for user_data_type, user_data in user_post_data.items():
-                    response = session.post(url_data, json={user_data_type: user_data}, params=auth_params)
+                    response = jupyterhub_session_post_request(session, url_data, json={user_data_type: user_data})
                     if response.status_code != 200:
                         logger.error(
                             "Jupyter: User %s failed to submit data %s to %s"
@@ -549,28 +567,36 @@ def main(client_id, user_arguments_dict):
 
     # Generate private/public keys
     (mount_private_key, mount_public_key) = generate_ssh_rsa_key_pair(
-        encode_utf8=True)
+        encode_utf8=True
+    )
+
+    logger.debug("User: %s - Creating a new jupyter mount keyset - "
+                 "private_key: %s public_key: %s "
+                 % (client_id, mount_private_key, mount_public_key))
 
     # Known hosts
     sftp_addresses = socket.gethostbyname_ex(
         configuration.user_sftp_show_address or socket.getfqdn())
 
+    # Write the authorization file
+    auth_content = []
+    str_mount_public_key = force_native_str(mount_public_key)
     # Subsys sftp support
     if configuration.site_enable_sftp_subsys:
         # Restrict possible mount agent
-        auth_content = []
         restrict_opts = 'no-agent-forwarding,no-port-forwarding,no-pty,'
         restrict_opts += 'no-user-rc,no-X11-forwarding'
         restrictions = '%s' % restrict_opts
-        auth_content.append('%s %s\n' % (restrictions, mount_public_key))
-        # Write auth file
-        write_file('\n'.join(auth_content),
-                   os.path.join(subsys_path, session_id
-                                + '.authorized_keys'), logger, umask=0o27)
-
-    logger.debug("User: %s - Creating a new jupyter mount keyset - "
-                 "private_key: %s public_key: %s "
-                 % (client_id, mount_private_key, mount_public_key))
+        auth_content.append('%s %s\n' % (restrictions, str_mount_public_key))
+    else:
+        auth_content.append('%s\n' % str_mount_public_key)
+
+    # Write auth file
+    write_file(
+        '\n'.join(auth_content),
+        os.path.join(subsys_path, session_id + '.authorized_keys'),
+        logger, umask=0o27
+    )
 
     jupyter_dict = {
         'MOUNT_HOST': configuration.short_title,
@@ -626,15 +652,12 @@ def main(client_id, user_arguments_dict):
     # First login
     with requests.session() as session:
         # Refresh cookies
-        session.get(url_auth)
-        auth_params = {}
-        if "_xsrf" in session.cookies:
-            auth_params = {"_xsrf": session.cookies['_xsrf']}
+        session.get(url_hub)
         # Authenticate
-        response = session.post(url_auth, headers=auth_header, params=auth_params)
+        response = jupyterhub_session_post_request(session, url_auth, headers=auth_header)
         if response.status_code == 200:
             for user_data_type, user_data in user_post_data.items():
-                response = session.post(url_data, json={user_data_type: user_data}, params=auth_params)
+                response = jupyterhub_session_post_request(session, url_data, json={user_data_type: user_data})
                 if response.status_code != 200:
                     logger.error("Jupyter: User %s failed to submit data %s to %s"
                                 % (client_id, user_data, url_data))

mig/shared/install.py

@@ -84,6 +84,22 @@ def abspath(path, start):
         return path
     return os.path.normpath(os.path.join(start, path))
 
+def transform_str_to_dict(input_str):
+    """
+    Transforms a string input into a Python literal or container.
+    The function will only return the transformed object if it becomes a dictionary.
+    input_str: The input string that is expected to be
+     structured as a dictionary. A valid input_str for example could be '{'hello': 'world'}'.
+    """
+    try:
+        output_dict = ast.literal_eval(input_str)
+    except SyntaxError as err:
+        return False
+
+    if not isinstance(output_dict, dict):
+        return False
+    return output_dict
+
 
 def determine_timezone(_environ=os.environ, _path_exists=os.path.exists, _print=print):
     """Attempt to detect the timezone in various known portable ways."""
@@ -318,6 +334,8 @@ def generate_confs(
     ftps_address='',
     davs_address='',
     jupyter_services='',
+    jupyter_services_enable_proxy_https=True,
+    jupyter_services_proxy_config='{}',
     jupyter_services_desc='{}',
     cloud_services='',
     cloud_services_desc='{}',
@@ -640,6 +658,8 @@ def _generate_confs_prepare(
     ftps_address,
     davs_address,
     jupyter_services,
+    jupyter_services_enable_proxy_https,
+    jupyter_services_proxy_config,
     jupyter_services_desc,
     cloud_services,
     cloud_services_desc,
@@ -1496,17 +1516,18 @@ def _generate_confs_prepare(
         jupyter_openids, jupyter_oidcs, jupyter_rewrites = [], [], []
         services = user_dict['__JUPYTER_SERVICES__'].split()
 
-        try:
-            descs = ast.literal_eval(jupyter_services_desc)
-        except SyntaxError as err:
-            print('Error: jupyter_services_desc '
-                  'could not be intepreted correctly. Double check that your '
+        jupyter_services_proxy_configs = transform_str_to_dict(jupyter_services_proxy_config)
+        if not isinstance(jupyter_services_proxy_configs, dict):
+            print('Error: jupyter_services_proxy_config '
+                  'could not be interpreted correctly. Double check that your '
                   'formatting is correct, a dictionary formatted string is expected.')
             sys.exit(1)
 
-        if not isinstance(descs, dict):
-            print('Error: %s was incorrectly formatted,'
-                  ' expects a string formatted as a dictionary' % descs)
+        jupyter_services_descs = transform_str_to_dict(jupyter_services_desc)
+        if not isinstance(jupyter_services_descs, dict):
+            print('Error: jupyter_services_desc '
+                  'could not be interpreted correctly. Double check that your '
+                  'formatting is correct, a dictionary formatted string is expected.')
             sys.exit(1)
 
         service_hosts = {}
@@ -1560,8 +1581,8 @@ def _generate_confs_prepare(
                 '__JUPYTER_%s_HOSTS__' % u_name: ' '.join(values['hosts'])
             }
 
-            if name in descs:
-                desc_value = descs[name] + "\n"
+            if name in jupyter_services_descs:
+                desc_value = jupyter_services_descs[name] + "\n"
             else:
                 desc_value = "\n"
 
@@ -1599,8 +1620,7 @@ def _generate_confs_prepare(
                 ws_host = host.replace(
                     "https://", "wss://").replace("http://", "ws://")
                 member_def = "Define JUPYTER_%s %s" % (name_index, host)
-                ws_member_def = "Define WS_JUPYTER_%s %s" % (name_index,
-                                                             ws_host)
+                ws_member_def = "Define WS_JUPYTER_%s %s" % (name_index, ws_host)
 
                 # No user supplied port, assign based on url prefix
                 if len(host.split(":")) < 3:
@@ -1617,9 +1637,14 @@ def _generate_confs_prepare(
                     ws_member_def += ":%s\n" % port
 
                 jupyter_defs.extend([member_def, ws_member_def])
+
+            service_proxy_config_kwargs = jupyter_services_proxy_configs.get(name, {})
             # Get proxy template and append to template conf
-            proxy_template = gen_balancer_proxy_template(url, def_name, name,
-                                                         hosts, ws_hosts)
+            proxy_template = gen_balancer_proxy_template(
+                url, def_name, name, hosts, ws_hosts,
+                enable_proxy_https=jupyter_services_enable_proxy_https,
+                proxy_balancer_template_kwargs=service_proxy_config_kwargs
+            )
             jupyter_proxies.append(proxy_template)
 
         user_dict['__JUPYTER_DEFS__'] = '\n'.join(jupyter_defs)

mig/shared/jupyter.py

@@ -34,7 +34,10 @@
 
 from past.builtins import basestring
 def gen_balancer_proxy_template(url, define, name, member_hosts,
-                                ws_member_hosts, timeout=600):
+                                ws_member_hosts,
+                                timeout=600,
+                                enable_proxy_https=True,
+                                proxy_balancer_template_kwargs=None):
     """ Generates an apache proxy balancer configuration section template
      for a particular jupyter service. Relies on the
      https://httpd.apache.org/docs/2.4/mod/mod_proxy_balancer.html module to
@@ -46,14 +49,26 @@ def gen_balancer_proxy_template(url, define, name, member_hosts,
      in balancer member defitions.
     ws_member_hosts: The list of unique identifiers that should be used to fill
      in websocket balancer member defitions.
+    timeout: The proxy timeout in seconds.
+    enable_proxy_https: Whether or not to enable SSL/TLS proxying.
+    proxy_balancer_template_kwargs: The optional extra apache config options that are used to
+     generate the proxy balancer templates. This for instance can be used to pass SSL options
+    such as a custom self-signed CA that should be used to establish a
+    trusted SSL/TLS connection to the designated jupyter service.
+    An example of this could be {'SSLProxyCACertificateFile': 'path/to/local/ca-certificate.pem'}.
     """
 
+    if not proxy_balancer_template_kwargs:
+        proxy_balancer_template_kwargs = {}
+
     assert isinstance(url, basestring)
     assert isinstance(define, basestring)
     assert isinstance(name, basestring)
     assert isinstance(member_hosts, list)
     assert isinstance(ws_member_hosts, list)
     assert isinstance(timeout, int)
+    assert isinstance(enable_proxy_https, bool)
+    assert isinstance(proxy_balancer_template_kwargs, dict)
 
     fill_helpers = {
         'url': url,
@@ -66,44 +81,61 @@ def gen_balancer_proxy_template(url, define, name, member_hosts,
         'ws_hosts': '',
         'timeout': timeout,
         'referer_fqdn': name.upper() + "_PROXY_FQDN=$1",
-        'referer_url': '%{' + name.upper() + "_PROXY_PROTOCOL}e://%{" + name.upper() + "_PROXY_FQDN}e/" + name + "/hub"
+        'referer_url': '%{' + name.upper() + "_PROXY_PROTOCOL}e://%{" + name.upper() + "_PROXY_FQDN}e/" + name + "/hub",
+        'proxy_balancer_template': ''
     }
 
     for host in member_hosts:
         fill_helpers['hosts'] += ''.join(['        ', host])
 
     for ws_host in ws_member_hosts:
         fill_helpers['ws_hosts'] += ''.join(['        ', ws_host])
-    print("filling in jupyter gen_balancer_proxy_template with helper: (%s)" %
-          fill_helpers)
+
+    proxy_balancer_options, proxy_balancer_template = {}, ''
+    if enable_proxy_https:
+        if "SSLProxyVerify" not in proxy_balancer_template_kwargs:
+            proxy_balancer_options["SSLProxyVerify"] = "require"
+        if "SSLProxyCheckPeerCN" not in proxy_balancer_template_kwargs:
+            proxy_balancer_options["SSLProxyCheckPeerCN"] = "on"
+        if "SSLProxyCheckPeerName" not in proxy_balancer_template_kwargs:
+            proxy_balancer_options["SSLProxyCheckPeerName"] = "on"
+
+    for key, value in proxy_balancer_template_kwargs.items():
+        proxy_balancer_options[key] = value
+
+    for key, value in proxy_balancer_options.items():
+        proxy_balancer_template += ''.join(['        ', '%s %s\n' % (key, value)])
+
+    fill_helpers["proxy_balancer_template"] = proxy_balancer_template
 
     template = """
 <IfDefine %(define)s>
     Header add Set-Cookie "%(route_cookie)s=%(balancer_worker_env)s; path=%(url)s" env=BALANCER_ROUTE_CHANGED
-    SetEnvIf Host (.*) %(referer_fqdn)s
 
     ProxyTimeout %(timeout)s
     <Proxy balancer://%(name)s_hosts>
+%(proxy_balancer_template)s
 %(hosts)s
         ProxySet stickysession=%(route_cookie)s
     </Proxy>
     # Websocket cluster
     <Proxy balancer://ws_%(name)s_hosts>
+%(proxy_balancer_template)s
 %(ws_hosts)s
         ProxySet stickysession=%(route_cookie)s
     </Proxy>
     <Location %(url)s>
-        ProxyPreserveHost on
         ProxyPass balancer://%(name)s_hosts%(url)s
         ProxyPassReverse balancer://%(name)s_hosts%(url)s
         RequestHeader set Remote-User %(remote_user_env)s
-        RequestHeader set Referer %(referer_url)s
+        RequestHeader set "X-Forwarded-Proto" expr=%%{REQUEST_SCHEME}
     </Location>
     <LocationMatch "%(url)s/(user/[^/]+)/(api/kernels/[^/]+/channels|terminals/websocket|api/events/subscribe)(/?|)">
         ProxyPass   balancer://ws_%(name)s_hosts
         ProxyPassReverse    balancer://ws_%(name)s_hosts
     </LocationMatch>
 </IfDefine>""" % fill_helpers
+
     return template