feat: improve DRS URI resolution (#200)

Author: uniqueg

cwl_wes/config/app_config.yaml

@@ -105,3 +105,7 @@ drs:
     port: Null  # use this port for resolving DRS URIs; set to `Null` to use default (443)
     base_path: Null  # use this base path for resolving DRS URIs; set to `Null` to use default (`ga4gh/drs/v1`)
     use_http: False  # use `http` for resolving DRS URIs; set to `False` to use default (`https`)
+    file_types:  # extensions of files to scan for DRS URI resolution
+        - cwl
+        - yaml
+        - yml

cwl_wes/ga4gh/wes/endpoints/run_workflow.py

@@ -249,6 +249,12 @@ def __create_run_environment(
         break
 
     # translate DRS URIs to access URLs
+    file_types: List[str] = get_conf_type(
+        current_app.config,
+        'drs',
+        'file_types',
+        types=(list),
+    )
     supported_access_methods: List[str] = get_conf_type(
         current_app.config,
         'service_info',
@@ -260,21 +266,22 @@ def __create_run_environment(
         'drs',
         'port',
     )
+    base_path: Optional[str] = get_conf(
+        current_app.config,
+        'drs',
+        'base_path',
+    )
     use_http: bool = get_conf(
         current_app.config,
         'drs',
         'use_http',
     )
     translate_drs_uris(
-        path=document['internal']['param_file_path'],
-        supported_access_methods=supported_access_methods,
-        port=port,
-        use_http=use_http,
-    )
-    translate_drs_uris(
-        path=document['internal']['cwl_path'],
+        path=document['internal']['workflow_files'],
+        file_types=file_types,
         supported_access_methods=supported_access_methods,
         port=port,
+        base_path=base_path,
         use_http=use_http,
     )
 
@@ -329,7 +336,7 @@ def __process_workflow_attachments(data: Dict) -> Dict:
     )
 
     # Create directory for storing workflow files
-    workflow_dir = os.path.abspath(
+    data['internal']['workflow_files'] = workflow_dir = os.path.abspath(
         os.path.join(
             data['internal']['out_dir'], 'workflow_files'
         )

cwl_wes/ga4gh/wes/endpoints/utils/drs.py

@@ -24,6 +24,7 @@
 
 def translate_drs_uris(
     path: str,
+    file_types: List[str],
     supported_access_methods: List[str],
     port: Optional[int] = None,
     base_path: Optional[str] = None,
@@ -37,6 +38,7 @@ def translate_drs_uris(
 
     Arguments:
         path: File or directory containing files.
+        file_types: Extensions of files to scan.
         supported_access_methods: List of access methods/file transfer
             protocols supported by this service, provided in the order of
             preference.
@@ -54,10 +56,15 @@ def translate_drs_uris(
     _RE_OBJECT_ID = rf"(?P<drs_uri>drs:\/\/{_RE_DOMAIN}\/\S+)"
 
     # get absolute paths of file or directory (including subdirectories)
-    files = abs_paths(dir=path) if os.path.isdir(path) else [path]
+    logger.debug(f"Collecting file(s) for provided path '{path}'...")
+    files = abs_paths(
+        dir=path,
+        file_ext=file_types,
+    ) if os.path.isdir(path) else [path]
 
     # replace any DRS URIs in any file in place
     for _file in files:
+        logger.debug(f"Scanning file '{_file}' for DRS URIs...")
         with FileInput(_file, inplace=True) as _f:
             for line in _f:
                 sys.stdout.write(
@@ -76,18 +83,24 @@ def translate_drs_uris(
                 )
 
 
-def abs_paths(dir: str) -> Iterator[str]:
-    """Yields absolute paths of all files in directory and subdirectories.
+def abs_paths(
+    dir: str,
+    file_ext: List[str],
+) -> Iterator[str]:
+    """Yields absolute paths of files with the indicated file extensions in
+    specified directory and subdirectories.
 
     Arguments:
         dir: Directory to search files in.
+        file_ext: List of file extensions for files to return.
 
     Returns:
         Generator yielding absolute file paths.
     """
     for dirpath, _, files in os.walk(dir):
         for _file in files:
-            yield os.path.abspath(os.path.join(dirpath, _file))
+            if _file.endswith(tuple(file_ext)):
+                yield os.path.abspath(os.path.join(dirpath, _file))
 
 
 def get_replacement_string(
@@ -166,6 +179,7 @@ def get_access_url_from_drs(
             use_http=use_http,
         )
     except InvalidURI:
+        logger.error(f"The provided DRS URI '{drs_uri}' is invalid.")
         raise BadRequest
 
     # get DRS object
@@ -174,11 +188,15 @@ def get_access_url_from_drs(
             object_id=drs_uri
         )
     except (ConnectionError, InvalidResponseError):
+        logger.error(f"Could not connect to DRS host for DRS URI '{drs_uri}'.")
         raise InternalServerError
     if isinstance(object, Error):
         if object.status_code == 404:
+            logger.error(f"Could not access DRS host for DRS URI '{drs_uri}'.")
             raise BadRequest
+        # TODO: handle 401 & 403
         else:
+            logger.error(f"DRS returned error: {object}'.")
             raise InternalServerError
 
     # get access methods and access method types/protocols
@@ -189,12 +207,19 @@ def get_access_url_from_drs(
     # TODO: add support for access URL headers
     for supported_method in supported_access_methods:
         try:
-            return str(
+            access_url = str(
                 available_methods
                 [available_types.index(supported_method)].access_url.url
             )
+            logger.info(
+                f"Resolved DRS URI '{drs_uri}' to access link '{access_url}'."
+            )
+            return access_url
         except ValueError:
             continue
 
     # no method was found
+    logger.error(
+        f"Could not find a supported access URL for DRS URI '{drs_uri}'."
+    )
     raise BadRequest