|
| 1 | +# ------------------------------------------------------------------------- |
| 2 | +# Copyright (c) Microsoft Corporation. All rights reserved. |
| 3 | +# Licensed under the MIT License. See License.txt in the project root for |
| 4 | +# license information. |
| 5 | +# -------------------------------------------------------------------------- |
| 6 | +import six.moves.urllib as urllib |
| 7 | +import re |
| 8 | +import logging |
| 9 | +import os |
| 10 | +from azure.core.pipeline import Pipeline |
| 11 | +from azure.core.tracing.decorator import distributed_trace |
| 12 | +from azure.core.pipeline.transport import RequestsTransport |
| 13 | +from azure.core.exceptions import ResourceNotFoundError |
| 14 | +from azure.core.pipeline.policies import ( |
| 15 | + UserAgentPolicy, |
| 16 | + HeadersPolicy, |
| 17 | + RetryPolicy, |
| 18 | + RedirectPolicy, |
| 19 | + NetworkTraceLoggingPolicy, |
| 20 | + ProxyPolicy, |
| 21 | +) |
| 22 | +from . import ( |
| 23 | + _resolver, |
| 24 | + _pseudo_parser, |
| 25 | + _constants, |
| 26 | +) |
| 27 | + |
| 28 | +_LOGGER = logging.getLogger(__name__) |
| 29 | + |
| 30 | + |
| 31 | +# Public constants exposed to consumers |
| 32 | +DEPENDENCY_MODE_TRY_FROM_EXPANDED = "tryFromExpanded" |
| 33 | +DEPENDENCY_MODE_DISABLED = "disabled" |
| 34 | +DEPENDENCY_MODE_ENABLED = "enabled" |
| 35 | + |
| 36 | + |
| 37 | +# Convention-private constants |
| 38 | +_DEFAULT_LOCATION = "https://devicemodels.azure.com" |
| 39 | +_REMOTE_PROTOCOLS = ["http", "https"] |
| 40 | +_TRACE_NAMESPACE = "modelsrepository" |
| 41 | + |
| 42 | + |
| 43 | +class ModelsRepositoryClient(object): |
| 44 | + """Client providing APIs for Models Repository operations""" |
| 45 | + |
| 46 | + def __init__(self, **kwargs): # pylint: disable=missing-client-constructor-parameter-credential |
| 47 | + # type: (Any) -> None |
| 48 | + """ |
| 49 | + :keyword str repository_location: Location of the Models Repository you wish to access. |
| 50 | + This location can be a remote HTTP/HTTPS URL, or a local filesystem path. |
| 51 | + If omitted, will default to using "https://devicemodels.azure.com". |
| 52 | + :keyword str dependency_resolution: Dependency resolution mode. |
| 53 | + Possible values: |
| 54 | + - "disabled": Do not resolve model dependencies |
| 55 | + - "enabled": Resolve model dependencies from the repository |
| 56 | + - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded |
| 57 | + model DTDL document in the repository. If this is not successful, will fall |
| 58 | + back on manually resolving dependencies in the repository |
| 59 | + If using the default repository location, the default dependency resolution mode will |
| 60 | + be "tryFromExpanded". If using a custom repository location, the default dependency |
| 61 | + resolution mode will be "enabled". |
| 62 | + :keyword str api_version: The API version for the Models Repository Service you wish to |
| 63 | + access. |
| 64 | +
|
| 65 | + For additional request configuration options, please see [core options](https://aka.ms/azsdk/python/options). |
| 66 | +
|
| 67 | + :raises: ValueError if an invalid argument is provided |
| 68 | + """ |
| 69 | + repository_location = kwargs.get("repository_location", _DEFAULT_LOCATION) |
| 70 | + _LOGGER.debug("Client configured for respository location %s", repository_location) |
| 71 | + |
| 72 | + self.resolution_mode = kwargs.get( |
| 73 | + "dependency_resolution", |
| 74 | + DEPENDENCY_MODE_TRY_FROM_EXPANDED |
| 75 | + if repository_location == _DEFAULT_LOCATION |
| 76 | + else DEPENDENCY_MODE_ENABLED, |
| 77 | + ) |
| 78 | + if self.resolution_mode not in [ |
| 79 | + DEPENDENCY_MODE_ENABLED, |
| 80 | + DEPENDENCY_MODE_DISABLED, |
| 81 | + DEPENDENCY_MODE_TRY_FROM_EXPANDED, |
| 82 | + ]: |
| 83 | + raise ValueError("Invalid dependency resolution mode: {}".format(self.resolution_mode)) |
| 84 | + _LOGGER.debug("Client configured for dependency mode %s", self.resolution_mode) |
| 85 | + |
| 86 | + # NOTE: depending on how this class develops over time, may need to adjust relationship |
| 87 | + # between some of these objects |
| 88 | + self.fetcher = _create_fetcher(location=repository_location, **kwargs) |
| 89 | + self.resolver = _resolver.DtmiResolver(self.fetcher) |
| 90 | + self._pseudo_parser = _pseudo_parser.PseudoParser(self.resolver) |
| 91 | + |
| 92 | + # Store api version here (for now). Currently doesn't do anything |
| 93 | + self._api_version = kwargs.get("api_version", _constants.DEFAULT_API_VERSION) |
| 94 | + |
| 95 | + def __enter__(self): |
| 96 | + self.fetcher.__enter__() |
| 97 | + return self |
| 98 | + |
| 99 | + def __exit__(self, *exc_details): |
| 100 | + self.fetcher.__exit__(*exc_details) |
| 101 | + |
| 102 | + def close(self): |
| 103 | + # type: () -> None |
| 104 | + """Close the client, preventing future operations""" |
| 105 | + self.__exit__() |
| 106 | + |
| 107 | + @distributed_trace |
| 108 | + def get_models(self, dtmis, **kwargs): |
| 109 | + # type: (Union[List[str], str], Any) -> Dict[str, Any] |
| 110 | + """Retrieve a model from the Models Repository. |
| 111 | +
|
| 112 | + :param dtmis: The DTMI(s) for the model(s) you wish to retrieve |
| 113 | + :type dtmis: str or list[str] |
| 114 | + :keyword str dependency_resolution: Dependency resolution mode override. This value takes |
| 115 | + precedence over the value set on the client. |
| 116 | + Possible values: |
| 117 | + - "disabled": Do not resolve model dependencies |
| 118 | + - "enabled": Resolve model dependencies from the repository |
| 119 | + - "tryFromExpanded": Attempt to resolve model and dependencies from an expanded |
| 120 | + model DTDL document in the repository. If this is not successful, will fall |
| 121 | + back on manually resolving dependencies in the repository |
| 122 | +
|
| 123 | + :raises: ValueError if given an invalid dependency resolution mode |
| 124 | + :raises: ~azure.iot.modelsrepository.ModelError if there is an error parsing the retrieved model(s) |
| 125 | + :raises: ~azure.core.exceptions.ResourceNotFoundError if the model(s) cannot be found in the repository |
| 126 | + :raises: ~azure.core.exceptions.ServiceRequestError if there is an error sending a request for the model(s) |
| 127 | + :raises: ~azure.core.exceptions.ServiceResponseError if the model(s) cannot be retrieved |
| 128 | + :raises: ~azure.core.exceptions.HttpResponseError if a failure response is received |
| 129 | +
|
| 130 | + :returns: Dictionary mapping DTMIs to models |
| 131 | + :rtype: dict |
| 132 | + """ |
| 133 | + if isinstance(dtmis, str): |
| 134 | + dtmis = [dtmis] |
| 135 | + |
| 136 | + dependency_resolution = kwargs.get("dependency_resolution", self.resolution_mode) |
| 137 | + |
| 138 | + if dependency_resolution == DEPENDENCY_MODE_DISABLED: |
| 139 | + # Simply retrieve the model(s) |
| 140 | + _LOGGER.debug("Getting models w/ dependency resolution mode: disabled") |
| 141 | + _LOGGER.debug("Retrieving model(s): %s...", dtmis) |
| 142 | + model_map = self.resolver.resolve(dtmis) |
| 143 | + elif dependency_resolution == DEPENDENCY_MODE_ENABLED: |
| 144 | + # Manually resolve dependencies using pseudo-parser |
| 145 | + _LOGGER.debug("Getting models w/ dependency resolution mode: enabled") |
| 146 | + _LOGGER.debug("Retrieving model(s): %s...", dtmis) |
| 147 | + base_model_map = self.resolver.resolve(dtmis) |
| 148 | + base_model_list = list(base_model_map.values()) |
| 149 | + _LOGGER.debug("Retrieving model dependencies for %s...", dtmis) |
| 150 | + model_map = self._pseudo_parser.expand(base_model_list) |
| 151 | + elif dependency_resolution == DEPENDENCY_MODE_TRY_FROM_EXPANDED: |
| 152 | + _LOGGER.debug("Getting models w/ dependency resolution mode: tryFromExpanded") |
| 153 | + # Try to use an expanded DTDL to resolve dependencies |
| 154 | + try: |
| 155 | + _LOGGER.debug("Retrieving expanded model(s): %s...", dtmis) |
| 156 | + model_map = self.resolver.resolve(dtmis, expanded_model=True) |
| 157 | + except ResourceNotFoundError: |
| 158 | + # Fallback to manual dependency resolution |
| 159 | + _LOGGER.debug( |
| 160 | + "Could not retrieve model(s) from expanded model DTDL - " |
| 161 | + "fallback to manual dependency resolution mode" |
| 162 | + ) |
| 163 | + _LOGGER.debug("Retrieving model(s): %s...", dtmis) |
| 164 | + base_model_map = self.resolver.resolve(dtmis) |
| 165 | + base_model_list = list(base_model_map.values()) |
| 166 | + _LOGGER.debug("Retrieving model dependencies for %s...", dtmis) |
| 167 | + model_map = self._pseudo_parser.expand(base_model_list) |
| 168 | + else: |
| 169 | + raise ValueError("Invalid dependency resolution mode: {}".format(dependency_resolution)) |
| 170 | + return model_map |
| 171 | + |
| 172 | + |
| 173 | +def _create_fetcher(location, **kwargs): |
| 174 | + """Return a Fetcher based upon the type of location""" |
| 175 | + scheme = urllib.parse.urlparse(location).scheme |
| 176 | + if scheme in _REMOTE_PROTOCOLS: |
| 177 | + # HTTP/HTTPS URL |
| 178 | + _LOGGER.debug("Repository Location identified as HTTP/HTTPS endpoint - using HttpFetcher") |
| 179 | + pipeline = _create_pipeline(**kwargs) |
| 180 | + fetcher = _resolver.HttpFetcher(location, pipeline) |
| 181 | + elif scheme == "file": |
| 182 | + # Filesystem URI |
| 183 | + _LOGGER.debug("Repository Location identified as filesystem URI - using FilesystemFetcher") |
| 184 | + location = location[len("file://") :] |
| 185 | + location = _sanitize_filesystem_path(location) |
| 186 | + fetcher = _resolver.FilesystemFetcher(location) |
| 187 | + elif scheme == "" and location.startswith("/"): |
| 188 | + # POSIX filesystem path |
| 189 | + _LOGGER.debug( |
| 190 | + "Repository Location identified as POSIX fileystem path - using FilesystemFetcher" |
| 191 | + ) |
| 192 | + location = _sanitize_filesystem_path(location) |
| 193 | + fetcher = _resolver.FilesystemFetcher(location) |
| 194 | + elif scheme == "" and re.search( |
| 195 | + r"\.[a-zA-z]{2,63}$", |
| 196 | + location[: location.find("/") if location.find("/") >= 0 else len(location)], |
| 197 | + ): |
| 198 | + # Web URL with protocol unspecified - default to HTTPS |
| 199 | + _LOGGER.debug( |
| 200 | + "Repository Location identified as remote endpoint without protocol specified - using HttpFetcher" |
| 201 | + ) |
| 202 | + location = "https://" + location |
| 203 | + pipeline = _create_pipeline(**kwargs) |
| 204 | + fetcher = _resolver.HttpFetcher(location, pipeline) |
| 205 | + elif scheme != "" and len(scheme) == 1 and scheme.isalpha(): |
| 206 | + # Filesystem path using drive letters (e.g. "C:", "D:", etc.) |
| 207 | + _LOGGER.debug( |
| 208 | + "Repository Location identified as drive letter fileystem path - using FilesystemFetcher" |
| 209 | + ) |
| 210 | + location = _sanitize_filesystem_path(location) |
| 211 | + fetcher = _resolver.FilesystemFetcher(location) |
| 212 | + else: |
| 213 | + raise ValueError("Unable to identify location: {}".format(location)) |
| 214 | + return fetcher |
| 215 | + |
| 216 | + |
| 217 | +def _create_pipeline(**kwargs): |
| 218 | + """Creates and returns a PipelineClient configured for the provided base_url and kwargs""" |
| 219 | + transport = kwargs.get("transport", RequestsTransport(**kwargs)) |
| 220 | + policies = [ |
| 221 | + kwargs.get("user_agent_policy", UserAgentPolicy(_constants.USER_AGENT, **kwargs)), |
| 222 | + kwargs.get("headers_policy", HeadersPolicy(**kwargs)), |
| 223 | + kwargs.get("authentication_policy"), |
| 224 | + kwargs.get("retry_policy", RetryPolicy(**kwargs)), |
| 225 | + kwargs.get("redirect_policy", RedirectPolicy(**kwargs)), |
| 226 | + kwargs.get("logging_policy", NetworkTraceLoggingPolicy(**kwargs)), |
| 227 | + kwargs.get("proxy_policy", ProxyPolicy(**kwargs)), |
| 228 | + ] |
| 229 | + return Pipeline(policies=policies, transport=transport) |
| 230 | + |
| 231 | + |
| 232 | +def _sanitize_filesystem_path(path): |
| 233 | + """Sanitize the filesystem path to be formatted correctly for the current OS""" |
| 234 | + path = os.path.normcase(path) |
| 235 | + path = os.path.normpath(path) |
| 236 | + return path |
0 commit comments