Added http_defaults and http_config_for_url_prefix configs

Author: sveinugu

src/omnipy/api/enums.py

@@ -65,3 +65,10 @@ class RunStateLogMessages(str, Enum):
     INITIALIZED = 'Initialized "{}"'
     RUNNING = 'Started running "{}"...'
     FINISHED = 'Finished running "{}"!'
+
+
+class BackoffStrategy(str, Enum):
+    EXPONENTIAL = 'exponential'
+    JITTER = 'jitter'
+    FIBONACCI = 'fibonacci'
+    RANDOM = 'random'

src/omnipy/api/protocols/public/config.py

@@ -1,6 +1,8 @@
+from collections import defaultdict
 from typing import Protocol, TextIO
 
-from omnipy.api.enums import (ConfigOutputStorageProtocolOptions,
+from omnipy.api.enums import (BackoffStrategy,
+                              ConfigOutputStorageProtocolOptions,
                               ConfigPersistOutputsOptions,
                               ConfigRestoreOutputsOptions)
 from omnipy.api.typedefs import LocaleType
@@ -32,6 +34,17 @@ class IsDataConfig(Protocol):
     dynamically_convert_elements_to_models: bool
     terminal_size_columns: int
     terminal_size_lines: int
+    http_defaults: 'IsHttpConfig'
+    http_config_for_url_prefix: defaultdict[str, 'IsHttpConfig']
+
+
+class IsHttpConfig(Protocol):
+    """"""
+    requests_per_time_period: float
+    time_period_in_secs: float
+    retry_http_statuses: tuple[int, ...]
+    retry_attempts: int
+    retry_backoff_strategy: BackoffStrategy
 
 
 class IsRootLogConfig(Protocol):

src/omnipy/config/data.py

@@ -1,12 +1,35 @@
-from dataclasses import dataclass
+from collections import defaultdict
+from copy import copy
+from dataclasses import dataclass, field
 import shutil
 
+from omnipy.api.enums import BackoffStrategy
+from omnipy.api.protocols.public.config import IsHttpConfig
+
 _terminal_size = shutil.get_terminal_size()
 
 
+@dataclass
+class HttpConfig:
+    # For RateLimitingClientSession helper class
+    requests_per_time_period: float = 60
+    time_period_in_secs: float = 60
+
+    # For get_*_from_api_endpoint tasks
+    retry_http_statuses: tuple[int, ...] = (408, 425, 429, 500, 502, 503, 504)
+    retry_attempts: int = 5
+    retry_backoff_strategy: BackoffStrategy = BackoffStrategy.EXPONENTIAL
+
+
 @dataclass
 class DataConfig:
     interactive_mode: bool = True
     dynamically_convert_elements_to_models: bool = False
     terminal_size_columns: int = _terminal_size.columns
     terminal_size_lines: int = _terminal_size.lines
+    http_defaults: IsHttpConfig = field(default_factory=HttpConfig)
+    http_config_for_url_prefix: defaultdict[str, IsHttpConfig] = field(
+        default_factory=lambda: defaultdict(HttpConfig))
+
+    def __post_init__(self):
+        self.http_config_for_url_prefix = defaultdict(lambda: copy(self.http_defaults))

src/omnipy/modules/remote/tasks.py

@@ -1,9 +1,9 @@
-from enum import Enum
 from typing import AsyncGenerator, cast
 
 from aiohttp import ClientResponse, ClientSession
 from aiohttp_retry import ExponentialRetry, FibonacciRetry, JitterRetry, RandomRetry, RetryClient
 
+from omnipy.api.enums import BackoffStrategy
 from omnipy.compute.task import TaskTemplate
 
 from ..json.datasets import JsonDataset
@@ -12,14 +12,6 @@
 from ..raw.models import BytesModel, StrModel
 from .models import HttpUrlModel
 
-
-class BackoffStrategy(str, Enum):
-    EXPONENTIAL = 'exponential'
-    JITTER = 'jitter'
-    FIBONACCI = 'fibonacci'
-    RANDOM = 'random'
-
-
 DEFAULT_RETRIES = 5
 DEFAULT_BACKOFF_STRATEGY = BackoffStrategy.EXPONENTIAL
 DEFAULT_RETRY_STATUSES = (408, 425, 429, 500, 502, 503, 504)

tests/hub/runtime/test_runtime.py

@@ -1,10 +1,12 @@
+from collections import defaultdict
 import os
 from pathlib import Path
 from typing import Annotated, Type
 
 import pytest
 
-from omnipy.api.enums import (ConfigOutputStorageProtocolOptions,
+from omnipy.api.enums import (BackoffStrategy,
+                              ConfigOutputStorageProtocolOptions,
                               ConfigPersistOutputsOptions,
                               ConfigRestoreOutputsOptions,
                               EngineChoice)
@@ -59,6 +61,12 @@ def _assert_runtime_config_default(config: IsRuntimeConfig, dir_path: Path):
     assert config.data.dynamically_convert_elements_to_models is False
     assert config.data.terminal_size_columns == 80
     assert config.data.terminal_size_lines == 24
+    assert config.data.http_defaults.requests_per_time_period == 60
+    assert config.data.http_defaults.time_period_in_secs == 60
+    assert config.data.http_defaults.retry_http_statuses == (408, 425, 429, 500, 502, 503, 504)
+    assert config.data.http_defaults.retry_attempts == 5
+    assert config.data.http_defaults.retry_backoff_strategy == BackoffStrategy.EXPONENTIAL
+    assert isinstance(config.data.http_config_for_url_prefix, defaultdict)
     assert config.engine == EngineChoice.LOCAL
     assert config.prefect.use_cached_results is False
 
@@ -104,6 +112,15 @@ def test_default_runtime(runtime: Annotated[IsRuntime, pytest.fixture],
     _assert_runtime_objects_default(runtime.objects)
 
 
+def test_data_config_http_config_for_url_prefix_default(
+        runtime: Annotated[IsRuntime, pytest.fixture]) -> None:
+    assert runtime.config.data.http_config_for_url_prefix['http://myserver.com']\
+        .requests_per_time_period == 60
+    runtime.config.data.http_defaults.requests_per_time_period = 30
+    assert runtime.config.data.http_config_for_url_prefix['http://myserver.com']\
+        .requests_per_time_period == 60
+
+
 def test_runtime_config_after_data_class_creator(
         runtime_cls: Annotated[Type[IsRuntime], pytest.fixture]) -> None:
     DataClassBase.data_class_creator.config.dynamically_convert_elements_to_models = True