Implement load_urls_into_new_dataset() convenience task

Author: sveinugu

src/omnipy/__init__.py

@@ -138,7 +138,8 @@
 from omnipy.modules.remote.models import HttpUrlModel, QueryParamsModel, UrlPathModel
 from omnipy.modules.remote.tasks import (get_bytes_from_api_endpoint,
                                          get_json_from_api_endpoint,
-                                         get_str_from_api_endpoint)
+                                         get_str_from_api_endpoint,
+                                         load_urls_into_new_dataset)
 from omnipy.modules.tables.datasets import (TableDictOfDictsOfJsonScalarsDataset,
                                             TableDictOfListsOfJsonScalarsDataset,
                                             TableListOfDictsOfJsonScalarsDataset,
@@ -318,6 +319,7 @@
     'get_bytes_from_api_endpoint',
     'get_json_from_api_endpoint',
     'get_str_from_api_endpoint',
+    'load_urls_into_new_dataset',
     'create_row_index_from_column',
     'remove_columns',
     'rename_col_names',

src/omnipy/modules/remote/tasks.py

@@ -2,14 +2,17 @@
 
 from aiohttp import ClientResponse, ClientSession
 from aiohttp_retry import ExponentialRetry, FibonacciRetry, JitterRetry, RandomRetry, RetryClient
+from typing_extensions import TypeVar
 
 from omnipy.api.enums import BackoffStrategy
 from omnipy.compute.task import TaskTemplate
+from omnipy.data.dataset import Dataset
 
 from ..json.datasets import JsonDataset
 from ..json.models import JsonModel
 from ..raw.datasets import BytesDataset, StrDataset
 from ..raw.models import BytesModel, StrModel
+from .datasets import HttpUrlDataset
 from .models import HttpUrlModel
 
 DEFAULT_RETRIES = 5
@@ -126,3 +129,16 @@ async def get_bytes_from_api_endpoint(
     ):
         async for response in _call_get(url, cast(ClientSession, retry_session)):
             return BytesModel(await response.read())
+
+
+JsonDatasetT = TypeVar('JsonDatasetT', bound=Dataset)
+
+
+@TaskTemplate()
+async def load_urls_into_new_dataset(
+    urls: HttpUrlDataset,
+    dataset_cls: type[JsonDatasetT] = JsonDataset,
+) -> JsonDatasetT:
+    dataset = dataset_cls()
+    await dataset.load(urls)
+    return dataset

tests/modules/remote/cases/request_types.py

@@ -2,16 +2,17 @@
 
 import pytest_cases as pc
 
-from omnipy import BytesDataset, Dataset, JsonDataset, StrDataset, TaskTemplate
+from omnipy import BytesDataset, Dataset, JsonDataset, JsonDictDataset, StrDataset, TaskTemplate
 from omnipy.modules.remote.tasks import (get_bytes_from_api_endpoint,
                                          get_json_from_api_endpoint,
-                                         get_str_from_api_endpoint)
+                                         get_str_from_api_endpoint,
+                                         load_urls_into_new_dataset)
 
 
 @dataclass
 class RequestTypeCase:
     job: TaskTemplate
-    kwargs: dict[str, str]
+    kwargs: dict[str, object]
     dataset_cls: type[Dataset]
 
 
@@ -28,3 +29,17 @@ def case_get_str_from_api_endpoint() -> RequestTypeCase:
 @pc.case
 def case_get_bytes_from_api_endpoint() -> RequestTypeCase:
     return RequestTypeCase(get_bytes_from_api_endpoint, dict(), BytesDataset)
+
+
+@pc.case
+def case_load_urls_into_new_dataset_default_json() -> RequestTypeCase:
+    return RequestTypeCase(load_urls_into_new_dataset, dict(), JsonDataset)
+
+
+@pc.case
+def case_load_urls_into_new_dataset_other_dataset_cls() -> RequestTypeCase:
+    return RequestTypeCase(
+        load_urls_into_new_dataset,
+        dict(dataset_cls=JsonDictDataset),
+        JsonDictDataset,
+    )

tests/modules/remote/test_tasks.py

@@ -133,7 +133,7 @@ def _assert_query_results(assert_model_if_dyn_conv_else_val, case: RequestTypeCa
         case omnipy.StrDataset | omnipy.BytesDataset:
             json_data = JsonDataset()
             json_data.from_json(data.to_data())
-        case omnipy.JsonDataset:
+        case omnipy.JsonDataset | omnipy.JsonDictDataset:
             json_data = data
         case _:
             raise ShouldNotOccurException()