- Upgraded dlt from ^0.4.12 to ^1.11.0

- Updated rest_api.py, utils.py, and other files
- Cleanup and minor fixes# This is a combination of 13 commits.

Update dlt dependency from ^0.4.12 to ^1.11.0

Updated 'update_rest_api.py'

Updated

Updated utils.py

updated

Updted

Updated

Updated

Updated

Updated

Updated

Updated

updated pokemon

Author: dat-a-man

CHANGELOG.md

@@ -1,5 +1,14 @@
 [Go to GitHub Releases](https://github.com/dlt-hub/dlt-init-openapi/releases)
 
+## Unreleased
+
+* Bump dlt to 1.11.0
+
+## 0.1.0
+
+* Bump dlt to 0.4.12
+* First beta release of REST API client generator
+
 0.1.0 - Initial Release
 * Bump dlt to 0.4.12
 

Makefile

@@ -13,12 +13,12 @@ lint: update-rest-api
 	poetry run flake8 dlt_init_openapi tests
 	poetry run mypy dlt_init_openapi tests
 	poetry run black tests dlt_init_openapi --check
-	poetry run isort black tests dlt_init_openapi --check --diff
+	poetry run isort --profile black tests dlt_init_openapi --check --diff
 
 # format the whole project
 format: update-rest-api
 	rm -rf tests/_local
-	poetry run isort black tests dlt_init_openapi
+	poetry run isort --profile black tests dlt_init_openapi
 	poetry run black tests dlt_init_openapi
 
 # all tests excluding the checks on e2e tests
@@ -42,7 +42,7 @@ create-pokemon-pipeline-interactive:
 
 # e2e test helpers
 create-e2e-pokemon-pipeline:
-	poetry run dlt-init-openapi pokemon --path tests/cases/e2e_specs/pokeapi.yml --global-limit 2 --no-interactive
+	poetry run dlt-init-openapi pokemon --path tests/cases/e2e_specs/pokeapi.yml --no-interactive --global-limit 2
 
 run-pokemon-pipeline:
 	cd pokemon_pipeline && poetry run python pokemon_pipeline.py

dlt_init_openapi/config.py

@@ -2,7 +2,7 @@
 import mimetypes
 import pathlib
 from pathlib import Path
-from typing import Any, List, Optional
+from typing import Any, Dict, List, Optional
 
 import yaml
 from pydantic import BaseModel
@@ -11,7 +11,12 @@
 
 from .typing import TEndpointFilter
 
-REST_API_SOURCE_LOCATION = str(pathlib.Path(__file__).parent.resolve() / "../rest_api")
+# Vendored rest_api source should be located inside the dlt_init_openapi package directory
+# e.g., dlt_init_openapi/rest_api/
+REST_API_SOURCE_LOCATION = str(pathlib.Path(__file__).parent / "rest_api")
+
+# Placeholder for SecretsTomlConfig to resolve ImportError in tests
+SecretsTomlConfig = Dict[str, Any]
 
 
 class Config(BaseModel):

dlt_init_openapi/renderer/default/__init__.py

@@ -4,7 +4,6 @@
 
 import shutil
 import subprocess
-from distutils.dir_util import copy_tree
 
 from jinja2 import Environment, PackageLoader
 from loguru import logger
@@ -69,7 +68,7 @@ def run(self, openapi: OpenapiParser, dry: bool = False) -> None:
         self._run_post_hooks()
 
         # copy rest api source into project dir
-        copy_tree(REST_API_SOURCE_LOCATION, str(self.config.project_dir / "rest_api"))
+        shutil.copytree(REST_API_SOURCE_LOCATION, str(self.config.project_dir / "rest_api"))
 
     def _build_meta_files(self) -> None:
         requirements_template = self.env.get_template("requirements.txt.j2")

dlt_init_openapi/renderer/default/templates/requirements.txt.j2

@@ -1 +1 @@
-dlt>=0.4.12
+dlt>=1.11.0

dlt_init_openapi/renderer/default/templates/source.py.j2

@@ -3,8 +3,8 @@ from typing import List
 import dlt
 from dlt.extract.source import DltResource
 
-from rest_api.typing import RESTAPIConfig
-from rest_api import rest_api_source
+from dlt.sources.rest_api.typing import RESTAPIConfig
+from dlt.sources.rest_api import rest_api_source
 
 
 @dlt.source(name="{{source_name}}", max_table_nesting=2)

dlt_init_openapi/utils/update_rest_api.py

@@ -3,29 +3,34 @@
 import requests
 from loguru import logger
 
-from dlt_init_openapi.config import REST_API_SOURCE_LOCATION
+# REST_API_SOURCE_LOCATION from config might resolve to site-packages.
+# For this script, we explicitly want to write into the source tree.
+# Assuming this script is in dlt_init_openapi/utils/update_rest_api.py
+# The target is dlt_init_openapi/dlt_init_openapi/rest_api/
+SCRIPT_DIR = pathlib.Path(__file__).parent.resolve()
+# Target path: dlt_init_openapi/dlt_init_openapi/rest_api/
+VENDOR_PATH = SCRIPT_DIR.parent / "rest_api"
 
 BASEPATH = "https://raw.githubusercontent.com/dlt-hub/verified-sources/master/sources/rest_api/"
-FILES = ["README.md", "__init__.py", "config_setup.py", "exceptions.py", "requirements.txt", "typing.py", "utils.py"]
+FILES = ["__init__.py", "requirements.txt"]
 
 
 def update_rest_api(force: bool = False) -> None:
     """updates local rest api"""
     logger.info("Syncing rest_api verified source")
 
-    path = pathlib.Path(REST_API_SOURCE_LOCATION)
-    if path.exists() and not force:
-        logger.info("rest_api verified source already present")
-        return
+    # Use the locally determined VENDOR_PATH for file operations
+    path = VENDOR_PATH
+    logger.info(f"Target absolute path for rest_api (script-determined): {path}")
 
-    path.mkdir(exist_ok=True)
+    path.mkdir(parents=True, exist_ok=True)
     for file in FILES:
         src_path = BASEPATH + file
-        dst_path = REST_API_SOURCE_LOCATION + "/" + file
-        logger.info(f"Copying {src_path}")
+        dst_file_path = path / file
+        logger.info(f"Copying {src_path} to {dst_file_path}")
         with requests.get(src_path, stream=True) as r:
             r.raise_for_status()
-            with open(dst_path, "wb") as f:
+            with open(dst_file_path, "wb") as f:
                 for chunk in r.iter_content(chunk_size=8192):
                     f.write(chunk)
     logger.success("rest_api verified source synced")