Update dlt dependency from ^0.4.12 to ^1.11.0

CHANGELOG.md

@@ -1,5 +1,14 @@
 [Go to GitHub Releases](https://github.com/dlt-hub/dlt-init-openapi/releases)
 
+## Unreleased
+
+* Bump dlt to 1.11.0
+
+## 0.1.0
+
+* Bump dlt to 0.4.12
+* First beta release of REST API client generator
+
 0.1.0 - Initial Release
 * Bump dlt to 0.4.12
 

Makefile

@@ -13,12 +13,12 @@ lint: update-rest-api
 	poetry run flake8 dlt_init_openapi tests
 	poetry run mypy dlt_init_openapi tests
 	poetry run black tests dlt_init_openapi --check
-	poetry run isort black tests dlt_init_openapi --check --diff
+	poetry run isort --profile black tests dlt_init_openapi --check --diff
 
 # format the whole project
 format: update-rest-api
 	rm -rf tests/_local
-	poetry run isort black tests dlt_init_openapi
+	poetry run isort --profile black tests dlt_init_openapi
 	poetry run black tests dlt_init_openapi
 
 # all tests excluding the checks on e2e tests
@@ -42,7 +42,7 @@ create-pokemon-pipeline-interactive:
 
 # e2e test helpers
 create-e2e-pokemon-pipeline:
-	poetry run dlt-init-openapi pokemon --path tests/cases/e2e_specs/pokeapi.yml --global-limit 2 --no-interactive
+	poetry run dlt-init-openapi pokemon --path tests/cases/e2e_specs/pokeapi.yml --no-interactive --global-limit 2
 
 run-pokemon-pipeline:
 	cd pokemon_pipeline && poetry run python pokemon_pipeline.py

dlt_init_openapi/config.py

@@ -2,7 +2,7 @@
 import mimetypes
 import pathlib
 from pathlib import Path
-from typing import Any, List, Optional
+from typing import Any, Dict, List, Optional
 
 import yaml
 from pydantic import BaseModel
@@ -11,7 +11,12 @@
 
 from .typing import TEndpointFilter
 
-REST_API_SOURCE_LOCATION = str(pathlib.Path(__file__).parent.resolve() / "../rest_api")
+# In dlt>=1.11.0, the rest_api is part of the main package
+# For backwards compatibility, we keep a stub directory
+REST_API_SOURCE_LOCATION = str(pathlib.Path(__file__).parent / "rest_api")
+
+# Placeholder for SecretsTomlConfig to resolve ImportError in tests
+SecretsTomlConfig = Dict[str, Any]
 
 
 class Config(BaseModel):

dlt_init_openapi/renderer/default/__init__.py

@@ -4,7 +4,6 @@
 
 import shutil
 import subprocess
-from distutils.dir_util import copy_tree
 
 from jinja2 import Environment, PackageLoader
 from loguru import logger
@@ -69,7 +68,7 @@ def run(self, openapi: OpenapiParser, dry: bool = False) -> None:
         self._run_post_hooks()
 
         # copy rest api source into project dir
-        copy_tree(REST_API_SOURCE_LOCATION, str(self.config.project_dir / "rest_api"))
+        shutil.copytree(REST_API_SOURCE_LOCATION, str(self.config.project_dir / "rest_api"))
 
     def _build_meta_files(self) -> None:
         requirements_template = self.env.get_template("requirements.txt.j2")

dlt_init_openapi/renderer/default/templates/requirements.txt.j2

@@ -1 +1 @@
-dlt>=0.4.12
+dlt>=1.11.0

dlt_init_openapi/renderer/default/templates/source.py.j2

@@ -3,8 +3,8 @@ from typing import List
 import dlt
 from dlt.extract.source import DltResource
 
-from rest_api.typing import RESTAPIConfig
-from rest_api import rest_api_source
+from dlt.sources.rest_api.typing import RESTAPIConfig
+from dlt.sources.rest_api import rest_api_source
 
 
 @dlt.source(name="{{source_name}}", max_table_nesting=2)

dlt_init_openapi/utils/update_rest_api.py

@@ -1,34 +1,24 @@
 import pathlib
 
-import requests
 from loguru import logger
 
-from dlt_init_openapi.config import REST_API_SOURCE_LOCATION
-
-BASEPATH = "https://raw.githubusercontent.com/dlt-hub/verified-sources/master/sources/rest_api/"
-FILES = ["README.md", "__init__.py", "config_setup.py", "exceptions.py", "requirements.txt", "typing.py", "utils.py"]
-
 
 def update_rest_api(force: bool = False) -> None:
-    """updates local rest api"""
-    logger.info("Syncing rest_api verified source")
-
-    path = pathlib.Path(REST_API_SOURCE_LOCATION)
-    if path.exists() and not force:
-        logger.info("rest_api verified source already present")
-        return
-
-    path.mkdir(exist_ok=True)
-    for file in FILES:
-        src_path = BASEPATH + file
-        dst_path = REST_API_SOURCE_LOCATION + "/" + file
-        logger.info(f"Copying {src_path}")
-        with requests.get(src_path, stream=True) as r:
-            r.raise_for_status()
-            with open(dst_path, "wb") as f:
-                for chunk in r.iter_content(chunk_size=8192):
-                    f.write(chunk)
-    logger.success("rest_api verified source synced")
+    """
+    This function is kept for backwards compatibility.
+    In dlt >=1.0.0, rest_api is part of the main package.
+    No need to vendor the files separately.
+    """
+    logger.info("Using built-in dlt.sources.rest_api (dlt >=1.11.0)")
+    # Create an empty rest_api directory to maintain compatibility
+    script_dir = pathlib.Path(__file__).parent.resolve()
+    vendor_path = script_dir.parent / "rest_api"
+    vendor_path.mkdir(parents=True, exist_ok=True)
+    # Create empty __init__.py to make it a proper package
+    init_file = vendor_path / "__init__.py"
+    if not init_file.exists():
+        with open(init_file, "w") as f:
+            f.write("# This is a compatibility package. Use dlt.sources.rest_api instead.\n")
 
 
 if __name__ == "__main__":