docs: document that path can also be a URL (#2099)

Author: joscha

dlt/sources/helpers/rest_client/client.py

@@ -96,18 +96,18 @@ def __init__(
 
     def _create_request(
         self,
-        path: str,
+        path_or_url: str,
         method: HTTPMethod,
         params: Optional[Dict[str, Any]] = None,
         json: Optional[Dict[str, Any]] = None,
         auth: Optional[AuthBase] = None,
         hooks: Optional[Hooks] = None,
     ) -> Request:
-        parsed_url = urlparse(path)
+        parsed_url = urlparse(path_or_url)
         if parsed_url.scheme in ("http", "https"):
-            url = path
+            url = path_or_url
         else:
-            url = join_url(self.base_url, path)
+            url = join_url(self.base_url, path_or_url)
 
         return Request(
             method=method,
@@ -140,7 +140,7 @@ def _send_request(self, request: Request, **kwargs: Any) -> Response:
 
     def request(self, path: str = "", method: HTTPMethod = "GET", **kwargs: Any) -> Response:
         prepared_request = self._create_request(
-            path=path,
+            path_or_url=path,
             method=method,
             params=kwargs.pop("params", None),
             json=kwargs.pop("json", None),
@@ -171,6 +171,8 @@ def paginate(
 
         Args:
             path (str): Endpoint path for the request, relative to `base_url`.
+                Can also be a fully qualified URL; if starting with http(s) it will
+                be used instead of the base_url + path.
             method (HTTPMethodBasic): HTTP method for the request, defaults to 'get'.
             params (Optional[Dict[str, Any]]): URL parameters for the request.
             json (Optional[Dict[str, Any]]): JSON payload for the request.
@@ -210,7 +212,7 @@ def paginate(
             hooks["response"] = [raise_for_status]
 
         request = self._create_request(
-            path=path, method=method, params=params, json=json, auth=auth, hooks=hooks
+            path_or_url=path, method=method, params=params, json=json, auth=auth, hooks=hooks
         )
 
         if paginator:

docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md

@@ -335,7 +335,8 @@ The endpoint configuration defines how to query the API endpoint. Quick example:
 
 The fields in the endpoint configuration are:
 
-- `path`: The path to the API endpoint.
+- `path`: The path to the API endpoint. By default this path is appended to the given `base_url`. If this is a fully qualified URL starting with `http:` or `https:` it will be
+used as-is and `base_url` will be ignored.
 - `method`: The HTTP method to be used. The default is `GET`.
 - `params`: Query parameters to be sent with each request. For example, `sort` to order the results or `since` to specify [incremental loading](#incremental-loading). This is also used to define [resource relationships](#define-resource-relationships).
 - `json`: The JSON payload to be sent with the request (for POST and PUT requests).

tests/sources/helpers/rest_client/test_client.py

@@ -7,6 +7,7 @@
 from requests import PreparedRequest, Request, Response
 from requests.auth import AuthBase
 from requests.exceptions import HTTPError
+import requests_mock
 
 from dlt.common import logger
 from dlt.common.typing import TSecretStrValue
@@ -512,3 +513,24 @@ def test_request_kwargs(self, mocker) -> None:
             "timeout": 432,
             "allow_redirects": False,
         }
+
+    @requests_mock.Mocker(kw="mock")
+    def test_overwrite_path(self, mocker, **kwargs) -> None:
+        expected = {"foo": "bar"}
+        kwargs["mock"].get("https://completely.different/endpoint", json=expected)
+        rest_client = RESTClient(
+            base_url="https://api.example.com",
+        )
+        response = rest_client.get("https://completely.different/endpoint")
+        assert response.json() == expected
+
+    @requests_mock.Mocker(kw="mock")
+    def test_overwrite_path_ignores_different_protocol(self, mocker, **kwargs) -> None:
+        expected = {"foo": "bar"}
+        base_url = "https://api.example.com"
+        kwargs["mock"].get(f"{base_url}/my://protocol", json=expected)
+        rest_client = RESTClient(
+            base_url=base_url,
+        )
+        response = rest_client.get("my://protocol")
+        assert response.json() == expected