Skip to content

Commit

Permalink
Delete code from another branch
Browse files Browse the repository at this point in the history
  • Loading branch information
@tolik0
tolik0 committed Feb 12, 2025
1 parent 342375c commit 05f4db7
Show file tree
Hide file tree
Showing 3 changed files with 0 additions and 279 deletions.
159 changes: 0 additions & 159 deletions unit_tests/sources/declarative/parsers/test_model_to_component_factory.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,6 @@
from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields
from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
from airbyte_cdk.sources.streams.call_rate import MovingWindowCallRatePolicy
from airbyte_cdk.sources.streams.concurrent.clamping import (
ClampingEndProvider,
DayClampingStrategy,
Expand Down Expand Up @@ -3685,161 +3684,3 @@ def test_create_async_retriever():
assert isinstance(selector, RecordSelector)
assert isinstance(extractor, DpathExtractor)
assert extractor.field_path == ["data"]


def test_api_budget():
manifest = {
"type": "DeclarativeSource",
"api_budget": {
"type": "HTTPAPIBudget",
"ratelimit_reset_header": "X-RateLimit-Reset",
"ratelimit_remaining_header": "X-RateLimit-Remaining",
"status_codes_for_ratelimit_hit": [429, 503],
"policies": [
{
"type": "MovingWindowCallRatePolicy",
"rates": [
{
"type": "Rate",
"limit": 3,
"interval": "PT0.1S", # 0.1 seconds
}
],
"matchers": [
{
"type": "HttpRequestRegexMatcher",
"method": "GET",
"url_base": "https://api.sendgrid.com",
"url_path_pattern": "/v3/marketing/lists",
}
],
}
],
},
"my_requester": {
"type": "HttpRequester",
"path": "/v3/marketing/lists",
"url_base": "https://api.sendgrid.com",
"http_method": "GET",
"authenticator": {
"type": "BasicHttpAuthenticator",
"username": "admin",
"password": "{{ config['password'] }}",
},
},
}

config = {
"password": "verysecrettoken",
}

factory = ModelToComponentFactory()
if "api_budget" in manifest:
factory.set_api_budget(manifest["api_budget"], config)

from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
HttpRequester as HttpRequesterModel,
)

requester_definition = manifest["my_requester"]
assert requester_definition["type"] == "HttpRequester"

http_requester = factory.create_component(
model_type=HttpRequesterModel,
component_definition=requester_definition,
config=config,
name="lists_stream",
decoder=None,
)

assert http_requester.api_budget is not None
assert http_requester.api_budget.ratelimit_reset_header == "X-RateLimit-Reset"
assert http_requester.api_budget.status_codes_for_ratelimit_hit == [429, 503]
assert len(http_requester.api_budget.policies) == 1

# The single policy is a MovingWindowCallRatePolicy
policy = http_requester.api_budget.policies[0]
assert isinstance(policy, MovingWindowCallRatePolicy)
assert policy._bucket.rates[0].limit == 3
# The 0.1s from 'PT0.1S' is stored in ms by PyRateLimiter internally
# but here just check that the limit and interval exist
assert policy._bucket.rates[0].interval == 100 # 100 ms


def test_api_budget_fixed_window_policy():
manifest = {
"type": "DeclarativeSource",
# Root-level api_budget referencing a FixedWindowCallRatePolicy
"api_budget": {
"type": "APIBudget",
"maximum_attempts_to_acquire": 9999,
"policies": [
{
"type": "FixedWindowCallRatePolicy",
"next_reset_ts": "2025-01-01T00:00:00Z",
"period": "PT1M", # 1 minute
"call_limit": 10,
"matchers": [
{
"type": "HttpRequestRegexMatcher",
"method": "GET",
"url_base": "https://example.org",
"url_path_pattern": "/v2/data",
}
],
}
],
},
# We'll define a single HttpRequester that references that base
"my_requester": {
"type": "HttpRequester",
"path": "/v2/data",
"url_base": "https://example.org",
"http_method": "GET",
"authenticator": {"type": "NoAuth"},
},
}

config = {}

factory = ModelToComponentFactory()
if "api_budget" in manifest:
factory.set_api_budget(manifest["api_budget"], config)

from airbyte_cdk.sources.declarative.models.declarative_component_schema import (
HttpRequester as HttpRequesterModel,
)

requester_definition = manifest["my_requester"]
assert requester_definition["type"] == "HttpRequester"
http_requester = factory.create_component(
model_type=HttpRequesterModel,
component_definition=requester_definition,
config=config,
name="my_stream",
decoder=None,
)

assert http_requester.api_budget is not None
assert http_requester.api_budget.maximum_attempts_to_acquire == 9999
assert len(http_requester.api_budget.policies) == 1

from airbyte_cdk.sources.streams.call_rate import FixedWindowCallRatePolicy

policy = http_requester.api_budget.policies[0]
assert isinstance(policy, FixedWindowCallRatePolicy)
assert policy._call_limit == 10
# The period is "PT1M" => 60 seconds
assert policy._offset.total_seconds() == 60

expected_reset_dt = datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
assert policy._next_reset_ts == expected_reset_dt

assert len(policy._matchers) == 1
matcher = policy._matchers[0]
from airbyte_cdk.sources.streams.call_rate import HttpRequestRegexMatcher

assert isinstance(matcher, HttpRequestRegexMatcher)
assert matcher._method == "GET"
assert matcher._url_base == "https://example.org"
assert matcher._url_path_pattern.pattern == "/v2/data"
32 changes: 0 additions & 32 deletions unit_tests/sources/declarative/requesters/test_http_requester.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,13 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#

from datetime import timedelta
from typing import Any, Mapping, Optional
from unittest import mock
from unittest.mock import MagicMock
from urllib.parse import parse_qs, urlparse

import pytest as pytest
import requests
import requests.sessions
from requests import PreparedRequest

from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
Expand All @@ -29,12 +27,6 @@
InterpolatedRequestOptionsProvider,
)
from airbyte_cdk.sources.message import MessageRepository
from airbyte_cdk.sources.streams.call_rate import (
AbstractAPIBudget,
HttpAPIBudget,
MovingWindowCallRatePolicy,
Rate,
)
from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction
from airbyte_cdk.sources.streams.http.exceptions import (
RequestBodyException,
Expand All @@ -53,7 +45,6 @@ def factory(
request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None,
authenticator: Optional[DeclarativeAuthenticator] = None,
error_handler: Optional[ErrorHandler] = None,
api_budget: Optional[HttpAPIBudget] = None,
config: Optional[Config] = None,
parameters: Mapping[str, Any] = None,
disable_retries: bool = False,
Expand All @@ -70,7 +61,6 @@ def factory(
http_method=http_method,
request_options_provider=request_options_provider,
error_handler=error_handler,
api_budget=api_budget,
disable_retries=disable_retries,
message_repository=message_repository or MagicMock(),
use_cache=use_cache,
Expand Down Expand Up @@ -944,25 +934,3 @@ def test_backoff_strategy_from_manifest_is_respected(http_requester_factory: Any
http_requester._http_client._request_attempt_count.get(request_mock)
== http_requester._http_client._max_retries + 1
)


def test_http_requester_with_mock_apibudget(http_requester_factory, monkeypatch):
mock_budget = MagicMock(spec=HttpAPIBudget)

requester = http_requester_factory(
url_base="https://example.com",
path="test",
api_budget=mock_budget,
)

dummy_response = requests.Response()
dummy_response.status_code = 200
send_mock = MagicMock(return_value=dummy_response)
monkeypatch.setattr(requests.Session, "send", send_mock)

response = requester.send_request()

assert send_mock.call_count == 1
assert response.status_code == 200

assert mock_budget.acquire_call.call_count == 1
88 changes: 0 additions & 88 deletions unit_tests/sources/streams/test_call_rate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
CallRateLimitHit,
FixedWindowCallRatePolicy,
HttpRequestMatcher,
HttpRequestRegexMatcher,
MovingWindowCallRatePolicy,
Rate,
UnlimitedCallRatePolicy,
Expand Down Expand Up @@ -358,90 +357,3 @@ def test_with_cache(self, mocker, requests_mock):
assert next(records) == {"data": "some_data"}

assert MovingWindowCallRatePolicy.try_acquire.call_count == 1


class TestHttpRequestRegexMatcher:
"""
Tests for the new regex-based logic:
- Case-insensitive HTTP method matching
- Optional url_base (scheme://netloc)
- Regex-based path matching
- Query params (must be present)
- Headers (case-insensitive keys)
"""

def test_case_insensitive_method(self):
matcher = HttpRequestRegexMatcher(method="GET")

req_ok = Request("get", "https://example.com/test/path")
req_wrong = Request("POST", "https://example.com/test/path")

assert matcher(req_ok)
assert not matcher(req_wrong)

def test_url_base(self):
matcher = HttpRequestRegexMatcher(url_base="https://example.com")

req_ok = Request("GET", "https://example.com/test/path?foo=bar")
req_wrong = Request("GET", "https://another.com/test/path?foo=bar")

assert matcher(req_ok)
assert not matcher(req_wrong)

def test_url_path_pattern(self):
matcher = HttpRequestRegexMatcher(url_path_pattern=r"/test/")

req_ok = Request("GET", "https://example.com/test/something")
req_wrong = Request("GET", "https://example.com/other/something")

assert matcher(req_ok)
assert not matcher(req_wrong)

def test_query_params(self):
matcher = HttpRequestRegexMatcher(params={"foo": "bar"})

req_ok = Request("GET", "https://example.com/api?foo=bar&extra=123")
req_missing = Request("GET", "https://example.com/api?not_foo=bar")

assert matcher(req_ok)
assert not matcher(req_missing)

def test_headers_case_insensitive(self):
matcher = HttpRequestRegexMatcher(headers={"X-Custom-Header": "abc"})

req_ok = Request(
"GET",
"https://example.com/api?foo=bar",
headers={"x-custom-header": "abc", "other": "123"},
)
req_wrong = Request("GET", "https://example.com/api", headers={"x-custom-header": "wrong"})

assert matcher(req_ok)
assert not matcher(req_wrong)

def test_combined_criteria(self):
matcher = HttpRequestRegexMatcher(
method="GET",
url_base="https://example.com",
url_path_pattern=r"/test/",
params={"foo": "bar"},
headers={"X-Test": "123"},
)

req_ok = Request("GET", "https://example.com/test/me?foo=bar", headers={"x-test": "123"})
req_bad_base = Request(
"GET", "https://other.com/test/me?foo=bar", headers={"x-test": "123"}
)
req_bad_path = Request("GET", "https://example.com/nope?foo=bar", headers={"x-test": "123"})
req_bad_param = Request(
"GET", "https://example.com/test/me?extra=xyz", headers={"x-test": "123"}
)
req_bad_header = Request(
"GET", "https://example.com/test/me?foo=bar", headers={"some-other-header": "xyz"}
)

assert matcher(req_ok)
assert not matcher(req_bad_base)
assert not matcher(req_bad_path)
assert not matcher(req_bad_param)
assert not matcher(req_bad_header)

0 comments on commit 05f4db7

Please sign in to comment.