diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/README.md b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/README.md
new file mode 100644
index 000000000..78505726c
--- /dev/null
+++ b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/README.md
@@ -0,0 +1,3 @@
+# PokeAPI with Custom `components.py` API Tests
+
+This test connector is a modified version of `source-pokeapi`. It has been modified to use custom `components.py` so we have a test case the completes quickly and _does not_ require any credentials.
diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components.py b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components.py
new file mode 100644
index 000000000..5e7e16f71
--- /dev/null
+++ b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components.py
@@ -0,0 +1,20 @@
+"""A sample implementation of custom components that does nothing but will cause syncs to fail if missing."""
+
+from typing import Any, Mapping
+
+import requests
+
+from airbyte_cdk.sources.declarative.extractors import DpathExtractor
+
+
+class IntentionalException(Exception):
+ """This exception is raised intentionally in order to test error handling."""
+
+
+class MyCustomExtractor(DpathExtractor):
+ """Dummy class, directly implements DPatchExtractor.
+
+ Used to prove that SDM can find the custom class by name.
+ """
+
+ pass
diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components_failing.py b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components_failing.py
new file mode 100644
index 000000000..5c05881e7
--- /dev/null
+++ b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components_failing.py
@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+"""A sample implementation of custom components that does nothing but will cause syncs to fail if missing."""
+
+from collections.abc import Iterable, MutableMapping
+from dataclasses import InitVar, dataclass
+from typing import Any, Mapping, Optional, Union
+
+import requests
+
+from airbyte_cdk.sources.declarative.extractors import DpathExtractor
+
+
+class IntentionalException(Exception):
+ """This exception is raised intentionally in order to test error handling."""
+
+
+class MyCustomExtractor(DpathExtractor):
+ def extract_records(
+ self,
+ response: requests.Response,
+ ) -> Iterable[MutableMapping[Any, Any]]:
+ raise IntentionalException
diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/manifest.yaml b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/manifest.yaml
new file mode 100644
index 000000000..af19485fa
--- /dev/null
+++ b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/manifest.yaml
@@ -0,0 +1,980 @@
+version: 6.30.0
+
+type: DeclarativeSource
+
+check:
+ type: CheckStream
+ stream_names:
+ - pokemon
+
+definitions:
+ streams:
+ pokemon:
+ type: DeclarativeStream
+ name: pokemon
+ retriever:
+ type: SimpleRetriever
+ requester:
+ $ref: "#/definitions/base_requester"
+ path: /{{config['pokemon_name']}}
+ http_method: GET
+ record_selector:
+ type: RecordSelector
+ extractor:
+ # Simple wrapper around `DpathExtractor`
+ type: CustomRecordExtractor
+ class_name: components.MyCustomExtractor
+ field_path: []
+ primary_key:
+ - id
+ schema_loader:
+ type: InlineSchemaLoader
+ # type: CustomSchemaLoader
+ schema:
+ $ref: "#/schemas/pokemon"
+ # class_name: components.MyCustomInlineSchemaLoader
+ base_requester:
+ type: HttpRequester
+ url_base: https://pokeapi.co/api/v2/pokemon
+
+streams:
+ - $ref: "#/definitions/streams/pokemon"
+
+spec:
+ type: Spec
+ connection_specification:
+ type: object
+ $schema: http://json-schema.org/draft-07/schema#
+ required:
+ - pokemon_name
+ properties:
+ pokemon_name:
+ type: string
+ description: Pokemon requested from the API.
+ enum:
+ - bulbasaur
+ - ivysaur
+ - venusaur
+ - charmander
+ - charmeleon
+ - charizard
+ - squirtle
+ - wartortle
+ - blastoise
+ - caterpie
+ - metapod
+ - butterfree
+ - weedle
+ - kakuna
+ - beedrill
+ - pidgey
+ - pidgeotto
+ - pidgeot
+ - rattata
+ - raticate
+ - spearow
+ - fearow
+ - ekans
+ - arbok
+ - pikachu
+ - raichu
+ - sandshrew
+ - sandslash
+ - nidoranf
+ - nidorina
+ - nidoqueen
+ - nidoranm
+ - nidorino
+ - nidoking
+ - clefairy
+ - clefable
+ - vulpix
+ - ninetales
+ - jigglypuff
+ - wigglytuff
+ - zubat
+ - golbat
+ - oddish
+ - gloom
+ - vileplume
+ - paras
+ - parasect
+ - venonat
+ - venomoth
+ - diglett
+ - dugtrio
+ - meowth
+ - persian
+ - psyduck
+ - golduck
+ - mankey
+ - primeape
+ - growlithe
+ - arcanine
+ - poliwag
+ - poliwhirl
+ - poliwrath
+ - abra
+ - kadabra
+ - alakazam
+ - machop
+ - machoke
+ - machamp
+ - bellsprout
+ - weepinbell
+ - victreebel
+ - tentacool
+ - tentacruel
+ - geodude
+ - graveler
+ - golem
+ - ponyta
+ - rapidash
+ - slowpoke
+ - slowbro
+ - magnemite
+ - magneton
+ - farfetchd
+ - doduo
+ - dodrio
+ - seel
+ - dewgong
+ - grimer
+ - muk
+ - shellder
+ - cloyster
+ - gastly
+ - haunter
+ - gengar
+ - onix
+ - drowzee
+ - hypno
+ - krabby
+ - kingler
+ - voltorb
+ - electrode
+ - exeggcute
+ - exeggutor
+ - cubone
+ - marowak
+ - hitmonlee
+ - hitmonchan
+ - lickitung
+ - koffing
+ - weezing
+ - rhyhorn
+ - rhydon
+ - chansey
+ - tangela
+ - kangaskhan
+ - horsea
+ - seadra
+ - goldeen
+ - seaking
+ - staryu
+ - starmie
+ - mrmime
+ - scyther
+ - jynx
+ - electabuzz
+ - magmar
+ - pinsir
+ - tauros
+ - magikarp
+ - gyarados
+ - lapras
+ - ditto
+ - eevee
+ - vaporeon
+ - jolteon
+ - flareon
+ - porygon
+ - omanyte
+ - omastar
+ - kabuto
+ - kabutops
+ - aerodactyl
+ - snorlax
+ - articuno
+ - zapdos
+ - moltres
+ - dratini
+ - dragonair
+ - dragonite
+ - mewtwo
+ - mew
+ - chikorita
+ - bayleef
+ - meganium
+ - cyndaquil
+ - quilava
+ - typhlosion
+ - totodile
+ - croconaw
+ - feraligatr
+ - sentret
+ - furret
+ - hoothoot
+ - noctowl
+ - ledyba
+ - ledian
+ - spinarak
+ - ariados
+ - crobat
+ - chinchou
+ - lanturn
+ - pichu
+ - cleffa
+ - igglybuff
+ - togepi
+ - togetic
+ - natu
+ - xatu
+ - mareep
+ - flaaffy
+ - ampharos
+ - bellossom
+ - marill
+ - azumarill
+ - sudowoodo
+ - politoed
+ - hoppip
+ - skiploom
+ - jumpluff
+ - aipom
+ - sunkern
+ - sunflora
+ - yanma
+ - wooper
+ - quagsire
+ - espeon
+ - umbreon
+ - murkrow
+ - slowking
+ - misdreavus
+ - unown
+ - wobbuffet
+ - girafarig
+ - pineco
+ - forretress
+ - dunsparce
+ - gligar
+ - steelix
+ - snubbull
+ - granbull
+ - qwilfish
+ - scizor
+ - shuckle
+ - heracross
+ - sneasel
+ - teddiursa
+ - ursaring
+ - slugma
+ - magcargo
+ - swinub
+ - piloswine
+ - corsola
+ - remoraid
+ - octillery
+ - delibird
+ - mantine
+ - skarmory
+ - houndour
+ - houndoom
+ - kingdra
+ - phanpy
+ - donphan
+ - porygon2
+ - stantler
+ - smeargle
+ - tyrogue
+ - hitmontop
+ - smoochum
+ - elekid
+ - magby
+ - miltank
+ - blissey
+ - raikou
+ - entei
+ - suicune
+ - larvitar
+ - pupitar
+ - tyranitar
+ - lugia
+ - ho-oh
+ - celebi
+ - treecko
+ - grovyle
+ - sceptile
+ - torchic
+ - combusken
+ - blaziken
+ - mudkip
+ - marshtomp
+ - swampert
+ - poochyena
+ - mightyena
+ - zigzagoon
+ - linoone
+ - wurmple
+ - silcoon
+ - beautifly
+ - cascoon
+ - dustox
+ - lotad
+ - lombre
+ - ludicolo
+ - seedot
+ - nuzleaf
+ - shiftry
+ - taillow
+ - swellow
+ - wingull
+ - pelipper
+ - ralts
+ - kirlia
+ - gardevoir
+ - surskit
+ - masquerain
+ - shroomish
+ - breloom
+ - slakoth
+ - vigoroth
+ - slaking
+ - nincada
+ - ninjask
+ - shedinja
+ - whismur
+ - loudred
+ - exploud
+ - makuhita
+ - hariyama
+ - azurill
+ - nosepass
+ - skitty
+ - delcatty
+ - sableye
+ - mawile
+ - aron
+ - lairon
+ - aggron
+ - meditite
+ - medicham
+ - electrike
+ - manectric
+ - plusle
+ - minun
+ - volbeat
+ - illumise
+ - roselia
+ - gulpin
+ - swalot
+ - carvanha
+ - sharpedo
+ - wailmer
+ - wailord
+ - numel
+ - camerupt
+ - torkoal
+ - spoink
+ - grumpig
+ - spinda
+ - trapinch
+ - vibrava
+ - flygon
+ - cacnea
+ - cacturne
+ - swablu
+ - altaria
+ - zangoose
+ - seviper
+ - lunatone
+ - solrock
+ - barboach
+ - whiscash
+ - corphish
+ - crawdaunt
+ - baltoy
+ - claydol
+ - lileep
+ - cradily
+ - anorith
+ - armaldo
+ - feebas
+ - milotic
+ - castform
+ - kecleon
+ - shuppet
+ - banette
+ - duskull
+ - dusclops
+ - tropius
+ - chimecho
+ - absol
+ - wynaut
+ - snorunt
+ - glalie
+ - spheal
+ - sealeo
+ - walrein
+ - clamperl
+ - huntail
+ - gorebyss
+ - relicanth
+ - luvdisc
+ - bagon
+ - shelgon
+ - salamence
+ - beldum
+ - metang
+ - metagross
+ - regirock
+ - regice
+ - registeel
+ - latias
+ - latios
+ - kyogre
+ - groudon
+ - rayquaza
+ - jirachi
+ - deoxys
+ - turtwig
+ - grotle
+ - torterra
+ - chimchar
+ - monferno
+ - infernape
+ - piplup
+ - prinplup
+ - empoleon
+ - starly
+ - staravia
+ - staraptor
+ - bidoof
+ - bibarel
+ - kricketot
+ - kricketune
+ - shinx
+ - luxio
+ - luxray
+ - budew
+ - roserade
+ - cranidos
+ - rampardos
+ - shieldon
+ - bastiodon
+ - burmy
+ - wormadam
+ - mothim
+ - combee
+ - vespiquen
+ - pachirisu
+ - buizel
+ - floatzel
+ - cherubi
+ - cherrim
+ - shellos
+ - gastrodon
+ - ambipom
+ - drifloon
+ - drifblim
+ - buneary
+ - lopunny
+ - mismagius
+ - honchkrow
+ - glameow
+ - purugly
+ - chingling
+ - stunky
+ - skuntank
+ - bronzor
+ - bronzong
+ - bonsly
+ - mimejr
+ - happiny
+ - chatot
+ - spiritomb
+ - gible
+ - gabite
+ - garchomp
+ - munchlax
+ - riolu
+ - lucario
+ - hippopotas
+ - hippowdon
+ - skorupi
+ - drapion
+ - croagunk
+ - toxicroak
+ - carnivine
+ - finneon
+ - lumineon
+ - mantyke
+ - snover
+ - abomasnow
+ - weavile
+ - magnezone
+ - lickilicky
+ - rhyperior
+ - tangrowth
+ - electivire
+ - magmortar
+ - togekiss
+ - yanmega
+ - leafeon
+ - glaceon
+ - gliscor
+ - mamoswine
+ - porygon-z
+ - gallade
+ - probopass
+ - dusknoir
+ - froslass
+ - rotom
+ - uxie
+ - mesprit
+ - azelf
+ - dialga
+ - palkia
+ - heatran
+ - regigigas
+ - giratina
+ - cresselia
+ - phione
+ - manaphy
+ - darkrai
+ - shaymin
+ - arceus
+ - victini
+ - snivy
+ - servine
+ - serperior
+ - tepig
+ - pignite
+ - emboar
+ - oshawott
+ - dewott
+ - samurott
+ - patrat
+ - watchog
+ - lillipup
+ - herdier
+ - stoutland
+ - purrloin
+ - liepard
+ - pansage
+ - simisage
+ - pansear
+ - simisear
+ - panpour
+ - simipour
+ - munna
+ - musharna
+ - pidove
+ - tranquill
+ - unfezant
+ - blitzle
+ - zebstrika
+ - roggenrola
+ - boldore
+ - gigalith
+ - woobat
+ - swoobat
+ - drilbur
+ - excadrill
+ - audino
+ - timburr
+ - gurdurr
+ - conkeldurr
+ - tympole
+ - palpitoad
+ - seismitoad
+ - throh
+ - sawk
+ - sewaddle
+ - swadloon
+ - leavanny
+ - venipede
+ - whirlipede
+ - scolipede
+ - cottonee
+ - whimsicott
+ - petilil
+ - lilligant
+ - basculin
+ - sandile
+ - krokorok
+ - krookodile
+ - darumaka
+ - darmanitan
+ - maractus
+ - dwebble
+ - crustle
+ - scraggy
+ - scrafty
+ - sigilyph
+ - yamask
+ - cofagrigus
+ - tirtouga
+ - carracosta
+ - archen
+ - archeops
+ - trubbish
+ - garbodor
+ - zorua
+ - zoroark
+ - minccino
+ - cinccino
+ - gothita
+ - gothorita
+ - gothitelle
+ - solosis
+ - duosion
+ - reuniclus
+ - ducklett
+ - swanna
+ - vanillite
+ - vanillish
+ - vanilluxe
+ - deerling
+ - sawsbuck
+ - emolga
+ - karrablast
+ - escavalier
+ - foongus
+ - amoonguss
+ - frillish
+ - jellicent
+ - alomomola
+ - joltik
+ - galvantula
+ - ferroseed
+ - ferrothorn
+ - klink
+ - klang
+ - klinklang
+ - tynamo
+ - eelektrik
+ - eelektross
+ - elgyem
+ - beheeyem
+ - litwick
+ - lampent
+ - chandelure
+ - axew
+ - fraxure
+ - haxorus
+ - cubchoo
+ - beartic
+ - cryogonal
+ - shelmet
+ - accelgor
+ - stunfisk
+ - mienfoo
+ - mienshao
+ - druddigon
+ - golett
+ - golurk
+ - pawniard
+ - bisharp
+ - bouffalant
+ - rufflet
+ - braviary
+ - vullaby
+ - mandibuzz
+ - heatmor
+ - durant
+ - deino
+ - zweilous
+ - hydreigon
+ - larvesta
+ - volcarona
+ - cobalion
+ - terrakion
+ - virizion
+ - tornadus
+ - thundurus
+ - reshiram
+ - zekrom
+ - landorus
+ - kyurem
+ - keldeo
+ - meloetta
+ - genesect
+ - chespin
+ - quilladin
+ - chesnaught
+ - fennekin
+ - braixen
+ - delphox
+ - froakie
+ - frogadier
+ - greninja
+ - bunnelby
+ - diggersby
+ - fletchling
+ - fletchinder
+ - talonflame
+ - scatterbug
+ - spewpa
+ - vivillon
+ - litleo
+ - pyroar
+ - flabebe
+ - floette
+ - florges
+ - skiddo
+ - gogoat
+ - pancham
+ - pangoro
+ - furfrou
+ - espurr
+ - meowstic
+ - honedge
+ - doublade
+ - aegislash
+ - spritzee
+ - aromatisse
+ - swirlix
+ - slurpuff
+ - inkay
+ - malamar
+ - binacle
+ - barbaracle
+ - skrelp
+ - dragalge
+ - clauncher
+ - clawitzer
+ - helioptile
+ - heliolisk
+ - tyrunt
+ - tyrantrum
+ - amaura
+ - aurorus
+ - sylveon
+ - hawlucha
+ - dedenne
+ - carbink
+ - goomy
+ - sliggoo
+ - goodra
+ - klefki
+ - phantump
+ - trevenant
+ - pumpkaboo
+ - gourgeist
+ - bergmite
+ - avalugg
+ - noibat
+ - noivern
+ - xerneas
+ - yveltal
+ - zygarde
+ - diancie
+ - hoopa
+ - volcanion
+ - rowlet
+ - dartrix
+ - decidueye
+ - litten
+ - torracat
+ - incineroar
+ - popplio
+ - brionne
+ - primarina
+ - pikipek
+ - trumbeak
+ - toucannon
+ - yungoos
+ - gumshoos
+ - grubbin
+ - charjabug
+ - vikavolt
+ - crabrawler
+ - crabominable
+ - oricorio
+ - cutiefly
+ - ribombee
+ - rockruff
+ - lycanroc
+ - wishiwashi
+ - mareanie
+ - toxapex
+ - mudbray
+ - mudsdale
+ - dewpider
+ - araquanid
+ - fomantis
+ - lurantis
+ - morelull
+ - shiinotic
+ - salandit
+ - salazzle
+ - stufful
+ - bewear
+ - bounsweet
+ - steenee
+ - tsareena
+ - comfey
+ - oranguru
+ - passimian
+ - wimpod
+ - golisopod
+ - sandygast
+ - palossand
+ - pyukumuku
+ - typenull
+ - silvally
+ - minior
+ - komala
+ - turtonator
+ - togedemaru
+ - mimikyu
+ - bruxish
+ - drampa
+ - dhelmise
+ - jangmo-o
+ - hakamo-o
+ - kommo-o
+ - tapukoko
+ - tapulele
+ - tapubulu
+ - tapufini
+ - cosmog
+ - cosmoem
+ - solgaleo
+ - lunala
+ - nihilego
+ - buzzwole
+ - pheromosa
+ - xurkitree
+ - celesteela
+ - kartana
+ - guzzlord
+ - necrozma
+ - magearna
+ - marshadow
+ - poipole
+ - naganadel
+ - stakataka
+ - blacephalon
+ - zeraora
+ - meltan
+ - melmetal
+ - grookey
+ - thwackey
+ - rillaboom
+ - scorbunny
+ - raboot
+ - cinderace
+ - sobble
+ - drizzile
+ - inteleon
+ - skwovet
+ - greedent
+ - rookidee
+ - corvisquire
+ - corviknight
+ - blipbug
+ - dottler
+ - orbeetle
+ - nickit
+ - thievul
+ - gossifleur
+ - eldegoss
+ - wooloo
+ - dubwool
+ - chewtle
+ - drednaw
+ - yamper
+ - boltund
+ - rolycoly
+ - carkol
+ - coalossal
+ - applin
+ - flapple
+ - appletun
+ - silicobra
+ - sandaconda
+ - cramorant
+ - arrokuda
+ - barraskewda
+ - toxel
+ - toxtricity
+ - sizzlipede
+ - centiskorch
+ - clobbopus
+ - grapploct
+ - sinistea
+ - polteageist
+ - hatenna
+ - hattrem
+ - hatterene
+ - impidimp
+ - morgrem
+ - grimmsnarl
+ - obstagoon
+ - perrserker
+ - cursola
+ - sirfetchd
+ - mrrime
+ - runerigus
+ - milcery
+ - alcremie
+ - falinks
+ - pincurchin
+ - snom
+ - frosmoth
+ - stonjourner
+ - eiscue
+ - indeedee
+ - morpeko
+ - cufant
+ - copperajah
+ - dracozolt
+ - arctozolt
+ - dracovish
+ - arctovish
+ - duraludon
+ - dreepy
+ - drakloak
+ - dragapult
+ - zacian
+ - zamazenta
+ - eternatus
+ - kubfu
+ - urshifu
+ - zarude
+ - regieleki
+ - regidrago
+ - glastrier
+ - spectrier
+ - calyrex
+ order: 0
+ title: Pokemon Name
+ pattern: ^[a-z0-9_\-]+$
+ examples:
+ - ditto
+ - luxray
+ - snorlax
+ additionalProperties: true
+
+metadata:
+ assist: {}
+ testedStreams:
+ pokemon:
+ hasRecords: true
+ streamHash: 71d50b057104f51772e5ef731e580332145d89dd
+ hasResponse: true
+ primaryKeysAreUnique: true
+ primaryKeysArePresent: true
+ responsesAreSuccessful: true
+ autoImportSchema:
+ pokemon: false
+
+schemas:
+ pokemon:
+ type: object
+ $schema: http://json-schema.org/draft-07/schema#
+ properties: {}
+ additionalProperties: true
diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/valid_config.yaml b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/valid_config.yaml
new file mode 100644
index 000000000..78af092bb
--- /dev/null
+++ b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/valid_config.yaml
@@ -0,0 +1 @@
+{ "start_date": "2024-01-01", "pokemon": "pikachu" }
diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/.gitignore b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/.gitignore
deleted file mode 100644
index c4ab49a30..000000000
--- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-secrets*
diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/README.md b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/README.md
deleted file mode 100644
index 403a4baba..000000000
--- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/README.md
+++ /dev/null
@@ -1,9 +0,0 @@
-# The Guardian API Tests
-
-For these tests to work, you'll need to create a `secrets.yaml` file in this directory that looks like this:
-
-```yml
-api_key: ******
-```
-
-The `.gitignore` file in this directory should ensure your file is not committed to git, but it's a good practice to double-check. 👀
diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components.py b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components.py
deleted file mode 100644
index 98a9f7ad5..000000000
--- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from dataclasses import InitVar, dataclass
-from typing import Any, Mapping, Optional, Union
-
-import requests
-
-from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
-from airbyte_cdk.sources.declarative.requesters.paginators import PaginationStrategy
-from airbyte_cdk.sources.declarative.types import Config, Record
-
-
-@dataclass
-class CustomPageIncrement(PaginationStrategy):
- """
- Starts page from 1 instead of the default value that is 0. Stops Pagination when currentPage is equal to totalPages.
- """
-
- config: Config
- page_size: Optional[Union[str, int]]
- parameters: InitVar[Mapping[str, Any]]
- start_from_page: int = 0
- inject_on_first_request: bool = False
-
- def __post_init__(self, parameters: Mapping[str, Any]) -> None:
- if isinstance(self.page_size, int) or (self.page_size is None):
- self._page_size = self.page_size
- else:
- page_size = InterpolatedString(self.page_size, parameters=parameters).eval(self.config)
- if not isinstance(page_size, int):
- raise Exception(f"{page_size} is of type {type(page_size)}. Expected {int}")
- self._page_size = page_size
-
- @property
- def initial_token(self) -> Optional[Any]:
- if self.inject_on_first_request:
- return self.start_from_page
- return None
-
- def next_page_token(
- self,
- response: requests.Response,
- last_page_size: int,
- last_record: Optional[Record],
- last_page_token_value: Optional[Any],
- ) -> Optional[Any]:
- res = response.json().get("response")
- current_page = res.get("currentPage")
- total_pages = res.get("pages")
-
- # The first request to the API does not include the page_token, so it comes in as None when determing whether to paginate
- last_page_token_value = last_page_token_value or 0
- if current_page < total_pages:
- return last_page_token_value + 1
- else:
- return None
-
- def get_page_size(self) -> Optional[int]:
- return self._page_size
diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components_failing.py b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components_failing.py
deleted file mode 100644
index 8655bdf2d..000000000
--- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components_failing.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from dataclasses import InitVar, dataclass
-from typing import Any, Mapping, Optional, Union
-
-import requests
-
-from airbyte_cdk.sources.declarative.interpolation import InterpolatedString
-from airbyte_cdk.sources.declarative.requesters.paginators import PaginationStrategy
-from airbyte_cdk.sources.declarative.types import Config, Record
-
-
-class IntentionalException(Exception):
- """This exception is raised intentionally in order to test error handling."""
-
-
-@dataclass
-class CustomPageIncrement(PaginationStrategy):
- """
- Starts page from 1 instead of the default value that is 0. Stops Pagination when currentPage is equal to totalPages.
- """
-
- config: Config
- page_size: Optional[Union[str, int]]
- parameters: InitVar[Mapping[str, Any]]
- start_from_page: int = 0
- inject_on_first_request: bool = False
-
- def __post_init__(self, parameters: Mapping[str, Any]) -> None:
- if isinstance(self.page_size, int) or (self.page_size is None):
- self._page_size = self.page_size
- else:
- page_size = InterpolatedString(self.page_size, parameters=parameters).eval(self.config)
- if not isinstance(page_size, int):
- raise Exception(f"{page_size} is of type {type(page_size)}. Expected {int}")
- self._page_size = page_size
-
- @property
- def initial_token(self) -> Optional[Any]:
- raise IntentionalException()
-
- def next_page_token(
- self,
- response: requests.Response,
- last_page_size: int,
- last_record: Optional[Record],
- last_page_token_value: Optional[Any],
- ) -> Optional[Any]:
- raise IntentionalException()
-
- def get_page_size(self) -> Optional[int]:
- return self._page_size
diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/manifest.yaml b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/manifest.yaml
deleted file mode 100644
index a42e0ebba..000000000
--- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/manifest.yaml
+++ /dev/null
@@ -1,376 +0,0 @@
-version: "4.3.2"
-definitions:
- selector:
- extractor:
- field_path:
- - response
- - results
- requester:
- url_base: "https://content.guardianapis.com"
- http_method: "GET"
- request_parameters:
- api-key: "{{ config['api_key'] }}"
- q: "{{ config['query'] }}"
- tag: "{{ config['tag'] }}"
- section: "{{ config['section'] }}"
- order-by: "oldest"
- incremental_sync:
- type: DatetimeBasedCursor
- start_datetime:
- datetime: "{{ config['start_date'] }}"
- datetime_format: "%Y-%m-%d"
- end_datetime:
- datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d') }}"
- datetime_format: "%Y-%m-%d"
- step: "P7D"
- datetime_format: "%Y-%m-%dT%H:%M:%SZ"
- cursor_granularity: "PT1S"
- cursor_field: "webPublicationDate"
- start_time_option:
- field_name: "from-date"
- inject_into: "request_parameter"
- end_time_option:
- field_name: "to-date"
- inject_into: "request_parameter"
- retriever:
- record_selector:
- extractor:
- field_path:
- - response
- - results
- paginator:
- type: DefaultPaginator
- pagination_strategy:
- type: CustomPaginationStrategy
- class_name: "CustomPageIncrement"
- page_size: 10
- page_token_option:
- type: RequestOption
- inject_into: "request_parameter"
- field_name: "page"
- page_size_option:
- inject_into: "body_data"
- field_name: "page_size"
- requester:
- url_base: "https://content.guardianapis.com"
- http_method: "GET"
- request_parameters:
- api-key: "{{ config['api_key'] }}"
- q: "{{ config['query'] }}"
- tag: "{{ config['tag'] }}"
- section: "{{ config['section'] }}"
- order-by: "oldest"
- base_stream:
- incremental_sync:
- type: DatetimeBasedCursor
- start_datetime:
- datetime: "{{ config['start_date'] }}"
- datetime_format: "%Y-%m-%d"
- end_datetime:
- datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d') }}"
- datetime_format: "%Y-%m-%d"
- step: "P7D"
- datetime_format: "%Y-%m-%dT%H:%M:%SZ"
- cursor_granularity: "PT1S"
- cursor_field: "webPublicationDate"
- start_time_option:
- field_name: "from-date"
- inject_into: "request_parameter"
- end_time_option:
- field_name: "to-date"
- inject_into: "request_parameter"
- retriever:
- record_selector:
- extractor:
- field_path:
- - response
- - results
- paginator:
- type: DefaultPaginator
- pagination_strategy:
- type: CustomPaginationStrategy
- class_name: "CustomPageIncrement"
- page_size: 10
- page_token_option:
- type: RequestOption
- inject_into: "request_parameter"
- field_name: "page"
- page_size_option:
- inject_into: "body_data"
- field_name: "page_size"
- requester:
- url_base: "https://content.guardianapis.com"
- http_method: "GET"
- request_parameters:
- api-key: "{{ config['api_key'] }}"
- q: "{{ config['query'] }}"
- tag: "{{ config['tag'] }}"
- section: "{{ config['section'] }}"
- order-by: "oldest"
- content_stream:
- incremental_sync:
- type: DatetimeBasedCursor
- start_datetime:
- datetime: "{{ config['start_date'] }}"
- datetime_format: "%Y-%m-%d"
- end_datetime:
- datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d') }}"
- datetime_format: "%Y-%m-%d"
- step: "P7D"
- datetime_format: "%Y-%m-%dT%H:%M:%SZ"
- cursor_granularity: "PT1S"
- cursor_field: "webPublicationDate"
- start_time_option:
- field_name: "from-date"
- inject_into: "request_parameter"
- end_time_option:
- field_name: "to-date"
- inject_into: "request_parameter"
- retriever:
- record_selector:
- extractor:
- field_path:
- - response
- - results
- paginator:
- type: "DefaultPaginator"
- pagination_strategy:
- type: CustomPaginationStrategy
- class_name: "components.CustomPageIncrement"
- page_size: 10
- page_token_option:
- type: RequestOption
- inject_into: "request_parameter"
- field_name: "page"
- page_size_option:
- inject_into: "body_data"
- field_name: "page_size"
- requester:
- url_base: "https://content.guardianapis.com"
- http_method: "GET"
- request_parameters:
- api-key: "{{ config['api_key'] }}"
- q: "{{ config['query'] }}"
- tag: "{{ config['tag'] }}"
- section: "{{ config['section'] }}"
- order-by: "oldest"
- schema_loader:
- type: InlineSchemaLoader
- schema:
- $schema: http://json-schema.org/draft-04/schema#
- type: object
- properties:
- id:
- type: string
- type:
- type: string
- sectionId:
- type: string
- sectionName:
- type: string
- webPublicationDate:
- type: string
- webTitle:
- type: string
- webUrl:
- type: string
- apiUrl:
- type: string
- isHosted:
- type: boolean
- pillarId:
- type: string
- pillarName:
- type: string
- required:
- - id
- - type
- - sectionId
- - sectionName
- - webPublicationDate
- - webTitle
- - webUrl
- - apiUrl
- - isHosted
- - pillarId
- - pillarName
-streams:
- - incremental_sync:
- type: DatetimeBasedCursor
- start_datetime:
- datetime: "{{ config['start_date'] }}"
- datetime_format: "%Y-%m-%d"
- type: MinMaxDatetime
- end_datetime:
- datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d') }}"
- datetime_format: "%Y-%m-%d"
- type: MinMaxDatetime
- step: "P7D"
- datetime_format: "%Y-%m-%dT%H:%M:%SZ"
- cursor_granularity: "PT1S"
- cursor_field: "webPublicationDate"
- start_time_option:
- field_name: "from-date"
- inject_into: "request_parameter"
- type: RequestOption
- end_time_option:
- field_name: "to-date"
- inject_into: "request_parameter"
- type: RequestOption
- retriever:
- record_selector:
- extractor:
- field_path:
- - response
- - results
- type: DpathExtractor
- type: RecordSelector
- paginator:
- type: "DefaultPaginator"
- pagination_strategy:
- class_name: components.CustomPageIncrement
- page_size: 10
- type: CustomPaginationStrategy
- page_token_option:
- type: RequestOption
- inject_into: "request_parameter"
- field_name: "page"
- page_size_option:
- inject_into: "body_data"
- field_name: "page_size"
- type: RequestOption
- requester:
- url_base: "https://content.guardianapis.com"
- http_method: "GET"
- request_parameters:
- api-key: "{{ config['api_key'] }}"
- q: "{{ config['query'] }}"
- tag: "{{ config['tag'] }}"
- section: "{{ config['section'] }}"
- order-by: "oldest"
- type: HttpRequester
- path: "/search"
- type: SimpleRetriever
- schema_loader:
- type: InlineSchemaLoader
- schema:
- $schema: http://json-schema.org/draft-04/schema#
- type: object
- properties:
- id:
- type: string
- type:
- type: string
- sectionId:
- type: string
- sectionName:
- type: string
- webPublicationDate:
- type: string
- webTitle:
- type: string
- webUrl:
- type: string
- apiUrl:
- type: string
- isHosted:
- type: boolean
- pillarId:
- type: string
- pillarName:
- type: string
- required:
- - id
- - type
- - sectionId
- - sectionName
- - webPublicationDate
- - webTitle
- - webUrl
- - apiUrl
- - isHosted
- - pillarId
- - pillarName
- type: DeclarativeStream
- name: "content"
- primary_key: "id"
-check:
- stream_names:
- - "content"
- type: CheckStream
-type: DeclarativeSource
-spec:
- type: Spec
- documentation_url: https://docs.airbyte.com/integrations/sources/the-guardian-api
- connection_specification:
- $schema: http://json-schema.org/draft-07/schema#
- title: The Guardian Api Spec
- type: object
- required:
- - api_key
- - start_date
- additionalProperties: true
- properties:
- api_key:
- title: API Key
- type: string
- description:
- Your API Key. See here.
- The key is case sensitive.
- airbyte_secret: true
- start_date:
- title: Start Date
- type: string
- description:
- Use this to set the minimum date (YYYY-MM-DD) of the results.
- Results older than the start_date will not be shown.
- pattern: ^([1-9][0-9]{3})\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$
- examples:
- - YYYY-MM-DD
- query:
- title: Query
- type: string
- description:
- (Optional) The query (q) parameter filters the results to only
- those that include that search term. The q parameter supports AND, OR and
- NOT operators.
- examples:
- - environment AND NOT water
- - environment AND political
- - amusement park
- - political
- tag:
- title: Tag
- type: string
- description:
- (Optional) A tag is a piece of data that is used by The Guardian
- to categorise content. Use this parameter to filter results by showing only
- the ones matching the entered tag. See here
- for a list of all tags, and here
- for the tags endpoint documentation.
- examples:
- - environment/recycling
- - environment/plasticbags
- - environment/energyefficiency
- section:
- title: Section
- type: string
- description:
- (Optional) Use this to filter the results by a particular section.
- See here
- for a list of all sections, and here
- for the sections endpoint documentation.
- examples:
- - media
- - technology
- - housing-network
- end_date:
- title: End Date
- type: string
- description:
- (Optional) Use this to set the maximum date (YYYY-MM-DD) of the
- results. Results newer than the end_date will not be shown. Default is set
- to the current date (today) for incremental syncs.
- pattern: ^([1-9][0-9]{3})\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$
- examples:
- - YYYY-MM-DD
diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/valid_config.yaml b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/valid_config.yaml
deleted file mode 100644
index b2f752ea1..000000000
--- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/valid_config.yaml
+++ /dev/null
@@ -1 +0,0 @@
-{ "start_date": "2024-01-01" }
diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py b/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py
index d608e7620..40bb6d40b 100644
--- a/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py
+++ b/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py
@@ -89,9 +89,8 @@ def test_components_module_from_string() -> None:
def get_py_components_config_dict(
*,
failing_components: bool = False,
- needs_secrets: bool = True,
) -> dict[str, Any]:
- connector_dir = Path(get_fixture_path("resources/source_the_guardian_api"))
+ connector_dir = Path(get_fixture_path("resources/source_pokeapi_w_components_py"))
manifest_yml_path: Path = connector_dir / "manifest.yaml"
custom_py_code_path: Path = connector_dir / (
"components.py" if not failing_components else "components_failing.py"
@@ -115,9 +114,6 @@ def get_py_components_config_dict(
},
}
combined_config_dict.update(yaml.safe_load(config_yaml_path.read_text()))
- if needs_secrets:
- combined_config_dict.update(yaml.safe_load(secrets_yaml_path.read_text()))
-
return combined_config_dict
@@ -127,9 +123,7 @@ def test_missing_checksum_fails_to_run(
"""Assert that missing checksum in the config will raise an error."""
monkeypatch.setenv(ENV_VAR_ALLOW_CUSTOM_CODE, "true")
- py_components_config_dict = get_py_components_config_dict(
- needs_secrets=False,
- )
+ py_components_config_dict = get_py_components_config_dict()
# Truncate the start_date to speed up tests
py_components_config_dict["start_date"] = (
datetime.datetime.now() - datetime.timedelta(days=2)
@@ -161,9 +155,7 @@ def test_invalid_checksum_fails_to_run(
"""Assert that an invalid checksum in the config will raise an error."""
monkeypatch.setenv(ENV_VAR_ALLOW_CUSTOM_CODE, "true")
- py_components_config_dict = get_py_components_config_dict(
- needs_secrets=False,
- )
+ py_components_config_dict = get_py_components_config_dict()
# Truncate the start_date to speed up tests
py_components_config_dict["start_date"] = (
datetime.datetime.now() - datetime.timedelta(days=2)
@@ -210,9 +202,7 @@ def test_fail_unless_custom_code_enabled_explicitly(
assert custom_code_execution_permitted() == (not should_raise)
- py_components_config_dict = get_py_components_config_dict(
- needs_secrets=False,
- )
+ py_components_config_dict = get_py_components_config_dict()
# Truncate the start_date to speed up tests
py_components_config_dict["start_date"] = (
datetime.datetime.now() - datetime.timedelta(days=2)
@@ -234,11 +224,6 @@ def test_fail_unless_custom_code_enabled_explicitly(
fn()
-# TODO: Create a new test source that doesn't require credentials to run.
-@pytest.mark.skipif(
- condition=not Path(get_fixture_path("resources/source_the_guardian_api/secrets.yaml")).exists(),
- reason="Skipped due to missing 'secrets.yaml'.",
-)
@pytest.mark.parametrize(
"failing_components",
[
@@ -288,17 +273,19 @@ def test_sync_with_injected_py_components(
]
)
- msg_iterator = source.read(
- logger=logging.getLogger(),
- config=py_components_config_dict,
- catalog=configured_catalog,
- state=None,
- )
- if failing_components:
- with pytest.raises(Exception):
- for msg in msg_iterator:
- assert msg
+ def _read_fn(*args, **kwargs):
+ msg_iterator = source.read(
+ logger=logging.getLogger(),
+ config=py_components_config_dict,
+ catalog=configured_catalog,
+ state=None,
+ )
+ for msg in msg_iterator:
+ assert msg
return
- for msg in msg_iterator:
- assert msg
+ if failing_components:
+ with pytest.raises(Exception):
+ _read_fn()
+ else:
+ _read_fn()