From e38f914bc49912ae99a7bb1d4a0b5f50393fee7f Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 6 Feb 2025 08:21:44 -0800 Subject: [PATCH] Chore: add new test using `source-pokeapi` and custom `components.py` (#317) --- .../source_pokeapi_w_components_py/README.md | 3 + .../components.py | 20 + .../components_failing.py | 24 + .../manifest.yaml | 980 ++++++++++++++++++ .../valid_config.yaml | 1 + .../source_the_guardian_api/.gitignore | 1 - .../source_the_guardian_api/README.md | 9 - .../source_the_guardian_api/components.py | 61 -- .../components_failing.py | 54 - .../source_the_guardian_api/manifest.yaml | 376 ------- .../source_the_guardian_api/valid_config.yaml | 1 - ..._source_declarative_w_custom_components.py | 49 +- 12 files changed, 1046 insertions(+), 533 deletions(-) create mode 100644 unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/README.md create mode 100644 unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components.py create mode 100644 unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components_failing.py create mode 100644 unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/manifest.yaml create mode 100644 unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/valid_config.yaml delete mode 100644 unit_tests/source_declarative_manifest/resources/source_the_guardian_api/.gitignore delete mode 100644 unit_tests/source_declarative_manifest/resources/source_the_guardian_api/README.md delete mode 100644 unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components.py delete mode 100644 unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components_failing.py delete mode 100644 unit_tests/source_declarative_manifest/resources/source_the_guardian_api/manifest.yaml delete mode 100644 unit_tests/source_declarative_manifest/resources/source_the_guardian_api/valid_config.yaml diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/README.md b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/README.md new file mode 100644 index 000000000..78505726c --- /dev/null +++ b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/README.md @@ -0,0 +1,3 @@ +# PokeAPI with Custom `components.py` API Tests + +This test connector is a modified version of `source-pokeapi`. It has been modified to use custom `components.py` so we have a test case the completes quickly and _does not_ require any credentials. diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components.py b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components.py new file mode 100644 index 000000000..5e7e16f71 --- /dev/null +++ b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components.py @@ -0,0 +1,20 @@ +"""A sample implementation of custom components that does nothing but will cause syncs to fail if missing.""" + +from typing import Any, Mapping + +import requests + +from airbyte_cdk.sources.declarative.extractors import DpathExtractor + + +class IntentionalException(Exception): + """This exception is raised intentionally in order to test error handling.""" + + +class MyCustomExtractor(DpathExtractor): + """Dummy class, directly implements DPatchExtractor. + + Used to prove that SDM can find the custom class by name. + """ + + pass diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components_failing.py b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components_failing.py new file mode 100644 index 000000000..5c05881e7 --- /dev/null +++ b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/components_failing.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +"""A sample implementation of custom components that does nothing but will cause syncs to fail if missing.""" + +from collections.abc import Iterable, MutableMapping +from dataclasses import InitVar, dataclass +from typing import Any, Mapping, Optional, Union + +import requests + +from airbyte_cdk.sources.declarative.extractors import DpathExtractor + + +class IntentionalException(Exception): + """This exception is raised intentionally in order to test error handling.""" + + +class MyCustomExtractor(DpathExtractor): + def extract_records( + self, + response: requests.Response, + ) -> Iterable[MutableMapping[Any, Any]]: + raise IntentionalException diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/manifest.yaml b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/manifest.yaml new file mode 100644 index 000000000..af19485fa --- /dev/null +++ b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/manifest.yaml @@ -0,0 +1,980 @@ +version: 6.30.0 + +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - pokemon + +definitions: + streams: + pokemon: + type: DeclarativeStream + name: pokemon + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /{{config['pokemon_name']}} + http_method: GET + record_selector: + type: RecordSelector + extractor: + # Simple wrapper around `DpathExtractor` + type: CustomRecordExtractor + class_name: components.MyCustomExtractor + field_path: [] + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + # type: CustomSchemaLoader + schema: + $ref: "#/schemas/pokemon" + # class_name: components.MyCustomInlineSchemaLoader + base_requester: + type: HttpRequester + url_base: https://pokeapi.co/api/v2/pokemon + +streams: + - $ref: "#/definitions/streams/pokemon" + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - pokemon_name + properties: + pokemon_name: + type: string + description: Pokemon requested from the API. + enum: + - bulbasaur + - ivysaur + - venusaur + - charmander + - charmeleon + - charizard + - squirtle + - wartortle + - blastoise + - caterpie + - metapod + - butterfree + - weedle + - kakuna + - beedrill + - pidgey + - pidgeotto + - pidgeot + - rattata + - raticate + - spearow + - fearow + - ekans + - arbok + - pikachu + - raichu + - sandshrew + - sandslash + - nidoranf + - nidorina + - nidoqueen + - nidoranm + - nidorino + - nidoking + - clefairy + - clefable + - vulpix + - ninetales + - jigglypuff + - wigglytuff + - zubat + - golbat + - oddish + - gloom + - vileplume + - paras + - parasect + - venonat + - venomoth + - diglett + - dugtrio + - meowth + - persian + - psyduck + - golduck + - mankey + - primeape + - growlithe + - arcanine + - poliwag + - poliwhirl + - poliwrath + - abra + - kadabra + - alakazam + - machop + - machoke + - machamp + - bellsprout + - weepinbell + - victreebel + - tentacool + - tentacruel + - geodude + - graveler + - golem + - ponyta + - rapidash + - slowpoke + - slowbro + - magnemite + - magneton + - farfetchd + - doduo + - dodrio + - seel + - dewgong + - grimer + - muk + - shellder + - cloyster + - gastly + - haunter + - gengar + - onix + - drowzee + - hypno + - krabby + - kingler + - voltorb + - electrode + - exeggcute + - exeggutor + - cubone + - marowak + - hitmonlee + - hitmonchan + - lickitung + - koffing + - weezing + - rhyhorn + - rhydon + - chansey + - tangela + - kangaskhan + - horsea + - seadra + - goldeen + - seaking + - staryu + - starmie + - mrmime + - scyther + - jynx + - electabuzz + - magmar + - pinsir + - tauros + - magikarp + - gyarados + - lapras + - ditto + - eevee + - vaporeon + - jolteon + - flareon + - porygon + - omanyte + - omastar + - kabuto + - kabutops + - aerodactyl + - snorlax + - articuno + - zapdos + - moltres + - dratini + - dragonair + - dragonite + - mewtwo + - mew + - chikorita + - bayleef + - meganium + - cyndaquil + - quilava + - typhlosion + - totodile + - croconaw + - feraligatr + - sentret + - furret + - hoothoot + - noctowl + - ledyba + - ledian + - spinarak + - ariados + - crobat + - chinchou + - lanturn + - pichu + - cleffa + - igglybuff + - togepi + - togetic + - natu + - xatu + - mareep + - flaaffy + - ampharos + - bellossom + - marill + - azumarill + - sudowoodo + - politoed + - hoppip + - skiploom + - jumpluff + - aipom + - sunkern + - sunflora + - yanma + - wooper + - quagsire + - espeon + - umbreon + - murkrow + - slowking + - misdreavus + - unown + - wobbuffet + - girafarig + - pineco + - forretress + - dunsparce + - gligar + - steelix + - snubbull + - granbull + - qwilfish + - scizor + - shuckle + - heracross + - sneasel + - teddiursa + - ursaring + - slugma + - magcargo + - swinub + - piloswine + - corsola + - remoraid + - octillery + - delibird + - mantine + - skarmory + - houndour + - houndoom + - kingdra + - phanpy + - donphan + - porygon2 + - stantler + - smeargle + - tyrogue + - hitmontop + - smoochum + - elekid + - magby + - miltank + - blissey + - raikou + - entei + - suicune + - larvitar + - pupitar + - tyranitar + - lugia + - ho-oh + - celebi + - treecko + - grovyle + - sceptile + - torchic + - combusken + - blaziken + - mudkip + - marshtomp + - swampert + - poochyena + - mightyena + - zigzagoon + - linoone + - wurmple + - silcoon + - beautifly + - cascoon + - dustox + - lotad + - lombre + - ludicolo + - seedot + - nuzleaf + - shiftry + - taillow + - swellow + - wingull + - pelipper + - ralts + - kirlia + - gardevoir + - surskit + - masquerain + - shroomish + - breloom + - slakoth + - vigoroth + - slaking + - nincada + - ninjask + - shedinja + - whismur + - loudred + - exploud + - makuhita + - hariyama + - azurill + - nosepass + - skitty + - delcatty + - sableye + - mawile + - aron + - lairon + - aggron + - meditite + - medicham + - electrike + - manectric + - plusle + - minun + - volbeat + - illumise + - roselia + - gulpin + - swalot + - carvanha + - sharpedo + - wailmer + - wailord + - numel + - camerupt + - torkoal + - spoink + - grumpig + - spinda + - trapinch + - vibrava + - flygon + - cacnea + - cacturne + - swablu + - altaria + - zangoose + - seviper + - lunatone + - solrock + - barboach + - whiscash + - corphish + - crawdaunt + - baltoy + - claydol + - lileep + - cradily + - anorith + - armaldo + - feebas + - milotic + - castform + - kecleon + - shuppet + - banette + - duskull + - dusclops + - tropius + - chimecho + - absol + - wynaut + - snorunt + - glalie + - spheal + - sealeo + - walrein + - clamperl + - huntail + - gorebyss + - relicanth + - luvdisc + - bagon + - shelgon + - salamence + - beldum + - metang + - metagross + - regirock + - regice + - registeel + - latias + - latios + - kyogre + - groudon + - rayquaza + - jirachi + - deoxys + - turtwig + - grotle + - torterra + - chimchar + - monferno + - infernape + - piplup + - prinplup + - empoleon + - starly + - staravia + - staraptor + - bidoof + - bibarel + - kricketot + - kricketune + - shinx + - luxio + - luxray + - budew + - roserade + - cranidos + - rampardos + - shieldon + - bastiodon + - burmy + - wormadam + - mothim + - combee + - vespiquen + - pachirisu + - buizel + - floatzel + - cherubi + - cherrim + - shellos + - gastrodon + - ambipom + - drifloon + - drifblim + - buneary + - lopunny + - mismagius + - honchkrow + - glameow + - purugly + - chingling + - stunky + - skuntank + - bronzor + - bronzong + - bonsly + - mimejr + - happiny + - chatot + - spiritomb + - gible + - gabite + - garchomp + - munchlax + - riolu + - lucario + - hippopotas + - hippowdon + - skorupi + - drapion + - croagunk + - toxicroak + - carnivine + - finneon + - lumineon + - mantyke + - snover + - abomasnow + - weavile + - magnezone + - lickilicky + - rhyperior + - tangrowth + - electivire + - magmortar + - togekiss + - yanmega + - leafeon + - glaceon + - gliscor + - mamoswine + - porygon-z + - gallade + - probopass + - dusknoir + - froslass + - rotom + - uxie + - mesprit + - azelf + - dialga + - palkia + - heatran + - regigigas + - giratina + - cresselia + - phione + - manaphy + - darkrai + - shaymin + - arceus + - victini + - snivy + - servine + - serperior + - tepig + - pignite + - emboar + - oshawott + - dewott + - samurott + - patrat + - watchog + - lillipup + - herdier + - stoutland + - purrloin + - liepard + - pansage + - simisage + - pansear + - simisear + - panpour + - simipour + - munna + - musharna + - pidove + - tranquill + - unfezant + - blitzle + - zebstrika + - roggenrola + - boldore + - gigalith + - woobat + - swoobat + - drilbur + - excadrill + - audino + - timburr + - gurdurr + - conkeldurr + - tympole + - palpitoad + - seismitoad + - throh + - sawk + - sewaddle + - swadloon + - leavanny + - venipede + - whirlipede + - scolipede + - cottonee + - whimsicott + - petilil + - lilligant + - basculin + - sandile + - krokorok + - krookodile + - darumaka + - darmanitan + - maractus + - dwebble + - crustle + - scraggy + - scrafty + - sigilyph + - yamask + - cofagrigus + - tirtouga + - carracosta + - archen + - archeops + - trubbish + - garbodor + - zorua + - zoroark + - minccino + - cinccino + - gothita + - gothorita + - gothitelle + - solosis + - duosion + - reuniclus + - ducklett + - swanna + - vanillite + - vanillish + - vanilluxe + - deerling + - sawsbuck + - emolga + - karrablast + - escavalier + - foongus + - amoonguss + - frillish + - jellicent + - alomomola + - joltik + - galvantula + - ferroseed + - ferrothorn + - klink + - klang + - klinklang + - tynamo + - eelektrik + - eelektross + - elgyem + - beheeyem + - litwick + - lampent + - chandelure + - axew + - fraxure + - haxorus + - cubchoo + - beartic + - cryogonal + - shelmet + - accelgor + - stunfisk + - mienfoo + - mienshao + - druddigon + - golett + - golurk + - pawniard + - bisharp + - bouffalant + - rufflet + - braviary + - vullaby + - mandibuzz + - heatmor + - durant + - deino + - zweilous + - hydreigon + - larvesta + - volcarona + - cobalion + - terrakion + - virizion + - tornadus + - thundurus + - reshiram + - zekrom + - landorus + - kyurem + - keldeo + - meloetta + - genesect + - chespin + - quilladin + - chesnaught + - fennekin + - braixen + - delphox + - froakie + - frogadier + - greninja + - bunnelby + - diggersby + - fletchling + - fletchinder + - talonflame + - scatterbug + - spewpa + - vivillon + - litleo + - pyroar + - flabebe + - floette + - florges + - skiddo + - gogoat + - pancham + - pangoro + - furfrou + - espurr + - meowstic + - honedge + - doublade + - aegislash + - spritzee + - aromatisse + - swirlix + - slurpuff + - inkay + - malamar + - binacle + - barbaracle + - skrelp + - dragalge + - clauncher + - clawitzer + - helioptile + - heliolisk + - tyrunt + - tyrantrum + - amaura + - aurorus + - sylveon + - hawlucha + - dedenne + - carbink + - goomy + - sliggoo + - goodra + - klefki + - phantump + - trevenant + - pumpkaboo + - gourgeist + - bergmite + - avalugg + - noibat + - noivern + - xerneas + - yveltal + - zygarde + - diancie + - hoopa + - volcanion + - rowlet + - dartrix + - decidueye + - litten + - torracat + - incineroar + - popplio + - brionne + - primarina + - pikipek + - trumbeak + - toucannon + - yungoos + - gumshoos + - grubbin + - charjabug + - vikavolt + - crabrawler + - crabominable + - oricorio + - cutiefly + - ribombee + - rockruff + - lycanroc + - wishiwashi + - mareanie + - toxapex + - mudbray + - mudsdale + - dewpider + - araquanid + - fomantis + - lurantis + - morelull + - shiinotic + - salandit + - salazzle + - stufful + - bewear + - bounsweet + - steenee + - tsareena + - comfey + - oranguru + - passimian + - wimpod + - golisopod + - sandygast + - palossand + - pyukumuku + - typenull + - silvally + - minior + - komala + - turtonator + - togedemaru + - mimikyu + - bruxish + - drampa + - dhelmise + - jangmo-o + - hakamo-o + - kommo-o + - tapukoko + - tapulele + - tapubulu + - tapufini + - cosmog + - cosmoem + - solgaleo + - lunala + - nihilego + - buzzwole + - pheromosa + - xurkitree + - celesteela + - kartana + - guzzlord + - necrozma + - magearna + - marshadow + - poipole + - naganadel + - stakataka + - blacephalon + - zeraora + - meltan + - melmetal + - grookey + - thwackey + - rillaboom + - scorbunny + - raboot + - cinderace + - sobble + - drizzile + - inteleon + - skwovet + - greedent + - rookidee + - corvisquire + - corviknight + - blipbug + - dottler + - orbeetle + - nickit + - thievul + - gossifleur + - eldegoss + - wooloo + - dubwool + - chewtle + - drednaw + - yamper + - boltund + - rolycoly + - carkol + - coalossal + - applin + - flapple + - appletun + - silicobra + - sandaconda + - cramorant + - arrokuda + - barraskewda + - toxel + - toxtricity + - sizzlipede + - centiskorch + - clobbopus + - grapploct + - sinistea + - polteageist + - hatenna + - hattrem + - hatterene + - impidimp + - morgrem + - grimmsnarl + - obstagoon + - perrserker + - cursola + - sirfetchd + - mrrime + - runerigus + - milcery + - alcremie + - falinks + - pincurchin + - snom + - frosmoth + - stonjourner + - eiscue + - indeedee + - morpeko + - cufant + - copperajah + - dracozolt + - arctozolt + - dracovish + - arctovish + - duraludon + - dreepy + - drakloak + - dragapult + - zacian + - zamazenta + - eternatus + - kubfu + - urshifu + - zarude + - regieleki + - regidrago + - glastrier + - spectrier + - calyrex + order: 0 + title: Pokemon Name + pattern: ^[a-z0-9_\-]+$ + examples: + - ditto + - luxray + - snorlax + additionalProperties: true + +metadata: + assist: {} + testedStreams: + pokemon: + hasRecords: true + streamHash: 71d50b057104f51772e5ef731e580332145d89dd + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + autoImportSchema: + pokemon: false + +schemas: + pokemon: + type: object + $schema: http://json-schema.org/draft-07/schema# + properties: {} + additionalProperties: true diff --git a/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/valid_config.yaml b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/valid_config.yaml new file mode 100644 index 000000000..78af092bb --- /dev/null +++ b/unit_tests/source_declarative_manifest/resources/source_pokeapi_w_components_py/valid_config.yaml @@ -0,0 +1 @@ +{ "start_date": "2024-01-01", "pokemon": "pikachu" } diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/.gitignore b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/.gitignore deleted file mode 100644 index c4ab49a30..000000000 --- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/.gitignore +++ /dev/null @@ -1 +0,0 @@ -secrets* diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/README.md b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/README.md deleted file mode 100644 index 403a4baba..000000000 --- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# The Guardian API Tests - -For these tests to work, you'll need to create a `secrets.yaml` file in this directory that looks like this: - -```yml -api_key: ****** -``` - -The `.gitignore` file in this directory should ensure your file is not committed to git, but it's a good practice to double-check. 👀 diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components.py b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components.py deleted file mode 100644 index 98a9f7ad5..000000000 --- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components.py +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from dataclasses import InitVar, dataclass -from typing import Any, Mapping, Optional, Union - -import requests - -from airbyte_cdk.sources.declarative.interpolation import InterpolatedString -from airbyte_cdk.sources.declarative.requesters.paginators import PaginationStrategy -from airbyte_cdk.sources.declarative.types import Config, Record - - -@dataclass -class CustomPageIncrement(PaginationStrategy): - """ - Starts page from 1 instead of the default value that is 0. Stops Pagination when currentPage is equal to totalPages. - """ - - config: Config - page_size: Optional[Union[str, int]] - parameters: InitVar[Mapping[str, Any]] - start_from_page: int = 0 - inject_on_first_request: bool = False - - def __post_init__(self, parameters: Mapping[str, Any]) -> None: - if isinstance(self.page_size, int) or (self.page_size is None): - self._page_size = self.page_size - else: - page_size = InterpolatedString(self.page_size, parameters=parameters).eval(self.config) - if not isinstance(page_size, int): - raise Exception(f"{page_size} is of type {type(page_size)}. Expected {int}") - self._page_size = page_size - - @property - def initial_token(self) -> Optional[Any]: - if self.inject_on_first_request: - return self.start_from_page - return None - - def next_page_token( - self, - response: requests.Response, - last_page_size: int, - last_record: Optional[Record], - last_page_token_value: Optional[Any], - ) -> Optional[Any]: - res = response.json().get("response") - current_page = res.get("currentPage") - total_pages = res.get("pages") - - # The first request to the API does not include the page_token, so it comes in as None when determing whether to paginate - last_page_token_value = last_page_token_value or 0 - if current_page < total_pages: - return last_page_token_value + 1 - else: - return None - - def get_page_size(self) -> Optional[int]: - return self._page_size diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components_failing.py b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components_failing.py deleted file mode 100644 index 8655bdf2d..000000000 --- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/components_failing.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from dataclasses import InitVar, dataclass -from typing import Any, Mapping, Optional, Union - -import requests - -from airbyte_cdk.sources.declarative.interpolation import InterpolatedString -from airbyte_cdk.sources.declarative.requesters.paginators import PaginationStrategy -from airbyte_cdk.sources.declarative.types import Config, Record - - -class IntentionalException(Exception): - """This exception is raised intentionally in order to test error handling.""" - - -@dataclass -class CustomPageIncrement(PaginationStrategy): - """ - Starts page from 1 instead of the default value that is 0. Stops Pagination when currentPage is equal to totalPages. - """ - - config: Config - page_size: Optional[Union[str, int]] - parameters: InitVar[Mapping[str, Any]] - start_from_page: int = 0 - inject_on_first_request: bool = False - - def __post_init__(self, parameters: Mapping[str, Any]) -> None: - if isinstance(self.page_size, int) or (self.page_size is None): - self._page_size = self.page_size - else: - page_size = InterpolatedString(self.page_size, parameters=parameters).eval(self.config) - if not isinstance(page_size, int): - raise Exception(f"{page_size} is of type {type(page_size)}. Expected {int}") - self._page_size = page_size - - @property - def initial_token(self) -> Optional[Any]: - raise IntentionalException() - - def next_page_token( - self, - response: requests.Response, - last_page_size: int, - last_record: Optional[Record], - last_page_token_value: Optional[Any], - ) -> Optional[Any]: - raise IntentionalException() - - def get_page_size(self) -> Optional[int]: - return self._page_size diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/manifest.yaml b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/manifest.yaml deleted file mode 100644 index a42e0ebba..000000000 --- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/manifest.yaml +++ /dev/null @@ -1,376 +0,0 @@ -version: "4.3.2" -definitions: - selector: - extractor: - field_path: - - response - - results - requester: - url_base: "https://content.guardianapis.com" - http_method: "GET" - request_parameters: - api-key: "{{ config['api_key'] }}" - q: "{{ config['query'] }}" - tag: "{{ config['tag'] }}" - section: "{{ config['section'] }}" - order-by: "oldest" - incremental_sync: - type: DatetimeBasedCursor - start_datetime: - datetime: "{{ config['start_date'] }}" - datetime_format: "%Y-%m-%d" - end_datetime: - datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d') }}" - datetime_format: "%Y-%m-%d" - step: "P7D" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" - cursor_granularity: "PT1S" - cursor_field: "webPublicationDate" - start_time_option: - field_name: "from-date" - inject_into: "request_parameter" - end_time_option: - field_name: "to-date" - inject_into: "request_parameter" - retriever: - record_selector: - extractor: - field_path: - - response - - results - paginator: - type: DefaultPaginator - pagination_strategy: - type: CustomPaginationStrategy - class_name: "CustomPageIncrement" - page_size: 10 - page_token_option: - type: RequestOption - inject_into: "request_parameter" - field_name: "page" - page_size_option: - inject_into: "body_data" - field_name: "page_size" - requester: - url_base: "https://content.guardianapis.com" - http_method: "GET" - request_parameters: - api-key: "{{ config['api_key'] }}" - q: "{{ config['query'] }}" - tag: "{{ config['tag'] }}" - section: "{{ config['section'] }}" - order-by: "oldest" - base_stream: - incremental_sync: - type: DatetimeBasedCursor - start_datetime: - datetime: "{{ config['start_date'] }}" - datetime_format: "%Y-%m-%d" - end_datetime: - datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d') }}" - datetime_format: "%Y-%m-%d" - step: "P7D" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" - cursor_granularity: "PT1S" - cursor_field: "webPublicationDate" - start_time_option: - field_name: "from-date" - inject_into: "request_parameter" - end_time_option: - field_name: "to-date" - inject_into: "request_parameter" - retriever: - record_selector: - extractor: - field_path: - - response - - results - paginator: - type: DefaultPaginator - pagination_strategy: - type: CustomPaginationStrategy - class_name: "CustomPageIncrement" - page_size: 10 - page_token_option: - type: RequestOption - inject_into: "request_parameter" - field_name: "page" - page_size_option: - inject_into: "body_data" - field_name: "page_size" - requester: - url_base: "https://content.guardianapis.com" - http_method: "GET" - request_parameters: - api-key: "{{ config['api_key'] }}" - q: "{{ config['query'] }}" - tag: "{{ config['tag'] }}" - section: "{{ config['section'] }}" - order-by: "oldest" - content_stream: - incremental_sync: - type: DatetimeBasedCursor - start_datetime: - datetime: "{{ config['start_date'] }}" - datetime_format: "%Y-%m-%d" - end_datetime: - datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d') }}" - datetime_format: "%Y-%m-%d" - step: "P7D" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" - cursor_granularity: "PT1S" - cursor_field: "webPublicationDate" - start_time_option: - field_name: "from-date" - inject_into: "request_parameter" - end_time_option: - field_name: "to-date" - inject_into: "request_parameter" - retriever: - record_selector: - extractor: - field_path: - - response - - results - paginator: - type: "DefaultPaginator" - pagination_strategy: - type: CustomPaginationStrategy - class_name: "components.CustomPageIncrement" - page_size: 10 - page_token_option: - type: RequestOption - inject_into: "request_parameter" - field_name: "page" - page_size_option: - inject_into: "body_data" - field_name: "page_size" - requester: - url_base: "https://content.guardianapis.com" - http_method: "GET" - request_parameters: - api-key: "{{ config['api_key'] }}" - q: "{{ config['query'] }}" - tag: "{{ config['tag'] }}" - section: "{{ config['section'] }}" - order-by: "oldest" - schema_loader: - type: InlineSchemaLoader - schema: - $schema: http://json-schema.org/draft-04/schema# - type: object - properties: - id: - type: string - type: - type: string - sectionId: - type: string - sectionName: - type: string - webPublicationDate: - type: string - webTitle: - type: string - webUrl: - type: string - apiUrl: - type: string - isHosted: - type: boolean - pillarId: - type: string - pillarName: - type: string - required: - - id - - type - - sectionId - - sectionName - - webPublicationDate - - webTitle - - webUrl - - apiUrl - - isHosted - - pillarId - - pillarName -streams: - - incremental_sync: - type: DatetimeBasedCursor - start_datetime: - datetime: "{{ config['start_date'] }}" - datetime_format: "%Y-%m-%d" - type: MinMaxDatetime - end_datetime: - datetime: "{{ config['end_date'] or now_utc().strftime('%Y-%m-%d') }}" - datetime_format: "%Y-%m-%d" - type: MinMaxDatetime - step: "P7D" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" - cursor_granularity: "PT1S" - cursor_field: "webPublicationDate" - start_time_option: - field_name: "from-date" - inject_into: "request_parameter" - type: RequestOption - end_time_option: - field_name: "to-date" - inject_into: "request_parameter" - type: RequestOption - retriever: - record_selector: - extractor: - field_path: - - response - - results - type: DpathExtractor - type: RecordSelector - paginator: - type: "DefaultPaginator" - pagination_strategy: - class_name: components.CustomPageIncrement - page_size: 10 - type: CustomPaginationStrategy - page_token_option: - type: RequestOption - inject_into: "request_parameter" - field_name: "page" - page_size_option: - inject_into: "body_data" - field_name: "page_size" - type: RequestOption - requester: - url_base: "https://content.guardianapis.com" - http_method: "GET" - request_parameters: - api-key: "{{ config['api_key'] }}" - q: "{{ config['query'] }}" - tag: "{{ config['tag'] }}" - section: "{{ config['section'] }}" - order-by: "oldest" - type: HttpRequester - path: "/search" - type: SimpleRetriever - schema_loader: - type: InlineSchemaLoader - schema: - $schema: http://json-schema.org/draft-04/schema# - type: object - properties: - id: - type: string - type: - type: string - sectionId: - type: string - sectionName: - type: string - webPublicationDate: - type: string - webTitle: - type: string - webUrl: - type: string - apiUrl: - type: string - isHosted: - type: boolean - pillarId: - type: string - pillarName: - type: string - required: - - id - - type - - sectionId - - sectionName - - webPublicationDate - - webTitle - - webUrl - - apiUrl - - isHosted - - pillarId - - pillarName - type: DeclarativeStream - name: "content" - primary_key: "id" -check: - stream_names: - - "content" - type: CheckStream -type: DeclarativeSource -spec: - type: Spec - documentation_url: https://docs.airbyte.com/integrations/sources/the-guardian-api - connection_specification: - $schema: http://json-schema.org/draft-07/schema# - title: The Guardian Api Spec - type: object - required: - - api_key - - start_date - additionalProperties: true - properties: - api_key: - title: API Key - type: string - description: - Your API Key. See here. - The key is case sensitive. - airbyte_secret: true - start_date: - title: Start Date - type: string - description: - Use this to set the minimum date (YYYY-MM-DD) of the results. - Results older than the start_date will not be shown. - pattern: ^([1-9][0-9]{3})\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$ - examples: - - YYYY-MM-DD - query: - title: Query - type: string - description: - (Optional) The query (q) parameter filters the results to only - those that include that search term. The q parameter supports AND, OR and - NOT operators. - examples: - - environment AND NOT water - - environment AND political - - amusement park - - political - tag: - title: Tag - type: string - description: - (Optional) A tag is a piece of data that is used by The Guardian - to categorise content. Use this parameter to filter results by showing only - the ones matching the entered tag. See here - for a list of all tags, and here - for the tags endpoint documentation. - examples: - - environment/recycling - - environment/plasticbags - - environment/energyefficiency - section: - title: Section - type: string - description: - (Optional) Use this to filter the results by a particular section. - See here - for a list of all sections, and here - for the sections endpoint documentation. - examples: - - media - - technology - - housing-network - end_date: - title: End Date - type: string - description: - (Optional) Use this to set the maximum date (YYYY-MM-DD) of the - results. Results newer than the end_date will not be shown. Default is set - to the current date (today) for incremental syncs. - pattern: ^([1-9][0-9]{3})\-(0?[1-9]|1[012])\-(0?[1-9]|[12][0-9]|3[01])$ - examples: - - YYYY-MM-DD diff --git a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/valid_config.yaml b/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/valid_config.yaml deleted file mode 100644 index b2f752ea1..000000000 --- a/unit_tests/source_declarative_manifest/resources/source_the_guardian_api/valid_config.yaml +++ /dev/null @@ -1 +0,0 @@ -{ "start_date": "2024-01-01" } diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py b/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py index d608e7620..40bb6d40b 100644 --- a/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py +++ b/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py @@ -89,9 +89,8 @@ def test_components_module_from_string() -> None: def get_py_components_config_dict( *, failing_components: bool = False, - needs_secrets: bool = True, ) -> dict[str, Any]: - connector_dir = Path(get_fixture_path("resources/source_the_guardian_api")) + connector_dir = Path(get_fixture_path("resources/source_pokeapi_w_components_py")) manifest_yml_path: Path = connector_dir / "manifest.yaml" custom_py_code_path: Path = connector_dir / ( "components.py" if not failing_components else "components_failing.py" @@ -115,9 +114,6 @@ def get_py_components_config_dict( }, } combined_config_dict.update(yaml.safe_load(config_yaml_path.read_text())) - if needs_secrets: - combined_config_dict.update(yaml.safe_load(secrets_yaml_path.read_text())) - return combined_config_dict @@ -127,9 +123,7 @@ def test_missing_checksum_fails_to_run( """Assert that missing checksum in the config will raise an error.""" monkeypatch.setenv(ENV_VAR_ALLOW_CUSTOM_CODE, "true") - py_components_config_dict = get_py_components_config_dict( - needs_secrets=False, - ) + py_components_config_dict = get_py_components_config_dict() # Truncate the start_date to speed up tests py_components_config_dict["start_date"] = ( datetime.datetime.now() - datetime.timedelta(days=2) @@ -161,9 +155,7 @@ def test_invalid_checksum_fails_to_run( """Assert that an invalid checksum in the config will raise an error.""" monkeypatch.setenv(ENV_VAR_ALLOW_CUSTOM_CODE, "true") - py_components_config_dict = get_py_components_config_dict( - needs_secrets=False, - ) + py_components_config_dict = get_py_components_config_dict() # Truncate the start_date to speed up tests py_components_config_dict["start_date"] = ( datetime.datetime.now() - datetime.timedelta(days=2) @@ -210,9 +202,7 @@ def test_fail_unless_custom_code_enabled_explicitly( assert custom_code_execution_permitted() == (not should_raise) - py_components_config_dict = get_py_components_config_dict( - needs_secrets=False, - ) + py_components_config_dict = get_py_components_config_dict() # Truncate the start_date to speed up tests py_components_config_dict["start_date"] = ( datetime.datetime.now() - datetime.timedelta(days=2) @@ -234,11 +224,6 @@ def test_fail_unless_custom_code_enabled_explicitly( fn() -# TODO: Create a new test source that doesn't require credentials to run. -@pytest.mark.skipif( - condition=not Path(get_fixture_path("resources/source_the_guardian_api/secrets.yaml")).exists(), - reason="Skipped due to missing 'secrets.yaml'.", -) @pytest.mark.parametrize( "failing_components", [ @@ -288,17 +273,19 @@ def test_sync_with_injected_py_components( ] ) - msg_iterator = source.read( - logger=logging.getLogger(), - config=py_components_config_dict, - catalog=configured_catalog, - state=None, - ) - if failing_components: - with pytest.raises(Exception): - for msg in msg_iterator: - assert msg + def _read_fn(*args, **kwargs): + msg_iterator = source.read( + logger=logging.getLogger(), + config=py_components_config_dict, + catalog=configured_catalog, + state=None, + ) + for msg in msg_iterator: + assert msg return - for msg in msg_iterator: - assert msg + if failing_components: + with pytest.raises(Exception): + _read_fn() + else: + _read_fn()