From 2256f0f0aaf5cdbdcf0c74d2ed9c8c869b774a58 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 21 Nov 2024 14:20:36 -0800 Subject: [PATCH 1/7] tests: add sdm unit tests from source-declarative-manifest connector directory --- .../source_declarative_manifest/__init__.py | 0 .../fixtures/__init__.py | 44 + .../fixtures/invalid_local_manifest.yaml | 8 + .../invalid_local_pokeapi_config.json | 1 + .../fixtures/invalid_remote_config.json | 1 + .../fixtures/valid_local_manifest.yaml | 1368 +++++++++++++++++ .../fixtures/valid_local_pokeapi_config.json | 3 + .../fixtures/valid_remote_config.json | 33 + .../test_source_declarative_local_manifest.py | 51 + ...test_source_declarative_remote_manifest.py | 28 + 10 files changed, 1537 insertions(+) create mode 100644 unit_tests/source_declarative_manifest/__init__.py create mode 100644 unit_tests/source_declarative_manifest/fixtures/__init__.py create mode 100644 unit_tests/source_declarative_manifest/fixtures/invalid_local_manifest.yaml create mode 100644 unit_tests/source_declarative_manifest/fixtures/invalid_local_pokeapi_config.json create mode 100644 unit_tests/source_declarative_manifest/fixtures/invalid_remote_config.json create mode 100644 unit_tests/source_declarative_manifest/fixtures/valid_local_manifest.yaml create mode 100644 unit_tests/source_declarative_manifest/fixtures/valid_local_pokeapi_config.json create mode 100644 unit_tests/source_declarative_manifest/fixtures/valid_remote_config.json create mode 100644 unit_tests/source_declarative_manifest/test_source_declarative_local_manifest.py create mode 100644 unit_tests/source_declarative_manifest/test_source_declarative_remote_manifest.py diff --git a/unit_tests/source_declarative_manifest/__init__.py b/unit_tests/source_declarative_manifest/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/unit_tests/source_declarative_manifest/fixtures/__init__.py b/unit_tests/source_declarative_manifest/fixtures/__init__.py new file mode 100644 index 000000000..18f7eb3da --- /dev/null +++ b/unit_tests/source_declarative_manifest/fixtures/__init__.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + +import pytest +import os +import yaml + +def get_fixture_path(file_name): + return os.path.join(os.path.dirname(__file__), file_name) + +@pytest.fixture +def valid_remote_config(): + return get_fixture_path("valid_remote_config.json") + +@pytest.fixture +def invalid_remote_config(): + return get_fixture_path("invalid_remote_config.json") + +@pytest.fixture +def valid_local_manifest(): + return get_fixture_path("valid_local_manifest.yaml") + +@pytest.fixture +def invalid_local_manifest(): + return get_fixture_path("invalid_local_manifest.yaml") + +@pytest.fixture +def valid_local_manifest_yaml(valid_local_manifest): + with open(valid_local_manifest, "r") as file: + return yaml.safe_load(file) + +@pytest.fixture +def invalid_local_manifest_yaml(invalid_local_manifest): + with open(invalid_local_manifest, "r") as file: + return yaml.safe_load(file) + +@pytest.fixture +def valid_local_config_file(): + return get_fixture_path("valid_local_pokeapi_config.json") + +@pytest.fixture +def invalid_local_config_file(): + return get_fixture_path("invalid_local_pokeapi_config.json") diff --git a/unit_tests/source_declarative_manifest/fixtures/invalid_local_manifest.yaml b/unit_tests/source_declarative_manifest/fixtures/invalid_local_manifest.yaml new file mode 100644 index 000000000..d73a692e9 --- /dev/null +++ b/unit_tests/source_declarative_manifest/fixtures/invalid_local_manifest.yaml @@ -0,0 +1,8 @@ +version: 0.78.5 + +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - pokemon diff --git a/unit_tests/source_declarative_manifest/fixtures/invalid_local_pokeapi_config.json b/unit_tests/source_declarative_manifest/fixtures/invalid_local_pokeapi_config.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/unit_tests/source_declarative_manifest/fixtures/invalid_local_pokeapi_config.json @@ -0,0 +1 @@ +{} diff --git a/unit_tests/source_declarative_manifest/fixtures/invalid_remote_config.json b/unit_tests/source_declarative_manifest/fixtures/invalid_remote_config.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/unit_tests/source_declarative_manifest/fixtures/invalid_remote_config.json @@ -0,0 +1 @@ +{} diff --git a/unit_tests/source_declarative_manifest/fixtures/valid_local_manifest.yaml b/unit_tests/source_declarative_manifest/fixtures/valid_local_manifest.yaml new file mode 100644 index 000000000..d77f33669 --- /dev/null +++ b/unit_tests/source_declarative_manifest/fixtures/valid_local_manifest.yaml @@ -0,0 +1,1368 @@ +version: 3.9.6 + +type: DeclarativeSource + +description: This is just a test + +check: + type: CheckStream + stream_names: + - pokemon + +definitions: + streams: + pokemon: + type: DeclarativeStream + name: pokemon + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /{{config['pokemon_name']}} + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/pokemon" + base_requester: + type: HttpRequester + url_base: https://pokeapi.co/api/v2/pokemon + +streams: + - $ref: "#/definitions/streams/pokemon" + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - pokemon_name + properties: + pokemon_name: + type: string + description: Pokemon requested from the API. + enum: + - bulbasaur + - ivysaur + - venusaur + - charmander + - charmeleon + - charizard + - squirtle + - wartortle + - blastoise + - caterpie + - metapod + - butterfree + - weedle + - kakuna + - beedrill + - pidgey + - pidgeotto + - pidgeot + - rattata + - raticate + - spearow + - fearow + - ekans + - arbok + - pikachu + - raichu + - sandshrew + - sandslash + - nidoranf + - nidorina + - nidoqueen + - nidoranm + - nidorino + - nidoking + - clefairy + - clefable + - vulpix + - ninetales + - jigglypuff + - wigglytuff + - zubat + - golbat + - oddish + - gloom + - vileplume + - paras + - parasect + - venonat + - venomoth + - diglett + - dugtrio + - meowth + - persian + - psyduck + - golduck + - mankey + - primeape + - growlithe + - arcanine + - poliwag + - poliwhirl + - poliwrath + - abra + - kadabra + - alakazam + - machop + - machoke + - machamp + - bellsprout + - weepinbell + - victreebel + - tentacool + - tentacruel + - geodude + - graveler + - golem + - ponyta + - rapidash + - slowpoke + - slowbro + - magnemite + - magneton + - farfetchd + - doduo + - dodrio + - seel + - dewgong + - grimer + - muk + - shellder + - cloyster + - gastly + - haunter + - gengar + - onix + - drowzee + - hypno + - krabby + - kingler + - voltorb + - electrode + - exeggcute + - exeggutor + - cubone + - marowak + - hitmonlee + - hitmonchan + - lickitung + - koffing + - weezing + - rhyhorn + - rhydon + - chansey + - tangela + - kangaskhan + - horsea + - seadra + - goldeen + - seaking + - staryu + - starmie + - mrmime + - scyther + - jynx + - electabuzz + - magmar + - pinsir + - tauros + - magikarp + - gyarados + - lapras + - ditto + - eevee + - vaporeon + - jolteon + - flareon + - porygon + - omanyte + - omastar + - kabuto + - kabutops + - aerodactyl + - snorlax + - articuno + - zapdos + - moltres + - dratini + - dragonair + - dragonite + - mewtwo + - mew + - chikorita + - bayleef + - meganium + - cyndaquil + - quilava + - typhlosion + - totodile + - croconaw + - feraligatr + - sentret + - furret + - hoothoot + - noctowl + - ledyba + - ledian + - spinarak + - ariados + - crobat + - chinchou + - lanturn + - pichu + - cleffa + - igglybuff + - togepi + - togetic + - natu + - xatu + - mareep + - flaaffy + - ampharos + - bellossom + - marill + - azumarill + - sudowoodo + - politoed + - hoppip + - skiploom + - jumpluff + - aipom + - sunkern + - sunflora + - yanma + - wooper + - quagsire + - espeon + - umbreon + - murkrow + - slowking + - misdreavus + - unown + - wobbuffet + - girafarig + - pineco + - forretress + - dunsparce + - gligar + - steelix + - snubbull + - granbull + - qwilfish + - scizor + - shuckle + - heracross + - sneasel + - teddiursa + - ursaring + - slugma + - magcargo + - swinub + - piloswine + - corsola + - remoraid + - octillery + - delibird + - mantine + - skarmory + - houndour + - houndoom + - kingdra + - phanpy + - donphan + - porygon2 + - stantler + - smeargle + - tyrogue + - hitmontop + - smoochum + - elekid + - magby + - miltank + - blissey + - raikou + - entei + - suicune + - larvitar + - pupitar + - tyranitar + - lugia + - ho-oh + - celebi + - treecko + - grovyle + - sceptile + - torchic + - combusken + - blaziken + - mudkip + - marshtomp + - swampert + - poochyena + - mightyena + - zigzagoon + - linoone + - wurmple + - silcoon + - beautifly + - cascoon + - dustox + - lotad + - lombre + - ludicolo + - seedot + - nuzleaf + - shiftry + - taillow + - swellow + - wingull + - pelipper + - ralts + - kirlia + - gardevoir + - surskit + - masquerain + - shroomish + - breloom + - slakoth + - vigoroth + - slaking + - nincada + - ninjask + - shedinja + - whismur + - loudred + - exploud + - makuhita + - hariyama + - azurill + - nosepass + - skitty + - delcatty + - sableye + - mawile + - aron + - lairon + - aggron + - meditite + - medicham + - electrike + - manectric + - plusle + - minun + - volbeat + - illumise + - roselia + - gulpin + - swalot + - carvanha + - sharpedo + - wailmer + - wailord + - numel + - camerupt + - torkoal + - spoink + - grumpig + - spinda + - trapinch + - vibrava + - flygon + - cacnea + - cacturne + - swablu + - altaria + - zangoose + - seviper + - lunatone + - solrock + - barboach + - whiscash + - corphish + - crawdaunt + - baltoy + - claydol + - lileep + - cradily + - anorith + - armaldo + - feebas + - milotic + - castform + - kecleon + - shuppet + - banette + - duskull + - dusclops + - tropius + - chimecho + - absol + - wynaut + - snorunt + - glalie + - spheal + - sealeo + - walrein + - clamperl + - huntail + - gorebyss + - relicanth + - luvdisc + - bagon + - shelgon + - salamence + - beldum + - metang + - metagross + - regirock + - regice + - registeel + - latias + - latios + - kyogre + - groudon + - rayquaza + - jirachi + - deoxys + - turtwig + - grotle + - torterra + - chimchar + - monferno + - infernape + - piplup + - prinplup + - empoleon + - starly + - staravia + - staraptor + - bidoof + - bibarel + - kricketot + - kricketune + - shinx + - luxio + - luxray + - budew + - roserade + - cranidos + - rampardos + - shieldon + - bastiodon + - burmy + - wormadam + - mothim + - combee + - vespiquen + - pachirisu + - buizel + - floatzel + - cherubi + - cherrim + - shellos + - gastrodon + - ambipom + - drifloon + - drifblim + - buneary + - lopunny + - mismagius + - honchkrow + - glameow + - purugly + - chingling + - stunky + - skuntank + - bronzor + - bronzong + - bonsly + - mimejr + - happiny + - chatot + - spiritomb + - gible + - gabite + - garchomp + - munchlax + - riolu + - lucario + - hippopotas + - hippowdon + - skorupi + - drapion + - croagunk + - toxicroak + - carnivine + - finneon + - lumineon + - mantyke + - snover + - abomasnow + - weavile + - magnezone + - lickilicky + - rhyperior + - tangrowth + - electivire + - magmortar + - togekiss + - yanmega + - leafeon + - glaceon + - gliscor + - mamoswine + - porygon-z + - gallade + - probopass + - dusknoir + - froslass + - rotom + - uxie + - mesprit + - azelf + - dialga + - palkia + - heatran + - regigigas + - giratina + - cresselia + - phione + - manaphy + - darkrai + - shaymin + - arceus + - victini + - snivy + - servine + - serperior + - tepig + - pignite + - emboar + - oshawott + - dewott + - samurott + - patrat + - watchog + - lillipup + - herdier + - stoutland + - purrloin + - liepard + - pansage + - simisage + - pansear + - simisear + - panpour + - simipour + - munna + - musharna + - pidove + - tranquill + - unfezant + - blitzle + - zebstrika + - roggenrola + - boldore + - gigalith + - woobat + - swoobat + - drilbur + - excadrill + - audino + - timburr + - gurdurr + - conkeldurr + - tympole + - palpitoad + - seismitoad + - throh + - sawk + - sewaddle + - swadloon + - leavanny + - venipede + - whirlipede + - scolipede + - cottonee + - whimsicott + - petilil + - lilligant + - basculin + - sandile + - krokorok + - krookodile + - darumaka + - darmanitan + - maractus + - dwebble + - crustle + - scraggy + - scrafty + - sigilyph + - yamask + - cofagrigus + - tirtouga + - carracosta + - archen + - archeops + - trubbish + - garbodor + - zorua + - zoroark + - minccino + - cinccino + - gothita + - gothorita + - gothitelle + - solosis + - duosion + - reuniclus + - ducklett + - swanna + - vanillite + - vanillish + - vanilluxe + - deerling + - sawsbuck + - emolga + - karrablast + - escavalier + - foongus + - amoonguss + - frillish + - jellicent + - alomomola + - joltik + - galvantula + - ferroseed + - ferrothorn + - klink + - klang + - klinklang + - tynamo + - eelektrik + - eelektross + - elgyem + - beheeyem + - litwick + - lampent + - chandelure + - axew + - fraxure + - haxorus + - cubchoo + - beartic + - cryogonal + - shelmet + - accelgor + - stunfisk + - mienfoo + - mienshao + - druddigon + - golett + - golurk + - pawniard + - bisharp + - bouffalant + - rufflet + - braviary + - vullaby + - mandibuzz + - heatmor + - durant + - deino + - zweilous + - hydreigon + - larvesta + - volcarona + - cobalion + - terrakion + - virizion + - tornadus + - thundurus + - reshiram + - zekrom + - landorus + - kyurem + - keldeo + - meloetta + - genesect + - chespin + - quilladin + - chesnaught + - fennekin + - braixen + - delphox + - froakie + - frogadier + - greninja + - bunnelby + - diggersby + - fletchling + - fletchinder + - talonflame + - scatterbug + - spewpa + - vivillon + - litleo + - pyroar + - flabebe + - floette + - florges + - skiddo + - gogoat + - pancham + - pangoro + - furfrou + - espurr + - meowstic + - honedge + - doublade + - aegislash + - spritzee + - aromatisse + - swirlix + - slurpuff + - inkay + - malamar + - binacle + - barbaracle + - skrelp + - dragalge + - clauncher + - clawitzer + - helioptile + - heliolisk + - tyrunt + - tyrantrum + - amaura + - aurorus + - sylveon + - hawlucha + - dedenne + - carbink + - goomy + - sliggoo + - goodra + - klefki + - phantump + - trevenant + - pumpkaboo + - gourgeist + - bergmite + - avalugg + - noibat + - noivern + - xerneas + - yveltal + - zygarde + - diancie + - hoopa + - volcanion + - rowlet + - dartrix + - decidueye + - litten + - torracat + - incineroar + - popplio + - brionne + - primarina + - pikipek + - trumbeak + - toucannon + - yungoos + - gumshoos + - grubbin + - charjabug + - vikavolt + - crabrawler + - crabominable + - oricorio + - cutiefly + - ribombee + - rockruff + - lycanroc + - wishiwashi + - mareanie + - toxapex + - mudbray + - mudsdale + - dewpider + - araquanid + - fomantis + - lurantis + - morelull + - shiinotic + - salandit + - salazzle + - stufful + - bewear + - bounsweet + - steenee + - tsareena + - comfey + - oranguru + - passimian + - wimpod + - golisopod + - sandygast + - palossand + - pyukumuku + - typenull + - silvally + - minior + - komala + - turtonator + - togedemaru + - mimikyu + - bruxish + - drampa + - dhelmise + - jangmo-o + - hakamo-o + - kommo-o + - tapukoko + - tapulele + - tapubulu + - tapufini + - cosmog + - cosmoem + - solgaleo + - lunala + - nihilego + - buzzwole + - pheromosa + - xurkitree + - celesteela + - kartana + - guzzlord + - necrozma + - magearna + - marshadow + - poipole + - naganadel + - stakataka + - blacephalon + - zeraora + - meltan + - melmetal + - grookey + - thwackey + - rillaboom + - scorbunny + - raboot + - cinderace + - sobble + - drizzile + - inteleon + - skwovet + - greedent + - rookidee + - corvisquire + - corviknight + - blipbug + - dottler + - orbeetle + - nickit + - thievul + - gossifleur + - eldegoss + - wooloo + - dubwool + - chewtle + - drednaw + - yamper + - boltund + - rolycoly + - carkol + - coalossal + - applin + - flapple + - appletun + - silicobra + - sandaconda + - cramorant + - arrokuda + - barraskewda + - toxel + - toxtricity + - sizzlipede + - centiskorch + - clobbopus + - grapploct + - sinistea + - polteageist + - hatenna + - hattrem + - hatterene + - impidimp + - morgrem + - grimmsnarl + - obstagoon + - perrserker + - cursola + - sirfetchd + - mrrime + - runerigus + - milcery + - alcremie + - falinks + - pincurchin + - snom + - frosmoth + - stonjourner + - eiscue + - indeedee + - morpeko + - cufant + - copperajah + - dracozolt + - arctozolt + - dracovish + - arctovish + - duraludon + - dreepy + - drakloak + - dragapult + - zacian + - zamazenta + - eternatus + - kubfu + - urshifu + - zarude + - regieleki + - regidrago + - glastrier + - spectrier + - calyrex + order: 0 + title: Pokemon Name + pattern: ^[a-z0-9_\-]+$ + examples: + - ditto + - luxray + - snorlax + additionalProperties: true + +metadata: + testedStreams: + pokemon: + hasRecords: true + streamHash: f619395f8c7a553f51cec2a7274a4ce517ab46c8 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + autoImportSchema: + pokemon: false + +schemas: + pokemon: + type: object + $schema: http://json-schema.org/draft-07/schema# + properties: + id: + type: + - "null" + - integer + name: + type: + - "null" + - string + forms: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + moves: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + move: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + version_group_details: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + version_group: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + level_learned_at: + type: + - "null" + - integer + move_learn_method: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + additionalProperties: true + additionalProperties: true + order: + type: + - "null" + - integer + stats: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + stat: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + effort: + type: + - "null" + - integer + base_stat: + type: + - "null" + - integer + additionalProperties: true + types: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + type: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + slot: + type: + - "null" + - integer + additionalProperties: true + height: + type: + - "null" + - integer + weight: + type: + - "null" + - integer + species: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + sprites: + type: + - "null" + - object + properties: + back_shiny: + type: + - "null" + - string + back_female: + type: + - "null" + - string + front_shiny: + type: + - "null" + - string + back_default: + type: + - "null" + - string + front_female: + type: + - "null" + - string + front_default: + type: + - "null" + - string + back_shiny_female: + type: + - "null" + - string + front_shiny_female: + type: + - "null" + - string + additionalProperties: true + abilities: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + slot: + type: + - "null" + - integer + ability: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + is_hidden: + type: + - "null" + - boolean + additionalProperties: true + held_items: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + item: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + version_details: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + version: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + rarity: + type: + - "null" + - integer + additionalProperties: true + additionalProperties: true + is_default: + type: + - "null" + - boolean + past_types: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + types: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + type: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + slot: + type: + - "null" + - integer + additionalProperties: true + generation: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + additionalProperties: true + game_indices: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + version: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + game_index: + type: + - "null" + - integer + additionalProperties: true + base_experience: + type: + - "null" + - integer + location_area_encounters: + type: + - "null" + - string + additionalProperties: true diff --git a/unit_tests/source_declarative_manifest/fixtures/valid_local_pokeapi_config.json b/unit_tests/source_declarative_manifest/fixtures/valid_local_pokeapi_config.json new file mode 100644 index 000000000..214fc684f --- /dev/null +++ b/unit_tests/source_declarative_manifest/fixtures/valid_local_pokeapi_config.json @@ -0,0 +1,3 @@ +{ + "pokemon_name": "blastoise" +} diff --git a/unit_tests/source_declarative_manifest/fixtures/valid_remote_config.json b/unit_tests/source_declarative_manifest/fixtures/valid_remote_config.json new file mode 100644 index 000000000..8280914dc --- /dev/null +++ b/unit_tests/source_declarative_manifest/fixtures/valid_remote_config.json @@ -0,0 +1,33 @@ +{ + "__injected_declarative_manifest": { + "version": "0.29.0", + "definitions": { + "selector": { "extractor": { "field_path": [] } }, + "requester": { "url_base": "https://test.com/api", "http_method": "GET" }, + "retriever": { + "record_selector": { "$ref": "#/definitions/selector" }, + "requester": { "$ref": "#/definitions/requester" } + }, + "base_stream": { "retriever": { "$ref": "#/definitions/retriever" } }, + "data_stream": { + "$ref": "#/definitions/base_stream", + "$parameters": { "name": "data", "path": "/data" } + } + }, + "streams": ["#/definitions/data_stream"], + "check": { + "stream_names": ["data"] + }, + "spec": { + "type": "Spec", + "documentation_url": "https://test.com/doc", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Test Spec", + "type": "object", + "additionalProperties": true, + "properties": {} + } + } + } +} diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_local_manifest.py b/unit_tests/source_declarative_manifest/test_source_declarative_local_manifest.py new file mode 100644 index 000000000..aa3603c01 --- /dev/null +++ b/unit_tests/source_declarative_manifest/test_source_declarative_local_manifest.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from unittest.mock import patch + +import pytest +from jsonschema import ValidationError +from airbyte_cdk.cli.source_declarative_manifest import run + +POKEAPI_JSON_SPEC_SUBSTRING = '"required":["pokemon_name"]' +SUCCESS_CHECK_SUBSTRING = '"connectionStatus":{"status":"SUCCEEDED"}' +FAILED_CHECK_SUBSTRING = '"connectionStatus":{"status":"FAILED"}' + + +@pytest.fixture(autouse=True) +def setup(valid_local_manifest_yaml): + with patch("source_declarative_manifest.run._is_local_manifest_command", return_value=True): + with patch( + "source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file", + return_value=valid_local_manifest_yaml, + ): + yield + + +def test_spec_is_poke_api(capsys): + run.handle_command(["spec"]) + stdout = capsys.readouterr() + assert POKEAPI_JSON_SPEC_SUBSTRING in stdout.out + + +def test_invalid_yaml_throws(capsys, invalid_local_manifest_yaml): + with patch( + "source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file", + return_value=invalid_local_manifest_yaml, + ): + with pytest.raises(ValidationError): + run.handle_command(["spec"]) + + +def test_given_invalid_config_then_unsuccessful_check(capsys, invalid_local_config_file): + run.handle_command(["check", "--config", str(invalid_local_config_file)]) + stdout = capsys.readouterr() + assert json.loads(stdout.out).get("connectionStatus").get("status") == "FAILED" + + +def test_given_valid_config_with_successful_check(capsys, valid_local_config_file): + run.handle_command(["check", "--config", str(valid_local_config_file)]) + stdout = capsys.readouterr() + assert SUCCESS_CHECK_SUBSTRING in stdout.out diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_remote_manifest.py b/unit_tests/source_declarative_manifest/test_source_declarative_remote_manifest.py new file mode 100644 index 000000000..4ae34fc93 --- /dev/null +++ b/unit_tests/source_declarative_manifest/test_source_declarative_remote_manifest.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource +from airbyte_cdk.cli.source_declarative_manifest._run import ( + create_declarative_source, + handle_command, +) + +REMOTE_MANIFEST_SPEC_SUBSTRING = '"required":["__injected_declarative_manifest"]' + + +def test_spec_does_not_raise_value_error(capsys): + handle_command(["spec"]) + stdout = capsys.readouterr() + assert REMOTE_MANIFEST_SPEC_SUBSTRING in stdout.out + + +def test_given_no_injected_declarative_manifest_then_raise_value_error(invalid_remote_config): + with pytest.raises(ValueError): + create_declarative_source(["check", "--config", str(invalid_remote_config)]) + + +def test_given_injected_declarative_manifest_then_return_declarative_manifest(valid_remote_config): + source = create_declarative_source(["check", "--config", str(valid_remote_config)]) + assert isinstance(source, ManifestDeclarativeSource) From 6a77a6d76cd34aca476bf1770c74e0e9854c9589 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 21 Nov 2024 14:29:10 -0800 Subject: [PATCH 2/7] move fixtures to conftest.py and update relative references --- .../{fixtures/__init__.py => conftest.py} | 21 +++++++++++++------ .../resources/__init__.py | 0 .../invalid_local_manifest.yaml | 0 .../invalid_local_pokeapi_config.json | 0 .../invalid_remote_config.json | 0 .../valid_local_manifest.yaml | 0 .../valid_local_pokeapi_config.json | 0 .../valid_remote_config.json | 0 .../test_source_declarative_local_manifest.py | 19 ++++++++++------- 9 files changed, 26 insertions(+), 14 deletions(-) rename unit_tests/source_declarative_manifest/{fixtures/__init__.py => conftest.py} (65%) create mode 100644 unit_tests/source_declarative_manifest/resources/__init__.py rename unit_tests/source_declarative_manifest/{fixtures => resources}/invalid_local_manifest.yaml (100%) rename unit_tests/source_declarative_manifest/{fixtures => resources}/invalid_local_pokeapi_config.json (100%) rename unit_tests/source_declarative_manifest/{fixtures => resources}/invalid_remote_config.json (100%) rename unit_tests/source_declarative_manifest/{fixtures => resources}/valid_local_manifest.yaml (100%) rename unit_tests/source_declarative_manifest/{fixtures => resources}/valid_local_pokeapi_config.json (100%) rename unit_tests/source_declarative_manifest/{fixtures => resources}/valid_remote_config.json (100%) diff --git a/unit_tests/source_declarative_manifest/fixtures/__init__.py b/unit_tests/source_declarative_manifest/conftest.py similarity index 65% rename from unit_tests/source_declarative_manifest/fixtures/__init__.py rename to unit_tests/source_declarative_manifest/conftest.py index 18f7eb3da..4d4a16aa0 100644 --- a/unit_tests/source_declarative_manifest/fixtures/__init__.py +++ b/unit_tests/source_declarative_manifest/conftest.py @@ -6,39 +6,48 @@ import os import yaml + def get_fixture_path(file_name): return os.path.join(os.path.dirname(__file__), file_name) + @pytest.fixture def valid_remote_config(): - return get_fixture_path("valid_remote_config.json") + return get_fixture_path("resources/valid_remote_config.json") + @pytest.fixture def invalid_remote_config(): - return get_fixture_path("invalid_remote_config.json") + return get_fixture_path("resources/invalid_remote_config.json") + @pytest.fixture def valid_local_manifest(): - return get_fixture_path("valid_local_manifest.yaml") + return get_fixture_path("resources/valid_local_manifest.yaml") + @pytest.fixture def invalid_local_manifest(): - return get_fixture_path("invalid_local_manifest.yaml") + return get_fixture_path("resources/invalid_local_manifest.yaml") + @pytest.fixture def valid_local_manifest_yaml(valid_local_manifest): with open(valid_local_manifest, "r") as file: return yaml.safe_load(file) + @pytest.fixture def invalid_local_manifest_yaml(invalid_local_manifest): with open(invalid_local_manifest, "r") as file: return yaml.safe_load(file) + @pytest.fixture def valid_local_config_file(): - return get_fixture_path("valid_local_pokeapi_config.json") + return get_fixture_path("resources/valid_local_pokeapi_config.json") + @pytest.fixture def invalid_local_config_file(): - return get_fixture_path("invalid_local_pokeapi_config.json") + return get_fixture_path("resources/invalid_local_pokeapi_config.json") diff --git a/unit_tests/source_declarative_manifest/resources/__init__.py b/unit_tests/source_declarative_manifest/resources/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/unit_tests/source_declarative_manifest/fixtures/invalid_local_manifest.yaml b/unit_tests/source_declarative_manifest/resources/invalid_local_manifest.yaml similarity index 100% rename from unit_tests/source_declarative_manifest/fixtures/invalid_local_manifest.yaml rename to unit_tests/source_declarative_manifest/resources/invalid_local_manifest.yaml diff --git a/unit_tests/source_declarative_manifest/fixtures/invalid_local_pokeapi_config.json b/unit_tests/source_declarative_manifest/resources/invalid_local_pokeapi_config.json similarity index 100% rename from unit_tests/source_declarative_manifest/fixtures/invalid_local_pokeapi_config.json rename to unit_tests/source_declarative_manifest/resources/invalid_local_pokeapi_config.json diff --git a/unit_tests/source_declarative_manifest/fixtures/invalid_remote_config.json b/unit_tests/source_declarative_manifest/resources/invalid_remote_config.json similarity index 100% rename from unit_tests/source_declarative_manifest/fixtures/invalid_remote_config.json rename to unit_tests/source_declarative_manifest/resources/invalid_remote_config.json diff --git a/unit_tests/source_declarative_manifest/fixtures/valid_local_manifest.yaml b/unit_tests/source_declarative_manifest/resources/valid_local_manifest.yaml similarity index 100% rename from unit_tests/source_declarative_manifest/fixtures/valid_local_manifest.yaml rename to unit_tests/source_declarative_manifest/resources/valid_local_manifest.yaml diff --git a/unit_tests/source_declarative_manifest/fixtures/valid_local_pokeapi_config.json b/unit_tests/source_declarative_manifest/resources/valid_local_pokeapi_config.json similarity index 100% rename from unit_tests/source_declarative_manifest/fixtures/valid_local_pokeapi_config.json rename to unit_tests/source_declarative_manifest/resources/valid_local_pokeapi_config.json diff --git a/unit_tests/source_declarative_manifest/fixtures/valid_remote_config.json b/unit_tests/source_declarative_manifest/resources/valid_remote_config.json similarity index 100% rename from unit_tests/source_declarative_manifest/fixtures/valid_remote_config.json rename to unit_tests/source_declarative_manifest/resources/valid_remote_config.json diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_local_manifest.py b/unit_tests/source_declarative_manifest/test_source_declarative_local_manifest.py index aa3603c01..871950b16 100644 --- a/unit_tests/source_declarative_manifest/test_source_declarative_local_manifest.py +++ b/unit_tests/source_declarative_manifest/test_source_declarative_local_manifest.py @@ -7,7 +7,7 @@ import pytest from jsonschema import ValidationError -from airbyte_cdk.cli.source_declarative_manifest import run +from airbyte_cdk.cli.source_declarative_manifest import _run POKEAPI_JSON_SPEC_SUBSTRING = '"required":["pokemon_name"]' SUCCESS_CHECK_SUBSTRING = '"connectionStatus":{"status":"SUCCEEDED"}' @@ -16,36 +16,39 @@ @pytest.fixture(autouse=True) def setup(valid_local_manifest_yaml): - with patch("source_declarative_manifest.run._is_local_manifest_command", return_value=True): + with patch( + "airbyte_cdk.cli.source_declarative_manifest._run._is_local_manifest_command", + return_value=True, + ): with patch( - "source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file", + "airbyte_cdk.cli.source_declarative_manifest._run.YamlDeclarativeSource._read_and_parse_yaml_file", return_value=valid_local_manifest_yaml, ): yield def test_spec_is_poke_api(capsys): - run.handle_command(["spec"]) + _run.handle_command(["spec"]) stdout = capsys.readouterr() assert POKEAPI_JSON_SPEC_SUBSTRING in stdout.out def test_invalid_yaml_throws(capsys, invalid_local_manifest_yaml): with patch( - "source_declarative_manifest.run.YamlDeclarativeSource._read_and_parse_yaml_file", + "airbyte_cdk.cli.source_declarative_manifest._run.YamlDeclarativeSource._read_and_parse_yaml_file", return_value=invalid_local_manifest_yaml, ): with pytest.raises(ValidationError): - run.handle_command(["spec"]) + _run.handle_command(["spec"]) def test_given_invalid_config_then_unsuccessful_check(capsys, invalid_local_config_file): - run.handle_command(["check", "--config", str(invalid_local_config_file)]) + _run.handle_command(["check", "--config", str(invalid_local_config_file)]) stdout = capsys.readouterr() assert json.loads(stdout.out).get("connectionStatus").get("status") == "FAILED" def test_given_valid_config_with_successful_check(capsys, valid_local_config_file): - run.handle_command(["check", "--config", str(valid_local_config_file)]) + _run.handle_command(["check", "--config", str(valid_local_config_file)]) stdout = capsys.readouterr() assert SUCCESS_CHECK_SUBSTRING in stdout.out From e4ef8b21ac9af719d1c95e63a0362cdb9d224639 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 21 Nov 2024 14:32:07 -0800 Subject: [PATCH 3/7] always run pytest-fast --- .github/workflows/pytest_fast.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml index c25116440..edbbe3520 100644 --- a/.github/workflows/pytest_fast.yml +++ b/.github/workflows/pytest_fast.yml @@ -4,15 +4,7 @@ on: push: branches: - main - paths: - - 'airbyte_cdk/**' - - 'poetry.lock' - - 'pyproject.toml' pull_request: - paths: - - 'airbyte_cdk/**' - - 'poetry.lock' - - 'pyproject.toml' jobs: pytest-fast: From 1b73806180a7373ba7357b67e754906a7f34651b Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 21 Nov 2024 14:33:59 -0800 Subject: [PATCH 4/7] ci: run pytest matrix if tests are changed --- .github/workflows/pytest_matrix.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest_matrix.yml b/.github/workflows/pytest_matrix.yml index 8417a08e6..c0a9e7bb7 100644 --- a/.github/workflows/pytest_matrix.yml +++ b/.github/workflows/pytest_matrix.yml @@ -13,11 +13,13 @@ on: - main paths: - 'airbyte_cdk/**' + - 'unit_tests/**' - 'poetry.lock' - 'pyproject.toml' pull_request: paths: - 'airbyte_cdk/**' + - 'unit_tests/**' - 'poetry.lock' - 'pyproject.toml' From 22c026ed25842a5d9ad5737dc1926416dc468455 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 21 Nov 2024 14:46:49 -0800 Subject: [PATCH 5/7] add 'tests' scope for semantic PR reviewer --- .github/release-drafter.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index 6bae11d61..8fc36adfd 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -15,6 +15,7 @@ categories: - "chore" - "ci" - "refactor" + - "testing" - title: "Documentation 📖" label: "docs" change-template: "- $TITLE (#$NUMBER)" @@ -38,6 +39,9 @@ autolabeler: - label: "chore" title: - '/^chore(\(.*\))?\:/i' + - label: "testing" + title: + - '/^tests(\(.*\))?\:/i' - label: "ci" title: - '/^ci(\(.*\))?\:/i' From f44234e9e7278618edde66191712032298464e85 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 21 Nov 2024 14:48:45 -0800 Subject: [PATCH 6/7] update workflow --- .github/workflows/semantic_pr_check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/semantic_pr_check.yml b/.github/workflows/semantic_pr_check.yml index ef9f4a3b0..e738d428b 100644 --- a/.github/workflows/semantic_pr_check.yml +++ b/.github/workflows/semantic_pr_check.yml @@ -37,5 +37,5 @@ jobs: Chore build Build - test - Test + tests + Tests From 11a3ace24d67c445a0b98b20738ed87fdf710d9c Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 21 Nov 2024 15:05:33 -0800 Subject: [PATCH 7/7] apply suggestion --- .../test_source_declarative_remote_manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_remote_manifest.py b/unit_tests/source_declarative_manifest/test_source_declarative_remote_manifest.py index 4ae34fc93..093f634a4 100644 --- a/unit_tests/source_declarative_manifest/test_source_declarative_remote_manifest.py +++ b/unit_tests/source_declarative_manifest/test_source_declarative_remote_manifest.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. # import pytest