From af7ab8f835e95e41bfbb29e9beb4709744aedf8b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Oct 2025 07:22:28 +0000 Subject: [PATCH] feat(metadata-service): Add schema support for smoke test scenarios Add JSON schema definitions to support the new smoke tests configuration format introduced in airbytehq/airbyte-python-cdk#775. Changes: - Created SmokeTestScenario.yaml schema defining the structure for individual smoke test scenarios with properties: name, config_file, config_settings, expect_failure, only_streams, exclude_streams, suggested_streams_only, and configured_catalog_path - Updated ConnectorTestSuiteOptions.yaml to add 'smokeTests' to the suite enum and added 'scenarios' property to support the new smoke test format - Regenerated Python models and bundled JSON schema This enables connectors to define smoke tests in their metadata.yaml file under data.connectorTestSuitesOptions with the following structure: connectorTestSuitesOptions: - suite: smokeTests scenarios: - name: default config_file: secrets/config_oauth.json config_settings: start_date: "2025-01-01T00:00:00Z" only_streams: - users Related: airbytehq/airbyte-python-cdk#775 Co-Authored-By: AJ Steers --- .../ConnectorMetadataDefinitionV0.json | 61 ++++++++++++++++++- .../ConnectorMetadataDefinitionV0.py | 44 ++++++++++++- .../generated/ConnectorTestSuiteOptions.py | 46 ++++++++++++-- .../models/generated/SmokeTestScenario.py | 42 +++++++++++++ .../models/generated/__init__.py | 1 + .../models/src/ConnectorTestSuiteOptions.yaml | 6 ++ .../models/src/SmokeTestScenario.yaml | 41 +++++++++++++ 7 files changed, 233 insertions(+), 8 deletions(-) create mode 100644 airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SmokeTestScenario.py create mode 100644 airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SmokeTestScenario.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.json b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.json index b6d1f54e6ab4..405b09481663 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.json +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.json @@ -72,7 +72,8 @@ "unitTests", "integrationTests", "acceptanceTests", - "liveTests" + "liveTests", + "smokeTests" ] }, "testSecrets": { @@ -147,6 +148,64 @@ } } } + }, + "scenarios": { + "description": "List of smoke test scenarios (only applicable when suite is 'smokeTests')", + "type": "array", + "items": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SmokeTestScenario.yaml", + "title": "SmokeTestScenario", + "description": "A single smoke test scenario configuration for a connector.", + "type": "object", + "required": [ + "name" + ], + "additionalProperties": false, + "properties": { + "name": { + "description": "Name of the test scenario (e.g., 'default', 'invalid_config', 'oauth_config')", + "type": "string" + }, + "config_file": { + "description": "Relative path to the config file to use for this scenario", + "type": "string" + }, + "config_settings": { + "description": "Optional dictionary of config settings to override or supplement config_file settings", + "type": "object", + "additionalProperties": true + }, + "expect_failure": { + "description": "Whether the scenario is expected to fail", + "type": "boolean", + "default": false + }, + "only_streams": { + "description": "List of stream names to include in the scenario (if specified, only these streams will be tested)", + "type": "array", + "items": { + "type": "string" + } + }, + "exclude_streams": { + "description": "List of stream names to exclude from the scenario", + "type": "array", + "items": { + "type": "string" + } + }, + "suggested_streams_only": { + "description": "Whether to limit testing to the connector's suggested streams list (from data.suggestedStreams)", + "type": "boolean", + "default": false + }, + "configured_catalog_path": { + "description": "Path to a pre-configured catalog file for the scenario", + "type": "string" + } + } + } } } } diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py index 32d20510cd6f..03bcbbefb5f3 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py @@ -38,6 +38,40 @@ class Config: id: str = Field(..., description="The connection ID") +class SmokeTestScenario(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field( + ..., + description="Name of the test scenario (e.g., 'default', 'invalid_config', 'oauth_config')", + ) + config_file: Optional[str] = Field( + None, description="Relative path to the config file to use for this scenario" + ) + config_settings: Optional[Dict[str, Any]] = Field( + None, + description="Optional dictionary of config settings to override or supplement config_file settings", + ) + expect_failure: Optional[bool] = Field( + False, description="Whether the scenario is expected to fail" + ) + only_streams: Optional[List[str]] = Field( + None, + description="List of stream names to include in the scenario (if specified, only these streams will be tested)", + ) + exclude_streams: Optional[List[str]] = Field( + None, description="List of stream names to exclude from the scenario" + ) + suggested_streams_only: Optional[bool] = Field( + False, + description="Whether to limit testing to the connector's suggested streams list (from data.suggestedStreams)", + ) + configured_catalog_path: Optional[str] = Field( + None, description="Path to a pre-configured catalog file for the scenario" + ) + + class ReleaseStage(BaseModel): __root__: Literal["alpha", "beta", "generally_available", "custom"] = Field( ..., @@ -278,9 +312,9 @@ class ConnectorTestSuiteOptions(BaseModel): class Config: extra = Extra.forbid - suite: Literal["unitTests", "integrationTests", "acceptanceTests", "liveTests"] = ( - Field(..., description="Name of the configured test suite") - ) + suite: Literal[ + "unitTests", "integrationTests", "acceptanceTests", "liveTests", "smokeTests" + ] = Field(..., description="Name of the configured test suite") testSecrets: Optional[List[Secret]] = Field( None, description="List of secrets required to run the test suite" ) @@ -288,6 +322,10 @@ class Config: None, description="List of sandbox cloud connections that tests can be run against", ) + scenarios: Optional[List[SmokeTestScenario]] = Field( + None, + description="List of smoke test scenarios (only applicable when suite is 'smokeTests')", + ) class ActorDefinitionResourceRequirements(BaseModel): diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorTestSuiteOptions.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorTestSuiteOptions.py index b1a3b408c9dc..f1e299e0496d 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorTestSuiteOptions.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorTestSuiteOptions.py @@ -3,7 +3,7 @@ from __future__ import annotations -from typing import List, Literal, Optional +from typing import Any, Dict, List, Literal, Optional from pydantic import BaseModel, Extra, Field @@ -29,6 +29,40 @@ class Config: id: str = Field(..., description="The connection ID") +class SmokeTestScenario(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field( + ..., + description="Name of the test scenario (e.g., 'default', 'invalid_config', 'oauth_config')", + ) + config_file: Optional[str] = Field( + None, description="Relative path to the config file to use for this scenario" + ) + config_settings: Optional[Dict[str, Any]] = Field( + None, + description="Optional dictionary of config settings to override or supplement config_file settings", + ) + expect_failure: Optional[bool] = Field( + False, description="Whether the scenario is expected to fail" + ) + only_streams: Optional[List[str]] = Field( + None, + description="List of stream names to include in the scenario (if specified, only these streams will be tested)", + ) + exclude_streams: Optional[List[str]] = Field( + None, description="List of stream names to exclude from the scenario" + ) + suggested_streams_only: Optional[bool] = Field( + False, + description="Whether to limit testing to the connector's suggested streams list (from data.suggestedStreams)", + ) + configured_catalog_path: Optional[str] = Field( + None, description="Path to a pre-configured catalog file for the scenario" + ) + + class Secret(BaseModel): class Config: extra = Extra.forbid @@ -45,9 +79,9 @@ class ConnectorTestSuiteOptions(BaseModel): class Config: extra = Extra.forbid - suite: Literal["unitTests", "integrationTests", "acceptanceTests", "liveTests"] = ( - Field(..., description="Name of the configured test suite") - ) + suite: Literal[ + "unitTests", "integrationTests", "acceptanceTests", "liveTests", "smokeTests" + ] = Field(..., description="Name of the configured test suite") testSecrets: Optional[List[Secret]] = Field( None, description="List of secrets required to run the test suite" ) @@ -55,3 +89,7 @@ class Config: None, description="List of sandbox cloud connections that tests can be run against", ) + scenarios: Optional[List[SmokeTestScenario]] = Field( + None, + description="List of smoke test scenarios (only applicable when suite is 'smokeTests')", + ) diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SmokeTestScenario.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SmokeTestScenario.py new file mode 100644 index 000000000000..4e7eb13a56d2 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SmokeTestScenario.py @@ -0,0 +1,42 @@ +# generated by datamodel-codegen: +# filename: SmokeTestScenario.yaml + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Extra, Field + + +class SmokeTestScenario(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field( + ..., + description="Name of the test scenario (e.g., 'default', 'invalid_config', 'oauth_config')", + ) + config_file: Optional[str] = Field( + None, description="Relative path to the config file to use for this scenario" + ) + config_settings: Optional[Dict[str, Any]] = Field( + None, + description="Optional dictionary of config settings to override or supplement config_file settings", + ) + expect_failure: Optional[bool] = Field( + False, description="Whether the scenario is expected to fail" + ) + only_streams: Optional[List[str]] = Field( + None, + description="List of stream names to include in the scenario (if specified, only these streams will be tested)", + ) + exclude_streams: Optional[List[str]] = Field( + None, description="List of stream names to exclude from the scenario" + ) + suggested_streams_only: Optional[bool] = Field( + False, + description="Whether to limit testing to the connector's suggested streams list (from data.suggestedStreams)", + ) + configured_catalog_path: Optional[str] = Field( + None, description="Path to a pre-configured catalog file for the scenario" + ) diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py index 3877faa22716..4ad0ab48805b 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py @@ -25,6 +25,7 @@ from .RolloutConfiguration import * from .Secret import * from .SecretStore import * +from .SmokeTestScenario import * from .SourceFileInfo import * from .SuggestedStreams import * from .SupportLevel import * diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorTestSuiteOptions.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorTestSuiteOptions.yaml index 2830332ee520..a0234256ebfd 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorTestSuiteOptions.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorTestSuiteOptions.yaml @@ -16,6 +16,7 @@ properties: - "integrationTests" - "acceptanceTests" - "liveTests" + - "smokeTests" testSecrets: description: "List of secrets required to run the test suite" type: array @@ -26,3 +27,8 @@ properties: type: array items: "$ref": "TestConnections.yaml" + scenarios: + description: "List of smoke test scenarios (only applicable when suite is 'smokeTests')" + type: array + items: + "$ref": "SmokeTestScenario.yaml" diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SmokeTestScenario.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SmokeTestScenario.yaml new file mode 100644 index 000000000000..4255529007fd --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SmokeTestScenario.yaml @@ -0,0 +1,41 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SmokeTestScenario.yaml +title: SmokeTestScenario +description: A single smoke test scenario configuration for a connector. +type: object +required: + - name +additionalProperties: false +properties: + name: + description: "Name of the test scenario (e.g., 'default', 'invalid_config', 'oauth_config')" + type: string + config_file: + description: "Relative path to the config file to use for this scenario" + type: string + config_settings: + description: "Optional dictionary of config settings to override or supplement config_file settings" + type: object + additionalProperties: true + expect_failure: + description: "Whether the scenario is expected to fail" + type: boolean + default: false + only_streams: + description: "List of stream names to include in the scenario (if specified, only these streams will be tested)" + type: array + items: + type: string + exclude_streams: + description: "List of stream names to exclude from the scenario" + type: array + items: + type: string + suggested_streams_only: + description: "Whether to limit testing to the connector's suggested streams list (from data.suggestedStreams)" + type: boolean + default: false + configured_catalog_path: + description: "Path to a pre-configured catalog file for the scenario" + type: string