feat: Add unprivileged and config-free discover for declarative static schemas

airbyte_cdk/cli/source_declarative_manifest/_run.py

@@ -235,8 +235,8 @@ def _parse_inputs_into_config_catalog_state(
 ]:
     config = (
         ConcurrentDeclarativeSource.read_config(parsed_args.config)
-        if hasattr(parsed_args, "config")
-        else None
+        if hasattr(parsed_args, "config") and parsed_args.config
+        else {}
     )
     catalog = (
         ConcurrentDeclarativeSource.read_catalog(parsed_args.catalog)

airbyte_cdk/connector.py

@@ -31,8 +31,15 @@ def load_optional_package_file(package: str, filename: str) -> Optional[bytes]:
 
 
 class BaseConnector(ABC, Generic[TConfig]):
-    # configure whether the `check_config_against_spec_or_exit()` needs to be called
     check_config_against_spec: bool = True
+    """Configure whether `check_config_against_spec_or_exit()` needs to be called."""
+
+    check_config_during_discover: bool = True
+    """Determines whether config validation should be skipped during discovery.
+
+    By default, config validation is not skipped during discovery. This can be overridden
+    by sources that can provide catalog information without requiring authentication.
+    """
 
     @abstractmethod
     def configure(self, config: Mapping[str, Any], temp_dir: str) -> TConfig:

airbyte_cdk/entrypoint.py

@@ -105,7 +105,7 @@ def parse_args(args: List[str]) -> argparse.Namespace:
         )
         required_discover_parser = discover_parser.add_argument_group("required named arguments")
         required_discover_parser.add_argument(
-            "--config", type=str, required=True, help="path to the json configuration file"
+            "--config", type=str, required=False, help="path to the json configuration file"
         )
         discover_parser.add_argument(
             "--manifest-path",
@@ -177,19 +177,37 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]:
                 )
                 if cmd == "spec":
                     message = AirbyteMessage(type=Type.SPEC, spec=source_spec)
-                    yield from [
+                    yield from (
                         self.airbyte_message_to_string(queued_message)
                         for queued_message in self._emit_queued_messages(self.source)
-                    ]
+                    )
                     yield self.airbyte_message_to_string(message)
+                elif (
+                    cmd == "discover"
+                    and not parsed_args.config
+                    and not self.source.check_config_during_discover
+                ):
+                    # Connector supports unprivileged discover
+                    empty_config: dict[str, Any] = {}
+                    yield from (
+                        self.airbyte_message_to_string(queued_message)
+                        for queued_message in self._emit_queued_messages(self.source)
+                    )
+                    yield from map(
+                        AirbyteEntrypoint.airbyte_message_to_string,
+                        self.discover(source_spec, empty_config),
+                    )
+                elif parsed_args.config is None:
+                    # Raise a helpful error message if we reach here with no config.
+                    raise ValueError("The '--config' arg is required but was not provided.")
                 else:
                     raw_config = self.source.read_config(parsed_args.config)
                     config = self.source.configure(raw_config, temp_dir)
 
-                    yield from [
+                    yield from (
                         self.airbyte_message_to_string(queued_message)
                         for queued_message in self._emit_queued_messages(self.source)
-                    ]
+                    )
                     if cmd == "check":
                         yield from map(
                             AirbyteEntrypoint.airbyte_message_to_string,
@@ -261,7 +279,7 @@ def discover(
         self, source_spec: ConnectorSpecification, config: TConfig
     ) -> Iterable[AirbyteMessage]:
         self.set_up_secret_filter(config, source_spec.connectionSpecification)
-        if self.source.check_config_against_spec:
+        if self.source.check_config_during_discover:
             self.validate_connection(source_spec, config)
         catalog = self.source.discover(self.logger, config)
 

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -58,6 +58,7 @@
 )
 from airbyte_cdk.sources.declarative.resolvers import COMPONENTS_RESOLVER_TYPE_MAPPING
 from airbyte_cdk.sources.message import MessageRepository
+from airbyte_cdk.sources.source import Source
 from airbyte_cdk.sources.streams.core import Stream
 from airbyte_cdk.sources.types import ConnectionDefinition
 from airbyte_cdk.sources.utils.slice_logger import (
@@ -89,6 +90,9 @@ def _get_declarative_component_schema() -> Dict[str, Any]:
 class ManifestDeclarativeSource(DeclarativeSource):
     """Declarative source defined by a manifest of low-code components that define source connector behavior"""
 
+    check_config_during_discover: bool = False
+    """Declarative sources default to not checking config before discovery."""
+
     def __init__(
         self,
         source_config: ConnectionDefinition,
@@ -139,6 +143,8 @@ def __init__(
         # apply additional post-processing to the manifest
         self._post_process_manifest()
 
+        self.check_config_during_discover = self._uses_dynamic_schema_loader()
+
     @property
     def resolved_manifest(self) -> Mapping[str, Any]:
         """
@@ -542,3 +548,35 @@ def _dynamic_stream_configs(
 
     def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None:
         self.logger.debug("declarative source created from manifest", extra=extra_args)
+
+    def _uses_dynamic_schema_loader(self) -> bool:
+        """
+        Determines if any stream in the source uses a DynamicSchemaLoader.
+
+        DynamicSchemaLoader makes a separate call to retrieve schema information,
+        which might not require authentication, so we can skip config validation
+        during discovery when it's used.
+
+        Returns:
+            bool: True if any stream uses a DynamicSchemaLoader, False otherwise.
+        """
+        for stream_config in self._stream_configs(self._source_config):
+            schema_loader = stream_config.get("schema_loader", {})
+            if (
+                isinstance(schema_loader, dict)
+                and schema_loader.get("type") == "DynamicSchemaLoader"
+            ):
+                return True
+
+        dynamic_streams = self._source_config.get("dynamic_streams", [])
+        if dynamic_streams:
+            for dynamic_stream in dynamic_streams:
+                stream_template = dynamic_stream.get("stream_template", {})
+                schema_loader = stream_template.get("schema_loader", {})
+                if (
+                    isinstance(schema_loader, dict)
+                    and schema_loader.get("type") == "DynamicSchemaLoader"
+                ):
+                    return True
+
+        return False

airbyte_cdk/test/standard_tests/_job_runner.py

@@ -148,15 +148,9 @@ def run_test_job(
 
         return result
 
-    # For all other verbs, we assert check that an exception is raised (or not).
-    if test_scenario.expect_exception:
-        if not result.errors:
-            raise AssertionError("Expected exception but got none.")
-
-        return result
-
-    assert not result.errors, (
-        f"Expected no errors but got {len(result.errors)}: \n" + _errors_to_str(result)
-    )
+    if not test_scenario.expect_exception:
+        assert not result.errors, (
+            f"Expected no errors but got {len(result.errors)}: \n" + _errors_to_str(result)
+        )
 
     return result

airbyte_cdk/test/standard_tests/connector_base.py

@@ -13,12 +13,10 @@
 from typing import cast
 
 import yaml
+from airbyte_protocol_dataclasses.models.airbyte_protocol import AirbyteConnectionStatus
 from boltons.typeutils import classproperty
 
-from airbyte_cdk.models import (
-    AirbyteMessage,
-    Type,
-)
+from airbyte_cdk.models import Status
 from airbyte_cdk.test import entrypoint_wrapper
 from airbyte_cdk.test.standard_tests._job_runner import IConnector, run_test_job
 from airbyte_cdk.test.standard_tests.models import (
@@ -117,12 +115,22 @@ def test_check(
             "check",
             test_scenario=scenario,
         )
-        conn_status_messages: list[AirbyteMessage] = [
-            msg for msg in result._messages if msg.type == Type.CONNECTION_STATUS
-        ]  # noqa: SLF001  # Non-public API
-        assert len(conn_status_messages) == 1, (
-            f"Expected exactly one CONNECTION_STATUS message. Got: {result._messages}"
+        assert len(result.connection_status_messages) == 1, (
+            f"Expected exactly one CONNECTION_STATUS message. Got {len(result.connection_status_messages)}: \n"
+            + "\n".join([str(m) for m in result._messages])
+            + "\nErrors: "
+            + str(result.errors)
+            or "None"
+        )
+        conn_status = cast(
+            AirbyteConnectionStatus, result.connection_status_messages[0].connectionStatus
         )
+        if (
+            scenario.expect_exception
+            and conn_status.status == Status.SUCCEEDED
+            and not result.errors
+        ):
+            raise AssertionError(f"Expected error in `check` but got success.")
 
     @classmethod
     def get_connector_root_dir(cls) -> Path:

airbyte_cdk/test/standard_tests/source_base.py

@@ -3,6 +3,8 @@
 
 from dataclasses import asdict
 
+import pytest
+
 from airbyte_cdk.models import (
     AirbyteMessage,
     AirbyteStream,
@@ -58,6 +60,10 @@ def test_discover(
         scenario: ConnectorTestScenario,
     ) -> None:
         """Standard test for `discover`."""
+        if scenario.expect_exception:
+            pytest.skip(
+                "Skipping `discover` test because the scenario is expected to raise an exception."
+            )
         run_test_job(
             self.create_connector(scenario),
             "discover",
@@ -99,13 +105,22 @@ def test_basic_read(
         obtain the catalog of streams, and then it runs a `read` job to fetch
         records from those streams.
         """
+        check_result: entrypoint_wrapper.EntrypointOutput = run_test_job(
+            self.create_connector(scenario),
+            "check",
+            test_scenario=scenario,
+        )
+        if scenario.expect_exception and check_result.errors:
+            # Expected failure and we got it. Return early.
+            return
+
         discover_result = run_test_job(
             self.create_connector(scenario),
             "discover",
             test_scenario=scenario,
         )
-        if scenario.expect_exception:
-            assert discover_result.errors, "Expected exception but got none."
+        if scenario.expect_exception and check_result.errors:
+            # Expected failure and we got it. Return early.
             return
 
         configured_catalog = ConfiguredAirbyteCatalog(
@@ -124,6 +139,9 @@ def test_basic_read(
             test_scenario=scenario,
             catalog=configured_catalog,
         )
+        if scenario.expect_exception and not result.errors:
+            # By now we should have raised an exception.
+            raise AssertionError("Expected an error but got none.")
 
         if not result.records:
             raise AssertionError("Expected records but got none.")  # noqa: TRY003
@@ -133,6 +151,11 @@ def test_fail_read_with_bad_catalog(
         scenario: ConnectorTestScenario,
     ) -> None:
         """Standard test for `read` when passed a bad catalog file."""
+        # Recreate the scenario with the same config but set the status to "failed".
+        scenario = ConnectorTestScenario(
+            config_dict=scenario.get_config_dict(empty_if_missing=False),
+            status="failed",
+        )
         invalid_configured_catalog = ConfiguredAirbyteCatalog(
             streams=[
                 # Create ConfiguredAirbyteStream which is deliberately invalid
@@ -153,7 +176,6 @@ def test_fail_read_with_bad_catalog(
             ]
         )
         # Set expected status to "failed" to ensure the test fails if the connector.
-        scenario.status = "failed"
         result: entrypoint_wrapper.EntrypointOutput = run_test_job(
             self.create_connector(scenario),
             "read",