Capture collection errors (e.g. import failures) in JSON report

When a test file fails to import, pytest fires pytest_collectreport
instead of the pytest_runtest_* hooks. The plugin had no handler for
this, so collection errors were silently dropped from the JSON report.

With pytest-xdist and bktec retries, this caused builds to report green
even when a test file failed to collect.

Add pytest_collectreport to BuildkitePlugin to capture collection
errors as failed TestData entries in the payload.

Fixes #106

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Jason Barnett

src/buildkite_test_collector/pytest_plugin/buildkite_plugin.py

@@ -48,6 +48,48 @@ def pytest_collection_modifyitems(self, config, items):
         config.hook.pytest_deselected(items=unfiltered_items)
         items[:] = filtered_items
 
+    def pytest_collectreport(self, report):
+        """Capture collection errors (e.g. import failures) as failed tests.
+
+        When a test file fails to import, pytest fires this hook instead of
+        the pytest_runtest_* hooks.  Without handling it, collection errors
+        are silently dropped from the JSON report.
+        """
+        if not report.failed:
+            return
+
+        if not report.nodeid:
+            return
+
+        logger.debug('hook=pytest_collectreport nodeid=%s outcome=%s', report.nodeid, report.outcome)
+
+        if not self.payload.is_started():
+            self.payload = self.payload.started()
+
+        chunks = report.nodeid.split("::")
+        scope = "::".join(chunks[:-1])
+        name = chunks[-1]
+        file_name = chunks[0]
+
+        test_data = TestData.start(
+            uuid4(),
+            scope=scope,
+            name=name,
+            file_name=file_name,
+            location=file_name,
+        )
+
+        failure_reason, failure_expanded = failure_reasons(longrepr=report.longrepr)
+        logger.debug('-> collection error: %s', failure_reason)
+
+        test_data = test_data.failed(
+            failure_reason=failure_reason,
+            failure_expanded=failure_expanded,
+        )
+        test_data = test_data.finish()
+
+        self.payload = self.payload.push_test_data(test_data)
+
     def pytest_runtest_logstart(self, nodeid, location):
         """pytest_runtest_logstart hook callback"""
         logger.debug('hook=pytest_runtest_logstart nodeid=%s', nodeid)

tests/buildkite_test_collector/data/test_sample_import_error.py

@@ -0,0 +1,13 @@
+"""Sample test file with an import error.
+
+This file deliberately imports a non-existent module to trigger a collection
+error in pytest — the same scenario as importing a removed/renamed symbol
+from a real package.
+"""
+
+from nonexistent_module import does_not_exist
+
+
+def test_should_be_reported_as_failed():
+    """This test can never run, but should still appear as a failure in the report."""
+    assert does_not_exist() == 42

tests/buildkite_test_collector/pytest_plugin/test_plugin.py

@@ -5,7 +5,7 @@
 from buildkite_test_collector.pytest_plugin import BuildkitePlugin
 
 from _pytest._code.code import ExceptionInfo
-from _pytest.reports import TestReport
+from _pytest.reports import CollectReport, TestReport
 
 def test_runtest_logstart_with_unstarted_payload(fake_env):
     payload = Payload.init(fake_env)
@@ -346,3 +346,110 @@ def test_save_json_payload_concurrent_merge(fake_env, tmp_path, successful_test)
 
     # All writers must have merged their entry
     assert len(result) == num_writers
+
+
+# ---------------------------------------------------------------------------
+# pytest_collectreport tests
+# ---------------------------------------------------------------------------
+
+def test_pytest_collectreport_import_error(fake_env):
+    """Collection errors are captured as failed tests."""
+    payload = Payload.init(fake_env)
+    plugin = BuildkitePlugin(payload)
+
+    report = CollectReport(
+        nodeid="tests/foo.py",
+        outcome="failed",
+        longrepr="ModuleNotFoundError: No module named 'bar'",
+        result=None,
+    )
+
+    plugin.pytest_collectreport(report)
+
+    assert plugin.payload.is_started()
+    assert len(plugin.in_flight) == 0
+    assert len(plugin.payload.data) == 1
+
+    test_data = plugin.payload.data[0]
+    assert test_data.name == "tests/foo.py"
+    assert test_data.file_name == "tests/foo.py"
+    assert test_data.scope == ""
+    assert isinstance(test_data.result, TestResultFailed)
+    assert "ModuleNotFoundError" in test_data.result.failure_reason
+    assert test_data.is_finished()
+
+
+def test_pytest_collectreport_passed_ignored(fake_env):
+    """Successful collection reports are ignored."""
+    payload = Payload.init(fake_env)
+    plugin = BuildkitePlugin(payload)
+
+    report = CollectReport(
+        nodeid="tests/foo.py",
+        outcome="passed",
+        longrepr=None,
+        result=[],
+    )
+
+    plugin.pytest_collectreport(report)
+
+    assert not plugin.payload.is_started()
+    assert len(plugin.payload.data) == 0
+
+
+def test_pytest_collectreport_empty_nodeid_ignored(fake_env):
+    """The root session collect report (empty nodeid) is ignored."""
+    payload = Payload.init(fake_env)
+    plugin = BuildkitePlugin(payload)
+
+    report = CollectReport(
+        nodeid="",
+        outcome="failed",
+        longrepr="some error",
+        result=None,
+    )
+
+    plugin.pytest_collectreport(report)
+
+    assert len(plugin.payload.data) == 0
+
+
+def test_pytest_collectreport_does_not_restart_payload(fake_env):
+    """If the payload is already started, pytest_collectreport does not restart it."""
+    payload = Payload.init(fake_env)
+    plugin = BuildkitePlugin(payload)
+
+    plugin.payload = plugin.payload.started()
+    original_started_at = plugin.payload.started_at
+
+    report = CollectReport(
+        nodeid="tests/foo.py",
+        outcome="failed",
+        longrepr="ModuleNotFoundError: No module named 'bar'",
+        result=None,
+    )
+
+    plugin.pytest_collectreport(report)
+
+    assert plugin.payload.started_at == original_started_at
+    assert len(plugin.payload.data) == 1
+
+
+def test_pytest_collectreport_tuple_longrepr(fake_env):
+    """Tuple longrepr (path, lineno, message) is handled correctly."""
+    payload = Payload.init(fake_env)
+    plugin = BuildkitePlugin(payload)
+
+    report = CollectReport(
+        nodeid="tests/foo.py",
+        outcome="failed",
+        longrepr=("tests/foo.py", 8, "ModuleNotFoundError: No module named 'bar'"),
+        result=None,
+    )
+
+    plugin.pytest_collectreport(report)
+
+    assert len(plugin.payload.data) == 1
+    test_data = plugin.payload.data[0]
+    assert isinstance(test_data.result, TestResultFailed)
+    assert "ModuleNotFoundError" in test_data.result.failure_reason

tests/buildkite_test_collector/test_integration_import_error.py

@@ -0,0 +1,88 @@
+"""Integration test: collection errors (import failures) are captured in the JSON report.
+
+When a test file has an import error, pytest fails during collection. The
+``pytest_collectreport`` hook on ``BuildkitePlugin`` captures the error and
+adds it to the JSON report as a failed test entry.
+
+See: https://github.com/buildkite/test-collector-python/issues/106
+"""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+
+BROKEN_FILE = Path(__file__).parent / "data" / "test_sample_import_error.py"
+PASSING_FILE = Path(__file__).parent / "data" / "test_sample_skip.py"
+
+
+class TestImportErrorReporting:
+    """Verify that import errors are captured in the JSON report."""
+
+    def _run_pytest(self, tmp_path, test_files, *extra_args):
+        json_output = tmp_path / "results.json"
+        cmd = [
+            sys.executable,
+            "-m",
+            "pytest",
+            *[str(f) for f in test_files],
+            f"--json={json_output}",
+            "-v",
+            *extra_args,
+        ]
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        return result, json_output
+
+    def test_pytest_exits_nonzero_on_import_error(self, tmp_path):
+        """Pytest itself correctly detects the import error and exits non-zero."""
+        result, _ = self._run_pytest(tmp_path, [BROKEN_FILE])
+
+        assert result.returncode != 0, (
+            "pytest should exit non-zero when a test file has an import error"
+        )
+        assert "ModuleNotFoundError" in result.stdout, (
+            "pytest output should mention the import error"
+        )
+
+    def test_import_error_captured_in_json_report(self, tmp_path):
+        """The JSON report captures the collection error as a failed test."""
+        result, json_output = self._run_pytest(tmp_path, [BROKEN_FILE])
+
+        assert result.returncode != 0
+
+        assert json_output.exists(), f"JSON not created. stderr:\n{result.stderr}"
+        data = json.loads(json_output.read_text())
+
+        assert len(data) == 1, (
+            f"Expected 1 entry for the collection error, got {len(data)}: "
+            f"{[t.get('name') for t in data]}"
+        )
+
+        entry = data[0]
+        assert entry["result"] == "failed"
+        assert "ImportError" in (entry.get("failure_reason") or "")
+
+    def test_import_error_reported_alongside_passing_tests(self, tmp_path):
+        """With --continue-on-collection-errors, both passing tests and the
+        collection error appear in the report."""
+        result, json_output = self._run_pytest(
+            tmp_path,
+            [BROKEN_FILE, PASSING_FILE],
+            "--continue-on-collection-errors",
+        )
+
+        assert result.returncode != 0, "pytest should still exit non-zero"
+
+        data = json.loads(json_output.read_text())
+        names = [t["name"] for t in data]
+
+        assert len(data) == 6, (
+            f"Expected 6 entries (5 passing + 1 collection error), got {len(data)}: {names}"
+        )
+
+        assert "test_passing" in names
+
+        failed = [t for t in data if t["result"] == "failed"]
+        assert len(failed) == 1
+        assert "ImportError" in (failed[0].get("failure_reason") or "")