ci: Remember failed workflows/td files

to be able to run them first in the next run on a PR

Author: def-

misc/python/materialize/mzcompose/composition.py

@@ -43,7 +43,7 @@
 import yaml
 from psycopg import Connection, Cursor
 
-from materialize import MZ_ROOT, mzbuild, spawn, ui
+from materialize import MZ_ROOT, buildkite, mzbuild, spawn, ui
 from materialize.mzcompose import cluster_replica_size_map, loader
 from materialize.mzcompose.service import Service
 from materialize.mzcompose.services.materialized import (
@@ -1564,3 +1564,30 @@ def cloud_hostname(self, quiet: bool = False) -> str:
         # It is necessary to append the 'https://' protocol; otherwise, urllib can't parse it correctly.
         cloud_hostname = urllib.parse.urlparse("https://" + cloud_url).hostname
         return str(cloud_hostname)
+
+    @contextmanager
+    def test_parts(self, parts: list) -> Iterator[str]:
+        from materialize.test_analytics.config.test_analytics_db_config import (
+            create_test_analytics_config,
+        )
+        from materialize.test_analytics.test_analytics_db import TestAnalyticsDb
+
+        priority: dict[str, int] = {}
+        test_analytics: TestAnalyticsDb | None = None
+
+        if buildkite.is_in_buildkite():
+            test_analytics_config = create_test_analytics_config(self)
+            test_analytics = TestAnalyticsDb(test_analytics_config)
+            priority = test_analytics.builds.get_part_priorities()
+
+        sorted_parts = sorted(
+            parts, key=lambda part: priority.get(str(part), 0), reverse=True
+        )
+        for part in sorted_parts:
+            try:
+                yield part
+            except:
+                if buildkite.is_in_buildkite():
+                    assert test_analytics
+                    test_analytics.builds.add_build_job_failure(part)
+                raise

misc/python/materialize/test_analytics/data/build/build_data_storage.py

@@ -173,3 +173,38 @@ def update_build_job_success(
         )
 
         self.database_connector.add_update_statements(sql_statements)
+
+    def add_build_job_failure(
+        self,
+        part: str,
+    ) -> None:
+        job_id = buildkite.get_var(BuildkiteEnvVar.BUILDKITE_JOB_ID)
+
+        sql_statements = []
+        sql_statements.append(
+            f"""
+            INSERT INTO build_job_failure
+            (
+                build_job_id,
+                part
+            )
+            VALUES
+            (
+                {as_sanitized_literal(job_id)},
+                {as_sanitized_literal(part)}
+            )
+            """
+        )
+
+        self.database_connector.add_update_statements(sql_statements)
+
+    def get_part_priorities(self) -> dict[str, int]:
+        # TODO: Join with the relevant ids
+        with self.database_connector.create_cursor() as cur:
+            # 2 for failures in this PR
+            # 1 for failed recently in CI
+            # 0 or nothing for rest
+            cur.execute(f"SELECT part, 2 FROM build_job_failure")
+            results = cur.fetchall()
+            print(results)
+            return {part: prio for part, prio in results}

misc/python/materialize/test_analytics/setup/tables/03-build-job-failure.sql

@@ -0,0 +1,17 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- part of a build job that failed, can be a testdrive file or a workflow
+CREATE TABLE build_job_failure (
+    build_job_id TEXT NOT NULL,
+    part TEXT NOT NULL
+);
+
+ALTER TABLE build_job_failure OWNER TO qa;
+GRANT SELECT, INSERT, UPDATE ON TABLE build_job_failure TO "hetzner-ci";

test/testdrive/mzcompose.py

@@ -214,7 +214,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
             print(f"Passing through arguments to testdrive {passthrough_args}\n")
             # do not set default args, they should be set in the td file using set-arg-default to easen the execution
             # without mzcompose
-            for file in args.files:
+            with c.test_parts(args.files) as file:
                 c.run_testdrive_files(
                     (
                         "--rewrite-results"