Add mz_cluster_blue_green_lineage

We need this view+index to determine the logical identity of clusters between blue green deployments in the Console.

Author: SangJunBak

doc/user/content/sql/system-catalog/mz_internal.md

@@ -131,6 +131,17 @@ the most recent status for each AWS PrivateLink connection in the system.
 | `last_status_change_at` | [`timestamp with time zone`] | Wall-clock timestamp of the connection status change.|
 | `status` | [`text`] | | The status of the connection: one of `pending-service-discovery`, `creating-endpoint`, `recreating-endpoint`, `updating-endpoint`, `available`, `deleted`, `deleting`, `expired`, `failed`, `pending`, `pending-acceptance`, `rejected`, or `unknown`. |
 
+## `mz_cluster_blue_green_lineage`
+
+The `mz_cluster_blue_green_lineage` table shows the blue/green deployment lineage of all clusters in [`mz_clusters`](../mz_catalog/#mz_clusters). It determines all cluster IDs that are logically the same cluster.
+
+<!-- RELATION_SPEC mz_internal.mz_cluster_blue_green_lineage -->
+| Field                               | Type         | Meaning                                                        |
+|-------------------------------------|--------------|----------------------------------------------------------------|
+| `cluster_id`                        | [`text`]     | The ID of the cluster. Corresponds to [`mz_clusters.id`](../mz_catalog/#mz_clusters). |
+| `current_deployment_cluster_id`                              | [`text`]     | The cluster ID of the last cluster in `cluster_id`'s blue/green lineage. |
+| `cluster_name`   | [`text`] | The name of the cluster |
+
 ## `mz_cluster_schedules`
 
 The `mz_cluster_schedules` table shows the `SCHEDULE` option specified for each cluster.

src/catalog/src/builtin.rs

@@ -8248,6 +8248,116 @@ FROM max_memory
     }
 });
 
+/**
+ * Traces the blue/green deployment lineage in the audit log to determine all cluster
+ * IDs that are logically the same cluster.
+ * cluster_id: The ID of a cluster.
+ * current_deployment_cluster_id: The cluster ID of the last cluster in
+ *   cluster_id's blue/green lineage.
+ * cluster_name: The name of the cluster.
+ * The approach taken is as follows. First, find all extant clusters and add them
+ * to the result set. Per cluster, we do the following:
+ * 1. Find the most recent create or rename event. This moment represents when the
+ *    cluster took on its final logical identity.
+ * 2. Look for a cluster that had the same name (or the same name with `_dbt_deploy`
+ *    appended) that was dropped within one minute of that moment. That cluster is
+ *    almost certainly the logical predecessor of the current cluster. Add the cluster
+ *    to the result set.
+ * 3. Repeat the procedure until a cluster with no logical predecessor is discovered.
+ * Limiting the search for a dropped cluster to a window of one minute is a heuristic,
+ * but one that's likely to be pretty good one. If a name is reused after more
+ * than one minute, that's a good sign that it wasn't an automatic blue/green
+ * process, but someone turning on a new use case that happens to have the same
+ * name as a previous but logically distinct use case.
+ */
+pub static MZ_CLUSTER_BLUE_GREEN_LINEAGE: LazyLock<BuiltinView> = LazyLock::new(|| BuiltinView {
+    name: "mz_cluster_blue_green_lineage",
+    schema: MZ_INTERNAL_SCHEMA,
+    oid: oid::VIEW_MZ_CLUSTER_BLUE_GREEN_LINEAGE_OID,
+    column_defs: Some(r#"cluster_id, current_deployment_cluster_id, cluster_name"#),
+    sql: r#"WITH MUTUALLY RECURSIVE cluster_events (
+  cluster_id text,
+  cluster_name text,
+  event_type text,
+  occurred_at timestamptz
+) AS (
+  SELECT coalesce(details->>'id', details->>'cluster_id') AS cluster_id,
+    coalesce(details->>'name', details->>'new_name') AS cluster_name,
+    event_type,
+    occurred_at
+  FROM mz_audit_events
+  WHERE (
+      event_type IN ('create', 'drop')
+      OR (
+        event_type = 'alter'
+        AND details ? 'new_name'
+      )
+    )
+    AND object_type = 'cluster'
+    AND mz_now() < occurred_at + INTERVAL '30 days'
+),
+mz_cluster_blue_green_lineage (
+  cluster_id text,
+  current_deployment_cluster_id text,
+  cluster_name text
+) AS (
+  SELECT c.id,
+    c.id,
+    c.name
+  FROM mz_clusters c
+  WHERE c.id LIKE 'u%'
+  UNION
+  SELECT *
+  FROM dropped_clusters
+),
+-- Closest create or rename event based on the current clusters in the result set
+most_recent_create_or_rename (
+  cluster_id text,
+  current_deployment_cluster_id text,
+  cluster_name text,
+  occurred_at timestamptz
+) AS (
+  SELECT DISTINCT ON (e.cluster_id) e.cluster_id,
+    c.current_deployment_cluster_id,
+    e.cluster_name,
+    e.occurred_at
+  FROM mz_cluster_blue_green_lineage c
+    JOIN cluster_events e ON c.cluster_id = e.cluster_id
+    AND c.cluster_name = e.cluster_name
+  WHERE e.event_type <> 'drop'
+  ORDER BY e.cluster_id,
+    e.occurred_at DESC
+),
+-- Clusters that were dropped most recently within 1 minute of most_recent_create_or_rename
+dropped_clusters (
+  cluster_id text,
+  current_deployment_cluster_id text,
+  cluster_name text
+) AS (
+  SELECT DISTINCT ON (cr.cluster_id) e.cluster_id,
+    cr.current_deployment_cluster_id,
+    cr.cluster_name
+  FROM most_recent_create_or_rename cr
+    JOIN cluster_events e ON e.occurred_at BETWEEN cr.occurred_at - interval '1 minute'
+    AND cr.occurred_at + interval '1 minute'
+    AND (
+      e.cluster_name = cr.cluster_name
+      OR e.cluster_name = cr.cluster_name || '_dbt_deploy'
+    )
+  WHERE e.event_type = 'drop'
+  ORDER BY cr.cluster_id,
+    abs(
+      extract(
+        epoch
+        FROM cr.occurred_at - e.occurred_at
+      )
+    )
+)
+SELECT *
+FROM mz_cluster_blue_green_lineage"#,
+    access: vec![PUBLIC_SELECT],
+});
+
 pub const MZ_SHOW_DATABASES_IND: BuiltinIndex = BuiltinIndex {
     name: "mz_show_databases_ind",
     schema: MZ_INTERNAL_SCHEMA,
@@ -8482,6 +8592,15 @@ ON mz_internal.mz_console_cluster_utilization_overview (cluster_id)",
     is_retained_metrics_object: false,
 };
 
+pub const MZ_CLUSTER_BLUE_GREEN_LINEAGE_IND: BuiltinIndex = BuiltinIndex {
+    name: "mz_cluster_blue_green_lineage_ind",
+    schema: MZ_INTERNAL_SCHEMA,
+    oid: oid::INDEX_MZ_CLUSTER_BLUE_GREEN_LINEAGE_IND_OID,
+    sql: "IN CLUSTER mz_catalog_server
+ON mz_internal.mz_cluster_blue_green_lineage (cluster_id)",
+    is_retained_metrics_object: false,
+};
+
 pub const MZ_CLUSTERS_IND: BuiltinIndex = BuiltinIndex {
     name: "mz_clusters_ind",
     schema: MZ_CATALOG_SCHEMA,
@@ -9420,6 +9539,7 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne
         Builtin::View(&MZ_HYDRATION_STATUSES),
         Builtin::View(&MZ_SHOW_CLUSTER_REPLICAS),
         Builtin::View(&MZ_SHOW_NETWORK_POLICIES),
+        Builtin::View(&MZ_CLUSTER_BLUE_GREEN_LINEAGE),
         Builtin::Index(&MZ_SHOW_DATABASES_IND),
         Builtin::Index(&MZ_SHOW_SCHEMAS_IND),
         Builtin::Index(&MZ_SHOW_CONNECTIONS_IND),
@@ -9475,6 +9595,7 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne
         Builtin::Index(&MZ_SECRETS_IND),
         Builtin::Index(&MZ_VIEWS_IND),
         Builtin::Index(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_IND),
+        Builtin::Index(&MZ_CLUSTER_BLUE_GREEN_LINEAGE_IND),
         Builtin::View(&MZ_RECENT_STORAGE_USAGE),
         Builtin::Index(&MZ_RECENT_STORAGE_USAGE_IND),
         Builtin::Connection(&MZ_ANALYTICS),

src/environmentd/tests/testdata/http/ws

@@ -768,3 +768,5 @@ pub const LOG_MZ_COMPUTE_DATAFLOW_GLOBAL_IDS_PER_WORKER_OID: u32 = 17045;
 pub const VIEW_MZ_LIR_MAPPING_OID: u32 = 17046;
 pub const VIEW_MZ_DATAFLOW_GLOBAL_IDS_OID: u32 = 17047;
 pub const NETWORK_POLICIES_DEFAULT_POLICY_OID: u32 = 17048;
+pub const VIEW_MZ_CLUSTER_BLUE_GREEN_LINEAGE_OID: u32 = 17049;
+pub const INDEX_MZ_CLUSTER_BLUE_GREEN_LINEAGE_IND_OID: u32 = 17050;

src/pgrepr-consts/src/oid.rs

@@ -425,15 +425,15 @@ CREATE CLUSTER test REPLICAS (foo (SIZE '1'));
 query I
 SELECT COUNT(name) FROM mz_indexes;
 ----
-285
+286
 
 statement ok
 DROP CLUSTER test CASCADE
 
 query T
 SELECT COUNT(name) FROM mz_indexes;
 ----
-254
+255
 
 simple conn=mz_system,user=mz_system
 ALTER CLUSTER quickstart OWNER TO materialize

test/sqllogictest/cluster.slt

@@ -281,6 +281,10 @@ mz_aws_privatelink_connection_statuses
 VIEW
 materialize
 mz_internal
+mz_cluster_blue_green_lineage
+VIEW
+materialize
+mz_internal
 mz_cluster_replica_history
 VIEW
 materialize

test/sqllogictest/information_schema_tables.slt

@@ -37,6 +37,7 @@ mz_arrangement_heap_capacity_raw_s2_primary_idx  CREATE␠INDEX␠"mz_arrangemen
 mz_arrangement_heap_size_raw_s2_primary_idx  CREATE␠INDEX␠"mz_arrangement_heap_size_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_heap_size_raw"␠("operator_id",␠"worker_id")
 mz_arrangement_records_raw_s2_primary_idx  CREATE␠INDEX␠"mz_arrangement_records_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_records_raw"␠("operator_id",␠"worker_id")
 mz_arrangement_sharing_raw_s2_primary_idx  CREATE␠INDEX␠"mz_arrangement_sharing_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_sharing_raw"␠("operator_id",␠"worker_id")
+mz_cluster_blue_green_lineage_ind  CREATE␠INDEX␠"mz_cluster_blue_green_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s717␠AS␠"mz_internal"."mz_cluster_blue_green_lineage"]␠("cluster_id")
 mz_cluster_replica_history_ind  CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s579␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at")
 mz_cluster_replica_metrics_history_ind  CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s492␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id")
 mz_cluster_replica_metrics_ind  CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s491␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id")
@@ -74,7 +75,7 @@ mz_message_batch_counts_received_raw_s2_primary_idx  CREATE␠INDEX␠"mz_messag
 mz_message_batch_counts_sent_raw_s2_primary_idx  CREATE␠INDEX␠"mz_message_batch_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id")
 mz_message_counts_received_raw_s2_primary_idx  CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id")
 mz_message_counts_sent_raw_s2_primary_idx  CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id")
-mz_notices_ind  CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s780␠AS␠"mz_internal"."mz_notices"]␠("id")
+mz_notices_ind  CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s782␠AS␠"mz_internal"."mz_notices"]␠("id")
 mz_object_dependencies_ind  CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s454␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id")
 mz_object_history_ind  CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s518␠AS␠"mz_internal"."mz_object_history"]␠("id")
 mz_object_lifetimes_ind  CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id")
@@ -83,7 +84,7 @@ mz_objects_ind  CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[
 mz_peek_durations_histogram_raw_s2_primary_idx  CREATE␠INDEX␠"mz_peek_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_peek_durations_histogram_raw"␠("worker_id",␠"type",␠"duration_ns")
 mz_recent_activity_log_thinned_ind  CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s678␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash")
 mz_recent_sql_text_ind  CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s674␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash")
-mz_recent_storage_usage_ind  CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s772␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id")
+mz_recent_storage_usage_ind  CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s774␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id")
 mz_roles_ind  CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s477␠AS␠"mz_catalog"."mz_roles"]␠("id")
 mz_scheduling_elapsed_raw_s2_primary_idx  CREATE␠INDEX␠"mz_scheduling_elapsed_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_elapsed_raw"␠("id",␠"worker_id")
 mz_scheduling_parks_histogram_raw_s2_primary_idx  CREATE␠INDEX␠"mz_scheduling_parks_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_parks_histogram_raw"␠("worker_id",␠"slept_for_ns",␠"requested_ns")
@@ -203,6 +204,9 @@ mz_audit_events  object_type
 mz_audit_events  occurred_at
 mz_audit_events  user
 mz_base_types  id
+mz_cluster_blue_green_lineage  cluster_id
+mz_cluster_blue_green_lineage  cluster_name
+mz_cluster_blue_green_lineage  current_deployment_cluster_id
 mz_cluster_replica_frontiers  object_id
 mz_cluster_replica_frontiers  replica_id
 mz_cluster_replica_frontiers  write_frontier

test/sqllogictest/mz_catalog_server_index_accounting.slt

@@ -1153,3 +1153,5 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid
 17045  mz_compute_dataflow_global_ids_per_worker
 17046  mz_lir_mapping
 17047  mz_dataflow_global_ids
+17049  mz_cluster_blue_green_lineage
+17050  mz_cluster_blue_green_lineage_ind