diff --git a/misc/python/materialize/cli/run.py b/misc/python/materialize/cli/run.py index 472dcfd7ed10b..d50ff84161a77 100644 --- a/misc/python/materialize/cli/run.py +++ b/misc/python/materialize/cli/run.py @@ -275,9 +275,10 @@ def main() -> int: if args.monitoring: command += ["--opentelemetry-endpoint=http://localhost:4317"] elif args.program == "sqllogictest": - params = get_default_system_parameters() - params["enable_columnation_lgalloc"] = "false" - formatted_params = [f"{key}={value}" for key, value in params.items()] + formatted_params = [ + f"{key}={value}" + for key, value in get_default_system_parameters().items() + ] system_parameter_default = ";".join(formatted_params) # Connect to the database to ensure it exists. _connect_sql(args.postgres) diff --git a/misc/python/materialize/mzcompose/services/sql_logic_test.py b/misc/python/materialize/mzcompose/services/sql_logic_test.py index 2dc370c29e785..e10c729c1ac73 100644 --- a/misc/python/materialize/mzcompose/services/sql_logic_test.py +++ b/misc/python/materialize/mzcompose/services/sql_logic_test.py @@ -28,12 +28,12 @@ def __init__( volumes: list[str] = ["../..:/workdir"], depends_on: list[str] = [METADATA_STORE], ) -> None: - params = get_default_system_parameters() - # Otherwise very noisy in SLT: lgalloc error: I/O error, falling back to heap - params["enable_columnation_lgalloc"] = "false" environment += [ "MZ_SYSTEM_PARAMETER_DEFAULT=" - + ";".join(f"{key}={value}" for key, value in params.items()) + + ";".join( + f"{key}={value}" + for key, value in get_default_system_parameters().items() + ) ] super().__init__( diff --git a/misc/python/materialize/parallel_benchmark/scenarios.py b/misc/python/materialize/parallel_benchmark/scenarios.py index 82a101f59eb7f..533ba6eb30cfa 100644 --- a/misc/python/materialize/parallel_benchmark/scenarios.py +++ b/misc/python/materialize/parallel_benchmark/scenarios.py @@ -746,7 +746,6 @@ def __init__(self, c: Composition, conn_infos: dict[str, PgConnInfo]): c, ), dist=Periodic(per_second=1), - report_regressions=False, # Don't care about this ), ClosedLoop( action=StandaloneQuery( diff --git a/src/catalog/tests/snapshots/debug__opened_trace.snap b/src/catalog/tests/snapshots/debug__opened_trace.snap index e3d7f0c1deb5e..378e0f3116da5 100644 --- a/src/catalog/tests/snapshots/debug__opened_trace.snap +++ b/src/catalog/tests/snapshots/debug__opened_trace.snap @@ -374,7 +374,7 @@ Trace { ), replica_name: "r1", logical_size: "1", - disk: true, + disk: false, billed_as: None, internal: false, reason: Some( @@ -558,7 +558,7 @@ Trace { ), }, ), - disk: true, + disk: false, optimizer_feature_overrides: [], schedule: Some( ClusterSchedule { @@ -628,7 +628,7 @@ Trace { ManagedLocation { size: "1", availability_zone: None, - disk: true, + disk: false, internal: false, billed_as: None, pending: false, diff --git a/src/catalog/tests/snapshots/open__initial_audit_log.snap b/src/catalog/tests/snapshots/open__initial_audit_log.snap index 66745556e338b..6a8a0cf500b89 100644 --- a/src/catalog/tests/snapshots/open__initial_audit_log.snap +++ b/src/catalog/tests/snapshots/open__initial_audit_log.snap @@ -1,7 +1,6 @@ --- source: src/catalog/tests/open.rs expression: audit_log -snapshot_kind: text --- [ V1( @@ -187,7 +186,7 @@ snapshot_kind: text ), replica_name: "r1", logical_size: "1", - disk: true, + disk: false, billed_as: None, internal: false, reason: System, diff --git a/src/catalog/tests/snapshots/open__initial_snapshot.snap b/src/catalog/tests/snapshots/open__initial_snapshot.snap index c435d8177d1e2..f75e57a75a5e5 100644 --- a/src/catalog/tests/snapshots/open__initial_snapshot.snap +++ b/src/catalog/tests/snapshots/open__initial_snapshot.snap @@ -1,7 +1,6 @@ --- source: src/catalog/tests/open.rs expression: test_snapshot -snapshot_kind: text --- Snapshot { databases: { @@ -1221,7 +1220,7 @@ Snapshot { ), }, ), - disk: true, + disk: false, optimizer_feature_overrides: [], schedule: Some( ClusterSchedule { @@ -1347,7 +1346,7 @@ Snapshot { ManagedLocation { size: "1", availability_zone: None, - disk: true, + disk: false, internal: false, billed_as: None, pending: false, diff --git a/test/cloudtest/test_compute.py b/test/cloudtest/test_compute.py index c72970552d44c..f488fa2270f76 100644 --- a/test/cloudtest/test_compute.py +++ b/test/cloudtest/test_compute.py @@ -133,21 +133,27 @@ def test_disk_label(mz: MaterializeApplication) -> None: user="mz_system", ) - mz.environmentd.sql("CREATE CLUSTER disk MANAGED, SIZE = '2-1', DISK = true") + for value in ("true", "false"): + mz.environmentd.sql( + f"CREATE CLUSTER disk_{value} MANAGED, SIZE = '2-1', DISK = {value}" + ) - (cluster_id, replica_id) = mz.environmentd.sql_query( - "SELECT mz_clusters.id, mz_cluster_replicas.id FROM mz_cluster_replicas JOIN mz_clusters ON mz_cluster_replicas.cluster_id = mz_clusters.id WHERE mz_clusters.name = 'disk'" - )[0] - assert cluster_id is not None - assert replica_id is not None + (cluster_id, replica_id) = mz.environmentd.sql_query( + f"SELECT mz_clusters.id, mz_cluster_replicas.id FROM mz_cluster_replicas JOIN mz_clusters ON mz_cluster_replicas.cluster_id = mz_clusters.id WHERE mz_clusters.name = 'disk_{value}'" + )[0] + assert cluster_id is not None + assert replica_id is not None - node_selectors = get_node_selector(mz, cluster_id, replica_id) - assert ( - node_selectors - == '\'{"materialize.cloud/disk":"true"} {"materialize.cloud/disk":"true"}\'' - ), node_selectors + node_selectors = get_node_selector(mz, cluster_id, replica_id) + if value == "true": + assert ( + node_selectors + == '\'{"materialize.cloud/disk":"true"} {"materialize.cloud/disk":"true"}\'' + ), node_selectors + else: + assert node_selectors == "''" - mz.environmentd.sql("DROP CLUSTER disk CASCADE") + mz.environmentd.sql(f"DROP CLUSTER disk_{value} CASCADE") # Reset mz.environmentd.sql( diff --git a/test/cloudtest/test_disk.py b/test/cloudtest/test_disk.py index dbfc9644166b4..d831f117122af 100644 --- a/test/cloudtest/test_disk.py +++ b/test/cloudtest/test_disk.py @@ -26,7 +26,7 @@ def test_disk_replica(mz: MaterializeApplication) -> None: > CREATE CLUSTER testdrive_no_reset_disk_cluster1 REPLICAS (r1 ( - SIZE '1' + SIZE '1', DISK = true )) > CREATE CONNECTION IF NOT EXISTS kafka TO KAFKA (BROKER '${testdrive.kafka-addr}', SECURITY PROTOCOL PLAINTEXT) @@ -83,7 +83,7 @@ def test_disk_replica(mz: MaterializeApplication) -> None: def test_always_use_disk_replica(mz: MaterializeApplication) -> None: - """Testing `cluster_always_use_disk = true` cluster replicas""" + """Testing `DISK = false, cluster_always_use_disk = true` cluster replicas""" mz.environmentd.sql( "ALTER SYSTEM SET cluster_always_use_disk = true", port="internal", @@ -93,43 +93,43 @@ def test_always_use_disk_replica(mz: MaterializeApplication) -> None: mz.testdrive.run( input=dedent( """ - $ kafka-create-topic topic=test + $ kafka-create-topic topic=test - $ kafka-ingest key-format=bytes format=bytes topic=test - key1:val1 - key2:val2 + $ kafka-ingest key-format=bytes format=bytes topic=test + key1:val1 + key2:val2 - > CREATE CLUSTER disk_cluster2 - REPLICAS (r1 (SIZE '1')) + > CREATE CLUSTER disk_cluster2 + REPLICAS (r1 (SIZE '1')) - > CREATE CONNECTION IF NOT EXISTS kafka TO KAFKA (BROKER '${testdrive.kafka-addr}', SECURITY PROTOCOL PLAINTEXT) + > CREATE CONNECTION IF NOT EXISTS kafka TO KAFKA (BROKER '${testdrive.kafka-addr}', SECURITY PROTOCOL PLAINTEXT) - > CREATE SOURCE source1 - IN CLUSTER disk_cluster2 - FROM KAFKA CONNECTION kafka - (TOPIC 'testdrive-test-${testdrive.seed}'); + > CREATE SOURCE source1 + IN CLUSTER disk_cluster2 + FROM KAFKA CONNECTION kafka + (TOPIC 'testdrive-test-${testdrive.seed}'); - > CREATE TABLE source1_tbl FROM SOURCE source1 (REFERENCE "testdrive-test-${testdrive.seed}") - KEY FORMAT TEXT - VALUE FORMAT TEXT - ENVELOPE UPSERT; + > CREATE TABLE source1_tbl FROM SOURCE source1 (REFERENCE "testdrive-test-${testdrive.seed}") + KEY FORMAT TEXT + VALUE FORMAT TEXT + ENVELOPE UPSERT; - > SELECT * FROM source1_tbl; - key text - ------------------ - key1 val1 - key2 val2 + > SELECT * FROM source1_tbl; + key text + ------------------ + key1 val1 + key2 val2 - $ kafka-ingest key-format=bytes format=bytes topic=test - key1:val3 + $ kafka-ingest key-format=bytes format=bytes topic=test + key1:val3 - > SELECT * FROM source1_tbl; - key text - ------------------ - key1 val3 - key2 val2 - """ + > SELECT * FROM source1_tbl; + key text + ------------------ + key1 val3 + key2 val2 + """ ) ) @@ -153,3 +153,54 @@ def test_always_use_disk_replica(mz: MaterializeApplication) -> None: "ls /scratch/storage/upsert", ) assert source_global_id in on_disk_sources + + +def test_no_disk_replica(mz: MaterializeApplication) -> None: + """Testing `DISK = false` cluster replicas""" + mz.testdrive.run( + input=dedent( + """ + $ kafka-create-topic topic=test-no-disk + + $ kafka-ingest key-format=bytes format=bytes topic=test-no-disk + key1:val1 + key2:val2 + + > CREATE CLUSTER no_disk_cluster1 + REPLICAS (r1 ( + SIZE '1', DISK = false + )) + + > CREATE CONNECTION IF NOT EXISTS kafka + TO KAFKA (BROKER '${testdrive.kafka-addr}', SECURITY PROTOCOL PLAINTEXT) + + > CREATE SOURCE no_disk_source1 + IN CLUSTER no_disk_cluster1 + FROM KAFKA CONNECTION kafka + (TOPIC 'testdrive-test-no-disk-${testdrive.seed}'); + + > CREATE TABLE no_disk_source1_tbl FROM SOURCE no_disk_source1 (REFERENCE "testdrive-test-no-disk-${testdrive.seed}") + KEY FORMAT TEXT + VALUE FORMAT TEXT + ENVELOPE UPSERT; + + + > SELECT * FROM no_disk_source1_tbl; + key text + ------------------ + key1 val1 + key2 val2 + + $ kafka-ingest key-format=bytes format=bytes topic=test-no-disk + key1:val3 + + > SELECT * FROM no_disk_source1_tbl; + key text + ------------------ + key1 val3 + key2 val2 + + > DROP CLUSTER no_disk_cluster1 CASCADE; + """ + ) + ) diff --git a/test/sqllogictest/audit_log.slt b/test/sqllogictest/audit_log.slt index ad8d996d61321..fdd613a3ffc86 100644 --- a/test/sqllogictest/audit_log.slt +++ b/test/sqllogictest/audit_log.slt @@ -151,7 +151,7 @@ SELECT id, event_type, object_type, details, user FROM mz_audit_events ORDER BY 12 create cluster {"id":"u1","name":"quickstart"} NULL 13 grant cluster {"grantee_id":"p","grantor_id":"s1","object_id":"Cu1","privileges":"U"} NULL 14 grant cluster {"grantee_id":"u1","grantor_id":"s1","object_id":"Cu1","privileges":"UC"} NULL -15 create cluster-replica {"billed_as":null,"cluster_id":"u1","cluster_name":"quickstart","disk":true,"internal":false,"logical_size":"2","reason":"system","replica_id":"u1","replica_name":"r1"} NULL +15 create cluster-replica {"billed_as":null,"cluster_id":"u1","cluster_name":"quickstart","disk":false,"internal":false,"logical_size":"2","reason":"system","replica_id":"u1","replica_name":"r1"} NULL 16 grant system {"grantee_id":"s1","grantor_id":"s1","object_id":"SYSTEM","privileges":"RBNP"} NULL 17 grant system {"grantee_id":"u1","grantor_id":"s1","object_id":"SYSTEM","privileges":"RBNP"} NULL 18 alter system {"name":"enable_reduce_mfp_fusion","value":"on"} mz_system diff --git a/test/sqllogictest/managed_cluster.slt b/test/sqllogictest/managed_cluster.slt index f0934f8efd8a8..972981f93d91c 100644 --- a/test/sqllogictest/managed_cluster.slt +++ b/test/sqllogictest/managed_cluster.slt @@ -420,24 +420,33 @@ ALTER SYSTEM SET enable_disk_cluster_replicas = true; ---- COMPLETE 0 -statement error db error: ERROR: DISK option not supported for non-legacy cluster sizes because disk is always enabled +statement ok CREATE CLUSTER foo REPLICAS (r1 (SIZE '1'), r2 (SIZE '1', DISK)) -statement error db error: ERROR: unknown cluster 'foo' +statement error db error: ERROR: Cluster replicas with DISK true do not match expected DISK false ALTER CLUSTER foo SET (MANAGED, DISK=False, SIZE '1') +statement ok +DROP CLUSTER foo + statement ok CREATE CLUSTER foo REPLICAS (r1 (SIZE '1')) -statement error db error: ERROR: DISK option not supported for modern cluster sizes because disk is always enabled +statement error db error: ERROR: Cluster replicas with DISK true do not match expected DISK false ALTER CLUSTER foo SET (MANAGED, SIZE '1', DISK=False) statement ok DROP CLUSTER foo -statement error db error: ERROR: DISK option not supported for non-legacy cluster sizes because disk is always enabled +statement ok CREATE CLUSTER foo REPLICAS (r1 (SIZE '1', DISK), r2 (SIZE '1', DISK)) +statement ok +ALTER CLUSTER foo SET (MANAGED, SIZE '1', DISK) + +statement ok +DROP CLUSTER foo + simple conn=mz_system,user=mz_system ALTER SYSTEM SET enable_graceful_cluster_reconfiguration = true; diff --git a/test/testdrive/cc_cluster_sizes.td b/test/testdrive/cc_cluster_sizes.td index 5b97508838790..f76f21149e4e9 100644 --- a/test/testdrive/cc_cluster_sizes.td +++ b/test/testdrive/cc_cluster_sizes.td @@ -56,9 +56,9 @@ true # Altering to a cc size with disk explicitly toggled is not allowed. ! ALTER CLUSTER c SET (SIZE = '1cc', DISK = true) -contains:DISK option not supported for modern cluster sizes because disk is always enabled +contains:DISK option not supported for cluster sizes ending in cc or C ! ALTER CLUSTER c SET (SIZE = '1cc', DISK = false) -contains:DISK option not supported for modern cluster sizes because disk is always enabled +contains:DISK option not supported for cluster sizes ending in cc or C # But it's fine as long as the ALTER command doesn't mention disk explicitly, # even though the cluster's initial creation specified disk explicitly. The @@ -68,31 +68,59 @@ contains:DISK option not supported for modern cluster sizes because disk is alwa true > DROP CLUSTER c -! CREATE CLUSTER c SIZE '1', DISK = false -contains:DISK option disabled is not supported for non-legacy cluster sizes because disk is always enabled +# Same test as before, except the legacy size cluster has disk explicitly +# disabled. +> CREATE CLUSTER c SIZE '1', DISK = false +> ALTER CLUSTER c SET (SIZE = '1cc') +> SELECT disk FROM mz_clusters WHERE name = 'c' +true +> DROP CLUSTER c # Same test as before, except the legacy size cluster has no disk explicitly # configured. > CREATE CLUSTER c SIZE = '1' > SELECT disk FROM mz_clusters WHERE name = 'c' -true +false > ALTER CLUSTER c SET (SIZE = '1cc') > SELECT disk FROM mz_clusters WHERE name = 'c' true # Cannot explicitly alter DISK option for new sizes. ! ALTER CLUSTER c SET (DISK = false) -contains:DISK option not supported for modern cluster sizes because disk is always enabled +contains:DISK option not supported for cluster sizes ending in cc or C ! ALTER CLUSTER c SET (DISK = true) -contains:DISK option not supported for modern cluster sizes because disk is always enabled +contains:DISK option not supported for cluster sizes ending in cc or C + +# But it's okay if you're going back to a legacy size. +> ALTER CLUSTER c SET (DISK = true, SIZE = '1') +> SELECT disk FROM mz_clusters WHERE name = 'c' +true > DROP CLUSTER c +# Ensure that altering from a legacy size to a legacy size does not enable disk. +> CREATE CLUSTER c SIZE = '1' +> SELECT disk FROM mz_clusters WHERE name = 'c' +false +> ALTER CLUSTER c SET (SIZE = '2') +> SELECT disk FROM mz_clusters WHERE name = 'c' +false +> DROP CLUSTER c + +# Ensure that disk isn't configurable for the new sizes (as it's force enabled). + +> CREATE CLUSTER c SIZE '1cc', DISK = true; + +> DROP CLUSTER c; + +! CREATE CLUSTER c SIZE '1cc', DISK = false; +contains:DISK option disabled is not supported for cluster sizes ending in cc or C + > CREATE CLUSTER c REPLICAS (r1 (SIZE '1cc')) > CREATE CLUSTER REPLICA c.r2 SIZE '1cc'; > CREATE CLUSTER REPLICA c.r3 SIZE '1C'; ! CREATE CLUSTER REPLICA c.r SIZE '1cc', DISK = true; -contains:DISK option not supported for non-legacy cluster sizes because disk is always enabled +contains:DISK option not supported for cluster sizes ending in cc or C ! CREATE CLUSTER REPLICA c.r SIZE '1cc', DISK = false; -contains:DISK option not supported for non-legacy cluster sizes because disk is always enabled +contains:DISK option not supported for cluster sizes ending in cc or C diff --git a/test/testdrive/disk-feature-flag.td b/test/testdrive/disk-feature-flag.td index 476ff00664a4c..b348094e6d12f 100644 --- a/test/testdrive/disk-feature-flag.td +++ b/test/testdrive/disk-feature-flag.td @@ -11,6 +11,16 @@ $ postgres-execute connection=postgres://mz_system:materialize@${testdrive.mater ALTER SYSTEM SET enable_disk_cluster_replicas = false ALTER SYSTEM SET disk_cluster_replicas_default = false +! CREATE CLUSTER no SIZE = '1', REPLICATION FACTOR 0, DISK; +exact:`WITH (DISK)` for cluster replicas is not supported + +> CREATE CLUSTER no SIZE = '1', REPLICATION FACTOR 0; + +! ALTER CLUSTER no SET (REPLICATION FACTOR 1, DISK); +exact:`WITH (DISK)` for cluster replicas is not supported + +> DROP CLUSTER no; + # Test that with `enable_create_source_denylist_with_options` off, if # `enable_disk_cluster_replicas` is on, we can use `WITH(DISK)`. $ postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr} @@ -20,14 +30,11 @@ ALTER SYSTEM SET enable_disk_cluster_replicas = true > DROP CLUSTER IF EXISTS c; # Can set unmanaged cluster replica options directly, mixing and matching disk -! CREATE CLUSTER c REPLICAS (r1 (SIZE '1', DISK), r2 (SIZE '1')) -contains: DISK option not supported for non-legacy cluster sizes because disk is always enabled - -> CREATE CLUSTER c REPLICAS (r1 (SIZE '1'), r2 (SIZE '1')) +> CREATE CLUSTER c REPLICAS (r1 (SIZE '1', DISK), r2 (SIZE '1')) > SELECT r.name, r.size, r.disk FROM mz_catalog.mz_clusters c, mz_catalog.mz_cluster_replicas r WHERE c.name = 'c' AND c.id = r.cluster_id; r1 1 true -r2 1 true +r2 1 false > DROP CLUSTER c; @@ -44,10 +51,7 @@ r2 1 true $ postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr} ALTER SYSTEM SET disk_cluster_replicas_default = true -! CREATE CLUSTER c REPLICAS (r1 (SIZE '1', DISK), r2 (SIZE '1')) -contains:DISK option not supported for non-legacy cluster sizes because disk is always enabled - -> CREATE CLUSTER c REPLICAS (r1 (SIZE '1'), r2 (SIZE '1')) +> CREATE CLUSTER c REPLICAS (r1 (SIZE '1', DISK), r2 (SIZE '1')) > SELECT r.name, r.size, r.disk FROM mz_catalog.mz_clusters c, mz_catalog.mz_cluster_replicas r WHERE c.name = 'c' AND c.id = r.cluster_id; r1 1 true