Skip to content

Commit c4d2649

Browse files
committed
source: Fix key-value load gen w/o primary export
This commit fixes the key-value load generator source when the primary export is removed via the force_source_table_syntax flag. Fixes #MaterializeInc/database-issues/issues/8904
1 parent 411e6c9 commit c4d2649

File tree

2 files changed

+44
-42
lines changed

2 files changed

+44
-42
lines changed

src/storage/src/source/generator.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,10 @@ impl GeneratorKind {
148148
) {
149149
// figure out which output types from the generator belong to which output indexes
150150
let mut output_map = BTreeMap::new();
151+
// Make sure that there's an entry for the default output, even if there are no exports
152+
// that need data output. Certain implementations rely on it (at the time of this comment
153+
// that includes the key-value load gen source).
154+
output_map.insert(LoadGeneratorOutput::Default, Vec::new());
151155
for (idx, (_, export)) in config.source_exports.iter().enumerate() {
152156
let output_type = match &export.details {
153157
SourceExportDetails::LoadGenerator(details) => details.output,

test/testdrive/force-source-tables.td

Lines changed: 40 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -464,49 +464,47 @@ avro_table_upsert ""
464464
# Key-value load generator source using source-fed tables
465465
#
466466

467-
# TODO(database-issues#8904): Re-enable when key-value load gen sources work with force_source_table_syntax.
467+
$ postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr}
468+
ALTER SYSTEM SET enable_create_table_from_source = true
469+
ALTER SYSTEM SET enable_load_generator_key_value = true
470+
ALTER SYSTEM SET force_source_table_syntax = true
468471

469-
# $ postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr}
470-
# ALTER SYSTEM SET enable_create_table_from_source = true
471-
# ALTER SYSTEM SET enable_load_generator_key_value = true
472-
# ALTER SYSTEM SET force_source_table_syntax = true
473-
#
474-
# > CREATE SOURCE keyvalue
475-
# IN CLUSTER ${arg.single-replica-cluster}
476-
# FROM LOAD GENERATOR KEY VALUE (
477-
# KEYS 16,
478-
# PARTITIONS 4,
479-
# SNAPSHOT ROUNDS 3,
480-
# SEED 123,
481-
# VALUE SIZE 10,
482-
# BATCH SIZE 2,
483-
# TICK INTERVAL '1s'
484-
# );
485-
#
486-
# > CREATE TABLE kv_1 FROM SOURCE keyvalue INCLUDE KEY ENVELOPE UPSERT;
487-
#
488-
# > CREATE TABLE kv_2 FROM SOURCE keyvalue INCLUDE KEY ENVELOPE NONE;
489-
#
490-
# > SELECT partition, count(*) FROM kv_1 GROUP BY partition
491-
# 0 4
492-
# 1 4
493-
# 2 4
494-
# 3 4
495-
#
496-
# > SELECT status FROM mz_internal.mz_source_statuses WHERE name = 'kv_1';
497-
# running
498-
#
499-
# > SELECT partition, count(*) > 10 FROM kv_2 GROUP BY partition
500-
# 0 true
501-
# 1 true
502-
# 2 true
503-
# 3 true
504-
#
505-
# > SHOW TABLES ON keyvalue;
506-
# kv_1 ""
507-
# kv_2 ""
508-
#
509-
# > DROP SOURCE keyvalue CASCADE;
472+
> CREATE SOURCE keyvalue
473+
IN CLUSTER ${arg.single-replica-cluster}
474+
FROM LOAD GENERATOR KEY VALUE (
475+
KEYS 16,
476+
PARTITIONS 4,
477+
SNAPSHOT ROUNDS 3,
478+
SEED 123,
479+
VALUE SIZE 10,
480+
BATCH SIZE 2,
481+
TICK INTERVAL '1s'
482+
);
483+
484+
> CREATE TABLE kv_1 FROM SOURCE keyvalue INCLUDE KEY ENVELOPE UPSERT;
485+
486+
> CREATE TABLE kv_2 FROM SOURCE keyvalue INCLUDE KEY ENVELOPE NONE;
487+
488+
> SELECT partition, count(*) FROM kv_1 GROUP BY partition
489+
0 4
490+
1 4
491+
2 4
492+
3 4
493+
494+
> SELECT status FROM mz_internal.mz_source_statuses WHERE name = 'kv_1';
495+
running
496+
497+
> SELECT partition, count(*) > 10 FROM kv_2 GROUP BY partition
498+
0 true
499+
1 true
500+
2 true
501+
3 true
502+
503+
> SHOW TABLES ON keyvalue;
504+
kv_1 ""
505+
kv_2 ""
506+
507+
> DROP SOURCE keyvalue CASCADE;
510508

511509
#
512510
# Force usage of the new syntax and check that old statements are disallowed

0 commit comments

Comments
 (0)