From c99b46b71b11d3f0a49ca3b1493163954d4ad059 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Wed, 20 May 2026 11:25:10 +0100 Subject: [PATCH] duckdb-bench --baseline option Signed-off-by: Mikhail Kot --- benchmarks/compress-bench/src/main.rs | 3 +- benchmarks/duckdb-bench/src/main.rs | 84 ++++++++++++++++++- benchmarks/random-access-bench/src/main.rs | 2 +- vortex-bench/src/display.rs | 94 ++++++++++++++++------ vortex-bench/src/runner.rs | 4 +- vortex-bench/src/v3.rs | 40 +++++++-- 6 files changed, 187 insertions(+), 40 deletions(-) diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index ce48001abb5..b665022a466 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -192,7 +192,7 @@ async fn run_compress( match display_format { DisplayFormat::Table => { - render_table(&mut writer, measurements.timings, &targets)?; + render_table(&mut writer, measurements.timings, &targets, None)?; render_table( &mut writer, measurements.ratios, @@ -201,6 +201,7 @@ async fn run_compress( } else { vec![] }, + None, ) } DisplayFormat::GhJson => { diff --git a/benchmarks/duckdb-bench/src/main.rs b/benchmarks/duckdb-bench/src/main.rs index b2a4ff1bd15..3faeda77e03 100644 --- a/benchmarks/duckdb-bench/src/main.rs +++ b/benchmarks/duckdb-bench/src/main.rs @@ -10,21 +10,26 @@ use clap::value_parser; use duckdb_bench::DuckClient; use tokio::runtime::Runtime; use vortex::metrics::tracing::set_global_labels; +use vortex::utils::aliases::hash_map::HashMap; +use vortex::utils::aliases::hash_set::HashSet; use vortex_bench::BenchmarkArg; use vortex_bench::CompactionStrategy; use vortex_bench::Engine; use vortex_bench::Format; use vortex_bench::Opt; use vortex_bench::Opts; +use vortex_bench::Target; use vortex_bench::conversions::convert_parquet_directory_to_vortex; use vortex_bench::create_benchmark; use vortex_bench::create_output_writer; use vortex_bench::display::DisplayFormat; +use vortex_bench::display::render_table; use vortex_bench::runner::BenchmarkMode; use vortex_bench::runner::SqlBenchmarkRunner; use vortex_bench::runner::filter_queries; use vortex_bench::setup_logging_and_tracing; use vortex_bench::v3; +use vortex_bench::v3::V3Record; /// Common arguments shared across benchmarks #[derive(Parser)] @@ -64,6 +69,11 @@ struct Args { #[arg(long)] gh_json_v3: Option, + /// Compare with a JSONL file exported with --gh-json-v3. + /// The intersection of queries, formats, and datasets is displayed. + #[arg(short, long)] + baseline: Option, + #[arg(long, default_value_t = false)] track_memory: bool, @@ -73,7 +83,7 @@ struct Args { #[arg(long, default_value = "unknown")] runner: String, - #[arg(long, value_delimiter = ',', value_parser = value_parser!(Format))] + #[arg(long, default_value = "vortex", value_delimiter = ',', value_parser = value_parser!(Format))] formats: Vec, #[arg(long = "opt", value_delimiter = ',', value_parser = value_parser!(Opt))] @@ -196,14 +206,80 @@ fn main() -> anyhow::Result<()> { )?; if !args.explain { + let current_records = runner.v3_records(); + if let Some(path) = args.gh_json_v3.as_ref() { - v3::write_jsonl_to_path(path, &runner.v3_records())?; + v3::write_jsonl_to_path(path, ¤t_records)?; } let benchmark_id = format!("duckdb-{}", benchmark.dataset_name()); - let writer = create_output_writer(&args.display_format, args.output_path, &benchmark_id)?; - runner.export_to(&args.display_format, writer)?; + let mut writer = + create_output_writer(&args.display_format, args.output_path, &benchmark_id)?; + + if let Some(baseline_path) = args.baseline + && matches!(args.display_format, DisplayFormat::Table) + { + let baseline_records = v3::read_jsonl_from_path(&baseline_path)?; + let baseline_map = build_query_baseline_map(&baseline_records, ¤t_records); + let targets = args + .formats + .iter() + .map(|f| Target::new(Engine::DuckDB, *f)) + .collect::>(); + let results = runner.into_results(); + if !results.memory_measurements.is_empty() { + render_table(&mut writer, results.memory_measurements, &targets, None)?; + } + render_table( + &mut writer, + results.query_measurements, + &targets, + Some(&baseline_map), + )?; + } else { + runner.export_to(&args.display_format, writer)?; + } } Ok(()) } + +fn build_query_baseline_map( + baseline: &[V3Record], + current: &[V3Record], +) -> HashMap<(u32, String), u64> { + let current_dims: HashSet<(String, Option, Option)> = current + .iter() + .filter_map(|r| { + if let V3Record::QueryMeasurement(qm) = r { + Some(( + qm.dataset.clone(), + qm.dataset_variant.clone(), + qm.scale_factor.clone(), + )) + } else { + None + } + }) + .collect(); + + baseline + .iter() + .filter_map(|r| { + if let V3Record::QueryMeasurement(qm) = r { + let dims = ( + qm.dataset.clone(), + qm.dataset_variant.clone(), + qm.scale_factor.clone(), + ); + if current_dims.contains(&dims) { + Some(((qm.query_idx, qm.format.clone()), qm.value_ns)) + } else { + None + } + } else { + None + } + }) + .collect() +} diff --git a/benchmarks/random-access-bench/src/main.rs b/benchmarks/random-access-bench/src/main.rs index 6543e221e33..9fef4f98127 100644 --- a/benchmarks/random-access-bench/src/main.rs +++ b/benchmarks/random-access-bench/src/main.rs @@ -468,7 +468,7 @@ async fn run_random_access( match display_format { DisplayFormat::Table => { - render_table(&mut writer, measurements, &targets)?; + render_table(&mut writer, measurements, &targets, None)?; } DisplayFormat::GhJson => { print_measurements_json(&mut writer, measurements)?; diff --git a/vortex-bench/src/display.rs b/vortex-bench/src/display.rs index a8e903df25a..c96ef1555de 100644 --- a/vortex-bench/src/display.rs +++ b/vortex-bench/src/display.rs @@ -4,6 +4,7 @@ use std::io::Write; use std::iter; +use anyhow::Result; use clap::ValueEnum; use itertools::Itertools; use tabled::builder::Builder; @@ -29,7 +30,8 @@ pub fn render_table( writer: &mut W, all_measurements: Vec, targets: &[Target], -) -> anyhow::Result<()> { + baseline: Option<&HashMap<(u32, String), u64>>, +) -> Result<()> { let mut measurements: HashMap> = HashMap::with_capacity(all_measurements.len().div_ceil(targets.len())); @@ -45,9 +47,8 @@ pub fn render_table( measurements.values_mut().sorted_unstable(); - // The first format serves as the baseline - let baseline_target = &targets[0]; - let baseline = measurements[baseline_target].clone(); + let first_target = &targets[0]; + let reference = measurements[first_target].clone(); let mut table_builder = Builder::default(); let mut colors = vec![]; @@ -57,37 +58,82 @@ pub fn render_table( if engines.len() > 1 { table_builder.push_record( iter::once("".to_owned()) - .chain(targets.iter().map(move |t| format!("{}", t.engine))) + .chain(targets.iter().flat_map(|t| { + let label = format!("{}", t.engine); + if baseline.is_some() { + vec![label, String::new()] + } else { + vec![label] + } + })) .collect::>(), ); } table_builder.push_record( iter::once("Benchmark".to_owned()) - .chain(targets.iter().map(|t| format!("{}", t.format))) + .chain(targets.iter().flat_map(|t| { + if baseline.is_some() { + vec![ + format!("{} (baseline)", t.format), + format!("{} (current)", t.format), + ] + } else { + vec![format!("{}", t.format)] + } + })) .collect::>(), ); - for (idx, baseline_measure) in baseline.iter().enumerate() { - let query_baseline = baseline_measure.value; - let mut row = vec![baseline_measure.name.clone()]; - for (col_idx, target) in targets.iter().enumerate() { - let measurement = &measurements[target][idx]; - let value = measurement.value; - - if target != baseline_target { - let color = color(query_baseline, value); - + let mut row = Vec::with_capacity(1 + targets.len() * (1 + baseline.is_some() as usize)); + for (row_idx, ref_m) in reference.iter().enumerate() { + row.clear(); + row.push(ref_m.name.clone()); + + if let Some(baseline) = baseline { + let query_id = ref_m.id.map(|i| i as u32); + for (target_col, target) in targets.iter().enumerate() { + let measurement = &measurements[target][row_idx]; + let value = measurement.value; + // baseline stores nanoseconds, TableValue uses microseconds. + let bv_us = query_id.and_then(|id| { + baseline + .get(&(id, target.format.name().to_string())) + .map(|&ns| ns / 1_000) + }); + let Some(bv_us) = bv_us else { + // We have already filtered missing values in + // build_query_baseline_map + anyhow::bail!("Query id or baseline value missing"); + }; + + assert!(bv_us > 0); + let bv = MeasurementValue::Int(bv_us as u128); + let ratio = value / bv; + row.push(format!("{bv:.2} {}", measurement.unit)); + row.push(format!("{value:.2} {} ({ratio:.2})", measurement.unit)); colors.push(Colorization::exact( - vec![color], - (idx + header_count, col_idx + 1), - )) + vec![color(bv, value)], + (row_idx + header_count, 2 + target_col * 2), + )); + } + } else { + let query_baseline = ref_m.value; + for (col_idx, target) in targets.iter().enumerate() { + let measurement = &measurements[target][row_idx]; + let value = measurement.value; + if target != first_target { + colors.push(Colorization::exact( + vec![color(query_baseline, value)], + (row_idx + header_count, col_idx + 1), + )); + } + let ratio = value / query_baseline; + row.push(format!("{value:.2} {} ({ratio:.2})", measurement.unit)); } - - let ratio = value / query_baseline; - row.push(format!("{value:.2} {} ({ratio:.2})", measurement.unit)); } - table_builder.push_record(row); + + table_builder.push_record(&row); } let mut table = table_builder.build(); @@ -105,7 +151,7 @@ pub fn render_table( pub fn print_measurements_json( writer: &mut dyn Write, all_measurements: Vec, -) -> anyhow::Result<()> { +) -> Result<()> { for measurement in all_measurements { writeln!(writer, "{}", measurement.to_json())?; } diff --git a/vortex-bench/src/runner.rs b/vortex-bench/src/runner.rs index b3ea8b5c1b3..2ad925595f7 100644 --- a/vortex-bench/src/runner.rs +++ b/vortex-bench/src/runner.rs @@ -465,13 +465,13 @@ pub fn export_results( if !memory.is_empty() { match display_format { - DisplayFormat::Table => render_table(&mut output, memory, &targets)?, + DisplayFormat::Table => render_table(&mut output, memory, &targets, None)?, DisplayFormat::GhJson => print_measurements_json(&mut output, memory)?, }; } match display_format { - DisplayFormat::Table => render_table(&mut output, queries, &targets)?, + DisplayFormat::Table => render_table(&mut output, queries, &targets, None)?, DisplayFormat::GhJson => print_measurements_json(&mut output, queries)?, }; diff --git a/vortex-bench/src/v3.rs b/vortex-bench/src/v3.rs index 99c85314fbe..8cd020374c8 100644 --- a/vortex-bench/src/v3.rs +++ b/vortex-bench/src/v3.rs @@ -69,9 +69,13 @@ //! [`vortex_bench_server::records`]: ../../../benchmarks-website/server/src/records.rs //! [`vortex_bench_migrate::classifier`]: ../../../benchmarks-website/migrate/src/classifier.rs +use std::io::BufRead; +use std::io::Result; use std::io::Write; +use std::path::Path; use std::sync::LazyLock; +use serde::Deserialize; use serde::Serialize; use target_lexicon::Triple; @@ -100,7 +104,7 @@ pub static ENV_TRIPLE: LazyLock = LazyLock::new(|| { /// /// Each variant flattens its inner record next to a `"kind"` field, matching the /// shape consumed by `/api/ingest`. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum V3Record { /// SQL query suite measurement (TPC-H/TPC-DS/ClickBench/...). @@ -120,7 +124,7 @@ pub enum V3Record { /// /// Memory fields are populated together (all four or none), matching the /// `--track-memory` instrumentation. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct QueryMeasurementRecord { /// 40-hex lowercase commit SHA. pub commit_sha: String, @@ -167,7 +171,7 @@ pub struct QueryMeasurementRecord { } /// A single encode-or-decode timing from `compress-bench`. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct CompressionTimeRecord { /// 40-hex lowercase commit SHA. pub commit_sha: String, @@ -190,7 +194,7 @@ pub struct CompressionTimeRecord { } /// On-disk size of a compressed file from `compress-bench`. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct CompressionSizeRecord { /// 40-hex lowercase commit SHA. pub commit_sha: String, @@ -206,7 +210,7 @@ pub struct CompressionSizeRecord { } /// A single take-time timing from `random-access-bench`. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct RandomAccessTimeRecord { /// 40-hex lowercase commit SHA. pub commit_sha: String, @@ -227,7 +231,7 @@ pub struct RandomAccessTimeRecord { /// /// Carries timing **and** the side counters in one row, mirroring the /// `vector_search_runs` fact table. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct VectorSearchRunRecord { /// 40-hex lowercase commit SHA. pub commit_sha: String, @@ -467,7 +471,7 @@ pub fn vector_search_record( /// Write `records` as JSONL (one JSON object per line) to `writer`. /// /// JSONL is the on-disk format consumed by `scripts/post-ingest.py`. -pub fn write_jsonl(writer: &mut W, records: &[V3Record]) -> std::io::Result<()> { +pub fn write_jsonl(writer: &mut W, records: &[V3Record]) -> Result<()> { for record in records { let line = serde_json::to_string(record).map_err(std::io::Error::other)?; writer.write_all(line.as_bytes())?; @@ -477,7 +481,7 @@ pub fn write_jsonl(writer: &mut W, records: &[V3Record]) -> std::io::R } /// Write `records` as JSONL to `path`, creating parent directories as needed. -pub fn write_jsonl_to_path(path: &std::path::Path, records: &[V3Record]) -> std::io::Result<()> { +pub fn write_jsonl_to_path(path: &Path, records: &[V3Record]) -> Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } @@ -485,6 +489,26 @@ pub fn write_jsonl_to_path(path: &std::path::Path, records: &[V3Record]) -> std: write_jsonl(&mut file, records) } +pub fn read_jsonl_from_path(path: &Path) -> Result> { + let file = std::fs::File::open(path)?; + let reader = std::io::BufReader::new(file); + let mut records = Vec::new(); + for (i, line) in reader.lines().enumerate() { + let line = line?; + if line.trim().is_empty() { + continue; + } + let record: V3Record = serde_json::from_str(&line).map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("{}:{}: {e}", path.display(), i + 1), + ) + })?; + records.push(record); + } + Ok(records) +} + fn duration_as_ns(d: std::time::Duration) -> u64 { u64::try_from(d.as_nanos()).unwrap_or(u64::MAX) }