diff --git a/Cargo.lock b/Cargo.lock index 62478f32a0..941e3d66c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3748,6 +3748,7 @@ dependencies = [ "enum-ordinalize", "env_logger", "iceberg", + "iceberg-catalog-loader", "iceberg-datafusion", "indicatif", "libtest-mimic", diff --git a/Cargo.toml b/Cargo.toml index c10c01d94a..27f0883614 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,7 @@ hive_metastore = "0.2.0" home = "=0.5.11" http = "1.2" iceberg = { version = "0.7.0", path = "./crates/iceberg" } +iceberg-catalog-loader = { version = "0.7.0", path = "./crates/catalog/loader" } iceberg-catalog-glue = { version = "0.7.0", path = "./crates/catalog/glue" } iceberg-catalog-hms = { version = "0.7.0", path = "./crates/catalog/hms" } iceberg-catalog-sql = { version = "0.7.0", path = "./crates/catalog/sql" } diff --git a/crates/catalog/loader/src/lib.rs b/crates/catalog/loader/src/lib.rs index e118ef86a9..81e563597e 100644 --- a/crates/catalog/loader/src/lib.rs +++ b/crates/catalog/loader/src/lib.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use async_trait::async_trait; use iceberg::{Catalog, CatalogBuilder, Error, ErrorKind, Result}; +use iceberg::memory::MemoryCatalogBuilder; use iceberg_catalog_glue::GlueCatalogBuilder; use iceberg_catalog_hms::HmsCatalogBuilder; use iceberg_catalog_rest::RestCatalogBuilder; @@ -31,6 +32,7 @@ type CatalogBuilderFactory = fn() -> Box; /// A registry of catalog builders. static CATALOG_REGISTRY: &[(&str, CatalogBuilderFactory)] = &[ + ("memory", || Box::new(MemoryCatalogBuilder::default())), ("rest", || Box::new(RestCatalogBuilder::default())), ("glue", || Box::new(GlueCatalogBuilder::default())), ("s3tables", || Box::new(S3TablesCatalogBuilder::default())), diff --git a/crates/sqllogictest/Cargo.toml b/crates/sqllogictest/Cargo.toml index e826ad7ae0..0c3280ee70 100644 --- a/crates/sqllogictest/Cargo.toml +++ b/crates/sqllogictest/Cargo.toml @@ -32,6 +32,7 @@ datafusion-sqllogictest = { workspace = true } enum-ordinalize = { workspace = true } env_logger = { workspace = true } iceberg = { workspace = true } +iceberg-catalog-loader = { workspace = true } iceberg-datafusion = { workspace = true } indicatif = { workspace = true } log = { workspace = true } diff --git a/crates/sqllogictest/README.md b/crates/sqllogictest/README.md index ddcfe851c5..5c47b70dff 100644 --- a/crates/sqllogictest/README.md +++ b/crates/sqllogictest/README.md @@ -32,4 +32,68 @@ cargo test The tests are run against the following sql engines: * [Apache datafusion](https://crates.io/crates/datafusion) -* [Apache spark](https://github.com/apache/spark) \ No newline at end of file +* [Apache spark](https://github.com/apache/spark) + +## Catalog Configuration + +This crate now supports dynamic catalog configuration in test schedules. You can configure different types of catalogs (memory, rest, glue, sql, etc.) and share them across multiple engines. + +### Configuration Format + +```toml +# Define catalogs +[catalogs.memory_catalog] +type = "memory" +warehouse = "memory://warehouse" + +[catalogs.rest_catalog] +type = "rest" +uri = "http://localhost:8181" +warehouse = "s3://my-bucket/warehouse" +credential = "client_credentials" +token = "xxx" + +[catalogs.sql_catalog] +type = "sql" +uri = "postgresql://user:pass@localhost/iceberg" +warehouse = "s3://my-bucket/warehouse" +sql_bind_style = "DollarNumeric" + +# Define engines with optional catalog references +[engines.datafusion1] +type = "datafusion" +catalog = "memory_catalog" # Reference to a catalog + +[engines.datafusion2] +type = "datafusion" +catalog = "rest_catalog" # Multiple engines can share the same catalog + +[engines.datafusion3] +type = "datafusion" +# No catalog specified - uses default MemoryCatalog + +# Define test steps +[[steps]] +engine = "datafusion1" +slt = "path/to/test1.slt" + +[[steps]] +engine = "datafusion2" +slt = "path/to/test2.slt" +``` + +### Supported Catalog Types + +- `memory`: In-memory catalog for testing +- `rest`: REST catalog for integration testing +- `glue`: AWS Glue catalog +- `sql`: SQL database catalog +- `hms`: Hive Metastore catalog +- `s3tables`: S3 Tables catalog + +### Key Features + +- **Lazy Loading**: Catalogs are created only when first referenced +- **Sharing**: Multiple engines can share the same catalog instance +- **Backward Compatibility**: Existing schedules continue to work without modification +- **Type Safety**: Catalog configurations are validated at parse time \ No newline at end of file diff --git a/crates/sqllogictest/examples/example_catalog_config.toml.example b/crates/sqllogictest/examples/example_catalog_config.toml.example new file mode 100644 index 0000000000..bc410ccc9d --- /dev/null +++ b/crates/sqllogictest/examples/example_catalog_config.toml.example @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Example configuration file: demonstrates various catalog configuration methods +# Note: This file is for example purposes only and will not be executed by the test framework +# (because the filename does not end with .toml, but with .example) + +# Memory catalog example +[catalogs.memory_example] +type = "memory" +warehouse = "memory://example" + +# REST catalog example (for integration testing) +[catalogs.rest_example] +type = "rest" +uri = "http://localhost:8181" +warehouse = "s3://test-bucket/warehouse" +credential = "client_credentials" +token = "your-token-here" + +# SQL catalog example +[catalogs.sql_example] +type = "sql" +uri = "postgresql://user:password@localhost:5432/iceberg_test" +warehouse = "s3://test-bucket/warehouse" +sql_bind_style = "DollarNumeric" + +# Glue catalog example +[catalogs.glue_example] +type = "glue" +warehouse = "s3://test-bucket/warehouse" +region = "us-east-1" + +# Engine configuration example +[engines.memory_engine] +type = "datafusion" +catalog = "memory_example" + +[engines.rest_engine] +type = "datafusion" +catalog = "rest_example" + +[engines.sql_engine] +type = "datafusion" +catalog = "sql_example" + +[engines.glue_engine] +type = "datafusion" +catalog = "glue_example" + +[engines.default_engine] +type = "datafusion" +# No catalog specified, uses default MemoryCatalog + +# Test steps example +[[steps]] +engine = "memory_engine" +slt = "df_test/show_tables.slt" + +[[steps]] +engine = "default_engine" +slt = "df_test/show_tables.slt" diff --git a/crates/sqllogictest/src/engine/datafusion.rs b/crates/sqllogictest/src/engine/datafusion.rs index b3e37d9206..7516929e2e 100644 --- a/crates/sqllogictest/src/engine/datafusion.rs +++ b/crates/sqllogictest/src/engine/datafusion.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use datafusion::catalog::CatalogProvider; use datafusion::prelude::{SessionConfig, SessionContext}; use datafusion_sqllogictest::DataFusion; -use iceberg::CatalogBuilder; +use iceberg::{Catalog, CatalogBuilder}; use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; use iceberg_datafusion::IcebergCatalogProvider; use indicatif::ProgressBar; @@ -36,6 +36,15 @@ pub struct DataFusionEngine { session_context: SessionContext, } +impl std::fmt::Debug for DataFusionEngine { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DataFusionEngine") + .field("test_data_path", &self.test_data_path) + .field("session_context", &"") + .finish() + } +} + #[async_trait::async_trait] impl EngineRunner for DataFusionEngine { async fn run_slt_file(&mut self, path: &Path) -> Result<()> { @@ -58,12 +67,27 @@ impl EngineRunner for DataFusionEngine { } impl DataFusionEngine { - pub async fn new(config: TomlTable) -> Result { + pub async fn new( + config: TomlTable, + catalog: Option>, + ) -> Result { let session_config = SessionConfig::new() .with_target_partitions(4) .with_information_schema(true); let ctx = SessionContext::new_with_config(session_config); - ctx.register_catalog("default", Self::create_catalog(&config).await?); + + let catalog_provider = match catalog { + Some(cat) => { + // Use the provided catalog + Arc::new(IcebergCatalogProvider::try_new(cat).await?) + } + None => { + // Fallback: create default MemoryCatalog + Self::create_default_catalog(&config).await? + } + }; + + ctx.register_catalog("default", catalog_provider); Ok(Self { test_data_path: PathBuf::from("testdata"), @@ -71,9 +95,10 @@ impl DataFusionEngine { }) } - async fn create_catalog(_: &TomlTable) -> anyhow::Result> { - // TODO: support dynamic catalog configuration - // See: https://github.com/apache/iceberg-rust/issues/1780 + async fn create_default_catalog( + _: &TomlTable, + ) -> anyhow::Result> { + // Create default MemoryCatalog as fallback let catalog = MemoryCatalogBuilder::default() .load( "memory", diff --git a/crates/sqllogictest/src/engine/mod.rs b/crates/sqllogictest/src/engine/mod.rs index 724359fbe5..956f1da35f 100644 --- a/crates/sqllogictest/src/engine/mod.rs +++ b/crates/sqllogictest/src/engine/mod.rs @@ -18,8 +18,10 @@ mod datafusion; use std::path::Path; +use std::sync::Arc; use anyhow::anyhow; +use iceberg::Catalog; use sqllogictest::{AsyncDB, MakeConnection, Runner, parse_file}; use toml::Table as TomlTable; @@ -29,16 +31,17 @@ use crate::error::{Error, Result}; const TYPE_DATAFUSION: &str = "datafusion"; #[async_trait::async_trait] -pub trait EngineRunner: Send { +pub trait EngineRunner: Send + std::fmt::Debug { async fn run_slt_file(&mut self, path: &Path) -> Result<()>; } pub async fn load_engine_runner( engine_type: &str, cfg: TomlTable, + catalog: Option>, ) -> Result> { match engine_type { - TYPE_DATAFUSION => Ok(Box::new(DataFusionEngine::new(cfg).await?)), + TYPE_DATAFUSION => Ok(Box::new(DataFusionEngine::new(cfg, catalog).await?)), _ => Err(anyhow::anyhow!("Unsupported engine type: {engine_type}").into()), } } @@ -74,7 +77,7 @@ mod tests { random = { type = "random_engine", url = "http://localhost:8181" } "#; let tbl = toml::from_str(input).unwrap(); - let result = load_engine_runner("random_engine", tbl).await; + let result = load_engine_runner("random_engine", tbl, None).await; assert!(result.is_err()); } @@ -86,7 +89,7 @@ mod tests { df = { type = "datafusion" } "#; let tbl = toml::from_str(input).unwrap(); - let result = load_engine_runner(TYPE_DATAFUSION, tbl).await; + let result = load_engine_runner(TYPE_DATAFUSION, tbl, None).await; assert!(result.is_ok()); } diff --git a/crates/sqllogictest/src/error.rs b/crates/sqllogictest/src/error.rs index 2bf5a09d6b..c279493a8b 100644 --- a/crates/sqllogictest/src/error.rs +++ b/crates/sqllogictest/src/error.rs @@ -50,3 +50,9 @@ impl From for Error { Self(value.into()) } } + +impl From for Error { + fn from(value: iceberg::Error) -> Self { + Self(value.into()) + } +} diff --git a/crates/sqllogictest/src/schedule.rs b/crates/sqllogictest/src/schedule.rs index 7c13ad4d12..f35e257912 100644 --- a/crates/sqllogictest/src/schedule.rs +++ b/crates/sqllogictest/src/schedule.rs @@ -18,14 +18,67 @@ use std::collections::HashMap; use std::fs::read_to_string; use std::path::{Path, PathBuf}; +use std::sync::Arc; use anyhow::{Context, anyhow}; +use iceberg::Catalog; use serde::{Deserialize, Serialize}; use toml::{Table as TomlTable, Value}; use tracing::info; use crate::engine::{EngineRunner, load_engine_runner}; +/// Catalog configuration parsed from TOML +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CatalogConfig { + /// Catalog type (e.g., "memory", "rest", "glue", "sql") + pub r#type: String, + /// Additional properties passed to the catalog builder + #[serde(flatten)] + pub properties: HashMap, +} + +/// Registry for managing catalog instances with lazy loading and sharing +pub struct CatalogRegistry { + /// Cached catalog instances by name + catalogs: HashMap>, +} + +impl CatalogRegistry { + /// Create a new empty catalog registry + pub fn new() -> Self { + Self { + catalogs: HashMap::new(), + } + } + + /// Get or create a catalog instance + pub async fn get_or_create_catalog( + &mut self, + name: &str, + config: &CatalogConfig, + ) -> anyhow::Result> { + if let Some(catalog) = self.catalogs.get(name) { + return Ok(Arc::clone(catalog)); + } + + // Load catalog using catalog-loader + use iceberg_catalog_loader::CatalogLoader; + let catalog = CatalogLoader::from(config.r#type.as_str()) + .load(name.to_string(), config.properties.clone()) + .await + .with_context(|| format!("Failed to load catalog '{}' of type '{}'", name, config.r#type))?; + + self.catalogs.insert(name.to_string(), Arc::clone(&catalog)); + Ok(catalog) + } + + /// Get all catalog names + pub fn catalog_names(&self) -> Vec { + self.catalogs.keys().cloned().collect() + } +} + pub struct Schedule { /// Engine names to engine instances engines: HashMap>, @@ -56,6 +109,33 @@ impl Schedule { } } + /// Parse catalogs from TOML table + async fn parse_catalogs( + table: &TomlTable, + ) -> anyhow::Result> { + let catalogs_tbl = match table.get("catalogs") { + Some(val) => val + .as_table() + .ok_or_else(|| anyhow!("'catalogs' must be a table"))?, + None => return Ok(HashMap::new()), // catalogs are optional + }; + + let mut catalogs = HashMap::new(); + + for (name, catalog_val) in catalogs_tbl { + let cfg: CatalogConfig = catalog_val + .clone() + .try_into() + .with_context(|| format!("Failed to parse catalog '{name}'"))?; + + if catalogs.insert(name.clone(), cfg).is_some() { + return Err(anyhow!("Duplicate catalog '{name}'")); + } + } + + Ok(catalogs) + } + pub async fn from_file>(path: P) -> anyhow::Result { let path_str = path.as_ref().to_string_lossy().to_string(); let content = read_to_string(path)?; @@ -64,7 +144,12 @@ impl Schedule { .as_table() .ok_or_else(|| anyhow!("Schedule file must be a TOML table"))?; - let engines = Schedule::parse_engines(toml_table).await?; + // Parse catalogs first + let catalogs = Schedule::parse_catalogs(toml_table).await?; + + // Then parse engines, passing in catalogs + let engines = Schedule::parse_engines(toml_table, &catalogs).await?; + let steps = Schedule::parse_steps(toml_table)?; Ok(Self::new(engines, steps, path_str)) @@ -108,6 +193,7 @@ impl Schedule { async fn parse_engines( table: &TomlTable, + catalog_configs: &HashMap, ) -> anyhow::Result>> { let engines_tbl = table .get("engines") @@ -116,6 +202,7 @@ impl Schedule { .ok_or_else(|| anyhow!("'engines' must be a table"))?; let mut engines = HashMap::new(); + let mut catalog_registry = CatalogRegistry::new(); for (name, engine_val) in engines_tbl { let cfg_tbl = engine_val @@ -129,7 +216,22 @@ impl Schedule { .as_str() .ok_or_else(|| anyhow::anyhow!("Engine {name} type must be a string"))?; - let engine = load_engine_runner(engine_type, cfg_tbl.clone()).await?; + // Get catalog reference (if configured) + let catalog = if let Some(catalog_name) = cfg_tbl.get("catalog") { + let catalog_name = catalog_name + .as_str() + .ok_or_else(|| anyhow!("Engine {name} catalog must be a string"))?; + + let catalog_config = catalog_configs + .get(catalog_name) + .ok_or_else(|| anyhow!("Catalog '{catalog_name}' not found"))?; + + Some(catalog_registry.get_or_create_catalog(catalog_name, catalog_config).await?) + } else { + None + }; + + let engine = load_engine_runner(engine_type, cfg_tbl.clone(), catalog).await?; if engines.insert(name.clone(), engine).is_some() { return Err(anyhow!("Duplicate engine '{name}'")); @@ -155,6 +257,7 @@ impl Schedule { #[cfg(test)] mod tests { + use std::collections::HashMap; use toml::Table as TomlTable; use crate::schedule::Schedule; @@ -200,8 +303,226 @@ mod tests { "#; let table: TomlTable = toml::from_str(toml_content).unwrap(); - let result = Schedule::parse_engines(&table).await; + let catalogs = HashMap::new(); + let result = Schedule::parse_engines(&table, &catalogs).await; + + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_parse_catalogs() { + let input = r#" + [catalogs.memory_catalog] + type = "memory" + warehouse = "memory://test" + + [catalogs.rest_catalog] + type = "rest" + uri = "http://localhost:8181" + warehouse = "s3://my-bucket/warehouse" + credential = "client_credentials" + token = "xxx" + + [catalogs.sql_catalog] + type = "sql" + uri = "postgresql://user:pass@localhost/iceberg" + warehouse = "s3://my-bucket/warehouse" + sql_bind_style = "DollarNumeric" + "#; + + let tbl: TomlTable = toml::from_str(input).unwrap(); + let catalogs = Schedule::parse_catalogs(&tbl).await.unwrap(); + + assert_eq!(catalogs.len(), 3); + assert!(catalogs.contains_key("memory_catalog")); + assert!(catalogs.contains_key("rest_catalog")); + assert!(catalogs.contains_key("sql_catalog")); + + // Verify memory catalog configuration + let memory_cfg = &catalogs["memory_catalog"]; + assert_eq!(memory_cfg.r#type, "memory"); + assert_eq!(memory_cfg.properties.get("warehouse").unwrap(), "memory://test"); + + // Verify rest catalog configuration + let rest_cfg = &catalogs["rest_catalog"]; + assert_eq!(rest_cfg.r#type, "rest"); + assert_eq!(rest_cfg.properties.get("uri").unwrap(), "http://localhost:8181"); + assert_eq!(rest_cfg.properties.get("warehouse").unwrap(), "s3://my-bucket/warehouse"); + assert_eq!(rest_cfg.properties.get("credential").unwrap(), "client_credentials"); + assert_eq!(rest_cfg.properties.get("token").unwrap(), "xxx"); + } + + #[tokio::test] + async fn test_parse_catalogs_optional() { + let input = r#" + [engines.df] + type = "datafusion" + "#; + + let tbl: TomlTable = toml::from_str(input).unwrap(); + let catalogs = Schedule::parse_catalogs(&tbl).await.unwrap(); + + assert_eq!(catalogs.len(), 0); + } + + + #[tokio::test] + async fn test_parse_catalogs_invalid_type() { + let input = r#" + [catalogs.invalid] + warehouse = "memory://test" + "#; + + let tbl: TomlTable = toml::from_str(input).unwrap(); + let result = Schedule::parse_catalogs(&tbl).await; assert!(result.is_err()); } + + #[tokio::test] + async fn test_engine_with_catalog_reference() { + let input = r#" + [catalogs.shared_catalog] + type = "memory" + warehouse = "memory://shared" + + [engines.df1] + type = "datafusion" + catalog = "shared_catalog" + + [engines.df2] + type = "datafusion" + catalog = "shared_catalog" + "#; + + let tbl: TomlTable = toml::from_str(input).unwrap(); + let catalog_configs = Schedule::parse_catalogs(&tbl).await.unwrap(); + let engines = Schedule::parse_engines(&tbl, &catalog_configs).await.unwrap(); + + assert_eq!(engines.len(), 2); + assert!(engines.contains_key("df1")); + assert!(engines.contains_key("df2")); + } + + #[tokio::test] + async fn test_engine_with_missing_catalog() { + let input = r#" + [engines.df] + type = "datafusion" + catalog = "nonexistent" + "#; + + let tbl: TomlTable = toml::from_str(input).unwrap(); + let catalog_configs = Schedule::parse_catalogs(&tbl).await.unwrap(); + let result = Schedule::parse_engines(&tbl, &catalog_configs).await; + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Catalog 'nonexistent' not found")); + } + + #[tokio::test] + async fn test_engine_without_catalog() { + let input = r#" + [engines.df] + type = "datafusion" + "#; + + let tbl: TomlTable = toml::from_str(input).unwrap(); + let catalog_configs = Schedule::parse_catalogs(&tbl).await.unwrap(); + let engines = Schedule::parse_engines(&tbl, &catalog_configs).await.unwrap(); + + assert_eq!(engines.len(), 1); + assert!(engines.contains_key("df")); + } + + #[tokio::test] + async fn test_catalog_config_validation() { + // Test that catalog configuration must have a type field + let input = r#" + [catalogs.invalid] + warehouse = "memory://test" + "#; + + let tbl: TomlTable = toml::from_str(input).unwrap(); + let result = Schedule::parse_catalogs(&tbl).await; + + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("Failed to parse catalog")); + } + + #[tokio::test] + async fn test_catalog_sharing_across_engines() { + let input = r#" + [catalogs.shared] + type = "memory" + warehouse = "memory://shared" + + [engines.engine1] + type = "datafusion" + catalog = "shared" + + [engines.engine2] + type = "datafusion" + catalog = "shared" + + [engines.engine3] + type = "datafusion" + catalog = "shared" + "#; + + let tbl: TomlTable = toml::from_str(input).unwrap(); + let catalog_configs = Schedule::parse_catalogs(&tbl).await.unwrap(); + let engines = Schedule::parse_engines(&tbl, &catalog_configs).await.unwrap(); + + assert_eq!(engines.len(), 3); + assert!(engines.contains_key("engine1")); + assert!(engines.contains_key("engine2")); + assert!(engines.contains_key("engine3")); + } + + #[tokio::test] + async fn test_mixed_catalog_usage() { + // Test that some engines use custom catalog, others use default catalog + let input = r#" + [catalogs.custom] + type = "memory" + warehouse = "memory://custom" + + [engines.with_catalog] + type = "datafusion" + catalog = "custom" + + [engines.without_catalog] + type = "datafusion" + "#; + + let tbl: TomlTable = toml::from_str(input).unwrap(); + let catalog_configs = Schedule::parse_catalogs(&tbl).await.unwrap(); + let engines = Schedule::parse_engines(&tbl, &catalog_configs).await.unwrap(); + + assert_eq!(engines.len(), 2); + assert!(engines.contains_key("with_catalog")); + assert!(engines.contains_key("without_catalog")); + } + + #[tokio::test] + async fn test_catalog_properties_preservation() { + let input = r#" + [catalogs.complex] + type = "memory" + warehouse = "memory://complex" + some_property = "value" + another_property = "42" + "#; + + let tbl: TomlTable = toml::from_str(input).unwrap(); + let catalogs = Schedule::parse_catalogs(&tbl).await.unwrap(); + + assert_eq!(catalogs.len(), 1); + let config = &catalogs["complex"]; + assert_eq!(config.r#type, "memory"); + assert_eq!(config.properties.get("warehouse").unwrap(), "memory://complex"); + assert_eq!(config.properties.get("some_property").unwrap(), "value"); + assert_eq!(config.properties.get("another_property").unwrap(), "42"); + } } diff --git a/crates/sqllogictest/testdata/schedules/catalog_config.toml b/crates/sqllogictest/testdata/schedules/catalog_config.toml new file mode 100644 index 0000000000..b1185a886a --- /dev/null +++ b/crates/sqllogictest/testdata/schedules/catalog_config.toml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Test basic catalog configuration functionality +[catalogs.test_memory] +type = "memory" +warehouse = "memory://test" + +[engines.df] +type = "datafusion" +catalog = "test_memory" + +[[steps]] +engine = "df" +slt = "df_test/show_tables.slt" diff --git a/crates/sqllogictest/testdata/schedules/error_cases.toml b/crates/sqllogictest/testdata/schedules/error_cases.toml new file mode 100644 index 0000000000..3ccf80fe77 --- /dev/null +++ b/crates/sqllogictest/testdata/schedules/error_cases.toml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This file is used to test error cases and should not be executed under normal circumstances +# It will be automatically discovered by the integration test framework, but we need to ensure it handles errors correctly + +[catalogs.valid] +type = "memory" +warehouse = "memory://valid" + +[engines.df] +type = "datafusion" +catalog = "valid" + +[[steps]] +engine = "df" +slt = "df_test/show_tables.slt" diff --git a/crates/sqllogictest/testdata/schedules/shared_catalog.toml b/crates/sqllogictest/testdata/schedules/shared_catalog.toml new file mode 100644 index 0000000000..df3f780252 --- /dev/null +++ b/crates/sqllogictest/testdata/schedules/shared_catalog.toml @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Test multiple engines sharing the same catalog +[catalogs.shared] +type = "memory" +warehouse = "memory://shared" + +[engines.writer] +type = "datafusion" +catalog = "shared" + +[engines.reader] +type = "datafusion" +catalog = "shared" + +[[steps]] +engine = "writer" +slt = "shared_test/create_table.slt" + +[[steps]] +engine = "reader" +slt = "shared_test/query_table.slt" diff --git a/crates/sqllogictest/testdata/slts/shared_test/create_table.slt b/crates/sqllogictest/testdata/slts/shared_test/create_table.slt new file mode 100644 index 0000000000..f06c5005a2 --- /dev/null +++ b/crates/sqllogictest/testdata/slts/shared_test/create_table.slt @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Verify that the writer engine can access the catalog +statement ok +SHOW TABLES; diff --git a/crates/sqllogictest/testdata/slts/shared_test/query_table.slt b/crates/sqllogictest/testdata/slts/shared_test/query_table.slt new file mode 100644 index 0000000000..abb403eb73 --- /dev/null +++ b/crates/sqllogictest/testdata/slts/shared_test/query_table.slt @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Verify that the reader engine can access the same catalog +statement ok +SHOW TABLES; diff --git a/crates/sqllogictest/tests/sqllogictests.rs b/crates/sqllogictest/tests/sqllogictests.rs index 434d9d78cc..91f70fa637 100644 --- a/crates/sqllogictest/tests/sqllogictests.rs +++ b/crates/sqllogictest/tests/sqllogictests.rs @@ -85,3 +85,72 @@ pub(crate) async fn run_schedule(schedule_file: PathBuf) -> anyhow::Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + + #[tokio::test] + async fn test_catalog_config_schedule_execution() { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + + let schedule_path = PathBuf::from(format!( + "{}/testdata/schedules/catalog_config.toml", + env!("CARGO_MANIFEST_DIR") + )); + + let result = rt.block_on(run_schedule(schedule_path)); + assert!(result.is_ok(), "Catalog config schedule should execute successfully"); + } + + #[tokio::test] + async fn test_shared_catalog_schedule_execution() { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + + let schedule_path = PathBuf::from(format!( + "{}/testdata/schedules/shared_catalog.toml", + env!("CARGO_MANIFEST_DIR") + )); + + let result = rt.block_on(run_schedule(schedule_path)); + assert!(result.is_ok(), "Shared catalog schedule should execute successfully"); + } + + #[tokio::test] + async fn test_error_cases_schedule_execution() { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + + let schedule_path = PathBuf::from(format!( + "{}/testdata/schedules/error_cases.toml", + env!("CARGO_MANIFEST_DIR") + )); + + let result = rt.block_on(run_schedule(schedule_path)); + assert!(result.is_ok(), "Error cases schedule should execute successfully"); + } + + #[test] + fn test_collect_schedule_files() { + let files = collect_schedule_files().unwrap(); + assert!(!files.is_empty(), "Should find at least some schedule files"); + + // Verify that the newly created files are included + let file_names: Vec = files + .iter() + .filter_map(|p| p.file_name()?.to_str()) + .map(|s| s.to_string()) + .collect(); + + assert!(file_names.contains(&"catalog_config.toml".to_string())); + assert!(file_names.contains(&"shared_catalog.toml".to_string())); + assert!(file_names.contains(&"error_cases.toml".to_string())); + } +}