From 2ec51784892e2ade5070e926f533327fbfc8a61d Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Thu, 10 Apr 2025 17:18:44 +0200 Subject: [PATCH 1/4] force import using gti --- Cargo.lock | 25 +++-- Cargo.toml | 6 +- datatypes/src/spatial_reference.rs | 2 +- operators/src/source/ogr_source/mod.rs | 25 +++-- services/examples/force-import.rs | 145 +++++++++++++++++++++++++ 5 files changed, 180 insertions(+), 23 deletions(-) create mode 100644 services/examples/force-import.rs diff --git a/Cargo.lock b/Cargo.lock index 20d4817f7..b613cea24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2214,27 +2214,36 @@ dependencies = [ [[package]] name = "gdal" -version = "0.17.1" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82ab834e8be6b54fee3d0141fce5e776ad405add1f9d0da054281926e0d35a9f" +checksum = "e721cea67b420fd4b5cb15ba8145f2f1d3a6931a27fdbfadb46cff02015e1cde" dependencies = [ "bitflags 2.9.0", "chrono", "gdal-sys", "geo-types", - "libc", - "once_cell", "semver", - "thiserror 1.0.69", + "thiserror 2.0.12", +] + +[[package]] +name = "gdal-src" +version = "0.2.0+3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfe065be4c59457d934487706b6ffa7fe91d238e20e42b8fae03f47a291f684" +dependencies = [ + "cmake", + "link-cplusplus", + "proj-sys", ] [[package]] name = "gdal-sys" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18ad5d608ee6726efcf6e1d91261eb6dec7da3ee7db6bda984cdfb8a7d65ebf9" +checksum = "febef67dc08a956a9ecb04de2b40dbd15ad56be49421aad9ae0cdcbe9a24166c" dependencies = [ - "libc", + "gdal-src", "pkg-config", "semver", ] diff --git a/Cargo.toml b/Cargo.toml index c473ee524..a99c658dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,8 +101,10 @@ flexi_logger = { version = "0.29", features = ["trc"] } float-cmp = "0.10" futures = "0.3" futures-util = "0.3" -gdal = "0.17" -gdal-sys = "0.10" +gdal = "0.18" +gdal-sys = { version = "0.11", features = ["bundled"] } +gdal-src = { version = "0.2.0+3.10.2", features = ["driver_gpkg"] } + # when changing geo version also adapt the Cargo.toml in the expression "deps-workspace"! geo = "0.29.3" geo-rand = "0.4" diff --git a/datatypes/src/spatial_reference.rs b/datatypes/src/spatial_reference.rs index 94e1f6458..fae01f7d8 100644 --- a/datatypes/src/spatial_reference.rs +++ b/datatypes/src/spatial_reference.rs @@ -222,7 +222,7 @@ impl TryFrom for SpatialReference { fn try_from(value: SpatialRef) -> Result { Ok(SpatialReference::new( - SpatialReferenceAuthority::from_str(&value.auth_name()?)?, + SpatialReferenceAuthority::from_str(&value.auth_name().unwrap())?, // TODO value.auth_code()? as u32, )) } diff --git a/operators/src/source/ogr_source/mod.rs b/operators/src/source/ogr_source/mod.rs index 341aaa0c0..ef1a43804 100644 --- a/operators/src/source/ogr_source/mod.rs +++ b/operators/src/source/ogr_source/mod.rs @@ -979,7 +979,7 @@ where let time_start_parser = Self::create_time_parser(start_format); Box::new(move |feature: &Feature| { - let field_value = feature.field(&start_field)?; + let field_value = feature.field(feature.field_index(&start_field)?)?; // TODO: resolve field index from field name only once if let Some(field_value) = field_value { let time_start = time_start_parser(field_value)?; TimeInterval::new(time_start, (time_start + duration)?).map_err(Into::into) @@ -999,8 +999,8 @@ where let time_end_parser = Self::create_time_parser(end_format); Box::new(move |feature: &Feature| { - let start_field_value = feature.field(&start_field)?; - let end_field_value = feature.field(&end_field)?; + let start_field_value = feature.field(feature.field_index(&start_field)?)?; + let end_field_value = feature.field(feature.field_index(&end_field)?)?; if let (Some(start_field_value), Some(end_field_value)) = (start_field_value, end_field_value) @@ -1023,8 +1023,9 @@ where let time_start_parser = Self::create_time_parser(start_format); Box::new(move |feature: &Feature| { - let start_field_value = feature.field(&start_field)?; - let duration_field_value = feature.field(&duration_field)?; + let start_field_value = feature.field(feature.field_index(&start_field)?)?; + let duration_field_value = + feature.field(feature.field_index(&duration_field)?)?; if let (Some(start_field_value), Some(duration_field_value)) = (start_field_value, duration_field_value) @@ -1300,7 +1301,7 @@ where builder.push_time_interval(time_interval); for (column, data_type) in data_types { - let field = feature.field(column); + let field = feature.field(feature.field_index(column)?); let value = Self::convert_field_value(*data_type, field, time_attribute_parser, error_spec)?; builder.push_data(column, value)?; @@ -1425,9 +1426,9 @@ impl TryFromOgrGeometry for MultiPoint { impl TryFromOgrGeometry for MultiLineString { fn try_from(geometry: Result<&gdal::vector::Geometry>) -> Result { fn coordinates(geometry: &gdal::vector::Geometry) -> Vec { - geometry - .get_point_vec() - .into_iter() + let mut vec = Vec::new(); // TODO capacity + geometry.get_points(&mut vec); + vec.into_iter() .map(|(x, y, _z)| Coordinate2D::new(x, y)) .collect() } @@ -1454,9 +1455,9 @@ impl TryFromOgrGeometry for MultiLineString { impl TryFromOgrGeometry for MultiPolygon { fn try_from(geometry: Result<&gdal::vector::Geometry>) -> Result { fn coordinates(geometry: &gdal::vector::Geometry) -> Vec { - geometry - .get_point_vec() - .into_iter() + let mut vec = Vec::new(); // TODO capacity + geometry.get_points(&mut vec); + vec.into_iter() .map(|(x, y, _z)| Coordinate2D::new(x, y)) .collect() } diff --git a/services/examples/force-import.rs b/services/examples/force-import.rs new file mode 100644 index 000000000..e00c1543b --- /dev/null +++ b/services/examples/force-import.rs @@ -0,0 +1,145 @@ +use gdal::vector::{Defn, Feature, FieldDefn, LayerOptions, OGRFieldType}; +use gdal::{Dataset as GdalDataset, DriverManager}; +use std::path::Path; + +/// Creates a tile index for the given datasets and writes it to a GeoJSON file. +/// uses the SRS from the first dataset +fn gdaltindex(datasets: &[&Path], gti_file: &Path, tile_index_file: &Path) { + let spatial_ref = { + let first_dataset = GdalDataset::open(datasets.get(0).expect("datasets must not be empty")) + .expect("Failed to open dataset"); + first_dataset + .spatial_ref() + .expect("Failed to get spatial reference") + }; + + let driver = + DriverManager::get_driver_by_name("GeoJSON").expect("Failed to get GeoJSON driver"); + let mut vector_ds = driver + .create_vector_only(tile_index_file) + .expect("Failed to create vector dataset"); + + let layer = vector_ds + .create_layer(LayerOptions { + name: "tile_index", + srs: Some(&spatial_ref), + ty: gdal::vector::OGRwkbGeometryType::wkbPolygon, + options: None, + }) + .expect("Failed to create layer"); + + let field_defn = + FieldDefn::new("location", OGRFieldType::OFTString).expect("Failed to create field defn"); + // field_defn.set_width(80); + field_defn + .add_to_layer(&layer) + .expect("Failed to add field to layer"); + + let defn = Defn::from_layer(&layer); + + let location_idx = defn + .field_index("location") + .expect("Failed to get field index"); + + for dataset_path in datasets { + let dataset = GdalDataset::open(dataset_path).expect("Failed to open dataset"); + let geo_transform = dataset + .geo_transform() + .expect("Failed to get geo-transform"); + let raster_size = dataset.raster_size(); + + // TODO: get bbox from gdal? + let min_x = geo_transform[0]; + let max_y = geo_transform[3]; + let max_x = min_x + geo_transform[1] * raster_size.0 as f64; + let min_y = max_y + geo_transform[5] * raster_size.1 as f64; + + let mut ring = + gdal::vector::Geometry::empty(gdal::vector::OGRwkbGeometryType::wkbLinearRing) + .expect("Failed to create ring"); + + // TODO: reproject bbox to output srs(?) + ring.add_point_2d((min_x, max_y)); + ring.add_point_2d((max_x, max_y)); + ring.add_point_2d((max_x, min_y)); + ring.add_point_2d((min_x, min_y)); + ring.add_point_2d((min_x, max_y)); + + let mut polygon = + gdal::vector::Geometry::empty(gdal::vector::OGRwkbGeometryType::wkbPolygon) + .expect("Failed to create polygon"); + polygon + .add_geometry(ring) + .expect("Failed to add ring to polygon"); + + let mut feature = Feature::new(&defn).expect("Failed to create feature"); + feature + .set_geometry(polygon) + .expect("Failed to set geometry"); + feature + .set_field_string( + location_idx, + dataset_path + .to_str() + .expect("Failed to convert path to string"), + ) + .expect("Failed to set field"); + + feature.create(&layer).expect("Failed to create feature"); + } + + let gti_xml = format!( + r" + {index_dataset} + tile_index + location +", + index_dataset = tile_index_file + .to_str() + .expect("Failed to convert path to string"), + ); + + std::fs::write(gti_file, gti_xml).expect("Failed to write GTI XML to file"); +} + +fn main() { + let datasets: [&Path; 2] = [ + Path::new( + "/mnt/data_raid/geo_data/force/gti/data/X0059_Y0049/20000124_LEVEL2_LND07_BOA.tif", + ), + Path::new( + "/mnt/data_raid/geo_data/force/gti/data/X0059_Y0049/20000124_LEVEL2_LND07_BOA.tif", + ), + ]; + + let gti_file = Path::new("./foo.gti"); + let tile_index_file = Path::new("./foo.tile_index.geojson"); + + gdaltindex(&datasets, gti_file, tile_index_file); + + let gti_dataset = GdalDataset::open(gti_file).expect("Failed to open GTI dataset"); + + let raster_band = gti_dataset.rasterband(1).unwrap(); + let shape = raster_band.size(); + + let mut data = raster_band + .read_as::((0, 0), shape, shape, None) + .unwrap(); + + let driver = DriverManager::get_driver_by_name("GTiff").unwrap(); + let mut new_ds = driver + .create_with_band_type::( + "output.tif", + shape.0, + shape.1, + 1, // Number of bands + ) + .unwrap(); + new_ds + .set_spatial_ref(>i_dataset.spatial_ref().unwrap()) + .unwrap(); + + // Write the data to the new dataset + let mut new_band = new_ds.rasterband(1).unwrap(); + new_band.write((0, 0), shape, &mut data).unwrap(); +} From 25d6cfdbf5db4608727603cabc1c9b5832d53165 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Tue, 29 Apr 2025 10:33:20 +0200 Subject: [PATCH 2/4] generation of gdal (gti) and geoengine dataset definitions --- operators/src/source/gdal_source/mod.rs | 4 +- services/examples/force-import.rs | 279 +++++++++++++++++++++++- services/src/api/handlers/wms.rs | 6 + test_data/api_calls/force.http | 47 ++++ 4 files changed, 332 insertions(+), 4 deletions(-) create mode 100644 test_data/api_calls/force.http diff --git a/operators/src/source/gdal_source/mod.rs b/operators/src/source/gdal_source/mod.rs index df3c4ffba..9d3d6c4e7 100644 --- a/operators/src/source/gdal_source/mod.rs +++ b/operators/src/source/gdal_source/mod.rs @@ -657,7 +657,9 @@ where let mut empty = false; if !tiling_based_pixel_bounds.intersects(&query_pixel_bounds) { - debug!("query does not intersect spatial data bounds"); + debug!( + "query {query_pixel_bounds:?} does not intersect spatial data bounds {tiling_based_pixel_bounds:?}" + ); empty = true; } diff --git a/services/examples/force-import.rs b/services/examples/force-import.rs index e00c1543b..651c29893 100644 --- a/services/examples/force-import.rs +++ b/services/examples/force-import.rs @@ -1,6 +1,19 @@ +use chrono::{NaiveDate, TimeZone, naive}; use gdal::vector::{Defn, Feature, FieldDefn, LayerOptions, OGRFieldType}; -use gdal::{Dataset as GdalDataset, DriverManager}; +use gdal::{Dataset as GdalDataset, DriverManager, Metadata}; +use geoengine_datatypes::primitives::{CacheTtlSeconds, DateTime, TimeInstance, TimeInterval}; +use geoengine_datatypes::raster::GdalGeoTransform; +use geoengine_operators::source::{ + FileNotFoundHandling, GdalDatasetParameters, GdalLoadingInfoTemporalSlice, GdalMetaDataList, +}; +use geoengine_operators::util::gdal::raster_descriptor_from_dataset; +use geoengine_services::datasets::storage::{DatasetDefinition, MetaDataDefinition}; +use geoengine_services::datasets::{AddDataset, DatasetName}; +use std::collections::HashMap; +use std::fs; use std::path::Path; +use std::path::PathBuf; +use std::str::FromStr; /// Creates a tile index for the given datasets and writes it to a GeoJSON file. /// uses the SRS from the first dataset @@ -102,7 +115,8 @@ fn gdaltindex(datasets: &[&Path], gti_file: &Path, tile_index_file: &Path) { std::fs::write(gti_file, gti_xml).expect("Failed to write GTI XML to file"); } -fn main() { +#[allow(dead_code)] +fn test() { let datasets: [&Path; 2] = [ Path::new( "/mnt/data_raid/geo_data/force/gti/data/X0059_Y0049/20000124_LEVEL2_LND07_BOA.tif", @@ -119,7 +133,9 @@ fn main() { let gti_dataset = GdalDataset::open(gti_file).expect("Failed to open GTI dataset"); - let raster_band = gti_dataset.rasterband(1).unwrap(); + let raster_band = gti_dataset + .rasterband(1) + .expect("Failed to get raster band"); let shape = raster_band.size(); let mut data = raster_band @@ -143,3 +159,260 @@ fn main() { let mut new_band = new_ds.rasterband(1).unwrap(); new_band.write((0, 0), shape, &mut data).unwrap(); } + +fn naive_date_to_time_instance(date: NaiveDate) -> TimeInstance { + let time: chrono::DateTime = chrono::Utc.from_utc_datetime( + &date + .and_hms_opt(0, 0, 0) + .expect("Failed to create datetime"), + ); + let time: DateTime = time.into(); + time.into() +} + +fn main() { + const FORCE_DATA_DIR: &str = "/home/michael/geodata/force/marburg"; + const GTI_OUTPUT_DIR: &str = "/home/michael/geodata/force/marburg/gti"; + const DATASET_OUTPUT_DIR: &str = "/home/michael/geodata/force/marburg/geoengine"; + + let mut tile_dirs = Vec::new(); + + let entries = fs::read_dir(&FORCE_DATA_DIR).expect("Failed to read FORCE_DATA_DIR"); + + for entry in entries { + let entry = entry.expect("Failed to read directory entry"); + if entry.file_type().expect("Failed to get file type").is_dir() { + let folder_name = entry.file_name(); + // TODO: parse pattern X0059_Y0049 + if folder_name.to_string_lossy().starts_with('X') { + tile_dirs.push(entry.path()); + } + } + } + + println!("Found tiles: {:?}", tile_dirs); + + let mut tif_files = Vec::new(); + + for tile_dir in tile_dirs { + let entries = fs::read_dir(&tile_dir).expect("Failed to read tile directory"); + for entry in entries { + let entry = entry.expect("Failed to read directory entry"); + if entry + .file_type() + .expect("Failed to get file type") + .is_file() + { + if let Some(extension) = entry.path().extension() { + if extension == "tif" { + tif_files.push(entry.path()); + } + } + } + } + } + + println!("Found {:?} tif files", tif_files.len()); + + let mut product_dataset_timesteps: HashMap<(String, String), HashMap>> = + HashMap::new(); + + for tif in &tif_files { + if let Some(filename) = tif.file_name().and_then(|f| f.to_str()) { + let parts: Vec<&str> = filename.split('_').collect(); + if parts.len() >= 4 { + if let Ok(date_obj) = NaiveDate::parse_from_str(parts[0], "%Y%m%d") { + let product = parts[2].to_string(); + let band = parts[3].split('.').next().unwrap_or("").to_string(); + let key = (product, band); + + product_dataset_timesteps + .entry(key) + .or_insert_with(HashMap::new) + .entry(date_obj) + .or_insert_with(Vec::new) + .push(tif.clone()); + } + } + } + } + + println!("Found {} products:", product_dataset_timesteps.len()); + for ((product, dataset), timesteps) in &product_dataset_timesteps { + println!( + " ({}, {}), #timesteps: {}", + product, + dataset, + timesteps.len() + ); + } + + let mut product_dataset_bands: HashMap<(String, String), Vec> = HashMap::new(); + + for ((product, dataset), timesteps) in &product_dataset_timesteps { + if let Some(first_file) = timesteps.values().flat_map(|v| v).next() { + let gdal_dataset = GdalDataset::open(first_file).expect("Failed to open dataset"); + let band_count = gdal_dataset.raster_count(); + + print!("Available bands for {product} {dataset}: ",); + + let mut bands = Vec::new(); + + for band_index in 1..=band_count { + let band = gdal_dataset + .rasterband(band_index) + .expect("Failed to get raster band"); + + let band_desc = band.description().unwrap_or("No description".to_string()); + + bands.push(band_desc.clone()); + + print!("{band_desc}"); + + if band_index < band_count { + print!(", "); + } + } + + product_dataset_bands.insert((product.clone(), dataset.clone()), bands); + println!(); + } + } + + let product_datasets = product_dataset_timesteps.keys().collect::>(); + + let mut product_dataset_gtis: HashMap<(String, String), Vec<(NaiveDate, String)>> = + HashMap::new(); + + for (product, dataset) in product_datasets { + let mut timestep_gtis = Vec::new(); + + for (timestep, tiles) in product_dataset_timesteps + .get(&(product.clone(), dataset.clone())) + .expect("Failed to get timesteps") + { + let gti_file = format!( + "{}/{}_{}_{}.gti", + GTI_OUTPUT_DIR, product, dataset, timestep + ); + let tile_index_file = format!( + "{}/{}_{}_{}.tile_index.geojson", + GTI_OUTPUT_DIR, product, dataset, timestep + ); + + let tile_refs: Vec<&Path> = tiles.iter().map(|tile| tile.as_path()).collect(); + gdaltindex(&tile_refs, gti_file.as_ref(), tile_index_file.as_ref()); + + timestep_gtis.push((timestep.clone(), gti_file)); + } + + // Sort the vector by NaiveDate + timestep_gtis.sort_by_key(|(timestep, _)| *timestep); + + println!( + "Created {} GTIs for {product} {dataset}", + timestep_gtis.len() + ); + + product_dataset_gtis.insert((product.clone(), dataset.clone()), timestep_gtis); + } + + for ((product, dataset), timesteps) in &product_dataset_gtis { + for (band_idx, band_name) in product_dataset_bands + .get(&(product.clone(), dataset.clone())) + .expect("Failed to get bands") + .iter() + .enumerate() + .map(|(i, band)| (i + 1, band)) + { + let gdal_dataset = GdalDataset::open(Path::new( + ×teps + .iter() + .next() + .expect("Failed to get first timestep") + .1, + )) + .expect("Failed to open dataset"); + + let geo_transform: GdalGeoTransform = gdal_dataset + .geo_transform() + .expect("Failed to get geo-transform") + .into(); + let geo_transform: geoengine_operators::source::GdalDatasetGeoTransform = + geo_transform.into(); + + let result_descriptor = raster_descriptor_from_dataset(&gdal_dataset, band_idx) + .expect("Could not get raster descriptor"); + + let mut slices = Vec::new(); + + for (i, (timestep, file)) in timesteps.iter().enumerate() { + let start_time = naive_date_to_time_instance(*timestep); + + let end_time: TimeInstance = if let Some((next_timestep, _)) = timesteps.get(i + 1) + { + naive_date_to_time_instance(*next_timestep) + } else { + TimeInstance::MAX + }; + + let time_interval: TimeInterval = TimeInterval::new(start_time, end_time) + .expect("Failed to create time interval"); + + let rasterband = gdal_dataset + .rasterband(band_idx) + .expect("Failed to get raster band"); + + slices.push(GdalLoadingInfoTemporalSlice { + time: time_interval, + params: Some(GdalDatasetParameters { + file_path: file.into(), + rasterband_channel: band_idx as usize, + geo_transform: geo_transform.try_into().unwrap(), + width: rasterband.size().0, + height: rasterband.size().1, + file_not_found_handling: FileNotFoundHandling::Error, + no_data_value: None, // TODO + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: false, + retry: None, + }), + cache_ttl: CacheTtlSeconds::new(0), + }); + } + + let dataset_def = DatasetDefinition { + properties: AddDataset { + name: Some( + DatasetName::from_str(&format!("{product}_{dataset}_{band_idx}")) + .expect("Failed to create dataset name"), + ), + display_name: format!("{product} {dataset} {band_name}"), + description: format!("{} {} {}", product, dataset, band_name), + source_operator: "GdalSource".to_string(), + symbology: None, + provenance: None, + tags: None, + }, + meta_data: MetaDataDefinition::GdalMetaDataList(GdalMetaDataList { + result_descriptor: result_descriptor.into(), + params: slices, + }), + }; + + let output_file = format!( + "{}/{}_{}_{}_{}.json", + DATASET_OUTPUT_DIR, product, dataset, band_name, band_idx + ); + + let json = serde_json::to_string_pretty(&dataset_def) + .expect("Failed to serialize dataset definition to JSON"); + + std::fs::write(&output_file, json).expect("Failed to write dataset definition to file"); + + println!("Saved dataset definition to {}", output_file); + } + } +} diff --git a/services/src/api/handlers/wms.rs b/services/src/api/handlers/wms.rs index 28e9e5f68..8047d95d7 100644 --- a/services/src/api/handlers/wms.rs +++ b/services/src/api/handlers/wms.rs @@ -28,6 +28,7 @@ use snafu::ensure; use std::str::FromStr; use std::time::Duration; use tracing::debug; +use tracing_subscriber::field::debug; use uuid::Uuid; pub(crate) fn init_wms_routes(cfg: &mut web::ServiceConfig) @@ -276,10 +277,15 @@ async fn wms_map_handler( request.crs.ok_or(error::Error::MissingSpatialReference)?; let request_bounds: SpatialPartition2D = request.bbox.bounds(request_spatial_ref)?; + + debug!("WMS request bounds: {:?}", request_bounds); + let x_request_res = request_bounds.size_x() / f64::from(request.width); let y_request_res = request_bounds.size_y() / f64::from(request.height); let request_resolution = SpatialResolution::new(x_request_res.abs(), y_request_res.abs())?; + debug!("WMS request resolution: {:?}", request_resolution); + let raster_colorizer = raster_colorizer_from_style(&request.styles)?; let ctx = app_ctx.session_context(session); diff --git a/test_data/api_calls/force.http b/test_data/api_calls/force.http new file mode 100644 index 000000000..fa8ad3515 --- /dev/null +++ b/test_data/api_calls/force.http @@ -0,0 +1,47 @@ + +### + +# @name anonymousSession +POST http://localhost:3030/api/anonymous +Content-Type: application/json + +### + +# @name workflow +POST http://localhost:3030/api/workflow +Authorization: Bearer {{anonymousSession.response.body.$.id}} +Content-Type: application/json + +{ + "type": "Raster", + "operator": { + "type": "GdalSource", + "params": { + "data": "LND05_BOA_1" + } + } +} + +### + +@workflowId = {{workflow.response.body.$.id}} +@width=1000 +@height=2000 +@time = 2017-07-04T00%3A00%3A00.000Z +@minx = 4226026.3630416505 +@miny = 3044919.6079648044 + +# pixel size: 30 +@maxx = 4256026.3630416505 +@maxy = 3104919.6079648044 + +@colorizer_min = -1000 +@colorizer_max = 10000 +@styles = custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A{{colorizer_min}}%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A{{colorizer_max}}%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B246%2C250%2C254%2C255%5D%2C%22underColor%22%3A%5B247%2C251%2C255%2C255%5D%7D%7D + + +### + +# native res, full bbox +GET http://localhost:3030/api/wms/{{workflowId}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES={{styles}}&TRANSPARENT=true&layers={{workflowId}}&time={{time}}&EXCEPTIONS=application%2Fjson&WIDTH={{width}}&HEIGHT={{height}}&CRS=EPSG:3035&BBOX={{miny}}%2C{{minx}}%2C{{maxy}}%2C{{maxx}} +Authorization: Bearer {{anonymousSession.response.body.$.id}} \ No newline at end of file From 70593756642f0d59bad0847706c8cdac9779f9b2 Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Fri, 9 May 2025 10:54:30 +0200 Subject: [PATCH 3/4] debugging error in filler adapter --- .../raster_subquery_adapter.rs | 8 +- .../raster_subquery_reprojection.rs | 65 +++++++-- .../src/adapters/sparse_tiles_fill_adapter.rs | 89 ++++++++++-- operators/src/error.rs | 2 +- operators/src/processing/downsample/mod.rs | 6 + operators/src/processing/reprojection.rs | 128 +++++++++++++++++- .../src/source/gdal_source/loading_info.rs | 4 + operators/src/source/gdal_source/mod.rs | 5 +- services/src/api/handlers/wms.rs | 2 +- test_data/api_calls/force/downsampled.http | 44 ++++++ .../api_calls/force/filladaptererror.http | 28 ++++ test_data/api_calls/{ => force}/force.http | 8 +- test_data/api_calls/force/reprojected.http | 43 ++++++ ...MB_20220129_0_L2A__B02.tif_downsampled.tif | Bin 0 -> 20484 bytes 14 files changed, 395 insertions(+), 37 deletions(-) create mode 100644 test_data/api_calls/force/downsampled.http create mode 100644 test_data/api_calls/force/filladaptererror.http rename test_data/api_calls/{ => force}/force.http (86%) create mode 100644 test_data/api_calls/force/reprojected.http create mode 100644 test_data/raster/sentinel2/S2B_32UMB_20220129_0_L2A__B02.tif_downsampled.tif diff --git a/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs b/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs index 64d6a6bc6..061a0a7f2 100644 --- a/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs +++ b/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs @@ -23,6 +23,7 @@ use geoengine_datatypes::{ primitives::TimeInstance, raster::{Pixel, RasterTile2D, TileInformation}, }; +use log::debug; use pin_project::pin_project; use rayon::ThreadPool; use std::marker::PhantomData; @@ -289,6 +290,7 @@ where } Ok(None) => this.state.set(StateInner::ReturnResult(None)), Err(e) => { + debug!(">>>> Tile query rectangle not valid: {e}"); this.state.set(StateInner::Ended); return Poll::Ready(Some(Err(e))); } @@ -382,6 +384,7 @@ where // If there is a tile, set the current_time_end option. if let Some(tile) = &tile_option { + debug_assert!(tile.time.end() > tile.time.start()); debug_assert!(*this.current_time_start >= tile.time.start()); *this.current_time_end = Some(tile.time.end()); } @@ -408,6 +411,7 @@ where (None, None) => { // end the stream since we never recieved a tile from any subquery. Should only happen if we end the first grid iteration. // NOTE: this assumes that the input operator produces no data tiles for queries where time and space are valid but no data is avalable. + debug!(">>>> Tile stream ended without any data"); debug_assert!(&tile_option.is_none()); debug_assert!( *this.current_time_start == this.query_rect_to_answer.time_interval.start() @@ -437,7 +441,9 @@ where } } } - + if let Some(tile) = &tile_option { + debug_assert!(tile.time.end() > tile.time.start()); + } Poll::Ready(Some(Ok(tile_option))) } } diff --git a/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs b/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs index 30e3168d3..a3f5b60ee 100644 --- a/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs +++ b/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs @@ -95,12 +95,22 @@ where self.state.out_spatial_grid.geo_transform() ); - let valid_pixel_bounds = self - .state - .out_spatial_grid - .grid_bounds() - .intersection(&tile_info.global_pixel_bounds()) - .and_then(|b| b.intersection(&query_rect.spatial_query.grid_bounds())); + // why is this none? + let valid_pixel_bounds = dbg!( + dbg!( + self.state + .out_spatial_grid + .grid_bounds() + .intersection(&tile_info.global_pixel_bounds()) + ) + .and_then(|b| b.intersection(&query_rect.spatial_query.grid_bounds())) + ); + + log::debug!( + "ÖÖÖÖÖ valid_pixel_bounds {:?} -> {:?}", + tile_info.global_pixel_bounds(), + valid_pixel_bounds + ); let valid_spatial_bounds = valid_pixel_bounds.map(|pb| { self.state @@ -109,25 +119,49 @@ where .grid_to_spatial_bounds(&pb) }); + log::debug!( + "ÖÖÖÖÖ valid_spatial_bounds {:?} -> {:?}", + query_rect.spatial_query.grid_bounds(), + valid_spatial_bounds + ); + if let Some(bounds) = valid_spatial_bounds { let proj = CoordinateProjector::from_known_srs(self.out_srs, self.in_srs)?; let projected_bounds = bounds.reproject(&proj); + log::debug!( + "ÖÖÖÖÖ projected_bounds {:?} -> {:?}", + bounds, + projected_bounds + ); + match projected_bounds { - Ok(pb) => Ok(Some(RasterQueryRectangle::new_with_grid_bounds( - self.state - .in_spatial_grid - .geo_transform() - .spatial_to_grid_bounds(&pb), - TimeInterval::new_instant(start_time)?, - band_idx.into(), - ))), + Ok(pb) => { + dbg!("produce something"); + Ok(Some(RasterQueryRectangle::new_with_grid_bounds( + self.state + .in_spatial_grid + .geo_transform() + .spatial_to_grid_bounds(&pb), + TimeInterval::new_instant(start_time)?, + band_idx.into(), + ))) + } // In some strange cases the reprojection can return an empty box. // We ignore it since it contains no pixels. Err(geoengine_datatypes::error::Error::OutputBboxEmpty { bbox: _ }) => Ok(None), Err(e) => Err(e.into()), } } else { + dbg!("output query rectangle is not valid in source projection => produce empty tile"); + log::debug!( + "ÖÖÖÖÖ output query rectangle is not valid in source projection => produce empty tile {:?}", + self.state + .out_spatial_grid + .geo_transform() + .grid_to_spatial_bounds(&query_rect.spatial_query.grid_bounds()) + ); + // output query rectangle is not valid in source projection => produce empty tile Ok(None) } @@ -352,6 +386,8 @@ impl FoldTileAccu for TileWithProjectionCoordinates { type RasterType = T; async fn into_tile(self) -> Result> { + debug_assert!(self.accu_tile.time.end() > self.accu_tile.time.start()); + debug_assert!(self.accu_tile.time.end() != self.accu_tile.time.start() + 1); Ok(self.accu_tile) } @@ -362,6 +398,7 @@ impl FoldTileAccu for TileWithProjectionCoordinates { impl FoldTileAccuMut for TileWithProjectionCoordinates { fn set_time(&mut self, time: TimeInterval) { + debug_assert!(time.end() > time.start()); self.accu_tile.time = time; } diff --git a/operators/src/adapters/sparse_tiles_fill_adapter.rs b/operators/src/adapters/sparse_tiles_fill_adapter.rs index b37eab04e..96783b267 100644 --- a/operators/src/adapters/sparse_tiles_fill_adapter.rs +++ b/operators/src/adapters/sparse_tiles_fill_adapter.rs @@ -10,6 +10,7 @@ use geoengine_datatypes::{ use pin_project::pin_project; use snafu::Snafu; use std::{pin::Pin, task::Poll}; +use tracing::debug; #[derive(Debug, Snafu)] pub enum SparseTilesFillAdapterError { @@ -61,7 +62,11 @@ impl From for FillerTimeBounds { fn from(time: TimeInterval) -> FillerTimeBounds { FillerTimeBounds { start: time.start(), - end: time.end(), + end: if time.is_instant() { + time.end() + 1 + } else { + time.end() + }, } } } @@ -74,11 +79,14 @@ impl FillerTimeBounds { self.end } + // TODO: return result pub fn new(start: TimeInstance, end: TimeInstance) -> Self { + debug_assert!(start < end); Self::new_unchecked(start, end) } pub fn new_unchecked(start: TimeInstance, end: TimeInstance) -> Self { + debug_assert!(start < end); Self { start, end } } } @@ -107,6 +115,12 @@ struct GridIdxAndBand { impl StateContainer { /// Create a new no-data `RasterTile2D` with `GridIdx` and time from the current state fn current_no_data_tile(&self) -> RasterTile2D { + // debug_assert!( + // self.current_time.unwrap().end() != self.current_time.unwrap().start() + 1, + // "current no data tile, current_time: {:?}, current tile time: {:?}", + // self.current_time, + // self.next_tile.as_ref().map(|t| t.time) + // ); RasterTile2D::new( self.current_time .expect("time must exist when a tile is stored."), @@ -231,6 +245,12 @@ impl StateContainer { } fn next_time_interval_from_stored_tile(&self) -> Option { + debug!( + "filladapter ding current time {:?}, stored tile time {:?}", + self.current_time, + self.next_tile.as_ref().map(|t| t.time) + ); + // we wrapped around. We need to do time progress. if let Some(tile) = &self.next_tile { let stored_tile_time = tile.time; @@ -263,6 +283,8 @@ impl StateContainer { } fn set_current_time_from_initial_tile(&mut self, first_tile_time: TimeInterval) { + debug_assert!(first_tile_time.end() > first_tile_time.start()); + debug_assert!(first_tile_time.end() != first_tile_time.start() + 1); // if we know a bound we must use it to set the current time let start_data_bound = self.data_time_bounds.start(); let requested_start = self.requested_time_bounds.start(); @@ -279,10 +301,9 @@ impl StateContainer { requested_start, start_data_bound ); - self.current_time = Some(TimeInterval::new_unchecked( - start_data_bound, - first_tile_time.start(), - )); + self.current_time = + Some(TimeInterval::new(start_data_bound, first_tile_time.start()).unwrap()); + debug_assert!(!self.current_time.unwrap().is_instant()); return; } if start_data_bound > first_tile_time.start() { @@ -297,10 +318,11 @@ impl StateContainer { fn set_current_time_from_data_time_bounds(&mut self) { assert!(self.state == State::FillToEnd); - self.current_time = Some(TimeInterval::new_unchecked( - self.data_time_bounds.start(), - self.data_time_bounds.end(), - )); + self.current_time = Some( + TimeInterval::new(self.data_time_bounds.start(), self.data_time_bounds.end()).unwrap(), + ); + debug_assert!(!self.current_time.unwrap().is_instant()); + // debug_assert!(self.current_time.unwrap().end() != self.current_time.unwrap().start() + 1); } fn update_current_time(&mut self, new_time: TimeInterval) { @@ -308,6 +330,7 @@ impl StateContainer { !new_time.is_instant(), "Tile time is the data validity and must not be an instant!" ); + debug_assert!(new_time.end() != new_time.start() + 1); if let Some(old_time) = self.current_time { if old_time == new_time { @@ -335,6 +358,8 @@ impl StateContainer { debug_assert!(current_time.end() < self.data_time_bounds.end()); + debug_assert!(self.requested_time_bounds.end() <= self.data_time_bounds.end()); + let new_time = if current_time.is_instant() { TimeInterval::new_unchecked(current_time.end() + 1, self.data_time_bounds.end()) } else { @@ -363,10 +388,15 @@ impl StateContainer { } fn store_tile(&mut self, tile: RasterTile2D) { + debug_assert!(tile.time.end() > tile.time.start()); + debug_assert!(self.next_tile.is_none()); let current_time = self .current_time .expect("Time must be set when the first tile arrives"); + + debug_assert!(current_time.end() > current_time.start()); + debug_assert!(current_time.start() <= tile.time.start()); debug_assert!( current_time.start() < tile.time.start() @@ -481,6 +511,11 @@ where // poll for a first (input) tile let result_tile = match ready!(this.stream.as_mut().poll_next(cx)) { Some(Ok(tile)) => { + debug!( + "Initial tile: {:?} with time interval {:?}, currentime: {:?}", + tile.tile_position, tile.time, this.sc.current_time + ); + debug_assert!(tile.time.end() > tile.time.start()); // now we have to inspect the time we got and the bound we need to fill. If there are bounds known, then we need to check if the tile starts with the bounds. this.sc.set_current_time_from_initial_tile(tile.time); @@ -494,11 +529,24 @@ where tile.band, ) { + debug!( + "AAA Initial tile: {:?} with time interval {:?}, currentime: {:?}", + tile.tile_position, tile.time, this.sc.current_time + ); this.sc.state = State::PollingForNextTile; // return the received tile and set state to polling for the next tile tile } else { - this.sc.store_tile(tile); + debug!( + "BBB Initial tile: {:?} with time interval {:?}, currentime: {:?}", + tile.tile_position, tile.time, this.sc.current_time + ); + this.sc.store_tile(tile.clone()); this.sc.state = State::FillAndProduceNextTile; // save the tile and go to fill mode + + debug!( + "CCC Initial tile: {:?} with time interval {:?}, currentime: {:?}", + tile.tile_position, tile.time, this.sc.current_time + ); this.sc.current_no_data_tile() } } @@ -508,6 +556,7 @@ where return Poll::Ready(Some(Err(e))); } // the source never produced a tile. + // TODO: this should never happen?? None => { debug_assert!(this.sc.current_idx == min_idx); this.sc.state = State::FillToEnd; @@ -515,6 +564,9 @@ where this.sc.current_no_data_tile() } }; + + debug_assert!(result_tile.time.end() > result_tile.time.start()); + // move the current_idx. There is no need to do time progress here. Either a new tile triggers that or it is never needed for an empty source. this.sc.current_idx = wrapped_next_idx; this.sc.current_band_idx = wrapped_next_band; @@ -535,6 +587,17 @@ where let res = match ready!(this.stream.as_mut().poll_next(cx)) { Some(Ok(tile)) => { + debug_assert!( + tile.time.end() > tile.time.start(), + "Tile time interval is invalid: {:?}", + tile.time + ); + + debug!( + "DDD next tile: {:?} with time interval {:?}, currentime: {:?}", + tile.tile_position, tile.time, this.sc.current_time + ); + // 1. The start of the recieved TimeInterval MUST NOT BE before the start of the current TimeInterval. if this.sc.time_starts_before_current_state(tile.time) { this.sc.state = State::Ended; @@ -548,9 +611,10 @@ where } if tile.time.start() >= this.sc.requested_time_bounds.end() { log::warn!( - "The tile time start ({}) is outside of the requested time bounds ({})!", + "The tile time start ({}) is outside of the requested time bounds ({})! end is {}", tile.time.start(), - this.sc.requested_time_bounds.end() + this.sc.requested_time_bounds.end(), + tile.time.end() ); } @@ -575,6 +639,7 @@ where if this.sc.time_starts_equals_current_state(tile.time) && !this.sc.time_duration_equals_current_state(tile.time) { + debug!("missmatch tile is empty: {}", tile.is_empty()); this.sc.state = State::Ended; return Poll::Ready(Some(Err( SparseTilesFillAdapterError::TileTimeIntervalLengthMissmatch { diff --git a/operators/src/error.rs b/operators/src/error.rs index f8990bf10..d99494e3f 100644 --- a/operators/src/error.rs +++ b/operators/src/error.rs @@ -360,7 +360,7 @@ pub enum Error { InterpolationOperator { source: crate::processing::InterpolationError, }, - #[snafu(context(false))] + #[snafu(display("Downsampling error: {source}"), context(false))] DownsampleOperator { source: crate::processing::DownsamplingError, }, diff --git a/operators/src/processing/downsample/mod.rs b/operators/src/processing/downsample/mod.rs index 51c5c408e..d9815036c 100644 --- a/operators/src/processing/downsample/mod.rs +++ b/operators/src/processing/downsample/mod.rs @@ -256,6 +256,8 @@ where query: RasterQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { + log::debug!("ÖÖÖÖÖÖ Downsampling query: {:?}", query); + // do not interpolate if the source resolution is already fine enough let in_spatial_grid = self.source.result_descriptor().spatial_grid_descriptor(); @@ -402,6 +404,8 @@ impl FoldTileAccu for DownsampleAccu { type RasterType = T; async fn into_tile(self) -> Result> { + debug_assert!(self.time.unwrap().end() > self.time.unwrap().start()); + debug_assert!(self.time.unwrap().end() != self.time.unwrap().start() + 1); // TODO: later do conversation of accu into tile here let output_tile = RasterTile2D::new_with_tile_info( @@ -471,7 +475,9 @@ pub fn fold_impl(mut accu: DownsampleAccu, tile: RasterTile2D) -> Downs where T: Pixel, { + debug_assert!(tile.time.end() > tile.time.start()); // get the time now because it is not known when the accu was created + accu.set_time(tile.time); accu.cache_hint.merge_with(&tile.cache_hint); diff --git a/operators/src/processing/reprojection.rs b/operators/src/processing/reprojection.rs index c492d4ea8..5716952a0 100644 --- a/operators/src/processing/reprojection.rs +++ b/operators/src/processing/reprojection.rs @@ -638,6 +638,8 @@ where query: RasterQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { + log::debug!("ÖÖÖÖÖÖ Reprojection query: {:?}", query); + let state = self.state; // setup the subquery @@ -680,8 +682,9 @@ mod tests { use crate::mock::MockFeatureCollectionSource; use crate::mock::{MockRasterSource, MockRasterSourceParams}; use crate::source::{ - FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular, - GdalMetaDataStatic, GdalSourceTimePlaceholder, TimeReference, + FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, + GdalLoadingInfoTemporalSlice, GdalMetaDataList, GdalMetaDataRegular, GdalMetaDataStatic, + GdalSourceTimePlaceholder, TimeReference, }; use crate::util::gdal::add_ndvi_dataset; use crate::{ @@ -1751,4 +1754,125 @@ mod tests { GridBoundingBox2D::new_min_max(-1405, 1405, -1410, 1409).unwrap() ); } + + #[tokio::test] + async fn it_sets_correct_temporal_validity_for_partially_undefined_source_regions() -> Result<()> + { + let tile_size_in_pixels = [600, 600].into(); //TODO ?? + let data_geo_transform = + GeoTransform::new(Coordinate2D::new(399_960.000, 5700_000.000), 1098., -1098.); + let data_bounds = GridBoundingBox2D::new([0, 0], [99, 99]).unwrap(); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632).into(), + time: None, + spatial_grid: SpatialGridDescriptor::source_from_parts(data_geo_transform, data_bounds), + bands: RasterBandDescriptors::new_single_band(), + }; + + dbg!(result_descriptor.spatial_grid); + + let m = GdalMetaDataList { + result_descriptor: result_descriptor.clone(), + params: vec![GdalLoadingInfoTemporalSlice { + time: TimeInterval::new_unchecked( + TimeInstance::from_str("2022-02-01T00:00:00.000Z").unwrap(), + TimeInstance::from_str("2022-03-01T00:00:00.000Z").unwrap(), + ), + params: Some(GdalDatasetParameters { + file_path: test_data!( + "raster/sentinel2/S2B_32UMB_20220129_0_L2A__B02.tif_downsampled.tif" + ) + .into(), + rasterband_channel: 1, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: data_geo_transform.origin_coordinate, + x_pixel_size: data_geo_transform.x_pixel_size(), + y_pixel_size: data_geo_transform.y_pixel_size(), + }, + width: data_bounds.axis_size_x(), + height: data_bounds.axis_size_y(), + file_not_found_handling: FileNotFoundHandling::Error, + no_data_value: Some(0.), + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + retry: None, + }), + cache_ttl: CacheTtlSeconds::default(), + }], + }; + + let tiling_spec = TilingSpecification::new(tile_size_in_pixels); + let mut exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_spec); + + let id: DataId = DatasetId::new().into(); + let name = NamedData::with_system_name("s2"); + exe_ctx.add_meta_data(id.clone(), name.clone(), Box::new(m)); + + let gdal_op = GdalSource { + params: GdalSourceParameters::new(name), + } + .boxed(); + + let initialized_operator = RasterOperator::boxed(Reprojection { + params: ReprojectionParams { + target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: DeriveOutRasterSpecsSource::DataBounds, + }, + sources: SingleRasterOrVectorSource { + source: gdal_op.into(), + }, + }) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await?; + + let qp = initialized_operator + .query_processor() + .unwrap() + .get_u8() + .unwrap(); + + let qr = qp.result_descriptor(); + + dbg!(qr.spatial_grid); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + + // query with Germany bbox which is partially outside of 32632 projection + let request_bounds = SpatialPartition2D::new( + Coordinate2D::new(5.98865807458, 54.983104153), + Coordinate2D::new(15.0169958839, 47.3024876979), + )?; + + let query_tiling_pixel_grid = qr + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec) + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(request_bounds); + + let query_rect = RasterQueryRectangle::new_with_grid_bounds( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new_instant(TimeInstance::from_str("2022-02-01T00:00:00.000Z").unwrap()) + .unwrap(), + BandSelection::first(), + ); + + dbg!(&query_rect); + + let qs = qp.raster_query(query_rect, &query_ctx).await.unwrap(); + + let tiles = qs + .map(Result::unwrap) + .collect::>>() + .await; + + for r in tiles { + dbg!(r.time, r.is_empty()); + } + + assert!(false); + + Ok(()) + } } diff --git a/operators/src/source/gdal_source/loading_info.rs b/operators/src/source/gdal_source/loading_info.rs index 9738b2f0d..599ff11f0 100644 --- a/operators/src/source/gdal_source/loading_info.rs +++ b/operators/src/source/gdal_source/loading_info.rs @@ -303,6 +303,10 @@ impl MetaData for let known_time_before = known_time_start.unwrap_or(TimeInstance::MIN); let known_time_after = known_time_end.unwrap_or(TimeInstance::MAX); + log::debug!( + "ÄÄÄÄÄÄ known_time_before: {known_time_before}, known_time_after: {known_time_after}", + ); + Ok(GdalLoadingInfo::new( GdalLoadingInfoTemporalSliceIterator::Static { parts: data.into_iter(), diff --git a/operators/src/source/gdal_source/mod.rs b/operators/src/source/gdal_source/mod.rs index bedbb4f9f..05437f71a 100644 --- a/operators/src/source/gdal_source/mod.rs +++ b/operators/src/source/gdal_source/mod.rs @@ -382,6 +382,7 @@ impl GdalRasterLoader { tile_time: TimeInterval, cache_hint: CacheHint, ) -> Result> { + debug_assert!(tile_time.end() > tile_time.start()); let tile_spatial_grid = tile_information.spatial_grid_definition(); match dataset_params { @@ -546,7 +547,7 @@ impl GdalRasterLoader { info.params.clone(), reader_mode, tile, - info.time, + dbg!(info.time), info.cache_ttl.into(), ) }), @@ -564,6 +565,7 @@ impl GdalRasterLoader { ) -> impl Stream>> + use { loading_info_stream .map_ok(move |info| { + debug!("Loading info: {:?}", info); GdalRasterLoader::temporal_slice_tile_future_stream( spatial_query, info, @@ -614,6 +616,7 @@ where query: RasterQueryRectangle, _ctx: &'a dyn crate::engine::QueryContext, ) -> Result>> { + log::debug!("GdalSource query: {:?}", query); ensure!( query.attributes.as_slice() == [0], crate::error::GdalSourceDoesNotSupportQueryingOtherBandsThanTheFirstOneYet diff --git a/services/src/api/handlers/wms.rs b/services/src/api/handlers/wms.rs index 8047d95d7..5919d1aef 100644 --- a/services/src/api/handlers/wms.rs +++ b/services/src/api/handlers/wms.rs @@ -357,7 +357,7 @@ async fn wms_map_handler( query_tiling_pixel_grid.grid_bounds(), request.time.unwrap_or_else(default_time_from_config).into(), attributes, - ); + ); // <-- this one debug!("WMS query rect: {:?}", query_rect); diff --git a/test_data/api_calls/force/downsampled.http b/test_data/api_calls/force/downsampled.http new file mode 100644 index 000000000..99c41090d --- /dev/null +++ b/test_data/api_calls/force/downsampled.http @@ -0,0 +1,44 @@ + +### + +# @name anonymousSession +POST http://localhost:3030/api/anonymous +Content-Type: application/json + +### + +# @name workflow +POST http://localhost:3030/api/workflow +Authorization: Bearer {{anonymousSession.response.body.$.id}} +Content-Type: application/json + +{ + "type": "Raster", + "operator": { + "type": "GdalSource", + "params": { + "data": "LND05_BOA_1" + } + } +} + +### + +@workflowId = {{workflow.response.body.$.id}} +@crs=EPSG:3035 +@width=500 +@height=1000 +@time = 2017-07-04T00%3A00%3A00.000Z +@minx = 4226026.3630416505 +@miny = 3044919.6079648044 + +# pixel size: 30 +@maxx = 4256026.3630416505 +@maxy = 3104919.6079648044 + +@colorizer_min = -1000 +@colorizer_max = 10000 +@styles = custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A{{colorizer_min}}%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A{{colorizer_max}}%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B246%2C250%2C254%2C255%5D%2C%22underColor%22%3A%5B247%2C251%2C255%2C255%5D%7D%7D + +GET http://localhost:3030/api/wms/{{workflowId}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES={{styles}}&TRANSPARENT=true&layers={{workflowId}}&time={{time}}&EXCEPTIONS=application%2Fjson&WIDTH={{width}}&HEIGHT={{height}}&CRS=EPSG:3035&BBOX={{miny}}%2C{{minx}}%2C{{maxy}}%2C{{maxx}} +Authorization: Bearer {{anonymousSession.response.body.$.id}} \ No newline at end of file diff --git a/test_data/api_calls/force/filladaptererror.http b/test_data/api_calls/force/filladaptererror.http new file mode 100644 index 000000000..98a993f42 --- /dev/null +++ b/test_data/api_calls/force/filladaptererror.http @@ -0,0 +1,28 @@ + +### + +# @name anonymousSession +POST http://localhost:3030/api/anonymous +Content-Type: application/json + +### + +# @name workflow +POST http://localhost:3030/api/workflow +Authorization: Bearer {{anonymousSession.response.body.$.id}} +Content-Type: application/json + +{ + "type": "Raster", + "operator": { + "type": "GdalSource", + "params": { + "data": "LND05_BOA_1" + } + } +} + +### + +GET http://localhost:3030/api/wms/c965662b-a5bf-57ce-aae2-e7a6036d8a99?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES=custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A1%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A255%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B255%2C255%2C255%2C127%5D%2C%22underColor%22%3A%5B0%2C0%2C0%2C127%5D%7D%7D&TRANSPARENT=true&layers=c965662b-a5bf-57ce-aae2-e7a6036d8a99&time=2011-09-20T00%3A00%3A00.000Z&EXCEPTIONS=application%2Fjson&WIDTH=256&HEIGHT=256&CRS=EPSG%3A4326&BBOX=47.8125%2C9.84375%2C49.21875%2C11.25 +Authorization: Bearer {{anonymousSession.response.body.$.id}} \ No newline at end of file diff --git a/test_data/api_calls/force.http b/test_data/api_calls/force/force.http similarity index 86% rename from test_data/api_calls/force.http rename to test_data/api_calls/force/force.http index fa8ad3515..bc45ba1d3 100644 --- a/test_data/api_calls/force.http +++ b/test_data/api_calls/force/force.http @@ -25,6 +25,7 @@ Content-Type: application/json ### @workflowId = {{workflow.response.body.$.id}} +@crs=EPSG:3035 @width=1000 @height=2000 @time = 2017-07-04T00%3A00%3A00.000Z @@ -39,9 +40,6 @@ Content-Type: application/json @colorizer_max = 10000 @styles = custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A{{colorizer_min}}%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A{{colorizer_max}}%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B246%2C250%2C254%2C255%5D%2C%22underColor%22%3A%5B247%2C251%2C255%2C255%5D%7D%7D +GET http://localhost:3030/api/wms/{{workflowId}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES={{styles}}&TRANSPARENT=true&layers={{workflowId}}&time={{time}}&EXCEPTIONS=application%2Fjson&WIDTH={{width}}&HEIGHT={{height}}&CRS={{crs}}&BBOX={{miny}}%2C{{minx}}%2C{{maxy}}%2C{{maxx}} +Authorization: Bearer {{anonymousSession.response.body.$.id}} -### - -# native res, full bbox -GET http://localhost:3030/api/wms/{{workflowId}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES={{styles}}&TRANSPARENT=true&layers={{workflowId}}&time={{time}}&EXCEPTIONS=application%2Fjson&WIDTH={{width}}&HEIGHT={{height}}&CRS=EPSG:3035&BBOX={{miny}}%2C{{minx}}%2C{{maxy}}%2C{{maxx}} -Authorization: Bearer {{anonymousSession.response.body.$.id}} \ No newline at end of file diff --git a/test_data/api_calls/force/reprojected.http b/test_data/api_calls/force/reprojected.http new file mode 100644 index 000000000..ad195c53f --- /dev/null +++ b/test_data/api_calls/force/reprojected.http @@ -0,0 +1,43 @@ + +### + +# @name anonymousSession +POST http://localhost:3030/api/anonymous +Content-Type: application/json + +### + +# @name workflow +POST http://localhost:3030/api/workflow +Authorization: Bearer {{anonymousSession.response.body.$.id}} +Content-Type: application/json + +{ + "type": "Raster", + "operator": { + "type": "GdalSource", + "params": { + "data": "LND05_BOA_1" + } + } +} + +### + +@workflowId = {{workflow.response.body.$.id}} +@crs=EPSG:4326 +@minx = 8.66114350470939 +@miny = 50.5083520752026 + +@maxx = 9.07337279440548 +@maxy = 51.0518257240638 +@width=1000 +@height=1000 +@time = 2017-07-04T00%3A00%3A00.000Z + +@colorizer_min = -1000 +@colorizer_max = 10000 +@styles = custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A{{colorizer_min}}%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A{{colorizer_max}}%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B246%2C250%2C254%2C255%5D%2C%22underColor%22%3A%5B247%2C251%2C255%2C255%5D%7D%7D + +GET http://localhost:3030/api/wms/{{workflowId}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES={{styles}}&TRANSPARENT=true&layers={{workflowId}}&time={{time}}&EXCEPTIONS=application%2Fjson&WIDTH={{width}}&HEIGHT={{height}}&CRS={{crs}}&BBOX={{miny}}%2C{{minx}}%2C{{maxy}}%2C{{maxx}} +Authorization: Bearer {{anonymousSession.response.body.$.id}} diff --git a/test_data/raster/sentinel2/S2B_32UMB_20220129_0_L2A__B02.tif_downsampled.tif b/test_data/raster/sentinel2/S2B_32UMB_20220129_0_L2A__B02.tif_downsampled.tif new file mode 100644 index 0000000000000000000000000000000000000000..57999440d14ac01637ffbe3f0b1874743598418e GIT binary patch literal 20484 zcmZ^~Wpo?Mwk_;1Cq_|IaaEHf+hJyAX4oNznVA`On3El5W@g-BW@cuGzdGmLZ;bnX zJdGi%+-^(iuBx@>nscr)Wvqzw5fKql5fLd;MWm3UX7ZWhe~nZA&p1*ZPxU|J|D2ch ze~qJ3q>e}>kKL2cxc_zB^FL!Dk9+0#Kj++#PxXJDQ|*6^vkuAp-*XDe4;VZ&b3|Hs zZcRB}HzHL8kmKQUykSJj2wjek%JIO(DI==L@hkZpB*!V`Ygj)2^C)Z-HWp-iB+nvH zURB|Ui2S9B7b#z%dDkY*nsjYaFe3o@%XDqt4zzF5u6Y4esb4{`2?r8%Csy@I<7QQ6@&D^*y*AK7n6C6F-G5z(1geeqaBHGtvr1K7Eryj8|F+q|=weGK}!jBDHme z2dqriAv29#+wN=~=dH{w<_PPNSk8LWg18Vojke*fn1~&yJ&1yd;H~jYFATcqT{Ny2 zQd@&E{=Wl(K)OJAZ}->%#Bz12ZQWzf<6KD#^;a+7SmT4jnGWI4mQVGtlY%VyZBQ@@AU3epBqD9COUwW zq=3I-mxL+)fLzRLKM+~XlpKp~{2bdMw$Yhn6|O>OkwSbHsfqI70J%<9!~CF*QeagPtMfxDem zU{%bX{3<;}kD+F`5`BjD&_oXLL}M~cg2@o6Pt_?}7cE(hSMw?RyzhODJVRsDs2^GX zirf;J9MvFebj;r__dEMzFEP zXiaOA!>wk{P5TK?ux?s&M38?F^XXh(*W6?=@)P@MdUBXo=S@j2I20s;O=?5jo9qG` zjSC<(o&$cVhn05fg}_u_tnaC3pyz(v)VO@nonz}o?~5v%ZF+S7jBBz^%y=o|wT!i6 zx5m`>)l_c!x2R{0eA;dOyLtsy!vpAAvc`;I1z>JeTg^rbv6I>#BaXD8?@1ryjNT1o z!!xwSB#bd|KyTDJ&=;!4Q2i{L38$0qbO_8Q(p&kg$Id-#s(HzNXzjGtnk{()(UEPo zmeMiI&+=21G$SfWPur63xD%`cdV)i6i%|sn;U;($<<_t3LyWcRR)2cmoYxmilZB*kp=z182VOxdJd(#|OdwZ29tlz>06n)DQA zY&~j?YZz?|)flNfQFUAcZ3Ahv?szkzYHFOG{?Rt-b@dC{ZC{ca0g`Yhy&x#1?=r&d zqC3TTMHZW@tk>2jr;6yq)0=boDDh0}V2$W2dXvv*7v%fwBikq?`)LQTOP{5+hJTFi z#yv0_w9~#T-}G@teWi1tn(tEV1G$d}Mm6zN@HX`4@Xd|Q71Pk8Wo;VuEA#v4Q_;z6CoqzgAhTVoXshtMUF;o|myTW6DIn zjhXGc5nC>DvbRE1J8$ursOU+7;?c*HoPh^Q9QdP-fklneB-S`cBj9*3)x3ZT;gYB& zS{leo<`}W+xj^>VUf=-GiN!9lN+3bg@ofAHjl;ER0F;Zp2cE<0B#Cc@y+DvGHNVgX z*6CoO>>aFIW=iX+^M|Y4w5r;1y zLTB=}e42eEIZ$={!E72|gWh2M%&#KAubIu<4NeWWqB%w^vo44W zd@f#0Td?!ogTmxC^q|(r!2`9SYP7tf3EDd?*6(W9jFL)v<+{38c^;D@dTV60YYRSfPZ6&|#B$&16B5XM2aaz)l&ZKR%XIP6O8LJr?+k zaq0r?x$k6Pm3AtSQo+6#p1GdxzIwjW-c7Mrqg%x#W^CkJ=bIM&EAAJ(i@_sj5y>BDMXGKS&Ym08}3P_85^{H#yNF@)?Q1aY*1p9zJWgeHlDOG&Eh6|izvX? zQj7AR^`wZJm9?C=mAAd88@QwwrDb6uxQvYDM`16gCuh6}T`kI!KQvCp&QfTq&j&~7 ztK!b6Qv<`*bxJArkCjJk6s6D+cuRaY^&rFBEYh zn8w}_>>3_rj&fjVsjHfb-HuO3<&7Jl3b{p^qQUwZloRFCitF{rU+fP36kquM9+ zht|-a%DAn*Hg+l#t{NMZPRf(GG3rw7Id8)Z@_{d>S#V3dlt(zV zopfR)KhA#O5Wawh7<+I7zRVrsxXy!+W#^4<091D#k){*L#z+sbplkY?h#J=LB|x3WbvPPCy9)osSVtPhVC(48HeX798s zh|cCDvB-%G&l5YW)Mg!_up2U(bk(!LVYm(%4d;O}Xn=l4`y1F;C-cE|JOowHO!bCR z#K?$dYMImvp3AXg;!F6#SvxF7(@qBmId>4_JBafig#5 zrRl{dTpu*W%Sd;6gEllXuzInwh-jybG_W;1sAbf@ zptj(X`dq05mw^&+5y^^Ny}RlL&D3V+KqF3{q%8^I| z6;TVwJ#mw!VoPZWR*g*(em+OEwyuhoG$UImk&;Vt;y*BqbMVY81#3qgGLS8ih$%Pe zfHt!`@T{>KWyVL1VMZ2Zl=@L$gJNk2HP_R@74ReaX&i&)z-n9)T+zE}e*L=ENo}NU z_8#!{_n!&8^oP}o>M(tQ^3%&BZ)6=4y-*pi>{lC*N+K&S%CZ5W9>aHGbm%J|NK&9? z7yS+o>X(-cY#Pd*-qp$SSc&P94%UjB31)Cx7D0?ww=%g zu}KV~OK>T1ieAGN@hX-~<5>Z68Rj66ucUP)`nUtfq9nN1xNdybnrU^6)^H|jZ#YIT za1vCf5jd+6qn`%T(J8~L?E%e=f3%mr^S*BWty)2eeUtRN@FhN^)(hsyu@WI z@eAa`jbViT0OmJNf?N2Du~^1Tji@`#$*T@AbN?dqIAFdt&k6|l}(QD{3pMNXLuP-?3<&c#acn|!yd zF}HZvXwk}wxX0epzE;^rWG(0`rwkP7g=Kv(%h-k4BA!QFV19G4b<%ESWflWO33~-S zz&^4~q$-~$(y?KrBAsY1rM1ZT$5o5of!9QpMAfz8@#BRy!3RvP)$v|zV>&G-mQcnf>~cpAm7@^4lf zijLx}%D6VW-eQlDhl z=mM$+)~F+3KK-1w2wc~DXobOSm`vW{rRb2p5!}Qh^aQP&_FT^mdx9GQqO5_dNnRMQ z&p`#)0DB<)?GG!bJ=0=m$NbiZ(qwj4l;D2RmaD8W<<1s~cm`VrDIp$eM|Hn{4zDPH z;ib!Cro$ifI{@u7Q_EA>%FHZ}h=M*i|x0oTn8*@z~taAa);|NWE69)s1g9r|}B3 zgwxSH;+(Y{F~kg@oc?aGg0aHPWf#W_V;4pJ_9XdE_#1$to~+`h4sWOJeQ)2xajjjbdKUb1cWx04+6SKQFJA^Iga zK^q0%dlPt1E9mBM=h(LS(oD_r^8(fxVVO(JVdh=2SKJc$ooD1StIKy0C7=>)RKycr5Q zQDTzS71yJs$x=Gq(#@G-K3NYhfqJkYYe3uJQ%V)1v(Z)E=g*}BlwjP&^Kd7lgHaq* zgNwmI$(W?0nMhjNOfoaCK?-~VW(Ep+4mJ`>?}@$ihv;PHuwRJc;yP`rbyn&bL~IFd z3ioj5SYap0-N>`@!A@PLx{Rf(QDd`~zSHe2y!s8%jZJaBsORh~h@!{Z>X_o@I=hdZ z-l;8Kv#H$T2@(%AB=P(XpDW|VZ&nPqMD=J2R+Z&9=i0OFOJoO34StgzYzG|!-{_t3 z8P%(9&<7iR;5DNHXl&#*F2U2rIq1jx&{cAt^uZHwX;xM;eHi6HB|sQnmO0}H8Efq% z%xrCTGc)p;G8#sarD%rr(h1pvf|=rr1wY!;`9#**+(~}<^PzDvY9U*Yui_T@>2w24 z{XNk@cQhXd8=}MFyxYpMoFc(KPGM__HJ?qSv*}rypRS~T@I$_Y)nk9)XWSK!W=M2) zi}2>oQv6Hn0LHKjv=UuMKfpBFd~KcHM=z|kRz&+t?cL!L7t#7m?UCrL0 zT_im@j=%E_d=kGXGMVeG*5)R6A6sfaWu0LQ{Sod=KjZ!UE1UuTXmzw}dS;oMiy0?i z3gZn}E%U_-;Giw|6B!8-NCdkf5ls!!hfrWq z=7|lUw{<-9(_R(q-JNly7H_W)BfrB3vw!7j{CVhUqpC3+eb$PxT1qX- zV&)jOQpmpM+z1tR4@s{Yr=l#tBEF1sVY-As@_M0#G82yWC z(-r)W-H+|V=g~VoukNb08mG=jX}|?-qft^*_4URhaF)ElmB}M!k=1;#S)1-8BUn6# zA~R`3vePYO1zjf!IlJsz&fehjP>Rsh;0vddb1uGVXl+8l;1MUEvjdkl^O_ZdSMd~9 zkgOCY%4tR<FD+v2&o-$YQy38*zK|z*D;(^t?EVaaFbA|cC%Hlo>JrAV~mkSRI zEe|yaUk&TQQchZ{yRGJ2r7y%ct^NAnG-zZOV@0Af-vP!_{Y~&SXz9xekJ3G!&Cy{c zlXcvklchJ`T@ghyf6|;G#o;wp#ddxT=f< zpR}}QH!>Jqv=^Z}_zv5HbDIm?xsEG(VZZe&FbUMKN*Znbg*6B#g1scKIb44Kf=&si zglKJ+aJK|&*_Y@~F;IM=BVid{QD)OIR(T3Ggg($-def7sdKs3S(9pdfDiK`5Dl0qnRn}zp zkHt|Oy+ocXKgnV?$y{gtwCaklVw7FZ8Di!z6U711lwBc9MGsQIOz-Ts(n(cd3aPHQ z(W51+b%Vvz9>~XbOD19$IR?+*E#S8?4s14_qM7U|&&8vhUM#1azs>9~XS$o)%@ouf zYGx3GzotdZ6i!O(qJ6_EZ7sry+9~izymU*4_Jt@c;Gsn%3 z9bl9J#ZmLn3p17~{086S3?Rb3&-bB5-j?V9y@kT6jJ;Oaz9bpKjm~)IFB90`++$Ws zE2mY8oVJofx$UF)8*k62$h*|WXebqg1!N3Mixs?&v}Ajzk0#@$=pwoz@%lITgRV8_ zTB+Rf&N23qHs#nd-8P}2;mS@|cb7BUEK0Mo13VxOn!`j#(U_ei%TN+|V3vx%8Co3v z?M|g7%>j57Yz>wg`ORT0fJ^&*-budWXgSr)>fwLsQrtM4Ro^Z?gXQ2FpAI&QK)9yX z2hX%NTC?ouW_xSC^TdX(H#o{R>_mICm>YT(e233EuR}Mj>a?7_pLsxalF}N^$Kot7 z6M4_?%DkCO=aWxlBQ4Aakyk8{wzC#l#q2M3oF(-gbBvifv^Ur`^eA}D!t@)Co{ArlBWv)tiSR|X4XgnG8gBh-&a#f%P2B}SCAh?{YnKi6kn1>( z*WxBy!zQ@PLQQD}aM>;`tAUL4DxOIy@I~Srtt0c?G;)e`H7}wil3Cd#GC5m>tReXt zp4F-otQ%Sz9}^m9l@e55<#IB|S!S)X57?Hqh;`x_$PBZ4uw^h`IAc(=D?1kmBG36# zC%e%W9VbzG5#>MU3Z8(+QIfSjv|bFO zlb9i{(G@s}ijD<-rFzOy5&5A3LUkzAJ?Xa!@jR05lm#dr|8!nT+{?QDp_XfzHci^uRh zd(Z2_TVM+*$n%<+z%$UB-!`9$KjNvFXBV{QIORjNLNnu^m_J!Zo|Dys1wBUUz-OZo z+v}|4ed#o734dqbGi$I!bO@d%cknedf@BCriyyQ$Pes47Q}{k>Eu)lAY;hX8d&PI_ zxgGCR4^#J>xaH)eS79=)!G7_7*ls>sJjN=UAc`=RU&6h_b34%)YUyH(%#^FyJT}sV z=58xP!nydnq2BTJY>sZz_D*Hzv(?!uBr$qJv!Puyq=X_uU*elPTdc))W4C)kbi&rq zmV{mj%iSfINlsyv{go~vh0t_e%G~Ze?25_uMJI{1cYC_M+(=dgO(m=7DVpAzVjc!N z3}qxc+ZWtvKBWSaq|U=qoF`UKRW^Iwj?q*i7%5foF z;m0lE-)Ic^7Y$(bc$}>CRWt+r!~hK>so7t`wz~&cS(U_l(MjACGpuJa#?=xHoU!ps z!o?E8p^o9a@vp+GLnoZ0rp-r4cf(}f%Qf6};l!XX)HnWFs7w5!VAkv%vgZxox3iN* z{59&p2Z;RP8TcOB9M}l6uzTj_&_B-Y&`R3Hy+c>Tm$H_`kLRszk9aAD;alRVa|u;c za)RGtpMKUXVf74F3f{I#nP{^R-C6+At#lHvC@U}g_?#AyT5~zMKWLOl(Y^CDE=WESy56PRfBs2 zJ*mQ8iW6+N2%<-*EUC{9vU-BcNOX;E!pHDs(iXV~Q_^y zcuDB86LevaI`gFVYst?HvSXHEC7K0MMNRcC(p941U-X37E~4xsq9AQz-L-Sri^(Tv zj96&zbNu!I$>+93mCbhATew+GPd4DD@C)g}4&bUrKN)4eh^l4@GMStJt;hi~1R_u$ zpNFZ*W?jIia4C3$a&gGpv0Y|6s}QRr)|qEyR%suOOdKCN6#O188vYS_8qOUwLKVZE zg1tpnSvP&+b)BE=l>ot`a;9=69t&-A>N(Gx&Gsz2qB+4**gX2l9%eS8#lao5pgvt$ zix;xmJhl0ky~=*bx6pF*G3tR(c!sNE+fDd^vA!rXln6BmQRkas3|@z@t$o z_^-AU|0{8}!VB@f+$$33FL)Fs;a6xXdX1Oixu6%=2b+)=bPs(m<4<14Hd9CpK2>r@ z{oFM1`$EiZ6>buK?3M{Pi?0-}8e)zubSp;O=TX)jHcni3tGFfOJH@w7co`hyRt=t( zF)~_wVOiNriRK=dJy|_{n$}OfZ_MSN@H|n5U1s@syBtEsy|a8eUB$}d-8`dNT}1Oj zv=1yuijpSqF?g?^U++iifkkIrS5YEcaxgfUVH>TMGIjKTiu)`;k}Zv=@a_sCI^GzWv&`tD|xYhty0c?Gs1i& zTJTGvnt3a9F+mFt2`&kBPDt;Rvo8f}xL#2UzJ}Y4ZK4M}2G>i5C@tCp{Ol=X65R@O zLhvSE!%hN%DX9^fntiC*%wQce(~wgr3A_ia#B8 z!ykjAf)AXz_ABR&ZA(OU(OGW~wvO`5BGMUUPqUN49fBvr2g9v{|AZR4ZJZNi364_B z8g-F@dcf!UM8k`6z>6d|gD|_5-E8f?fg|ZubOKkEPOsXcxc%Ck>!gv{{jD(>G}1WB z1h4rk`Fd&vaXQqW9tD+1H4-P+^I|*}|Cah|TkXE_)M&0ORKFV|)V{ba`UHKbn&gJ+ z@h0YG_5)?WFUdmo#rh_{`@B$O{Nr$ngcV^K+CuxR2i9x%y#2(k>3%b(NS3Mt4_YIv z0(O$yEj}`#aE{>#597PI)TP1dRuRo_499z6PpR%+RI}kz#!fVZWMMjQhCbm1_^~k_ zeur7CNwT8<#;&uW?3QFyUSMH-Qfm1eCBv$YS{ENLf$5?K(Wf z-WYH7P9P@=N*B*eF_W!B8O2bsmLihJD27gmSMd$?^1z}!!&`Oh7u5J4n2e7 zL%pRVCan>#f0OE=sXkXjMm(r0wXw`#F&b&y)?29O)L3nUc0mn*TE+=YfgB%Wan^4% z9>i;Pu`XVTB=>Z%e<)KhA~-+3X?R&^dT@jlLNGfyz1*#&dAosw4&Pw9P`6;@e) zczbe}#X247C^{N)&{53+-R>0u7#Rv?E| zLO#e!;jUFP6ijTI_$Gc!cy%~u{KsHPcbhxUnqVGhr_Hgvj2O-$?L2mRH@{mW1mll} z-UJWZC9Ek{Ly=8*oF}ZdtOCpML=-C(i$SL9w6WKSwX8QgBXi6I*Ci#v7euUAY%D*I z^0IS!EzlXPL6gB3qaJbrE`^!;~)8B@sh7a-;LaXfyQ_vt-;B6_(^&@M<_ScZQ3!ViaJ0^ zQeUd$^fCHf^{u+x_*)yIUxo>|5j~^5Q=1wEK|gTP$SUX3CL^phV562gP!c5Z>LP{2 z!)y6XH+TFniJLs(Q3?BlSKP6(!h2x$Gv?Ec=4QK6sDN9`6k-4$9-Qt@wsr=iLuo_% z?JiayYlzqk?(3LbLk&q=vI?ePUGQhTfFHIWIXP_3&dHqJg|@T3e7n`k&0?*G_x1O_ zEy@b52AX2jqWe%wd_~O}=o4tD#%Q$yPyCyGRiy)MfSOrp2f7&7)ZY4i*bY}zm#S%` z%DMm^Fj5&*v(RupL>cDYuC>Goq%GY={W1=Jv(LK7B~H2c@viCqZFjU^m^b8oPEXcY zDedcSqfpymFIw4d<5mp6mHS&H3`$70sojv@p&zsz`s4p7UviBe)>lZq@d-_mEa)3j zf&6WmRPaX53La(l7N&KL{WAN*=ju;?g8mVVgI#5w&d&E5sya(8qqJ8}1-AS8`2)%k z6?Y}PYom$e%MOrR z(j8X>4a8lvk=hgWy|0rJR;u}?drSD=2NL{m)Vxw(t!re|+iHorZrs(6YkLDL{I|U= zyb<14enU^|o9>D6wpTZ3d5xQR6TU=?ntJe()586W9l;3IwI;}lttq*{_DCimGI%&K zYw(?D#is{r+e&twe-e%ag>V8C$F7x z?poJ$lB|QYJ*#GZ&>MRez)oViWR#yvJ}FD6qqD;~=oWHH#t#p63C*xyn$1mW^)uT$ z{m447(6=AvK^V=Hm?A4&Lgt#0qz~)}%hDSB5<7}j5hWV(O5`#g2^IhYFCru8e(7NI z$m{ro(<@p0K_$lf(lb1^xA&Ltm4BXpl%7eys4dbm>(}(Z^$2~v76Ce`elW{8;lCQ2 z#XCeft)>e!2xRnU(DLa|KqY*_C@4}olbq5{ee*Q0z$@8#@o8gxOapx{{vc+GMov5X zWoWsb&dKJS4%G_B1+UuMrME7h)nBAb=xSxrXT~kiPs1Rb<1jHBrbmsX#xq(Rh3UvI zew?3RkI)s`7Z#D|wi_4?&Y|t74w*{0R3xUe>hu-P9M~HuX)s7t=Y|NnXnwu*F&^7CPVEEuyN#T!Wkv34Xh;^^iHZBtI>g#a6IV zz)$9oWvnA!AvO1Lpdc7+%WUY{~%XqDK z#D(w}>B=o&j4<+&4|tAQ&)#Hh3RZWimCQ}F6#syQ)E;^P(h+B1mBe8F)tYLTvBOT& zAa*|o@6rTvQ%`49v`RB*#@HW?C8&txbw`tec&yPD)P}G56_MYrEn4CH=68b7C3w|{ zfkE&_RzUl}3AqMVvU;?W=*^SGN|DB_5t!}k8My0Psbu!&^uP7y_2yPH=`Z|L$?jk4 zg=$cV3kY9h{g@hQgpD=oqQG_I4yuag$#^#ll*RK&EbB;TNOV2kIT-5d+~f&N#fh+; z_FTmFYt?wjQO`N%*77GEv-hpN_ia|RzvpZI?I>-vtz z)$#Q3gFqhNeNUuzFz`v8soYd(ps>0{|EWc3#q^HSfBn{2rk7T8>G{wk)Y~YEN}>@s zm33bflN})EtlW;r)vU?nFX=cg4+bj_j8*z>tp)lGr&+bkJ@~LWCw#+x>@IO81uOBf zYuHeSvGU=lSX+vEaLs%}<&?qOF-Qn~$>uZk`Pf0O;lo(DlS5aOtRS#*ZM1xZd zj53iyvI=Ny4HPN)K-v$V^$v@xS=dp*H9VPULB%<1A4e7N_C{%RY?e{zcx5 z-Z|b{N~~u1Gf3aIPs!{Zb zAOmVHUFrYYs~z3?O{V#>2Uhua#+2|}3k>r{#@+L0_nlFPD(U2IudQVXJPhOutn^Ix z&iCh3S{VhvTKzAWAU$v?=~$SaB}z8wHZSE=w%?o0M95lW=|&3C+s z)x&zm+nJ~Ae0Fhq-S;BOidv@L(9YueG>f&utW6%Wl_VdjFI_%`Xg;ekX>Y{&&&ukv zJ^ZANfxiOn^|1cTU(Wc<4zpxf7Jq~rjdb+8RmYCA32#UV7!JOnsFKu9KY%bpiPYvm0a+DID9-g(n+cM@Z4(Ne7fn44xo{(l31N{#J&yBz3 zYI+MBlS=5W|BZeP{0pv=qj(NWCo-|}k~3>8d95lu8LtG};yf`c{fW}&C&a(Jzje=C zgX_XFB%O7F%;c5j>b@ma>bxdpn*!q?B0{UL?U0>dk@PS4H(pNv#h;8avXXVIX`yIm zvE5v<969uRYCP=DL;MWA&LVhPo}G7er}4iff3`CEO6@F#ND z?~i^Im)&doPI&73%X_5ob}LJ;nXr`^8aM* zlalfKi$9heT3_krT04{<$!mC z$MO|btI2mgNiy;NY$%8H4akFv!K0GRKLQlBHHk9gB|ANeq|qlyFYjnIQE#afMU~k- zkV`rds+g0^R-%Aa(On^G@`6?ic9WDb-{UGMnWdAxWDlitHIjc;76cX>GvxbUi!C30 z($mD#);}ze%?mwyeK`W-{k8QWxD6;vdVn5iIjKiavl6r{7=%|MLN)kMe}*@T?5qiX zgR8RiBBi*`w&0?s7dL<(;U=ZBub%(4Ur6;hH(G|ef|Vq|^GO!D4T&eCXiatx6$X8& zAs&kvB8OE94b$5Ns(?~po8&_$7%j*l;@D3u*XB-7^BhgIYFV{;E}~%*z9sS#$yF!G z$>8+1zrr)dGO1Mdq1^=WJc(NzJJQoRP+9pC`?v3r|8C%(@tpoE*jd3$H3WG1a|vb;D>sZ6j6KNzUYi} z`c;QjQEr$Ht7JBO&4$RzcrQ#~?RijYh?}LSW1G>2xwJn_K)0Z#N8%N7?Iz3at2A;4 z{~~8l3+a*`#~QKB;EwDnXvA~y>~>KnHy!1pKpiK-2adN zeq2+DNjk`S@i06DO6u*5y0AA~2Djl-tQOlzvJ1#s;~w+{cdR&30p^6WP(@Zc{sGsd z8l4kG2i|$h`AY|`2dd);J`xXP3$-hHcKy5l3wv2M_7yxb4xu*in5f7{aGRB9?eTv! zQjISoRgIHINbgO787DCNPTEU9MQi?zmo^WI$4@?E0yBzJPVs=t}-VDQurTxK6}>rR>zJEO!7N{H2$`FYfuB% zlQ{31T2uDu9weq@?=a%z6y0f!cJi4M`FZn@g{&(42=5(^N+@Hu(&vDc+8Nn-iUTqF zMQ|231PhexAQpZ{-()vgVOUZ$WW7-;W0SE~U(H&H<@OcX8&MdfL8IAOnwK~*2cBcJ z!qr(e>na~bI^b{YiXG*|h`o3ioFaQE_Q`IhH_~AkVWp)S87H0mV|iB^W1kn1b~bfK zAR?+kOj=L>sBgZTO0?JYjZleH3ECT{~C!hdt5E23-BMlAxXsRMaoyL135vf`6ehfOZ1ojFr;mdJjIAwcKa4M$S$; z!w-NCoOUo$Q)?rqet*z_@X0!srI{M|h^XsK71HOYJB zlcB<)MoYbidPR>hj=;vOi>NHS0ft(SLe~-|yAh$!((7B6rjy>$QlOtYLmi`T)GHYI z1NGvLs4ewN*kNBuN$I7!W1J%8P*&8JGy%4Z+GAxFyGj;;>dG6KACELvXr<&8_hD%y z51e2=v!>Hxyq`!#c8CXb9gdRSyepi}tR?n}FzId%<{e=#>GWBSy0FsDP3x`f#rPHL zjp-JXBJzl@fiJIrOkkBhOE0UHlND5c^$Qq_55oPhrLkTSl3l+Jn-NBK@MmtCP?9@c z`kebo?)wsw7!F;5Kh^zej=*EJr#iu_``apheK~N5D_Ysf(rxj8bmR4CJ~$OeI)8}) zxC*-l5XEyWFQObxo6kRiVT+A;2rMN)~2sHKo(N^l8v{w3ZEw{D-Snz~4 zOIx8GRUt?Nn!^#a8m&QN%_jVi-OMU$cjuoO<{9uAq#GTSV(M3YlTiiVfkxn$Z;$3k z2cK-wmyVreoKEzjUFmH;lD&pw@Fw(+fsKVY2i;&SB{_5$=mpE7T2g~ON>|GcyPZO{ z9E&>R?3KKpC?R_X>PsicGTD`So@YU_SJ5PZiaXX=e1UxC-(&kn?~bh%`@lQGn=Ozk za8tRWW>(h&k1+&YMG$p?H*f~ynC8=#$@S6#WruwMM3?D28G+j~gFc`)%>^_qzCrTp z$!exRTCEOfkKaoMCt4d~GzPoTFYAJ}f!8BF#YNPRzP2FCiQ3uw%-39z+4(LGae_YU z<&*`m3pgo;i(<4M-cKVqpu_CkQr+Jvb;V13G;e6nw=;{2kV$=^Jo!wjqsQo#olf@R zmE<+))tF{c(>z6DyLtx2(!evVj`Rnl!<%SNe4Heq8srf_hU@FSjE#W;s`NL|OsEXn zEwiJIy=E&moMK)>_)s3>1E{7P3JeOE>RH({myfna9c0H^M^m>_g|fKO)=!)l<{|0q z)zXE%Tly3ypqA73BGVp2UL4w&a z>6p2WhRIknno7S3O#%L*9@d3+U{#n`gkn3yZHZ|flMq=e>i?_YOx&s}*C;M#H|Z&5 z+UJ}-e)G2n4x~uVAOdz%a7@(H98gl~QNym~^=5`y4pat~prBVNtxV0~I;1%vX_zLc zIeQQjoBP|uDgU9Ilsm&&(#3hO`z((OcHlkFfV>H7rG zH9#Ea@cVWvxu2Y*E#=R26W<|HfKCm#du4@mz1<>=El>t3ca(!+Ya?4k_Kr+e*O`~C zVb)xx3C%uF_lsWQd6CNclJR`72mzlr5vGV;XamuCH@1mRf%9623{I(GS&8~D+5znq z^8vZTM?vEK6E@Fz;1&g0o9ofeSlF98kQPj zeQe#(N8?Yy+pa+eM5-7ipOT-*T9zQnc^P{^&hm2j4O{$MEMgBK>pvLu6wYgTb1^64 zrf-7pp|U5kM5&Iv8dwQzb$4?=TOz&?V}T>T#Zu%J7SE!L{iK}7sdjs$_lJ4ZmhauS@;&I+$aKJ>=91DquJ6-#9O*#O#J zTo(VbJ37aBICO(FS~a{9_18uN&HohgfjiU029eXW0_N0oNMs*DFIeli;I()fPlxII z8t+06!LPSeS={W73ID-{a2OkYK%JHx7-ZD ztD%L2@ZVWi9xlT8bY`&%yNE@B*7A5CNfrk{9UD(>)A^v>_hk$w=t1PDH5}D~>Wajd z;wbNf0+0?K=SFyW-ZwFOB?0bgbucwJ49?7TIF#2}9qD4cmh?ii>5KF)>k%1&{{g&J zKQRle=m6vF3;74)hwjVYNptu)Mi+AgBX5bPG=upYjSCBIha3M(tLP zfg%#8K}qB+(Nz(KaRt9eJ1#VXlsvDfPmJ+F<7dD5Nf6}g>)d)W};Iv?@fB-=_g z|1f{T4{!#BmKf`ff0&+RylSP8)pR{jByEX_4KkRG=J(8wT9cMxw$nRX>7dq(;5*17 z{3Sk0CyHD05ZCP$;+Xn+_%z?&K1aQvr259HL-ZfDJ!TWW0d(3mUQY9ABUE5!Mv>&<7~=#xYK}Fl6?C}(1@Mfwnxl& zKtMEjJDf-2p!;+1gw=_ckhh?dBLSS%>m-NPSS!q1S}b_3chMatXaX?nElDA1#awR) z{v6k8BhBTop`IfX;x~ml! zUjfJQ8hMu%lT`B}j$`5S9;rr6B#KnWDV8+pdny0$R8r2$Qnc+r16m-(gluD%pM76Hljkc!rU0_yW_+L+ZWA zLi2Ajk~N2n(O{HdF2l8~feyAV;+D|YW8&rFIdav?p>=K@O!pDM{R}02uBRx7g7oQ0LgPmw_51*wAV9@G!fNi3AP&@VGb z+OmLSXcH;p5fqcYw9Go?v=^t*QSS0zy_e!91v9;2P7>1WF_8G28$HSH#pj}>cm_%1 z%V`$yopHu})PZj|PWg^To%e<6htV>WWBKt+^s=a@F>(_L5enpHz7=QjXS6NxL1WT- zmghpqfn0J5xXq@DH@x@sT6MU8x<5+IQ~&Eb9$2scpl;O_{Yf($S{zm(mozu}8cWb= zbPLa*@6u#>R+Njoyo@`18CwX>-BnQ>eSkkA$7LwWG*#pz@K z8AUmI9(0!ba1Nd#S3oP+&mXZHI9?9*xOa_D6(7L+dLS&rAL7pl+*e)?9M)6x-det| zLX`napQ~s1+iJR2s`fH8>mtl1+sQj@Jov#+LWf8e4HwVYS$sZlwzXs^`UpQ{d+;*Q zRW6c97D{W_%j|9E!{{q^F+Ky|Zi7#gyF`=oJXP2Qe1f{rjAYaPpojM`cH)HVf{ryg(=N z-<&3yFW$q=jeX$nWSKLqdj2&zDK|qSPA|xF*T(E)Psm!)g^uC7>?H64XUNv9n!O7t zoH=498$_$kt)^%8u#!loY;34;f%B+t1QqZRp0-Mzb;HB?U>wU~$sAMP$eaY%WXn>fY zO;iW!ra6){gDhDz8^kN=M%GVs6sM&kFB&=K8vU`RIUP5`G*$_xZaiDTcLe?3PG_gp zU47a}HkR-a@)q=Hbf6J-He}OV3yv1TuKqd3v=TJRcLJ%ti@sD6kB_4xkTY6~*O|Yt zk+uNJw=?9|UchH~7;DC(Q7HLQw1)KU$Ifta74ksO%D#NGmtZAob(&vapx^Y(Hb$r? z^rglb|AxRDTCA_DGRVKmf6S-q@g{+^iL~y+xq1}tRTWO6i7-X>0HwQxoU&fjCA`sR z%(c`L*)RpR5o4T%F;}9QGt^E-+e~KO#%pAnC}Um748(CEE@5$?I*cdN&{XRRNye9S zC-N!Hk6p5s=AhoBg^Z1Ea5j-YG#zgQJ#Pr??fb01;0_*y9)*>V(n}FfvvckSQAsDW zMeh4#ivAk3*W3Z;d_-WZFD-C6(A+;Nd`r}%h;G`(K$$LF9qsHum cH1sw8fL)M3^Emk@@IAxDO8gm+o0Iwf0aK$5N&o-= literal 0 HcmV?d00001 From 437d1a1705877e1e00cf6b969c161736b1bf219b Mon Sep 17 00:00:00 2001 From: Michael Mattig Date: Fri, 16 May 2025 14:32:38 +0200 Subject: [PATCH 4/4] clean up --- .../raster_subquery_reprojection.rs | 36 ++--- .../src/adapters/sparse_tiles_fill_adapter.rs | 3 - operators/src/processing/downsample/mod.rs | 3 - operators/src/processing/reprojection.rs | 129 +----------------- .../src/source/gdal_source/loading_info.rs | 4 - services/examples/force-import.rs | 2 +- services/src/api/handlers/wms.rs | 8 +- 7 files changed, 15 insertions(+), 170 deletions(-) diff --git a/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs b/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs index a3f5b60ee..bcd4dc6a1 100644 --- a/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs +++ b/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs @@ -95,7 +95,16 @@ where self.state.out_spatial_grid.geo_transform() ); - // why is this none? + // TODO: instead of producing an empty stream if the query does not intersect the data or projection, + // we need to actually perform the query in order to give the nodata tiles the correct time interval + // because all tiles in a time step need the same temporal validity, even across diferent quereis + + // so we need a way to reliably query the source in such cases that ensures no data tiles are produced. + // what to do in cases where the bbox coordinates cannot be reprojected? + // - query the data bbox but discard the data? + // - implement a special empty bbox query that makes sure to produce empty tiles but correct time intervals? + // - ideally: implement a way to query the time intervals of the source data and produce empty tiles accordingly + let valid_pixel_bounds = dbg!( dbg!( self.state @@ -106,12 +115,6 @@ where .and_then(|b| b.intersection(&query_rect.spatial_query.grid_bounds())) ); - log::debug!( - "ÖÖÖÖÖ valid_pixel_bounds {:?} -> {:?}", - tile_info.global_pixel_bounds(), - valid_pixel_bounds - ); - let valid_spatial_bounds = valid_pixel_bounds.map(|pb| { self.state .out_spatial_grid @@ -119,22 +122,10 @@ where .grid_to_spatial_bounds(&pb) }); - log::debug!( - "ÖÖÖÖÖ valid_spatial_bounds {:?} -> {:?}", - query_rect.spatial_query.grid_bounds(), - valid_spatial_bounds - ); - if let Some(bounds) = valid_spatial_bounds { let proj = CoordinateProjector::from_known_srs(self.out_srs, self.in_srs)?; let projected_bounds = bounds.reproject(&proj); - log::debug!( - "ÖÖÖÖÖ projected_bounds {:?} -> {:?}", - bounds, - projected_bounds - ); - match projected_bounds { Ok(pb) => { dbg!("produce something"); @@ -154,13 +145,6 @@ where } } else { dbg!("output query rectangle is not valid in source projection => produce empty tile"); - log::debug!( - "ÖÖÖÖÖ output query rectangle is not valid in source projection => produce empty tile {:?}", - self.state - .out_spatial_grid - .geo_transform() - .grid_to_spatial_bounds(&query_rect.spatial_query.grid_bounds()) - ); // output query rectangle is not valid in source projection => produce empty tile Ok(None) diff --git a/operators/src/adapters/sparse_tiles_fill_adapter.rs b/operators/src/adapters/sparse_tiles_fill_adapter.rs index 96783b267..a9579a396 100644 --- a/operators/src/adapters/sparse_tiles_fill_adapter.rs +++ b/operators/src/adapters/sparse_tiles_fill_adapter.rs @@ -284,7 +284,6 @@ impl StateContainer { fn set_current_time_from_initial_tile(&mut self, first_tile_time: TimeInterval) { debug_assert!(first_tile_time.end() > first_tile_time.start()); - debug_assert!(first_tile_time.end() != first_tile_time.start() + 1); // if we know a bound we must use it to set the current time let start_data_bound = self.data_time_bounds.start(); let requested_start = self.requested_time_bounds.start(); @@ -322,7 +321,6 @@ impl StateContainer { TimeInterval::new(self.data_time_bounds.start(), self.data_time_bounds.end()).unwrap(), ); debug_assert!(!self.current_time.unwrap().is_instant()); - // debug_assert!(self.current_time.unwrap().end() != self.current_time.unwrap().start() + 1); } fn update_current_time(&mut self, new_time: TimeInterval) { @@ -330,7 +328,6 @@ impl StateContainer { !new_time.is_instant(), "Tile time is the data validity and must not be an instant!" ); - debug_assert!(new_time.end() != new_time.start() + 1); if let Some(old_time) = self.current_time { if old_time == new_time { diff --git a/operators/src/processing/downsample/mod.rs b/operators/src/processing/downsample/mod.rs index d9815036c..7135d58f2 100644 --- a/operators/src/processing/downsample/mod.rs +++ b/operators/src/processing/downsample/mod.rs @@ -256,8 +256,6 @@ where query: RasterQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { - log::debug!("ÖÖÖÖÖÖ Downsampling query: {:?}", query); - // do not interpolate if the source resolution is already fine enough let in_spatial_grid = self.source.result_descriptor().spatial_grid_descriptor(); @@ -405,7 +403,6 @@ impl FoldTileAccu for DownsampleAccu { async fn into_tile(self) -> Result> { debug_assert!(self.time.unwrap().end() > self.time.unwrap().start()); - debug_assert!(self.time.unwrap().end() != self.time.unwrap().start() + 1); // TODO: later do conversation of accu into tile here let output_tile = RasterTile2D::new_with_tile_info( diff --git a/operators/src/processing/reprojection.rs b/operators/src/processing/reprojection.rs index 5716952a0..144e98efa 100644 --- a/operators/src/processing/reprojection.rs +++ b/operators/src/processing/reprojection.rs @@ -638,8 +638,6 @@ where query: RasterQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { - log::debug!("ÖÖÖÖÖÖ Reprojection query: {:?}", query); - let state = self.state; // setup the subquery @@ -682,9 +680,8 @@ mod tests { use crate::mock::MockFeatureCollectionSource; use crate::mock::{MockRasterSource, MockRasterSourceParams}; use crate::source::{ - FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, - GdalLoadingInfoTemporalSlice, GdalMetaDataList, GdalMetaDataRegular, GdalMetaDataStatic, - GdalSourceTimePlaceholder, TimeReference, + FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular, + GdalMetaDataStatic, GdalSourceTimePlaceholder, TimeReference, }; use crate::util::gdal::add_ndvi_dataset; use crate::{ @@ -1483,6 +1480,7 @@ mod tests { for r in result { assert!(r.is_empty()); + assert_eq!(r.time, time_interval); } } @@ -1754,125 +1752,4 @@ mod tests { GridBoundingBox2D::new_min_max(-1405, 1405, -1410, 1409).unwrap() ); } - - #[tokio::test] - async fn it_sets_correct_temporal_validity_for_partially_undefined_source_regions() -> Result<()> - { - let tile_size_in_pixels = [600, 600].into(); //TODO ?? - let data_geo_transform = - GeoTransform::new(Coordinate2D::new(399_960.000, 5700_000.000), 1098., -1098.); - let data_bounds = GridBoundingBox2D::new([0, 0], [99, 99]).unwrap(); - let result_descriptor = RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32632).into(), - time: None, - spatial_grid: SpatialGridDescriptor::source_from_parts(data_geo_transform, data_bounds), - bands: RasterBandDescriptors::new_single_band(), - }; - - dbg!(result_descriptor.spatial_grid); - - let m = GdalMetaDataList { - result_descriptor: result_descriptor.clone(), - params: vec![GdalLoadingInfoTemporalSlice { - time: TimeInterval::new_unchecked( - TimeInstance::from_str("2022-02-01T00:00:00.000Z").unwrap(), - TimeInstance::from_str("2022-03-01T00:00:00.000Z").unwrap(), - ), - params: Some(GdalDatasetParameters { - file_path: test_data!( - "raster/sentinel2/S2B_32UMB_20220129_0_L2A__B02.tif_downsampled.tif" - ) - .into(), - rasterband_channel: 1, - geo_transform: GdalDatasetGeoTransform { - origin_coordinate: data_geo_transform.origin_coordinate, - x_pixel_size: data_geo_transform.x_pixel_size(), - y_pixel_size: data_geo_transform.y_pixel_size(), - }, - width: data_bounds.axis_size_x(), - height: data_bounds.axis_size_y(), - file_not_found_handling: FileNotFoundHandling::Error, - no_data_value: Some(0.), - properties_mapping: None, - gdal_open_options: None, - gdal_config_options: None, - allow_alphaband_as_mask: true, - retry: None, - }), - cache_ttl: CacheTtlSeconds::default(), - }], - }; - - let tiling_spec = TilingSpecification::new(tile_size_in_pixels); - let mut exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_spec); - - let id: DataId = DatasetId::new().into(); - let name = NamedData::with_system_name("s2"); - exe_ctx.add_meta_data(id.clone(), name.clone(), Box::new(m)); - - let gdal_op = GdalSource { - params: GdalSourceParameters::new(name), - } - .boxed(); - - let initialized_operator = RasterOperator::boxed(Reprojection { - params: ReprojectionParams { - target_spatial_reference: SpatialReference::epsg_4326(), - derive_out_spec: DeriveOutRasterSpecsSource::DataBounds, - }, - sources: SingleRasterOrVectorSource { - source: gdal_op.into(), - }, - }) - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await?; - - let qp = initialized_operator - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - let qr = qp.result_descriptor(); - - dbg!(qr.spatial_grid); - let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); - - // query with Germany bbox which is partially outside of 32632 projection - let request_bounds = SpatialPartition2D::new( - Coordinate2D::new(5.98865807458, 54.983104153), - Coordinate2D::new(15.0169958839, 47.3024876979), - )?; - - let query_tiling_pixel_grid = qr - .spatial_grid_descriptor() - .tiling_grid_definition(tiling_spec) - .tiling_spatial_grid_definition() - .spatial_bounds_to_compatible_spatial_grid(request_bounds); - - let query_rect = RasterQueryRectangle::new_with_grid_bounds( - query_tiling_pixel_grid.grid_bounds(), - TimeInterval::new_instant(TimeInstance::from_str("2022-02-01T00:00:00.000Z").unwrap()) - .unwrap(), - BandSelection::first(), - ); - - dbg!(&query_rect); - - let qs = qp.raster_query(query_rect, &query_ctx).await.unwrap(); - - let tiles = qs - .map(Result::unwrap) - .collect::>>() - .await; - - for r in tiles { - dbg!(r.time, r.is_empty()); - } - - assert!(false); - - Ok(()) - } } diff --git a/operators/src/source/gdal_source/loading_info.rs b/operators/src/source/gdal_source/loading_info.rs index 599ff11f0..9738b2f0d 100644 --- a/operators/src/source/gdal_source/loading_info.rs +++ b/operators/src/source/gdal_source/loading_info.rs @@ -303,10 +303,6 @@ impl MetaData for let known_time_before = known_time_start.unwrap_or(TimeInstance::MIN); let known_time_after = known_time_end.unwrap_or(TimeInstance::MAX); - log::debug!( - "ÄÄÄÄÄÄ known_time_before: {known_time_before}, known_time_after: {known_time_after}", - ); - Ok(GdalLoadingInfo::new( GdalLoadingInfoTemporalSliceIterator::Static { parts: data.into_iter(), diff --git a/services/examples/force-import.rs b/services/examples/force-import.rs index 651c29893..95d100c40 100644 --- a/services/examples/force-import.rs +++ b/services/examples/force-import.rs @@ -1,4 +1,4 @@ -use chrono::{NaiveDate, TimeZone, naive}; +use chrono::{NaiveDate, TimeZone}; use gdal::vector::{Defn, Feature, FieldDefn, LayerOptions, OGRFieldType}; use gdal::{Dataset as GdalDataset, DriverManager, Metadata}; use geoengine_datatypes::primitives::{CacheTtlSeconds, DateTime, TimeInstance, TimeInterval}; diff --git a/services/src/api/handlers/wms.rs b/services/src/api/handlers/wms.rs index 5919d1aef..28e9e5f68 100644 --- a/services/src/api/handlers/wms.rs +++ b/services/src/api/handlers/wms.rs @@ -28,7 +28,6 @@ use snafu::ensure; use std::str::FromStr; use std::time::Duration; use tracing::debug; -use tracing_subscriber::field::debug; use uuid::Uuid; pub(crate) fn init_wms_routes(cfg: &mut web::ServiceConfig) @@ -277,15 +276,10 @@ async fn wms_map_handler( request.crs.ok_or(error::Error::MissingSpatialReference)?; let request_bounds: SpatialPartition2D = request.bbox.bounds(request_spatial_ref)?; - - debug!("WMS request bounds: {:?}", request_bounds); - let x_request_res = request_bounds.size_x() / f64::from(request.width); let y_request_res = request_bounds.size_y() / f64::from(request.height); let request_resolution = SpatialResolution::new(x_request_res.abs(), y_request_res.abs())?; - debug!("WMS request resolution: {:?}", request_resolution); - let raster_colorizer = raster_colorizer_from_style(&request.styles)?; let ctx = app_ctx.session_context(session); @@ -357,7 +351,7 @@ async fn wms_map_handler( query_tiling_pixel_grid.grid_bounds(), request.time.unwrap_or_else(default_time_from_config).into(), attributes, - ); // <-- this one + ); debug!("WMS query rect: {:?}", query_rect);